1 /* Enumerate available IFUNC implementations of a function. x86-64 version.
2 Copyright (C) 2012-2021 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19 #include <assert.h>
20 #include <string.h>
21 #include <wchar.h>
22 #include <ifunc-impl-list.h>
23 #include <sysdep.h>
24 #include "init-arch.h"
25
26 /* Maximum number of IFUNC implementations. */
27 #define MAX_IFUNC 5
28
29 /* Fill ARRAY of MAX elements with IFUNC implementations for function
30 NAME supported on target machine and return the number of valid
31 entries. */
32
33 size_t
__libc_ifunc_impl_list(const char * name,struct libc_ifunc_impl * array,size_t max)34 __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
35 size_t max)
36 {
37 assert (max >= MAX_IFUNC);
38
39 size_t i = 0;
40
41 /* Support sysdeps/x86_64/multiarch/memcmpeq.c. */
42 IFUNC_IMPL (i, name, __memcmpeq,
43 IFUNC_IMPL_ADD (array, i, __memcmpeq,
44 (CPU_FEATURE_USABLE (AVX2)
45 && CPU_FEATURE_USABLE (BMI2)),
46 __memcmpeq_avx2)
47 IFUNC_IMPL_ADD (array, i, __memcmpeq,
48 (CPU_FEATURE_USABLE (AVX2)
49 && CPU_FEATURE_USABLE (BMI2)
50 && CPU_FEATURE_USABLE (RTM)),
51 __memcmpeq_avx2_rtm)
52 IFUNC_IMPL_ADD (array, i, __memcmpeq,
53 (CPU_FEATURE_USABLE (AVX512VL)
54 && CPU_FEATURE_USABLE (AVX512BW)
55 && CPU_FEATURE_USABLE (BMI2)),
56 __memcmpeq_evex)
57 IFUNC_IMPL_ADD (array, i, __memcmpeq, 1, __memcmpeq_sse2))
58
59 /* Support sysdeps/x86_64/multiarch/memchr.c. */
60 IFUNC_IMPL (i, name, memchr,
61 IFUNC_IMPL_ADD (array, i, memchr,
62 CPU_FEATURE_USABLE (AVX2),
63 __memchr_avx2)
64 IFUNC_IMPL_ADD (array, i, memchr,
65 (CPU_FEATURE_USABLE (AVX2)
66 && CPU_FEATURE_USABLE (RTM)),
67 __memchr_avx2_rtm)
68 IFUNC_IMPL_ADD (array, i, memchr,
69 (CPU_FEATURE_USABLE (AVX512VL)
70 && CPU_FEATURE_USABLE (AVX512BW)
71 && CPU_FEATURE_USABLE (BMI2)),
72 __memchr_evex)
73 IFUNC_IMPL_ADD (array, i, memchr,
74 (CPU_FEATURE_USABLE (AVX512VL)
75 && CPU_FEATURE_USABLE (AVX512BW)
76 && CPU_FEATURE_USABLE (BMI2)),
77 __memchr_evex_rtm)
78 IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2))
79
80 /* Support sysdeps/x86_64/multiarch/memcmp.c. */
81 IFUNC_IMPL (i, name, memcmp,
82 IFUNC_IMPL_ADD (array, i, memcmp,
83 (CPU_FEATURE_USABLE (AVX2)
84 && CPU_FEATURE_USABLE (BMI2)
85 && CPU_FEATURE_USABLE (MOVBE)),
86 __memcmp_avx2_movbe)
87 IFUNC_IMPL_ADD (array, i, memcmp,
88 (CPU_FEATURE_USABLE (AVX2)
89 && CPU_FEATURE_USABLE (BMI2)
90 && CPU_FEATURE_USABLE (MOVBE)
91 && CPU_FEATURE_USABLE (RTM)),
92 __memcmp_avx2_movbe_rtm)
93 IFUNC_IMPL_ADD (array, i, memcmp,
94 (CPU_FEATURE_USABLE (AVX512VL)
95 && CPU_FEATURE_USABLE (AVX512BW)
96 && CPU_FEATURE_USABLE (BMI2)
97 && CPU_FEATURE_USABLE (MOVBE)),
98 __memcmp_evex_movbe)
99 IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSE4_1),
100 __memcmp_sse4_1)
101 IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSSE3),
102 __memcmp_ssse3)
103 IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_sse2))
104
105 #ifdef SHARED
106 /* Support sysdeps/x86_64/multiarch/memmove_chk.c. */
107 IFUNC_IMPL (i, name, __memmove_chk,
108 IFUNC_IMPL_ADD (array, i, __memmove_chk,
109 CPU_FEATURE_USABLE (AVX512F),
110 __memmove_chk_avx512_no_vzeroupper)
111 IFUNC_IMPL_ADD (array, i, __memmove_chk,
112 CPU_FEATURE_USABLE (AVX512VL),
113 __memmove_chk_avx512_unaligned)
114 IFUNC_IMPL_ADD (array, i, __memmove_chk,
115 CPU_FEATURE_USABLE (AVX512VL),
116 __memmove_chk_avx512_unaligned_erms)
117 IFUNC_IMPL_ADD (array, i, __memmove_chk,
118 CPU_FEATURE_USABLE (AVX),
119 __memmove_chk_avx_unaligned)
120 IFUNC_IMPL_ADD (array, i, __memmove_chk,
121 CPU_FEATURE_USABLE (AVX),
122 __memmove_chk_avx_unaligned_erms)
123 IFUNC_IMPL_ADD (array, i, __memmove_chk,
124 (CPU_FEATURE_USABLE (AVX)
125 && CPU_FEATURE_USABLE (RTM)),
126 __memmove_chk_avx_unaligned_rtm)
127 IFUNC_IMPL_ADD (array, i, __memmove_chk,
128 (CPU_FEATURE_USABLE (AVX)
129 && CPU_FEATURE_USABLE (RTM)),
130 __memmove_chk_avx_unaligned_erms_rtm)
131 IFUNC_IMPL_ADD (array, i, __memmove_chk,
132 CPU_FEATURE_USABLE (AVX512VL),
133 __memmove_chk_evex_unaligned)
134 IFUNC_IMPL_ADD (array, i, __memmove_chk,
135 CPU_FEATURE_USABLE (AVX512VL),
136 __memmove_chk_evex_unaligned_erms)
137 IFUNC_IMPL_ADD (array, i, __memmove_chk,
138 CPU_FEATURE_USABLE (SSSE3),
139 __memmove_chk_ssse3_back)
140 IFUNC_IMPL_ADD (array, i, __memmove_chk,
141 CPU_FEATURE_USABLE (SSSE3),
142 __memmove_chk_ssse3)
143 IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
144 __memmove_chk_sse2_unaligned)
145 IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
146 __memmove_chk_sse2_unaligned_erms)
147 IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
148 __memmove_chk_erms))
149 #endif
150
151 /* Support sysdeps/x86_64/multiarch/memmove.c. */
152 IFUNC_IMPL (i, name, memmove,
153 IFUNC_IMPL_ADD (array, i, memmove,
154 CPU_FEATURE_USABLE (AVX),
155 __memmove_avx_unaligned)
156 IFUNC_IMPL_ADD (array, i, memmove,
157 CPU_FEATURE_USABLE (AVX),
158 __memmove_avx_unaligned_erms)
159 IFUNC_IMPL_ADD (array, i, memmove,
160 (CPU_FEATURE_USABLE (AVX)
161 && CPU_FEATURE_USABLE (RTM)),
162 __memmove_avx_unaligned_rtm)
163 IFUNC_IMPL_ADD (array, i, memmove,
164 (CPU_FEATURE_USABLE (AVX)
165 && CPU_FEATURE_USABLE (RTM)),
166 __memmove_avx_unaligned_erms_rtm)
167 IFUNC_IMPL_ADD (array, i, memmove,
168 CPU_FEATURE_USABLE (AVX512VL),
169 __memmove_evex_unaligned)
170 IFUNC_IMPL_ADD (array, i, memmove,
171 CPU_FEATURE_USABLE (AVX512VL),
172 __memmove_evex_unaligned_erms)
173 IFUNC_IMPL_ADD (array, i, memmove,
174 CPU_FEATURE_USABLE (AVX512F),
175 __memmove_avx512_no_vzeroupper)
176 IFUNC_IMPL_ADD (array, i, memmove,
177 CPU_FEATURE_USABLE (AVX512VL),
178 __memmove_avx512_unaligned)
179 IFUNC_IMPL_ADD (array, i, memmove,
180 CPU_FEATURE_USABLE (AVX512VL),
181 __memmove_avx512_unaligned_erms)
182 IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3),
183 __memmove_ssse3_back)
184 IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3),
185 __memmove_ssse3)
186 IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_erms)
187 IFUNC_IMPL_ADD (array, i, memmove, 1,
188 __memmove_sse2_unaligned)
189 IFUNC_IMPL_ADD (array, i, memmove, 1,
190 __memmove_sse2_unaligned_erms))
191
192 /* Support sysdeps/x86_64/multiarch/memrchr.c. */
193 IFUNC_IMPL (i, name, memrchr,
194 IFUNC_IMPL_ADD (array, i, memrchr,
195 CPU_FEATURE_USABLE (AVX2),
196 __memrchr_avx2)
197 IFUNC_IMPL_ADD (array, i, memrchr,
198 (CPU_FEATURE_USABLE (AVX2)
199 && CPU_FEATURE_USABLE (RTM)),
200 __memrchr_avx2_rtm)
201 IFUNC_IMPL_ADD (array, i, memrchr,
202 (CPU_FEATURE_USABLE (AVX512VL)
203 && CPU_FEATURE_USABLE (AVX512BW)),
204 __memrchr_evex)
205
206 IFUNC_IMPL_ADD (array, i, memrchr, 1, __memrchr_sse2))
207
208 #ifdef SHARED
209 /* Support sysdeps/x86_64/multiarch/memset_chk.c. */
210 IFUNC_IMPL (i, name, __memset_chk,
211 IFUNC_IMPL_ADD (array, i, __memset_chk, 1,
212 __memset_chk_erms)
213 IFUNC_IMPL_ADD (array, i, __memset_chk, 1,
214 __memset_chk_sse2_unaligned)
215 IFUNC_IMPL_ADD (array, i, __memset_chk, 1,
216 __memset_chk_sse2_unaligned_erms)
217 IFUNC_IMPL_ADD (array, i, __memset_chk,
218 CPU_FEATURE_USABLE (AVX2),
219 __memset_chk_avx2_unaligned)
220 IFUNC_IMPL_ADD (array, i, __memset_chk,
221 CPU_FEATURE_USABLE (AVX2),
222 __memset_chk_avx2_unaligned_erms)
223 IFUNC_IMPL_ADD (array, i, __memset_chk,
224 (CPU_FEATURE_USABLE (AVX2)
225 && CPU_FEATURE_USABLE (RTM)),
226 __memset_chk_avx2_unaligned_rtm)
227 IFUNC_IMPL_ADD (array, i, __memset_chk,
228 (CPU_FEATURE_USABLE (AVX2)
229 && CPU_FEATURE_USABLE (RTM)),
230 __memset_chk_avx2_unaligned_erms_rtm)
231 IFUNC_IMPL_ADD (array, i, __memset_chk,
232 (CPU_FEATURE_USABLE (AVX512VL)
233 && CPU_FEATURE_USABLE (AVX512BW)
234 && CPU_FEATURE_USABLE (BMI2)),
235 __memset_chk_evex_unaligned)
236 IFUNC_IMPL_ADD (array, i, __memset_chk,
237 (CPU_FEATURE_USABLE (AVX512VL)
238 && CPU_FEATURE_USABLE (AVX512BW)
239 && CPU_FEATURE_USABLE (BMI2)),
240 __memset_chk_evex_unaligned_erms)
241 IFUNC_IMPL_ADD (array, i, __memset_chk,
242 (CPU_FEATURE_USABLE (AVX512VL)
243 && CPU_FEATURE_USABLE (AVX512BW)
244 && CPU_FEATURE_USABLE (BMI2)),
245 __memset_chk_avx512_unaligned_erms)
246 IFUNC_IMPL_ADD (array, i, __memset_chk,
247 (CPU_FEATURE_USABLE (AVX512VL)
248 && CPU_FEATURE_USABLE (AVX512BW)
249 && CPU_FEATURE_USABLE (BMI2)),
250 __memset_chk_avx512_unaligned)
251 IFUNC_IMPL_ADD (array, i, __memset_chk,
252 CPU_FEATURE_USABLE (AVX512F),
253 __memset_chk_avx512_no_vzeroupper)
254 )
255 #endif
256
257 /* Support sysdeps/x86_64/multiarch/memset.c. */
258 IFUNC_IMPL (i, name, memset,
259 IFUNC_IMPL_ADD (array, i, memset, 1,
260 __memset_sse2_unaligned)
261 IFUNC_IMPL_ADD (array, i, memset, 1,
262 __memset_sse2_unaligned_erms)
263 IFUNC_IMPL_ADD (array, i, memset, 1, __memset_erms)
264 IFUNC_IMPL_ADD (array, i, memset,
265 CPU_FEATURE_USABLE (AVX2),
266 __memset_avx2_unaligned)
267 IFUNC_IMPL_ADD (array, i, memset,
268 CPU_FEATURE_USABLE (AVX2),
269 __memset_avx2_unaligned_erms)
270 IFUNC_IMPL_ADD (array, i, memset,
271 (CPU_FEATURE_USABLE (AVX2)
272 && CPU_FEATURE_USABLE (RTM)),
273 __memset_avx2_unaligned_rtm)
274 IFUNC_IMPL_ADD (array, i, memset,
275 (CPU_FEATURE_USABLE (AVX2)
276 && CPU_FEATURE_USABLE (RTM)),
277 __memset_avx2_unaligned_erms_rtm)
278 IFUNC_IMPL_ADD (array, i, memset,
279 (CPU_FEATURE_USABLE (AVX512VL)
280 && CPU_FEATURE_USABLE (AVX512BW)
281 && CPU_FEATURE_USABLE (BMI2)),
282 __memset_evex_unaligned)
283 IFUNC_IMPL_ADD (array, i, memset,
284 (CPU_FEATURE_USABLE (AVX512VL)
285 && CPU_FEATURE_USABLE (AVX512BW)
286 && CPU_FEATURE_USABLE (BMI2)),
287 __memset_evex_unaligned_erms)
288 IFUNC_IMPL_ADD (array, i, memset,
289 (CPU_FEATURE_USABLE (AVX512VL)
290 && CPU_FEATURE_USABLE (AVX512BW)
291 && CPU_FEATURE_USABLE (BMI2)),
292 __memset_avx512_unaligned_erms)
293 IFUNC_IMPL_ADD (array, i, memset,
294 (CPU_FEATURE_USABLE (AVX512VL)
295 && CPU_FEATURE_USABLE (AVX512BW)
296 && CPU_FEATURE_USABLE (BMI2)),
297 __memset_avx512_unaligned)
298 IFUNC_IMPL_ADD (array, i, memset,
299 CPU_FEATURE_USABLE (AVX512F),
300 __memset_avx512_no_vzeroupper)
301 )
302
303 /* Support sysdeps/x86_64/multiarch/rawmemchr.c. */
304 IFUNC_IMPL (i, name, rawmemchr,
305 IFUNC_IMPL_ADD (array, i, rawmemchr,
306 CPU_FEATURE_USABLE (AVX2),
307 __rawmemchr_avx2)
308 IFUNC_IMPL_ADD (array, i, rawmemchr,
309 (CPU_FEATURE_USABLE (AVX2)
310 && CPU_FEATURE_USABLE (RTM)),
311 __rawmemchr_avx2_rtm)
312 IFUNC_IMPL_ADD (array, i, rawmemchr,
313 (CPU_FEATURE_USABLE (AVX512VL)
314 && CPU_FEATURE_USABLE (AVX512BW)
315 && CPU_FEATURE_USABLE (BMI2)),
316 __rawmemchr_evex)
317 IFUNC_IMPL_ADD (array, i, rawmemchr,
318 (CPU_FEATURE_USABLE (AVX512VL)
319 && CPU_FEATURE_USABLE (AVX512BW)
320 && CPU_FEATURE_USABLE (BMI2)),
321 __rawmemchr_evex_rtm)
322 IFUNC_IMPL_ADD (array, i, rawmemchr, 1, __rawmemchr_sse2))
323
324 /* Support sysdeps/x86_64/multiarch/strlen.c. */
325 IFUNC_IMPL (i, name, strlen,
326 IFUNC_IMPL_ADD (array, i, strlen,
327 (CPU_FEATURE_USABLE (AVX2)
328 && CPU_FEATURE_USABLE (BMI2)),
329 __strlen_avx2)
330 IFUNC_IMPL_ADD (array, i, strlen,
331 (CPU_FEATURE_USABLE (AVX2)
332 && CPU_FEATURE_USABLE (BMI2)
333 && CPU_FEATURE_USABLE (RTM)),
334 __strlen_avx2_rtm)
335 IFUNC_IMPL_ADD (array, i, strlen,
336 (CPU_FEATURE_USABLE (AVX512VL)
337 && CPU_FEATURE_USABLE (AVX512BW)
338 && CPU_FEATURE_USABLE (BMI2)),
339 __strlen_evex)
340 IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_sse2))
341
342 /* Support sysdeps/x86_64/multiarch/strnlen.c. */
343 IFUNC_IMPL (i, name, strnlen,
344 IFUNC_IMPL_ADD (array, i, strnlen,
345 (CPU_FEATURE_USABLE (AVX2)
346 && CPU_FEATURE_USABLE (BMI2)),
347 __strnlen_avx2)
348 IFUNC_IMPL_ADD (array, i, strnlen,
349 (CPU_FEATURE_USABLE (AVX2)
350 && CPU_FEATURE_USABLE (BMI2)
351 && CPU_FEATURE_USABLE (RTM)),
352 __strnlen_avx2_rtm)
353 IFUNC_IMPL_ADD (array, i, strnlen,
354 (CPU_FEATURE_USABLE (AVX512VL)
355 && CPU_FEATURE_USABLE (AVX512BW)
356 && CPU_FEATURE_USABLE (BMI2)),
357 __strnlen_evex)
358 IFUNC_IMPL_ADD (array, i, strnlen, 1, __strnlen_sse2))
359
360 /* Support sysdeps/x86_64/multiarch/stpncpy.c. */
361 IFUNC_IMPL (i, name, stpncpy,
362 IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (SSSE3),
363 __stpncpy_ssse3)
364 IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2),
365 __stpncpy_avx2)
366 IFUNC_IMPL_ADD (array, i, stpncpy,
367 (CPU_FEATURE_USABLE (AVX2)
368 && CPU_FEATURE_USABLE (RTM)),
369 __stpncpy_avx2_rtm)
370 IFUNC_IMPL_ADD (array, i, stpncpy,
371 (CPU_FEATURE_USABLE (AVX512VL)
372 && CPU_FEATURE_USABLE (AVX512BW)),
373 __stpncpy_evex)
374 IFUNC_IMPL_ADD (array, i, stpncpy, 1,
375 __stpncpy_sse2_unaligned)
376 IFUNC_IMPL_ADD (array, i, stpncpy, 1, __stpncpy_sse2))
377
378 /* Support sysdeps/x86_64/multiarch/stpcpy.c. */
379 IFUNC_IMPL (i, name, stpcpy,
380 IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (SSSE3),
381 __stpcpy_ssse3)
382 IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2),
383 __stpcpy_avx2)
384 IFUNC_IMPL_ADD (array, i, stpcpy,
385 (CPU_FEATURE_USABLE (AVX2)
386 && CPU_FEATURE_USABLE (RTM)),
387 __stpcpy_avx2_rtm)
388 IFUNC_IMPL_ADD (array, i, stpcpy,
389 (CPU_FEATURE_USABLE (AVX512VL)
390 && CPU_FEATURE_USABLE (AVX512BW)),
391 __stpcpy_evex)
392 IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2_unaligned)
393 IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2))
394
395 /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */
396 IFUNC_IMPL (i, name, strcasecmp,
397 IFUNC_IMPL_ADD (array, i, strcasecmp,
398 CPU_FEATURE_USABLE (AVX),
399 __strcasecmp_avx)
400 IFUNC_IMPL_ADD (array, i, strcasecmp,
401 CPU_FEATURE_USABLE (SSE4_2),
402 __strcasecmp_sse42)
403 IFUNC_IMPL_ADD (array, i, strcasecmp,
404 CPU_FEATURE_USABLE (SSSE3),
405 __strcasecmp_ssse3)
406 IFUNC_IMPL_ADD (array, i, strcasecmp, 1, __strcasecmp_sse2))
407
408 /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */
409 IFUNC_IMPL (i, name, strcasecmp_l,
410 IFUNC_IMPL_ADD (array, i, strcasecmp_l,
411 CPU_FEATURE_USABLE (AVX),
412 __strcasecmp_l_avx)
413 IFUNC_IMPL_ADD (array, i, strcasecmp_l,
414 CPU_FEATURE_USABLE (SSE4_2),
415 __strcasecmp_l_sse42)
416 IFUNC_IMPL_ADD (array, i, strcasecmp_l,
417 CPU_FEATURE_USABLE (SSSE3),
418 __strcasecmp_l_ssse3)
419 IFUNC_IMPL_ADD (array, i, strcasecmp_l, 1,
420 __strcasecmp_l_sse2))
421
422 /* Support sysdeps/x86_64/multiarch/strcat.c. */
423 IFUNC_IMPL (i, name, strcat,
424 IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2),
425 __strcat_avx2)
426 IFUNC_IMPL_ADD (array, i, strcat,
427 (CPU_FEATURE_USABLE (AVX2)
428 && CPU_FEATURE_USABLE (RTM)),
429 __strcat_avx2_rtm)
430 IFUNC_IMPL_ADD (array, i, strcat,
431 (CPU_FEATURE_USABLE (AVX512VL)
432 && CPU_FEATURE_USABLE (AVX512BW)),
433 __strcat_evex)
434 IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (SSSE3),
435 __strcat_ssse3)
436 IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2_unaligned)
437 IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2))
438
439 /* Support sysdeps/x86_64/multiarch/strchr.c. */
440 IFUNC_IMPL (i, name, strchr,
441 IFUNC_IMPL_ADD (array, i, strchr,
442 (CPU_FEATURE_USABLE (AVX2)
443 && CPU_FEATURE_USABLE (BMI2)),
444 __strchr_avx2)
445 IFUNC_IMPL_ADD (array, i, strchr,
446 (CPU_FEATURE_USABLE (AVX2)
447 && CPU_FEATURE_USABLE (BMI2)
448 && CPU_FEATURE_USABLE (RTM)),
449 __strchr_avx2_rtm)
450 IFUNC_IMPL_ADD (array, i, strchr,
451 (CPU_FEATURE_USABLE (AVX512VL)
452 && CPU_FEATURE_USABLE (AVX512BW)
453 && CPU_FEATURE_USABLE (BMI2)),
454 __strchr_evex)
455 IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2_no_bsf)
456 IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2))
457
458 /* Support sysdeps/x86_64/multiarch/strchrnul.c. */
459 IFUNC_IMPL (i, name, strchrnul,
460 IFUNC_IMPL_ADD (array, i, strchrnul,
461 (CPU_FEATURE_USABLE (AVX2)
462 && CPU_FEATURE_USABLE (BMI2)),
463 __strchrnul_avx2)
464 IFUNC_IMPL_ADD (array, i, strchrnul,
465 (CPU_FEATURE_USABLE (AVX2)
466 && CPU_FEATURE_USABLE (BMI2)
467 && CPU_FEATURE_USABLE (RTM)),
468 __strchrnul_avx2_rtm)
469 IFUNC_IMPL_ADD (array, i, strchrnul,
470 (CPU_FEATURE_USABLE (AVX512VL)
471 && CPU_FEATURE_USABLE (AVX512BW)
472 && CPU_FEATURE_USABLE (BMI2)),
473 __strchrnul_evex)
474 IFUNC_IMPL_ADD (array, i, strchrnul, 1, __strchrnul_sse2))
475
476 /* Support sysdeps/x86_64/multiarch/strrchr.c. */
477 IFUNC_IMPL (i, name, strrchr,
478 IFUNC_IMPL_ADD (array, i, strrchr,
479 CPU_FEATURE_USABLE (AVX2),
480 __strrchr_avx2)
481 IFUNC_IMPL_ADD (array, i, strrchr,
482 (CPU_FEATURE_USABLE (AVX2)
483 && CPU_FEATURE_USABLE (RTM)),
484 __strrchr_avx2_rtm)
485 IFUNC_IMPL_ADD (array, i, strrchr,
486 (CPU_FEATURE_USABLE (AVX512VL)
487 && CPU_FEATURE_USABLE (AVX512BW)),
488 __strrchr_evex)
489 IFUNC_IMPL_ADD (array, i, strrchr, 1, __strrchr_sse2))
490
491 /* Support sysdeps/x86_64/multiarch/strcmp.c. */
492 IFUNC_IMPL (i, name, strcmp,
493 IFUNC_IMPL_ADD (array, i, strcmp,
494 CPU_FEATURE_USABLE (AVX2),
495 __strcmp_avx2)
496 IFUNC_IMPL_ADD (array, i, strcmp,
497 (CPU_FEATURE_USABLE (AVX2)
498 && CPU_FEATURE_USABLE (RTM)),
499 __strcmp_avx2_rtm)
500 IFUNC_IMPL_ADD (array, i, strcmp,
501 (CPU_FEATURE_USABLE (AVX512VL)
502 && CPU_FEATURE_USABLE (AVX512BW)
503 && CPU_FEATURE_USABLE (BMI2)),
504 __strcmp_evex)
505 IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSE4_2),
506 __strcmp_sse42)
507 IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSSE3),
508 __strcmp_ssse3)
509 IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_sse2_unaligned)
510 IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_sse2))
511
512 /* Support sysdeps/x86_64/multiarch/strcpy.c. */
513 IFUNC_IMPL (i, name, strcpy,
514 IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2),
515 __strcpy_avx2)
516 IFUNC_IMPL_ADD (array, i, strcpy,
517 (CPU_FEATURE_USABLE (AVX2)
518 && CPU_FEATURE_USABLE (RTM)),
519 __strcpy_avx2_rtm)
520 IFUNC_IMPL_ADD (array, i, strcpy,
521 (CPU_FEATURE_USABLE (AVX512VL)
522 && CPU_FEATURE_USABLE (AVX512BW)),
523 __strcpy_evex)
524 IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (SSSE3),
525 __strcpy_ssse3)
526 IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2_unaligned)
527 IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2))
528
529 /* Support sysdeps/x86_64/multiarch/strcspn.c. */
530 IFUNC_IMPL (i, name, strcspn,
531 IFUNC_IMPL_ADD (array, i, strcspn, CPU_FEATURE_USABLE (SSE4_2),
532 __strcspn_sse42)
533 IFUNC_IMPL_ADD (array, i, strcspn, 1, __strcspn_sse2))
534
535 /* Support sysdeps/x86_64/multiarch/strncase_l.c. */
536 IFUNC_IMPL (i, name, strncasecmp,
537 IFUNC_IMPL_ADD (array, i, strncasecmp,
538 CPU_FEATURE_USABLE (AVX),
539 __strncasecmp_avx)
540 IFUNC_IMPL_ADD (array, i, strncasecmp,
541 CPU_FEATURE_USABLE (SSE4_2),
542 __strncasecmp_sse42)
543 IFUNC_IMPL_ADD (array, i, strncasecmp,
544 CPU_FEATURE_USABLE (SSSE3),
545 __strncasecmp_ssse3)
546 IFUNC_IMPL_ADD (array, i, strncasecmp, 1,
547 __strncasecmp_sse2))
548
549 /* Support sysdeps/x86_64/multiarch/strncase_l.c. */
550 IFUNC_IMPL (i, name, strncasecmp_l,
551 IFUNC_IMPL_ADD (array, i, strncasecmp_l,
552 CPU_FEATURE_USABLE (AVX),
553 __strncasecmp_l_avx)
554 IFUNC_IMPL_ADD (array, i, strncasecmp_l,
555 CPU_FEATURE_USABLE (SSE4_2),
556 __strncasecmp_l_sse42)
557 IFUNC_IMPL_ADD (array, i, strncasecmp_l,
558 CPU_FEATURE_USABLE (SSSE3),
559 __strncasecmp_l_ssse3)
560 IFUNC_IMPL_ADD (array, i, strncasecmp_l, 1,
561 __strncasecmp_l_sse2))
562
563 /* Support sysdeps/x86_64/multiarch/strncat.c. */
564 IFUNC_IMPL (i, name, strncat,
565 IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2),
566 __strncat_avx2)
567 IFUNC_IMPL_ADD (array, i, strncat,
568 (CPU_FEATURE_USABLE (AVX2)
569 && CPU_FEATURE_USABLE (RTM)),
570 __strncat_avx2_rtm)
571 IFUNC_IMPL_ADD (array, i, strncat,
572 (CPU_FEATURE_USABLE (AVX512VL)
573 && CPU_FEATURE_USABLE (AVX512BW)),
574 __strncat_evex)
575 IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (SSSE3),
576 __strncat_ssse3)
577 IFUNC_IMPL_ADD (array, i, strncat, 1,
578 __strncat_sse2_unaligned)
579 IFUNC_IMPL_ADD (array, i, strncat, 1, __strncat_sse2))
580
581 /* Support sysdeps/x86_64/multiarch/strncpy.c. */
582 IFUNC_IMPL (i, name, strncpy,
583 IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2),
584 __strncpy_avx2)
585 IFUNC_IMPL_ADD (array, i, strncpy,
586 (CPU_FEATURE_USABLE (AVX2)
587 && CPU_FEATURE_USABLE (RTM)),
588 __strncpy_avx2_rtm)
589 IFUNC_IMPL_ADD (array, i, strncpy,
590 (CPU_FEATURE_USABLE (AVX512VL)
591 && CPU_FEATURE_USABLE (AVX512BW)),
592 __strncpy_evex)
593 IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (SSSE3),
594 __strncpy_ssse3)
595 IFUNC_IMPL_ADD (array, i, strncpy, 1,
596 __strncpy_sse2_unaligned)
597 IFUNC_IMPL_ADD (array, i, strncpy, 1, __strncpy_sse2))
598
599 /* Support sysdeps/x86_64/multiarch/strpbrk.c. */
600 IFUNC_IMPL (i, name, strpbrk,
601 IFUNC_IMPL_ADD (array, i, strpbrk, CPU_FEATURE_USABLE (SSE4_2),
602 __strpbrk_sse42)
603 IFUNC_IMPL_ADD (array, i, strpbrk, 1, __strpbrk_sse2))
604
605
606 /* Support sysdeps/x86_64/multiarch/strspn.c. */
607 IFUNC_IMPL (i, name, strspn,
608 IFUNC_IMPL_ADD (array, i, strspn, CPU_FEATURE_USABLE (SSE4_2),
609 __strspn_sse42)
610 IFUNC_IMPL_ADD (array, i, strspn, 1, __strspn_sse2))
611
612 /* Support sysdeps/x86_64/multiarch/strstr.c. */
613 IFUNC_IMPL (i, name, strstr,
614 IFUNC_IMPL_ADD (array, i, strstr, 1, __strstr_sse2_unaligned)
615 IFUNC_IMPL_ADD (array, i, strstr, 1, __strstr_sse2))
616
617 /* Support sysdeps/x86_64/multiarch/wcschr.c. */
618 IFUNC_IMPL (i, name, wcschr,
619 IFUNC_IMPL_ADD (array, i, wcschr,
620 (CPU_FEATURE_USABLE (AVX2)
621 && CPU_FEATURE_USABLE (BMI2)),
622 __wcschr_avx2)
623 IFUNC_IMPL_ADD (array, i, wcschr,
624 (CPU_FEATURE_USABLE (AVX2)
625 && CPU_FEATURE_USABLE (BMI2)
626 && CPU_FEATURE_USABLE (RTM)),
627 __wcschr_avx2_rtm)
628 IFUNC_IMPL_ADD (array, i, wcschr,
629 (CPU_FEATURE_USABLE (AVX512VL)
630 && CPU_FEATURE_USABLE (AVX512BW)
631 && CPU_FEATURE_USABLE (BMI2)),
632 __wcschr_evex)
633 IFUNC_IMPL_ADD (array, i, wcschr, 1, __wcschr_sse2))
634
635 /* Support sysdeps/x86_64/multiarch/wcsrchr.c. */
636 IFUNC_IMPL (i, name, wcsrchr,
637 IFUNC_IMPL_ADD (array, i, wcsrchr,
638 CPU_FEATURE_USABLE (AVX2),
639 __wcsrchr_avx2)
640 IFUNC_IMPL_ADD (array, i, wcsrchr,
641 (CPU_FEATURE_USABLE (AVX2)
642 && CPU_FEATURE_USABLE (RTM)),
643 __wcsrchr_avx2_rtm)
644 IFUNC_IMPL_ADD (array, i, wcsrchr,
645 (CPU_FEATURE_USABLE (AVX512VL)
646 && CPU_FEATURE_USABLE (AVX512BW)
647 && CPU_FEATURE_USABLE (BMI2)),
648 __wcsrchr_evex)
649 IFUNC_IMPL_ADD (array, i, wcsrchr, 1, __wcsrchr_sse2))
650
651 /* Support sysdeps/x86_64/multiarch/wcscmp.c. */
652 IFUNC_IMPL (i, name, wcscmp,
653 IFUNC_IMPL_ADD (array, i, wcscmp,
654 CPU_FEATURE_USABLE (AVX2),
655 __wcscmp_avx2)
656 IFUNC_IMPL_ADD (array, i, wcscmp,
657 (CPU_FEATURE_USABLE (AVX2)
658 && CPU_FEATURE_USABLE (RTM)),
659 __wcscmp_avx2_rtm)
660 IFUNC_IMPL_ADD (array, i, wcscmp,
661 (CPU_FEATURE_USABLE (AVX512VL)
662 && CPU_FEATURE_USABLE (AVX512BW)
663 && CPU_FEATURE_USABLE (BMI2)),
664 __wcscmp_evex)
665 IFUNC_IMPL_ADD (array, i, wcscmp, 1, __wcscmp_sse2))
666
667 /* Support sysdeps/x86_64/multiarch/wcsncmp.c. */
668 IFUNC_IMPL (i, name, wcsncmp,
669 IFUNC_IMPL_ADD (array, i, wcsncmp,
670 CPU_FEATURE_USABLE (AVX2),
671 __wcsncmp_avx2)
672 IFUNC_IMPL_ADD (array, i, wcsncmp,
673 (CPU_FEATURE_USABLE (AVX2)
674 && CPU_FEATURE_USABLE (RTM)),
675 __wcsncmp_avx2_rtm)
676 IFUNC_IMPL_ADD (array, i, wcsncmp,
677 (CPU_FEATURE_USABLE (AVX512VL)
678 && CPU_FEATURE_USABLE (AVX512BW)
679 && CPU_FEATURE_USABLE (BMI2)),
680 __wcsncmp_evex)
681 IFUNC_IMPL_ADD (array, i, wcsncmp, 1, __wcsncmp_sse2))
682
683 /* Support sysdeps/x86_64/multiarch/wcscpy.c. */
684 IFUNC_IMPL (i, name, wcscpy,
685 IFUNC_IMPL_ADD (array, i, wcscpy, CPU_FEATURE_USABLE (SSSE3),
686 __wcscpy_ssse3)
687 IFUNC_IMPL_ADD (array, i, wcscpy, 1, __wcscpy_sse2))
688
689 /* Support sysdeps/x86_64/multiarch/wcslen.c. */
690 IFUNC_IMPL (i, name, wcslen,
691 IFUNC_IMPL_ADD (array, i, wcslen,
692 (CPU_FEATURE_USABLE (AVX2)
693 && CPU_FEATURE_USABLE (BMI2)),
694 __wcslen_avx2)
695 IFUNC_IMPL_ADD (array, i, wcslen,
696 (CPU_FEATURE_USABLE (AVX2)
697 && CPU_FEATURE_USABLE (BMI2)
698 && CPU_FEATURE_USABLE (RTM)),
699 __wcslen_avx2_rtm)
700 IFUNC_IMPL_ADD (array, i, wcslen,
701 (CPU_FEATURE_USABLE (AVX512VL)
702 && CPU_FEATURE_USABLE (AVX512BW)
703 && CPU_FEATURE_USABLE (BMI2)),
704 __wcslen_evex)
705 IFUNC_IMPL_ADD (array, i, wcslen,
706 CPU_FEATURE_USABLE (SSE4_1),
707 __wcslen_sse4_1)
708 IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2))
709
710 /* Support sysdeps/x86_64/multiarch/wcsnlen.c. */
711 IFUNC_IMPL (i, name, wcsnlen,
712 IFUNC_IMPL_ADD (array, i, wcsnlen,
713 (CPU_FEATURE_USABLE (AVX2)
714 && CPU_FEATURE_USABLE (BMI2)),
715 __wcsnlen_avx2)
716 IFUNC_IMPL_ADD (array, i, wcsnlen,
717 (CPU_FEATURE_USABLE (AVX2)
718 && CPU_FEATURE_USABLE (BMI2)
719 && CPU_FEATURE_USABLE (RTM)),
720 __wcsnlen_avx2_rtm)
721 IFUNC_IMPL_ADD (array, i, wcsnlen,
722 (CPU_FEATURE_USABLE (AVX512VL)
723 && CPU_FEATURE_USABLE (AVX512BW)
724 && CPU_FEATURE_USABLE (BMI2)),
725 __wcsnlen_evex)
726 IFUNC_IMPL_ADD (array, i, wcsnlen,
727 CPU_FEATURE_USABLE (SSE4_1),
728 __wcsnlen_sse4_1)
729 IFUNC_IMPL_ADD (array, i, wcsnlen, 1, __wcsnlen_sse2))
730
731 /* Support sysdeps/x86_64/multiarch/wmemchr.c. */
732 IFUNC_IMPL (i, name, wmemchr,
733 IFUNC_IMPL_ADD (array, i, wmemchr,
734 CPU_FEATURE_USABLE (AVX2),
735 __wmemchr_avx2)
736 IFUNC_IMPL_ADD (array, i, wmemchr,
737 (CPU_FEATURE_USABLE (AVX2)
738 && CPU_FEATURE_USABLE (RTM)),
739 __wmemchr_avx2_rtm)
740 IFUNC_IMPL_ADD (array, i, wmemchr,
741 (CPU_FEATURE_USABLE (AVX512VL)
742 && CPU_FEATURE_USABLE (AVX512BW)
743 && CPU_FEATURE_USABLE (BMI2)),
744 __wmemchr_evex)
745 IFUNC_IMPL_ADD (array, i, wmemchr,
746 (CPU_FEATURE_USABLE (AVX512VL)
747 && CPU_FEATURE_USABLE (AVX512BW)
748 && CPU_FEATURE_USABLE (BMI2)),
749 __wmemchr_evex_rtm)
750 IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2))
751
752 /* Support sysdeps/x86_64/multiarch/wmemcmp.c. */
753 IFUNC_IMPL (i, name, wmemcmp,
754 IFUNC_IMPL_ADD (array, i, wmemcmp,
755 (CPU_FEATURE_USABLE (AVX2)
756 && CPU_FEATURE_USABLE (BMI2)
757 && CPU_FEATURE_USABLE (MOVBE)),
758 __wmemcmp_avx2_movbe)
759 IFUNC_IMPL_ADD (array, i, wmemcmp,
760 (CPU_FEATURE_USABLE (AVX2)
761 && CPU_FEATURE_USABLE (BMI2)
762 && CPU_FEATURE_USABLE (MOVBE)
763 && CPU_FEATURE_USABLE (RTM)),
764 __wmemcmp_avx2_movbe_rtm)
765 IFUNC_IMPL_ADD (array, i, wmemcmp,
766 (CPU_FEATURE_USABLE (AVX512VL)
767 && CPU_FEATURE_USABLE (AVX512BW)
768 && CPU_FEATURE_USABLE (BMI2)
769 && CPU_FEATURE_USABLE (MOVBE)),
770 __wmemcmp_evex_movbe)
771 IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSE4_1),
772 __wmemcmp_sse4_1)
773 IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSSE3),
774 __wmemcmp_ssse3)
775 IFUNC_IMPL_ADD (array, i, wmemcmp, 1, __wmemcmp_sse2))
776
777 /* Support sysdeps/x86_64/multiarch/wmemset.c. */
778 IFUNC_IMPL (i, name, wmemset,
779 IFUNC_IMPL_ADD (array, i, wmemset, 1,
780 __wmemset_sse2_unaligned)
781 IFUNC_IMPL_ADD (array, i, wmemset,
782 CPU_FEATURE_USABLE (AVX2),
783 __wmemset_avx2_unaligned)
784 IFUNC_IMPL_ADD (array, i, wmemset,
785 (CPU_FEATURE_USABLE (AVX2)
786 && CPU_FEATURE_USABLE (RTM)),
787 __wmemset_avx2_unaligned_rtm)
788 IFUNC_IMPL_ADD (array, i, wmemset,
789 (CPU_FEATURE_USABLE (AVX512VL)
790 && CPU_FEATURE_USABLE (AVX512BW)
791 && CPU_FEATURE_USABLE (BMI2)),
792 __wmemset_evex_unaligned)
793 IFUNC_IMPL_ADD (array, i, wmemset,
794 (CPU_FEATURE_USABLE (AVX512VL)
795 && CPU_FEATURE_USABLE (AVX512BW)
796 && CPU_FEATURE_USABLE (BMI2)),
797 __wmemset_avx512_unaligned))
798
799 #ifdef SHARED
800 /* Support sysdeps/x86_64/multiarch/memcpy_chk.c. */
801 IFUNC_IMPL (i, name, __memcpy_chk,
802 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
803 CPU_FEATURE_USABLE (AVX512F),
804 __memcpy_chk_avx512_no_vzeroupper)
805 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
806 CPU_FEATURE_USABLE (AVX512VL),
807 __memcpy_chk_avx512_unaligned)
808 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
809 CPU_FEATURE_USABLE (AVX512VL),
810 __memcpy_chk_avx512_unaligned_erms)
811 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
812 CPU_FEATURE_USABLE (AVX),
813 __memcpy_chk_avx_unaligned)
814 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
815 CPU_FEATURE_USABLE (AVX),
816 __memcpy_chk_avx_unaligned_erms)
817 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
818 (CPU_FEATURE_USABLE (AVX)
819 && CPU_FEATURE_USABLE (RTM)),
820 __memcpy_chk_avx_unaligned_rtm)
821 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
822 (CPU_FEATURE_USABLE (AVX)
823 && CPU_FEATURE_USABLE (RTM)),
824 __memcpy_chk_avx_unaligned_erms_rtm)
825 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
826 CPU_FEATURE_USABLE (AVX512VL),
827 __memcpy_chk_evex_unaligned)
828 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
829 CPU_FEATURE_USABLE (AVX512VL),
830 __memcpy_chk_evex_unaligned_erms)
831 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
832 CPU_FEATURE_USABLE (SSSE3),
833 __memcpy_chk_ssse3_back)
834 IFUNC_IMPL_ADD (array, i, __memcpy_chk,
835 CPU_FEATURE_USABLE (SSSE3),
836 __memcpy_chk_ssse3)
837 IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
838 __memcpy_chk_sse2_unaligned)
839 IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
840 __memcpy_chk_sse2_unaligned_erms)
841 IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
842 __memcpy_chk_erms))
843 #endif
844
845 /* Support sysdeps/x86_64/multiarch/memcpy.c. */
846 IFUNC_IMPL (i, name, memcpy,
847 IFUNC_IMPL_ADD (array, i, memcpy,
848 CPU_FEATURE_USABLE (AVX),
849 __memcpy_avx_unaligned)
850 IFUNC_IMPL_ADD (array, i, memcpy,
851 CPU_FEATURE_USABLE (AVX),
852 __memcpy_avx_unaligned_erms)
853 IFUNC_IMPL_ADD (array, i, memcpy,
854 (CPU_FEATURE_USABLE (AVX)
855 && CPU_FEATURE_USABLE (RTM)),
856 __memcpy_avx_unaligned_rtm)
857 IFUNC_IMPL_ADD (array, i, memcpy,
858 (CPU_FEATURE_USABLE (AVX)
859 && CPU_FEATURE_USABLE (RTM)),
860 __memcpy_avx_unaligned_erms_rtm)
861 IFUNC_IMPL_ADD (array, i, memcpy,
862 CPU_FEATURE_USABLE (AVX512VL),
863 __memcpy_evex_unaligned)
864 IFUNC_IMPL_ADD (array, i, memcpy,
865 CPU_FEATURE_USABLE (AVX512VL),
866 __memcpy_evex_unaligned_erms)
867 IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3),
868 __memcpy_ssse3_back)
869 IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3),
870 __memcpy_ssse3)
871 IFUNC_IMPL_ADD (array, i, memcpy,
872 CPU_FEATURE_USABLE (AVX512F),
873 __memcpy_avx512_no_vzeroupper)
874 IFUNC_IMPL_ADD (array, i, memcpy,
875 CPU_FEATURE_USABLE (AVX512VL),
876 __memcpy_avx512_unaligned)
877 IFUNC_IMPL_ADD (array, i, memcpy,
878 CPU_FEATURE_USABLE (AVX512VL),
879 __memcpy_avx512_unaligned_erms)
880 IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned)
881 IFUNC_IMPL_ADD (array, i, memcpy, 1,
882 __memcpy_sse2_unaligned_erms)
883 IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_erms))
884
885 #ifdef SHARED
886 /* Support sysdeps/x86_64/multiarch/mempcpy_chk.c. */
887 IFUNC_IMPL (i, name, __mempcpy_chk,
888 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
889 CPU_FEATURE_USABLE (AVX512F),
890 __mempcpy_chk_avx512_no_vzeroupper)
891 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
892 CPU_FEATURE_USABLE (AVX512VL),
893 __mempcpy_chk_avx512_unaligned)
894 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
895 CPU_FEATURE_USABLE (AVX512VL),
896 __mempcpy_chk_avx512_unaligned_erms)
897 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
898 CPU_FEATURE_USABLE (AVX),
899 __mempcpy_chk_avx_unaligned)
900 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
901 CPU_FEATURE_USABLE (AVX),
902 __mempcpy_chk_avx_unaligned_erms)
903 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
904 (CPU_FEATURE_USABLE (AVX)
905 && CPU_FEATURE_USABLE (RTM)),
906 __mempcpy_chk_avx_unaligned_rtm)
907 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
908 (CPU_FEATURE_USABLE (AVX)
909 && CPU_FEATURE_USABLE (RTM)),
910 __mempcpy_chk_avx_unaligned_erms_rtm)
911 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
912 CPU_FEATURE_USABLE (AVX512VL),
913 __mempcpy_chk_evex_unaligned)
914 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
915 CPU_FEATURE_USABLE (AVX512VL),
916 __mempcpy_chk_evex_unaligned_erms)
917 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
918 CPU_FEATURE_USABLE (SSSE3),
919 __mempcpy_chk_ssse3_back)
920 IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
921 CPU_FEATURE_USABLE (SSSE3),
922 __mempcpy_chk_ssse3)
923 IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
924 __mempcpy_chk_sse2_unaligned)
925 IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
926 __mempcpy_chk_sse2_unaligned_erms)
927 IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
928 __mempcpy_chk_erms))
929 #endif
930
931 /* Support sysdeps/x86_64/multiarch/mempcpy.c. */
932 IFUNC_IMPL (i, name, mempcpy,
933 IFUNC_IMPL_ADD (array, i, mempcpy,
934 CPU_FEATURE_USABLE (AVX512F),
935 __mempcpy_avx512_no_vzeroupper)
936 IFUNC_IMPL_ADD (array, i, mempcpy,
937 CPU_FEATURE_USABLE (AVX512VL),
938 __mempcpy_avx512_unaligned)
939 IFUNC_IMPL_ADD (array, i, mempcpy,
940 CPU_FEATURE_USABLE (AVX512VL),
941 __mempcpy_avx512_unaligned_erms)
942 IFUNC_IMPL_ADD (array, i, mempcpy,
943 CPU_FEATURE_USABLE (AVX),
944 __mempcpy_avx_unaligned)
945 IFUNC_IMPL_ADD (array, i, mempcpy,
946 CPU_FEATURE_USABLE (AVX),
947 __mempcpy_avx_unaligned_erms)
948 IFUNC_IMPL_ADD (array, i, mempcpy,
949 (CPU_FEATURE_USABLE (AVX)
950 && CPU_FEATURE_USABLE (RTM)),
951 __mempcpy_avx_unaligned_rtm)
952 IFUNC_IMPL_ADD (array, i, mempcpy,
953 (CPU_FEATURE_USABLE (AVX)
954 && CPU_FEATURE_USABLE (RTM)),
955 __mempcpy_avx_unaligned_erms_rtm)
956 IFUNC_IMPL_ADD (array, i, mempcpy,
957 CPU_FEATURE_USABLE (AVX512VL),
958 __mempcpy_evex_unaligned)
959 IFUNC_IMPL_ADD (array, i, mempcpy,
960 CPU_FEATURE_USABLE (AVX512VL),
961 __mempcpy_evex_unaligned_erms)
962 IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3),
963 __mempcpy_ssse3_back)
964 IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3),
965 __mempcpy_ssse3)
966 IFUNC_IMPL_ADD (array, i, mempcpy, 1,
967 __mempcpy_sse2_unaligned)
968 IFUNC_IMPL_ADD (array, i, mempcpy, 1,
969 __mempcpy_sse2_unaligned_erms)
970 IFUNC_IMPL_ADD (array, i, mempcpy, 1, __mempcpy_erms))
971
972 /* Support sysdeps/x86_64/multiarch/strncmp.c. */
973 IFUNC_IMPL (i, name, strncmp,
974 IFUNC_IMPL_ADD (array, i, strncmp,
975 CPU_FEATURE_USABLE (AVX2),
976 __strncmp_avx2)
977 IFUNC_IMPL_ADD (array, i, strncmp,
978 (CPU_FEATURE_USABLE (AVX2)
979 && CPU_FEATURE_USABLE (RTM)),
980 __strncmp_avx2_rtm)
981 IFUNC_IMPL_ADD (array, i, strncmp,
982 (CPU_FEATURE_USABLE (AVX512VL)
983 && CPU_FEATURE_USABLE (AVX512BW)),
984 __strncmp_evex)
985 IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSE4_2),
986 __strncmp_sse42)
987 IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSSE3),
988 __strncmp_ssse3)
989 IFUNC_IMPL_ADD (array, i, strncmp, 1, __strncmp_sse2))
990
991 #ifdef SHARED
992 /* Support sysdeps/x86_64/multiarch/wmemset_chk.c. */
993 IFUNC_IMPL (i, name, __wmemset_chk,
994 IFUNC_IMPL_ADD (array, i, __wmemset_chk, 1,
995 __wmemset_chk_sse2_unaligned)
996 IFUNC_IMPL_ADD (array, i, __wmemset_chk,
997 CPU_FEATURE_USABLE (AVX2),
998 __wmemset_chk_avx2_unaligned)
999 IFUNC_IMPL_ADD (array, i, __wmemset_chk,
1000 (CPU_FEATURE_USABLE (AVX512VL)
1001 && CPU_FEATURE_USABLE (AVX512BW)
1002 && CPU_FEATURE_USABLE (BMI2)),
1003 __wmemset_chk_evex_unaligned)
1004 IFUNC_IMPL_ADD (array, i, __wmemset_chk,
1005 (CPU_FEATURE_USABLE (AVX512VL)
1006 && CPU_FEATURE_USABLE (AVX512BW)
1007 && CPU_FEATURE_USABLE (BMI2)),
1008 __wmemset_chk_avx512_unaligned))
1009 #endif
1010
1011 return i;
1012 }
1013