diff options
| -rw-r--r-- | ChangeLog | 26 | ||||
| -rw-r--r-- | sysdeps/x86_64/memchr.S | 65 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/Makefile | 2 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/ifunc-avx2.h | 36 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/ifunc-impl-list.c | 21 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/memchr-avx2.S | 340 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/memchr-sse2.S | 28 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/memchr.c | 35 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/rawmemchr-avx2.S | 4 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/rawmemchr-sse2.S | 29 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/rawmemchr.c | 38 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/wmemchr-avx2.S | 4 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/wmemchr-sse2.S | 4 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/wmemchr.c | 39 |
14 files changed, 646 insertions, 25 deletions
@@ -1,3 +1,29 @@ +2017-06-09 H.J. Lu <hongjiu.lu@intel.com> + + * sysdeps/x86_64/memchr.S (MEMCHR): New. Depending on if + USE_AS_WMEMCHR is defined. + (PCMPEQ): Likewise. + (memchr): Renamed to ... + (MEMCHR): This. Support wmemchr if USE_AS_WMEMCHR is defined. + Replace pcmpeqb with PCMPEQ. + * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add + memchr-sse2, rawmemchr-sse2, memchr-avx2, rawmemchr-avx2, + wmemchr-sse4_1, wmemchr-avx2 and wmemchr-c. + * sysdeps/x86_64/multiarch/ifunc-avx2.h: New file. + * sysdeps/x86_64/multiarch/memchr-avx2.S: Likewise. + * sysdeps/x86_64/multiarch/memchr-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/memchr.c: Likewise. + * sysdeps/x86_64/multiarch/rawmemchr-avx2.S: Likewise. + * sysdeps/x86_64/multiarch/rawmemchr-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/rawmemchr.c: Likewise. + * sysdeps/x86_64/multiarch/wmemchr-avx2.S: Likewise. + * sysdeps/x86_64/multiarch/wmemchr-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/wmemchr.c: Likewise. + * sysdeps/x86_64/multiarch/ifunc-impl-list.c + (__libc_ifunc_impl_list): Test __memchr_avx2, __memchr_sse2, + __rawmemchr_avx2, __rawmemchr_sse2, __wmemchr_avx2 and + __wmemchr_sse2. + 2017-06-09 Florian Weimer <fweimer@redhat.com> * malloc/Makefile (test-srcs): Remove tst-dynarray, diff --git a/sysdeps/x86_64/memchr.S b/sysdeps/x86_64/memchr.S index d3be012..f5f05f6 100644 --- a/sysdeps/x86_64/memchr.S +++ b/sysdeps/x86_64/memchr.S @@ -18,17 +18,31 @@ #include <sysdep.h> +#ifdef USE_AS_WMEMCHR +# define MEMCHR wmemchr +# define PCMPEQ pcmpeqd +#else +# define MEMCHR memchr +# define PCMPEQ pcmpeqb +#endif + /* fast SSE2 version with using pmaxub and 64 byte loop */ .text -ENTRY(memchr) +ENTRY(MEMCHR) movd %esi, %xmm1 mov %edi, %ecx +#ifdef USE_AS_WMEMCHR + test %rdx, %rdx + jz L(return_null) + shl $2, %rdx +#else punpcklbw %xmm1, %xmm1 test %rdx, %rdx jz L(return_null) punpcklbw %xmm1, %xmm1 +#endif and $63, %ecx pshufd $0, %xmm1, %xmm1 @@ -37,7 +51,7 @@ ENTRY(memchr) ja L(crosscache) movdqu (%rdi), %xmm0 - pcmpeqb %xmm1, %xmm0 + PCMPEQ %xmm1, %xmm0 pmovmskb %xmm0, %eax test %eax, %eax @@ -58,7 +72,7 @@ L(crosscache): and $-16, %rdi movdqa (%rdi), %xmm0 - pcmpeqb %xmm1, %xmm0 + PCMPEQ %xmm1, %xmm0 /* Check if there is a match. */ pmovmskb %xmm0, %eax /* Remove the leading bytes. */ @@ -90,25 +104,25 @@ L(unaligned_no_match): .p2align 4 L(loop_prolog): movdqa (%rdi), %xmm0 - pcmpeqb %xmm1, %xmm0 + PCMPEQ %xmm1, %xmm0 pmovmskb %xmm0, %eax test %eax, %eax jnz L(matches) movdqa 16(%rdi), %xmm2 - pcmpeqb %xmm1, %xmm2 + PCMPEQ %xmm1, %xmm2 pmovmskb %xmm2, %eax test %eax, %eax jnz L(matches16) movdqa 32(%rdi), %xmm3 - pcmpeqb %xmm1, %xmm3 + PCMPEQ %xmm1, %xmm3 pmovmskb %xmm3, %eax test %eax, %eax jnz L(matches32) movdqa 48(%rdi), %xmm4 - pcmpeqb %xmm1, %xmm4 + PCMPEQ %xmm1, %xmm4 add $64, %rdi pmovmskb %xmm4, %eax test %eax, %eax @@ -121,25 +135,25 @@ L(loop_prolog): jbe L(exit_loop) movdqa (%rdi), %xmm0 - pcmpeqb %xmm1, %xmm0 + PCMPEQ %xmm1, %xmm0 pmovmskb %xmm0, %eax test %eax, %eax jnz L(matches) movdqa 16(%rdi), %xmm2 - pcmpeqb %xmm1, %xmm2 + PCMPEQ %xmm1, %xmm2 pmovmskb %xmm2, %eax test %eax, %eax jnz L(matches16) movdqa 32(%rdi), %xmm3 - pcmpeqb %xmm1, %xmm3 + PCMPEQ %xmm1, %xmm3 pmovmskb %xmm3, %eax test %eax, %eax jnz L(matches32) movdqa 48(%rdi), %xmm3 - pcmpeqb %xmm1, %xmm3 + PCMPEQ %xmm1, %xmm3 pmovmskb %xmm3, %eax add $64, %rdi @@ -160,10 +174,10 @@ L(align64_loop): movdqa 32(%rdi), %xmm3 movdqa 48(%rdi), %xmm4 - pcmpeqb %xmm1, %xmm0 - pcmpeqb %xmm1, %xmm2 - pcmpeqb %xmm1, %xmm3 - pcmpeqb %xmm1, %xmm4 + PCMPEQ %xmm1, %xmm0 + PCMPEQ %xmm1, %xmm2 + PCMPEQ %xmm1, %xmm3 + PCMPEQ %xmm1, %xmm4 pmaxub %xmm0, %xmm3 pmaxub %xmm2, %xmm4 @@ -186,9 +200,9 @@ L(align64_loop): jnz L(matches16) movdqa 32(%rdi), %xmm3 - pcmpeqb %xmm1, %xmm3 + PCMPEQ %xmm1, %xmm3 - pcmpeqb 48(%rdi), %xmm1 + PCMPEQ 48(%rdi), %xmm1 pmovmskb %xmm3, %eax test %eax, %eax jnz L(matches32) @@ -204,26 +218,26 @@ L(exit_loop): jle L(exit_loop_32) movdqa (%rdi), %xmm0 - pcmpeqb %xmm1, %xmm0 + PCMPEQ %xmm1, %xmm0 pmovmskb %xmm0, %eax test %eax, %eax jnz L(matches) movdqa 16(%rdi), %xmm2 - pcmpeqb %xmm1, %xmm2 + PCMPEQ %xmm1, %xmm2 pmovmskb %xmm2, %eax test %eax, %eax jnz L(matches16) movdqa 32(%rdi), %xmm3 - pcmpeqb %xmm1, %xmm3 + PCMPEQ %xmm1, %xmm3 pmovmskb %xmm3, %eax test %eax, %eax jnz L(matches32_1) sub $16, %edx jle L(return_null) - pcmpeqb 48(%rdi), %xmm1 + PCMPEQ 48(%rdi), %xmm1 pmovmskb %xmm1, %eax test %eax, %eax jnz L(matches48_1) @@ -234,14 +248,14 @@ L(exit_loop): L(exit_loop_32): add $32, %edx movdqa (%rdi), %xmm0 - pcmpeqb %xmm1, %xmm0 + PCMPEQ %xmm1, %xmm0 pmovmskb %xmm0, %eax test %eax, %eax jnz L(matches_1) sub $16, %edx jbe L(return_null) - pcmpeqb 16(%rdi), %xmm1 + PCMPEQ 16(%rdi), %xmm1 pmovmskb %xmm1, %eax test %eax, %eax jnz L(matches16_1) @@ -308,8 +322,9 @@ L(matches48_1): L(return_null): xor %eax, %eax ret -END(memchr) +END(MEMCHR) +#ifndef USE_AS_WMEMCHR strong_alias (memchr, __memchr) - libc_hidden_builtin_def(memchr) +#endif diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index 310a3a4..22c3903 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -6,6 +6,7 @@ ifeq ($(subdir),string) sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \ strcmp-sse2-unaligned strncmp-ssse3 \ + memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \ memcmp-avx2-movbe \ memcmp-sse4 memcpy-ssse3 \ memmove-ssse3 \ @@ -33,6 +34,7 @@ endif ifeq ($(subdir),wcsmbs) sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \ wmemcmp-avx2-movbe \ + wmemchr-sse2 wmemchr-avx2 \ wcscpy-ssse3 wcscpy-c \ wcsnlen-sse4_1 wcsnlen-c endif diff --git a/sysdeps/x86_64/multiarch/ifunc-avx2.h b/sysdeps/x86_64/multiarch/ifunc-avx2.h new file mode 100644 index 0000000..25432b8 --- /dev/null +++ b/sysdeps/x86_64/multiarch/ifunc-avx2.h @@ -0,0 +1,36 @@ +/* Common definition for ifunc selections optimized with SSE2 and AVX2. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <init-arch.h> + +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) + && CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + return OPTIMIZE (avx2); + + return OPTIMIZE (sse2); +} diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c index 5627183..e4e7466 100644 --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c @@ -38,6 +38,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, size_t i = 0; + /* Support sysdeps/x86_64/multiarch/memchr.S. */ + IFUNC_IMPL (i, name, memchr, + IFUNC_IMPL_ADD (array, i, memchr, + HAS_ARCH_FEATURE (AVX2_Usable), + __memchr_avx2) + IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2)) + /* Support sysdeps/x86_64/multiarch/memcmp.S. */ IFUNC_IMPL (i, name, memcmp, IFUNC_IMPL_ADD (array, i, memcmp, @@ -152,6 +159,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __memset_avx512_no_vzeroupper) ) + /* Support sysdeps/x86_64/multiarch/rawmemchr.S. */ + IFUNC_IMPL (i, name, rawmemchr, + IFUNC_IMPL_ADD (array, i, rawmemchr, + HAS_ARCH_FEATURE (AVX2_Usable), + __rawmemchr_avx2) + IFUNC_IMPL_ADD (array, i, rawmemchr, 1, __rawmemchr_sse2)) + /* Support sysdeps/x86_64/multiarch/stpncpy.S. */ IFUNC_IMPL (i, name, stpncpy, IFUNC_IMPL_ADD (array, i, stpncpy, HAS_CPU_FEATURE (SSSE3), @@ -303,6 +317,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __wcsnlen_sse4_1) IFUNC_IMPL_ADD (array, i, wcsnlen, 1, __wcsnlen_sse2)) + /* Support sysdeps/x86_64/multiarch/wmemchr.c. */ + IFUNC_IMPL (i, name, wmemchr, + IFUNC_IMPL_ADD (array, i, wmemchr, + HAS_ARCH_FEATURE (AVX2_Usable), + __wmemchr_avx2) + IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2)) + /* Support sysdeps/x86_64/multiarch/wmemcmp.S. */ IFUNC_IMPL (i, name, wmemcmp, IFUNC_IMPL_ADD (array, i, wmemcmp, diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S new file mode 100644 index 0000000..a7275ed --- /dev/null +++ b/sysdeps/x86_64/multiarch/memchr-avx2.S @@ -0,0 +1,340 @@ +/* memchr/wmemchr optimized with AVX2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) + +# include <sysdep.h> + +# ifndef MEMCHR +# define MEMCHR __memchr_avx2 +# endif + +# ifdef USE_AS_WMEMCHR +# define VPCMPEQ vpcmpeqd +# else +# define VPCMPEQ vpcmpeqb +# endif + +# ifndef VZEROUPPER +# define VZEROUPPER vzeroupper +# endif + +# define VEC_SIZE 32 + + .section .text.avx,"ax",@progbits +ENTRY (MEMCHR) +# ifndef USE_AS_RAWMEMCHR + /* Check for zero length. */ + testq %rdx, %rdx + jz L(null) +# endif + movl %edi, %ecx + /* Broadcast CHAR to YMM0. */ + vmovd %esi, %xmm0 +# ifdef USE_AS_WMEMCHR + shl $2, %rdx + vpbroadcastd %xmm0, %ymm0 +# else + vpbroadcastb %xmm0, %ymm0 +# endif + /* Check if we may cross page boundary with one vector load. */ + andl $(2 * VEC_SIZE - 1), %ecx + cmpl $VEC_SIZE, %ecx + ja L(cros_page_boundary) + + /* Check the first VEC_SIZE bytes. */ + VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + +# ifndef USE_AS_RAWMEMCHR + jnz L(first_vec_x0_check) + /* Adjust length and check the end of data. */ + subq $VEC_SIZE, %rdx + jbe L(zero) +# else + jnz L(first_vec_x0) +# endif + + /* Align data for aligned loads in the loop. */ + addq $VEC_SIZE, %rdi + andl $(VEC_SIZE - 1), %ecx + andq $-VEC_SIZE, %rdi + +# ifndef USE_AS_RAWMEMCHR + /* Adjust length. */ + addq %rcx, %rdx + + subq $(VEC_SIZE * 4), %rdx + jbe L(last_4x_vec_or_less) +# endif + jmp L(more_4x_vec) + + .p2align 4 +L(cros_page_boundary): + andl $(VEC_SIZE - 1), %ecx + andq $-VEC_SIZE, %rdi + VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + /* Remove the leading bytes. */ + sarl %cl, %eax + testl %eax, %eax + jz L(aligned_more) + tzcntl %eax, %eax +# ifndef USE_AS_RAWMEMCHR + /* Check the end of data. */ + cmpq %rax, %rdx + jbe L(zero) +# endif + addq %rdi, %rax + addq %rcx, %rax + VZEROUPPER + ret + + .p2align 4 +L(aligned_more): +# ifndef USE_AS_RAWMEMCHR + /* Calculate "rdx + rcx - VEC_SIZE" with "rdx - (VEC_SIZE - rcx)" + instead of "(rdx + rcx) - VEC_SIZE" to void possible addition + overflow. */ + negq %rcx + addq $VEC_SIZE, %rcx + + /* Check the end of data. */ + subq %rcx, %rdx + jbe L(zero) +# endif + + addq $VEC_SIZE, %rdi + +# ifndef USE_AS_RAWMEMCHR + subq $(VEC_SIZE * 4), %rdx + jbe L(last_4x_vec_or_less) +# endif + +L(more_4x_vec): + /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time + since data is only aligned to VEC_SIZE. */ + VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x0) + + VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x1) + + VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x2) + + VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x3) + + addq $(VEC_SIZE * 4), %rdi + +# ifndef USE_AS_RAWMEMCHR + subq $(VEC_SIZE * 4), %rdx + jbe L(last_4x_vec_or_less) +# endif + + /* Align data to 4 * VEC_SIZE. */ + movq %rdi, %rcx + andl $(4 * VEC_SIZE - 1), %ecx + andq $-(4 * VEC_SIZE), %rdi + +# ifndef USE_AS_RAWMEMCHR + /* Adjust length. */ + addq %rcx, %rdx +# endif + + .p2align 4 +L(loop_4x_vec): + /* Compare 4 * VEC at a time forward. */ + VPCMPEQ (%rdi), %ymm0, %ymm1 + VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm2 + VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm3 + VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm4 + + vpor %ymm1, %ymm2, %ymm5 + vpor %ymm3, %ymm4, %ymm6 + vpor %ymm5, %ymm6, %ymm5 + + vpmovmskb %ymm5, %eax + testl %eax, %eax + jnz L(4x_vec_end) + + addq $(VEC_SIZE * 4), %rdi + +# ifdef USE_AS_RAWMEMCHR + jmp L(loop_4x_vec) +# else + subq $(VEC_SIZE * 4), %rdx + ja L(loop_4x_vec) + +L(last_4x_vec_or_less): + /* Less than 4 * VEC and aligned to VEC_SIZE. */ + addl $(VEC_SIZE * 2), %edx + jle L(last_2x_vec) + + VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x0) + + VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x1) + + VPCMPEQ (VEC_SIZE * 2)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + + jnz L(first_vec_x2_check) + subl $VEC_SIZE, %edx + jle L(zero) + + VPCMPEQ (VEC_SIZE * 3)(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + + jnz L(first_vec_x3_check) + xorl %eax, %eax + VZEROUPPER + ret + + .p2align 4 +L(last_2x_vec): + addl $(VEC_SIZE * 2), %edx + VPCMPEQ (%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + + jnz L(first_vec_x0_check) + subl $VEC_SIZE, %edx + jle L(zero) + + VPCMPEQ VEC_SIZE(%rdi), %ymm0, %ymm1 + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x1_check) + xorl %eax, %eax + VZEROUPPER + ret + + .p2align 4 +L(first_vec_x0_check): + tzcntl %eax, %eax + /* Check the end of data. */ + cmpq %rax, %rdx + jbe L(zero) + addq %rdi, %rax + VZEROUPPER + ret + + .p2align 4 +L(first_vec_x1_check): + tzcntl %eax, %eax + /* Check the end of data. */ + cmpq %rax, %rdx + jbe L(zero) + addq $VEC_SIZE, %rax + addq %rdi, %rax + VZEROUPPER + ret + + .p2align 4 +L(first_vec_x2_check): + tzcntl %eax, %eax + /* Check the end of data. */ + cmpq %rax, %rdx + jbe L(zero) + addq $(VEC_SIZE * 2), %rax + addq %rdi, %rax + VZEROUPPER + ret + + .p2align 4 +L(first_vec_x3_check): + tzcntl %eax, %eax + /* Check the end of data. */ + cmpq %rax, %rdx + jbe L(zero) + addq $(VEC_SIZE * 3), %rax + addq %rdi, %rax + VZEROUPPER + ret + + .p2align 4 +L(zero): + VZEROUPPER +L(null): + xorl %eax, %eax + ret +# endif + + .p2align 4 +L(first_vec_x0): + tzcntl %eax, %eax + addq %rdi, %rax + VZEROUPPER + ret + + .p2align 4 +L(first_vec_x1): + tzcntl %eax, %eax + addq $VEC_SIZE, %rax + addq %rdi, %rax + VZEROUPPER + ret + + .p2align 4 +L(first_vec_x2): + tzcntl %eax, %eax + addq $(VEC_SIZE * 2), %rax + addq %rdi, %rax + VZEROUPPER + ret + + .p2align 4 +L(4x_vec_end): + vpmovmskb %ymm1, %eax + testl %eax, %eax + jnz L(first_vec_x0) + vpmovmskb %ymm2, %eax + testl %eax, %eax + jnz L(first_vec_x1) + vpmovmskb %ymm3, %eax + testl %eax, %eax + jnz L(first_vec_x2) + vpmovmskb %ymm4, %eax + testl %eax, %eax +L(first_vec_x3): + tzcntl %eax, %eax + addq $(VEC_SIZE * 3), %rax + addq %rdi, %rax + VZEROUPPER + ret + +END (MEMCHR) +#endif diff --git a/sysdeps/x86_64/multiarch/memchr-sse2.S b/sysdeps/x86_64/multiarch/memchr-sse2.S new file mode 100644 index 0000000..bbcc62d --- /dev/null +++ b/sysdeps/x86_64/multiarch/memchr-sse2.S @@ -0,0 +1,28 @@ +/* memchr optimized with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) +# define memchr __memchr_sse2 + +# undef strong_alias +# define strong_alias(memchr, __memchr) +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(memchr) +#endif + +#include "../memchr.S" diff --git a/sysdeps/x86_64/multiarch/memchr.c b/sysdeps/x86_64/multiarch/memchr.c new file mode 100644 index 0000000..92aaf61 --- /dev/null +++ b/sysdeps/x86_64/multiarch/memchr.c @@ -0,0 +1,35 @@ +/* Multiple versions of memchr + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define memchr __redirect_memchr +# include <string.h> +# undef memchr + +# define SYMBOL_NAME memchr +# include "ifunc-avx2.h" + +libc_ifunc_redirected (__redirect_memchr, memchr, IFUNC_SELECTOR ()); +strong_alias (memchr, __memchr) +# ifdef SHARED +__hidden_ver1 (memchr, __GI_memchr, __redirect_memchr) + __attribute__((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/rawmemchr-avx2.S b/sysdeps/x86_64/multiarch/rawmemchr-avx2.S new file mode 100644 index 0000000..128f9ea --- /dev/null +++ b/sysdeps/x86_64/multiarch/rawmemchr-avx2.S @@ -0,0 +1,4 @@ +#define MEMCHR __rawmemchr_avx2 +#define USE_AS_RAWMEMCHR 1 + +#include "memchr-avx2.S" diff --git a/sysdeps/x86_64/multiarch/rawmemchr-sse2.S b/sysdeps/x86_64/multiarch/rawmemchr-sse2.S new file mode 100644 index 0000000..233bbc7 --- /dev/null +++ b/sysdeps/x86_64/multiarch/rawmemchr-sse2.S @@ -0,0 +1,29 @@ +/* rawmemchr optimized with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define __rawmemchr __rawmemchr_sse2 + +# undefine weak_alias +# define weak_alias(__rawmemchr, rawmemchr) +# undef libc_hidden_def +# define libc_hidden_def(__rawmemchr) +#endif + +#include "../rawmemchr.S" diff --git a/sysdeps/x86_64/multiarch/rawmemchr.c b/sysdeps/x86_64/multiarch/rawmemchr.c new file mode 100644 index 0000000..c97cfa2 --- /dev/null +++ b/sysdeps/x86_64/multiarch/rawmemchr.c @@ -0,0 +1,38 @@ +/* Multiple versions of rawmemchr + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define rawmemchr __redirect_rawmemchr +# define __rawmemchr __redirect___rawmemchr +# include <string.h> +# undef rawmemchr +# undef __rawmemchr + +# define SYMBOL_NAME rawmemchr +# include "ifunc-avx2.h" + +libc_ifunc_redirected (__redirect_rawmemchr, __rawmemchr, + IFUNC_SELECTOR ()); +weak_alias (__rawmemchr, rawmemchr) +# ifdef SHARED +__hidden_ver1 (__rawmemchr, __GI___rawmemchr, __redirect___rawmemchr) + __attribute__((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/wmemchr-avx2.S b/sysdeps/x86_64/multiarch/wmemchr-avx2.S new file mode 100644 index 0000000..282854f --- /dev/null +++ b/sysdeps/x86_64/multiarch/wmemchr-avx2.S @@ -0,0 +1,4 @@ +#define MEMCHR __wmemchr_avx2 +#define USE_AS_WMEMCHR 1 + +#include "memchr-avx2.S" diff --git a/sysdeps/x86_64/multiarch/wmemchr-sse2.S b/sysdeps/x86_64/multiarch/wmemchr-sse2.S new file mode 100644 index 0000000..70a965d --- /dev/null +++ b/sysdeps/x86_64/multiarch/wmemchr-sse2.S @@ -0,0 +1,4 @@ +#define USE_AS_WMEMCHR 1 +#define wmemchr __wmemchr_sse2 + +#include "../memchr.S" diff --git a/sysdeps/x86_64/multiarch/wmemchr.c b/sysdeps/x86_64/multiarch/wmemchr.c new file mode 100644 index 0000000..bde0ec4 --- /dev/null +++ b/sysdeps/x86_64/multiarch/wmemchr.c @@ -0,0 +1,39 @@ +/* Multiple versions of wmemchr + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define wmemchr __redirect_wmemchr +# define __wmemchr __redirect___wmemchr +# include <wchar.h> +# undef wmemchr +# undef __wmemchr + +# define SYMBOL_NAME wmemchr +# include "ifunc-avx2.h" + +libc_ifunc_redirected (__redirect_wmemchr, __wmemchr, IFUNC_SELECTOR ()); +weak_alias (__wmemchr, wmemchr) +# ifdef SHARED +__hidden_ver1 (__wmemchr, __GI___wmemchr, __redirect___wmemchr) + __attribute__((visibility ("hidden"))); +__hidden_ver1 (wmemchr, __GI_wmemchr, __redirect_wmemchr) + __attribute__((weak, visibility ("hidden"))); +# endif +#endif |