diff options
| -rw-r--r-- | ChangeLog | 11 | ||||
| -rw-r--r-- | sysdeps/powerpc/powerpc64/multiarch/Makefile | 3 | ||||
| -rw-r--r-- | sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c | 3 | ||||
| -rw-r--r-- | sysdeps/powerpc/powerpc64/multiarch/memchr-power8.S | 28 | ||||
| -rw-r--r-- | sysdeps/powerpc/powerpc64/multiarch/memchr.c | 3 | ||||
| -rw-r--r-- | sysdeps/powerpc/powerpc64/power8/memchr.S | 335 |
6 files changed, 382 insertions, 1 deletions
@@ -1,5 +1,16 @@ 2017-06-21 Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com> + * sysdeps/powerpc/powerpc64/multiarch/Makefile + (sysdep_routines): Add memchr_power8. + * sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c + (memchr): Add __memchr_power8 to list of memchr functions. + * sysdeps/powerpc/powerpc64/multiarch/memchr-power8.S: New file. + * sysdeps/powerpc/powerpc64/multiarch/memchr.c + (memchr): Add __memchr_power8 to ifunc list. + * sysdeps/powerpc/powerpc64/power8/memchr.S: New file. + +2017-06-21 Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com> + * sysdeps/powerpc/powerpc64/fpu/s_llrint.S (__llrintf): Define as strong alias of __llrint. (llrintf): Define as weak alias of __llrint. diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile index 5da9052..f6491a2 100644 --- a/sysdeps/powerpc/powerpc64/multiarch/Makefile +++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile @@ -4,7 +4,8 @@ sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \ memcmp-power8 memcmp-power7 memcmp-power4 memcmp-ppc64 \ memset-power7 memset-power6 memset-power4 \ memset-ppc64 memset-power8 \ - mempcpy-power7 mempcpy-ppc64 memchr-power7 memchr-ppc64 \ + mempcpy-power7 mempcpy-ppc64 \ + memchr-power8 memchr-power7 memchr-ppc64 \ memrchr-power7 memrchr-ppc64 rawmemchr-power7 \ rawmemchr-ppc64 strlen-power7 strlen-ppc64 \ strnlen-power8 strnlen-power7 strnlen-ppc64 \ diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c index eb173f8..a1d3ca0 100644 --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c @@ -183,6 +183,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, /* Support sysdeps/powerpc/powerpc64/multiarch/memchr.c. */ IFUNC_IMPL (i, name, memchr, IFUNC_IMPL_ADD (array, i, memchr, + hwcap2 & PPC_FEATURE2_ARCH_2_07, + __memchr_power8) + IFUNC_IMPL_ADD (array, i, memchr, hwcap & PPC_FEATURE_HAS_VSX, __memchr_power7) IFUNC_IMPL_ADD (array, i, memchr, 1, diff --git a/sysdeps/powerpc/powerpc64/multiarch/memchr-power8.S b/sysdeps/powerpc/powerpc64/multiarch/memchr-power8.S new file mode 100644 index 0000000..30f8fe7 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/multiarch/memchr-power8.S @@ -0,0 +1,28 @@ +/* Optimized memchr implementation for PowerPC64/POWER8. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +#define MEMCHR __memchr_power8 + +#undef libc_hidden_builtin_def +#define libc_hidden_builtin_def(name) +#undef weak_alias +#define weak_alias(name,alias) + +#include <sysdeps/powerpc/powerpc64/power8/memchr.S> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memchr.c b/sysdeps/powerpc/powerpc64/multiarch/memchr.c index f6f4bab..fa2cfd5 100644 --- a/sysdeps/powerpc/powerpc64/multiarch/memchr.c +++ b/sysdeps/powerpc/powerpc64/multiarch/memchr.c @@ -23,10 +23,13 @@ extern __typeof (__memchr) __memchr_ppc attribute_hidden; extern __typeof (__memchr) __memchr_power7 attribute_hidden; +extern __typeof (__memchr) __memchr_power8 attribute_hidden; /* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle ifunc symbol properly. */ libc_ifunc (__memchr, + (hwcap2 & PPC_FEATURE2_ARCH_2_07) + ? __memchr_power8 : (hwcap & PPC_FEATURE_HAS_VSX) ? __memchr_power7 : __memchr_ppc); diff --git a/sysdeps/powerpc/powerpc64/power8/memchr.S b/sysdeps/powerpc/powerpc64/power8/memchr.S new file mode 100644 index 0000000..06abc47 --- /dev/null +++ b/sysdeps/powerpc/powerpc64/power8/memchr.S @@ -0,0 +1,335 @@ +/* Optimized memchr implementation for POWER8. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +/* void *[r3] memchr (const void *s [r3], int c [r4], size_t n [r5]) */ + +/* TODO: change these to the actual instructions when the minimum required + binutils allows it. */ +#define MTVRD(v, r) .long (0x7c000167 | ((v)<<(32-11)) | ((r)<<(32-16))) +#define MFVRD(r, v) .long (0x7c000067 | ((v)<<(32-11)) | ((r)<<(32-16))) +#define VBPERMQ(t, a, b) .long (0x1000054c \ + | ((t)<<(32-11)) \ + | ((a)<<(32-16)) \ + | ((b)<<(32-21)) ) + +#ifndef MEMCHR +# define MEMCHR __memchr +#endif +/* TODO: change this to .machine power8 when the minimum required binutils + allows it. */ + .machine power7 +ENTRY_TOCLESS (MEMCHR) + CALL_MCOUNT 3 + dcbt 0, r3 + clrrdi r8, r3, 3 + insrdi r4, r4, 8, 48 + + /* Calculate the last acceptable address and check for possible + addition overflow by using satured math: + r7 = r3 + r5 + r7 |= -(r7 < x) */ + add r7, r3, r5 + subfc r6, r3, r7 + subfe r9, r9, r9 + extsw r6, r9 + or r7, r7, r6 + + insrdi r4, r4, 16, 32 + cmpldi r5, 32 + li r9, -1 + rlwinm r6, r3, 3, 26, 28 /* Calculate padding. */ + insrdi r4, r4, 32, 0 + mr r10, r7 + addi r7, r7, -1 +#ifdef __LITTLE_ENDIAN__ + sld r9, r9, r6 +#else + srd r9, r9, r6 +#endif + ble L(small_range) + andi. r11, r3, 63 + beq cr0, L(align_qw) + clrldi r11, r3, 61 + ld r12, 0(r8) /* Load doubleword from memory. */ + cmpb r3, r12, r4 /* Check for BYTEs in DWORD1. */ + and r3, r3, r9 + clrldi r6, r7, 61 /* Byte count - 1 in last dword. */ + clrrdi r7, r7, 3 /* Address of last doubleword. */ + cmpldi cr7, r3, 0 /* Does r3 indicate we got a hit? */ + bne cr7, L(done) + addi r8, r8, 8 + addi r5, r5, -8 + add r5, r5, r11 + + /* Are we now aligned to a quadword boundary? */ + andi. r11, r8, 15 + beq cr0, L(align_qw) + + /* Handle DWORD to make it QW aligned. */ + ld r12, 0(r8) + cmpb r3, r12, r4 + cmpldi cr7, r3, 0 + bne cr7, L(done) + addi r5, r5, -8 + addi r8, r8, 8 + /* At this point, r8 is 16B aligned. */ +L(align_qw): + vspltisb v0, 0 + /* Precompute vbpermq constant. */ + vspltisb v10, 3 + li r0, 0 + lvsl v11, r0, r0 + vslb v10, v11, v10 + MTVRD(v1, r4) + vspltb v1, v1, 7 + cmpldi r5, 64 + ble L(tail64) + /* Are we 64-byte aligned? If so, jump to the vectorized loop. + Note: aligning to 64-byte will necessarily slow down performance for + strings around 64 bytes in length due to the extra comparisons + required to check alignment for the vectorized loop. This is a + necessary tradeoff we are willing to take in order to speed up the + calculation for larger strings. */ + andi. r11, r8, 63 + beq cr0, L(preloop_64B) + /* In order to begin the 64B loop, it needs to be 64 + bytes aligned. So read until it is 64B aligned. */ + lvx v4, 0, r8 + vcmpequb v6, v1, v4 + vcmpequb. v11, v0, v6 + bnl cr6, L(found_16B) + addi r8, r8, 16 + addi r5, r5, -16 + + andi. r11, r8, 63 + beq cr0, L(preloop_64B) + lvx v4, 0, r8 + vcmpequb v6, v1, v4 + vcmpequb. v11, v0, v6 + bnl cr6, L(found_16B) + addi r8, r8, 16 + addi r5, r5, -16 + + andi. r11, r8, 63 + beq cr0, L(preloop_64B) + lvx v4, 0, r8 + vcmpequb v6, v1, v4 + vcmpequb. v11, v0, v6 + bnl cr6, L(found_16B) + addi r8, r8, 16 + addi r5, r5, -16 + /* At this point it should be 64B aligned. + Prepare for the 64B loop. */ +L(preloop_64B): + cmpldi r5, 64 /* Check if r5 < 64. */ + ble L(tail64) + sub r6, r10, r8 + srdi r9, r6, 6 /* Number of loop iterations. */ + mtctr r9 /* Setup the counter. */ + li r11, 16 /* Load required offsets. */ + li r9, 32 + li r7, 48 + + /* Handle r5 > 64. Loop over the bytes in strides of 64B. */ + .align 4 +L(loop): + lvx v2, 0, r8 /* Load 4 quadwords. */ + lvx v3, r8, r11 + lvx v4, v8, r9 + lvx v5, v8, r7 + vcmpequb v6, v1, v2 + vcmpequb v7, v1, v3 + vcmpequb v8, v1, v4 + vcmpequb v9, v1, v5 + vor v11, v6, v7 + vor v12, v8, v9 + vor v11, v11, v12 /* Compare and merge into one VR for speed. */ + vcmpequb. v11, v0, v11 + bnl cr6, L(found) + addi r8, r8, 64 /* Adjust address for the next iteration. */ + bdnz L(loop) + clrldi r5, r6, 58 + + /* Handle remainder of 64B loop or r5 > 64. */ + .align 4 +L(tail64): + cmpldi r5, 0 + beq L(null) + lvx v4, 0, r8 + vcmpequb v6, v1, v4 + vcmpequb. v11, v0, v6 + bnl cr6, L(found_16B) + addi r8, r8, 16 + cmpldi cr6, r5, 16 + ble cr6, L(null) + addi r5, r5, -16 + + lvx v4, 0, r8 + vcmpequb v6, v1, v4 + vcmpequb. v11, v0, v6 + bnl cr6, L(found_16B) + addi r8, r8, 16 + cmpldi cr6, r5, 16 + ble cr6, L(null) + addi r5, r5, -16 + + lvx v4, 0, r8 + vcmpequb v6, v1, v4 + vcmpequb. v11, v0, v6 + bnl cr6, L(found_16B) + addi r8, r8, 16 + cmpldi cr6, r5, 16 + ble cr6, L(null) + addi r5, r5, -16 + + lvx v4, 0, r8 + vcmpequb v6, v1, v4 + vcmpequb. v11, v0, v6 + bnl cr6, L(found_16B) + li r3, 0 + blr + + /* Found a match in 64B loop. */ + .align 4 +L(found): + /* Permute the first bit of each byte into bits 48-63. */ + VBPERMQ(v6, v6, v10) + VBPERMQ(v7, v7, v10) + VBPERMQ(v8, v8, v10) + VBPERMQ(v9, v9, v10) + /* Shift each component into its correct position for merging. */ +#ifdef __LITTLE_ENDIAN__ + vsldoi v7, v7, v7, 2 + vsldoi v8, v8, v8, 4 + vsldoi v9, v9, v9, 6 +#else + vsldoi v6, v6, v6, 6 + vsldoi v7, v7, v7, 4 + vsldoi v8, v8, v8, 2 +#endif + /* Merge the results and move to a GPR. */ + vor v11, v6, v7 + vor v4, v9, v8 + vor v4, v11, v4 + MFVRD(r5, v4) +#ifdef __LITTLE_ENDIAN__ + addi r6, r5, -1 + andc r6, r6, r5 + popcntd r6, r6 +#else + cntlzd r6, r5 /* Count leading zeros before the match. */ +#endif + add r3, r8, r6 /* Compute final length. */ + blr + + /* Found a match in last 16 bytes. */ + .align 4 +L(found_16B): + /* Permute the first bit of each byte into bits 48-63. */ + VBPERMQ(v6, v6, v10) + /* Shift each component into its correct position for merging. */ +#ifdef __LITTLE_ENDIAN__ + MFVRD(r7, v6) + addi r6, r7, -1 + andc r6, r6, r7 + popcntd r6, r6 +#else + vsldoi v6, v6, v6, 6 + MFVRD(r7, v6) + cntlzd r6, r7 /* Count leading zeros before the match. */ +#endif + add r3, r8, r6 /* Compute final length. */ + cmpld r6, r5 + bltlr + li r3, 0 + blr + + .align 4 + /* r3 has the output of the cmpb instruction, that is, it contains + 0xff in the same position as BYTE in the original + doubleword from the string. Use that to calculate the pointer. + We need to make sure BYTE is *before* the end of the range. */ +L(done): +#ifdef __LITTLE_ENDIAN__ + addi r0, r3, -1 + andc r0, r0, r3 + popcntd r0, r0 /* Count trailing zeros. */ +#else + cntlzd r0, r3 /* Count leading zeros before the match. */ +#endif + cmpld r8, r7 /* Are we on the last dword? */ + srdi r0, r0, 3 /* Convert leading/trailing zeros to bytes. */ + add r3, r8, r0 + cmpld cr7, r0, r6 /* If on the last dword, check byte offset. */ + bnelr + blelr cr7 + li r3, 0 + blr + + .align 4 +L(null): + li r3, 0 + blr + +/* Deals with size <= 32. */ + .align 4 +L(small_range): + cmpldi r5, 0 + beq L(null) + ld r12, 0(r8) /* Load word from memory. */ + cmpb r3, r12, r4 /* Check for BYTE in DWORD1. */ + and r3, r3, r9 + cmpldi cr7, r3, 0 + clrldi r6, r7, 61 /* Byte count - 1 in last dword. */ + clrrdi r7, r7, 3 /* Address of last doubleword. */ + cmpld r8, r7 /* Are we done already? */ + bne cr7, L(done) + beqlr + + ldu r12, 8(r8) + cmpb r3, r12, r4 + cmpldi cr6, r3, 0 + cmpld r8, r7 + bne cr6, L(done) /* Found something. */ + beqlr /* Hit end of string (length). */ + + ldu r12, 8(r8) + cmpb r3, r12, r4 + cmpldi cr6, r3, 0 + cmpld r8, r7 + bne cr6, L(done) + beqlr + + ldu r12, 8(r8) + cmpb r3, r12, r4 + cmpldi cr6, r3, 0 + cmpld r8, r7 + bne cr6, L(done) + beqlr + + ldu r12, 8(r8) + cmpb r3, r12, r4 + cmpldi cr6, r3, 0 + bne cr6, L(done) + blr + +END (MEMCHR) +weak_alias (__memchr, memchr) +libc_hidden_builtin_def (memchr) |