S390: Add z13 memmove ifunc variant.

This patch introduces a z13 specific ifunc variant for memmove.
As the common code implementation, it checks if we can copy from
the beginning to the end - with z196 memcpy implementation - or
if we have to copy from the end to the beginning.
The latter case is done by using vector load/store instructions.

If vector instructions are not available, the common-code is
used as fallback.  Therefore it is implemented in memmove-c with
a different name.
Furthermore the ifunc logic decides if we need the common-code
implementation at all.  If vector instructions are supported
due to the minimum architecture level set we can skip the
common-code ifunc variant.

ChangeLog:

	* sysdeps/s390/Makefile (sysdep_routines): Add memmove-c.
	* sysdeps/s390/ifunc-memcpy.h (HAVE_MEMMOVE_IFUNC,
	HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT, MEMMOVE_DEFAULT,
	HAVE_MEMMOVE_C, MEMMOVE_C,  HAVE_MEMMOVE_Z13, MEMMOVE_Z13):
	New defines.
	* sysdeps/s390/memcpy-z900.S: Add z13 memmove implementation.
	* sysdeps/s390/memmove-c.c: New file.
	* sysdeps/s390/memmove.c: Likewise.
	* sysdeps/s390/multiarch/ifunc-impl-list.c
	(__libc_ifunc_impl_list): Add ifunc variants for memmove.

7 files changed, 230 insertions, 1 deletions

diff --git a/ChangeLog b/ChangeLog
index 52dbc37..539e653 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,18 @@
 2018-12-18  Stefan Liebler  <stli@linux.ibm.com>
 
+	* sysdeps/s390/Makefile (sysdep_routines): Add memmove-c.
+	* sysdeps/s390/ifunc-memcpy.h (HAVE_MEMMOVE_IFUNC,
+	HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT, MEMMOVE_DEFAULT,
+	HAVE_MEMMOVE_C, MEMMOVE_C,  HAVE_MEMMOVE_Z13, MEMMOVE_Z13):
+	New defines.
+	* sysdeps/s390/memcpy-z900.S: Add z13 memmove implementation.
+	* sysdeps/s390/memmove-c.c: New file.
+	* sysdeps/s390/memmove.c: Likewise.
+	* sysdeps/s390/multiarch/ifunc-impl-list.c
+	(__libc_ifunc_impl_list): Add ifunc variants for memmove.
+
+2018-12-18  Stefan Liebler  <stli@linux.ibm.com>
+
 	* config.h.in (HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT): New undefine.
 	* sysdeps/s390/configure.ac: Add check for z13 support.
 	* sysdeps/s390/configure: Regenerated.
diff --git a/sysdeps/s390/Makefile b/sysdeps/s390/Makefile
index c196a19..859f36f 100644
--- a/sysdeps/s390/Makefile
+++ b/sysdeps/s390/Makefile
@@ -56,5 +56,6 @@ endif
 ifeq ($(subdir),string)
 sysdep_routines += bzero memset memset-z900 \
 		   memcmp memcmp-z900 \
-		   mempcpy memcpy memcpy-z900
+		   mempcpy memcpy memcpy-z900 \
+		   memmove memmove-c
 endif
diff --git a/sysdeps/s390/ifunc-memcpy.h b/sysdeps/s390/ifunc-memcpy.h
index 51c71ba..0e70196 100644
--- a/sysdeps/s390/ifunc-memcpy.h
+++ b/sysdeps/s390/ifunc-memcpy.h
@@ -43,6 +43,29 @@
 # define HAVE_MEMCPY_Z196	HAVE_MEMCPY_IFUNC
 #endif
 
+#if defined SHARED && defined USE_MULTIARCH && IS_IN (libc)	\
+  && ! defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
+# define HAVE_MEMMOVE_IFUNC	1
+#else
+# define HAVE_MEMMOVE_IFUNC	0
+#endif
+
+#ifdef HAVE_S390_VX_ASM_SUPPORT
+# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT HAVE_MEMMOVE_IFUNC
+#else
+# define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT 0
+#endif
+
+#if defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT
+# define MEMMOVE_DEFAULT	MEMMOVE_Z13
+# define HAVE_MEMMOVE_C		0
+# define HAVE_MEMMOVE_Z13	1
+#else
+# define MEMMOVE_DEFAULT	MEMMOVE_C
+# define HAVE_MEMMOVE_C		1
+# define HAVE_MEMMOVE_Z13	HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT
+#endif
+
 #if HAVE_MEMCPY_Z900_G5
 # define MEMCPY_Z900_G5		__memcpy_default
 # define MEMPCPY_Z900_G5	__mempcpy_default
@@ -66,3 +89,15 @@
 # define MEMCPY_Z196		NULL
 # define MEMPCPY_Z196		NULL
 #endif
+
+#if HAVE_MEMMOVE_C
+# define MEMMOVE_C		__memmove_c
+#else
+# define MEMMOVE_C		NULL
+#endif
+
+#if HAVE_MEMMOVE_Z13
+# define MEMMOVE_Z13		__memmove_z13
+#else
+# define MEMMOVE_Z13		NULL
+#endif
diff --git a/sysdeps/s390/memcpy-z900.S b/sysdeps/s390/memcpy-z900.S
index 3a50cf4..bd3b195 100644
--- a/sysdeps/s390/memcpy-z900.S
+++ b/sysdeps/s390/memcpy-z900.S
@@ -182,6 +182,7 @@ ENTRY(MEMCPY_Z196)
 # endif /* !defined __s390x__  */
 	ltgr    %r4,%r4
 	je      .L_Z196_4
+.L_Z196_start2:
 	aghi    %r4,-1
 	srlg    %r5,%r4,8
 	ltgr    %r5,%r5
@@ -207,6 +208,75 @@ ENTRY(MEMCPY_Z196)
 END(MEMCPY_Z196)
 #endif /* HAVE_MEMCPY_Z196  */
 
+#if HAVE_MEMMOVE_Z13
+ENTRY(MEMMOVE_Z13)
+	.machine "z13"
+	.machinemode "zarch_nohighgprs"
+# if !defined __s390x__
+	/* Note: The 31bit dst and src pointers are prefixed with zeroes.  */
+	llgfr	%r4,%r4
+	llgfr	%r3,%r3
+	llgfr	%r2,%r2
+# endif /* !defined __s390x__ */
+	sgrk	%r0,%r2,%r3
+	clgijh	%r4,16,.L_MEMMOVE_Z13_LARGE
+	aghik	%r5,%r4,-1
+.L_MEMMOVE_Z13_SMALL:
+	jl .L_MEMMOVE_Z13_END		/* Jump away if len was zero.  */
+	/* Store up to 16 bytes with vll/vstl which needs the index
+	   instead of lengths.  */
+	vll	%v16,%r5,0(%r3)
+	vstl	%v16,%r5,0(%r2)
+.L_MEMMOVE_Z13_END:
+	br      %r14
+.L_MEMMOVE_Z13_LARGE:
+	lgr     %r1,%r2			/* For memcpy: r1: Use as dest ;
+					   r2: Return dest  */
+	/* The unsigned comparison (dst - src >= len) determines if we can
+	   execute the forward case with memcpy.  */
+#if ! HAVE_MEMCPY_Z196
+# error The z13 variant of memmove needs the z196 variant of memcpy!
+#endif
+	clgrjhe %r0,%r4,.L_Z196_start2
+	risbgn	%r5,%r4,4,128+63,60	/* r5 = r4 / 16  */
+	aghi	%r4,-16
+	clgijhe	%r5,8,.L_MEMMOVE_Z13_LARGE_64B
+.L_MEMMOVE_Z13_LARGE_16B_LOOP:
+	/* Store at least 16 bytes with vl/vst. The number of 16byte blocks
+	   is stored in r5.  */
+	vl	%v16,0(%r4,%r3)
+	vst	%v16,0(%r4,%r2)
+	aghi	%r4,-16
+	brctg	%r5,.L_MEMMOVE_Z13_LARGE_16B_LOOP
+	aghik	%r5,%r4,15
+	j	.L_MEMMOVE_Z13_SMALL
+.L_MEMMOVE_Z13_LARGE_64B:
+	/* Store at least 128 bytes with 4x vl/vst. The number of 64byte blocks
+	   will be stored in r0.  */
+	aghi	%r4,-48
+	srlg	%r0,%r5,2		/* r5 = %r0 / 4
+					   => Number of 64byte blocks.  */
+.L_MEMMOVE_Z13_LARGE_64B_LOOP:
+	vl	%v20,48(%r4,%r3)
+	vl	%v19,32(%r4,%r3)
+	vl	%v18,16(%r4,%r3)
+	vl	%v17,0(%r4,%r3)
+	vst	%v20,48(%r4,%r2)
+	vst	%v19,32(%r4,%r2)
+	vst	%v18,16(%r4,%r2)
+	vst	%v17,0(%r4,%r2)
+	aghi	%r4,-64
+	brctg	%r0,.L_MEMMOVE_Z13_LARGE_64B_LOOP
+	aghi	%r4,48
+	/* Recalculate the number of 16byte blocks.  */
+	risbg	%r5,%r5,62,128+63,0	/* r5 = r5 & 3
+					   => Remaining 16byte blocks.  */
+	jne	.L_MEMMOVE_Z13_LARGE_16B_LOOP
+	aghik	%r5,%r4,15
+	j	.L_MEMMOVE_Z13_SMALL
+END(MEMMOVE_Z13)
+#endif /* HAVE_MEMMOVE_Z13  */
+
 #if ! HAVE_MEMCPY_IFUNC
 /* If we don't use ifunc, define an alias for mem[p]cpy here.
    Otherwise see sysdeps/s390/mem[p]cpy.c.  */
@@ -215,10 +285,27 @@ strong_alias (MEMPCPY_DEFAULT, __mempcpy)
 weak_alias (__mempcpy, mempcpy)
 #endif
 
+#if ! HAVE_MEMMOVE_IFUNC
+/* If we don't use ifunc, define an alias for memmove here.
+   Otherwise see sysdeps/s390/memmove.c.  */
+# if ! HAVE_MEMMOVE_C
+/* If the c variant is needed, then sysdeps/s390/memmove-c.c
+   defines memmove.
+   Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it.  */
+strong_alias (MEMMOVE_DEFAULT, memmove)
+# endif
+#endif
+
 #if defined SHARED && IS_IN (libc)
 /* Defines the internal symbols.
    Compare to libc_hidden_[builtin_]def (mem[p]cpy) in string/mem[p]cpy.c.  */
 strong_alias (MEMCPY_DEFAULT, __GI_memcpy)
 strong_alias (MEMPCPY_DEFAULT, __GI_mempcpy)
 strong_alias (MEMPCPY_DEFAULT, __GI___mempcpy)
+# if ! HAVE_MEMMOVE_C
+/* If the c variant is needed, then sysdeps/s390/memmove-c.c
+   defines the internal symbol.
+   Otherwise MEMMOVE_DEFAULT is implemented here and we have to define it.  */
+strong_alias (MEMMOVE_DEFAULT, __GI_memmove)
+# endif
 #endif
diff --git a/sysdeps/s390/memmove-c.c b/sysdeps/s390/memmove-c.c
new file mode 100644
index 0000000..be57109
--- /dev/null
+++ b/sysdeps/s390/memmove-c.c
@@ -0,0 +1,37 @@
+/* Fallback C version of memmove.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <ifunc-memcpy.h>
+
+#if HAVE_MEMMOVE_C
+# if HAVE_MEMMOVE_IFUNC
+/* If we use ifunc, then the memmove symbol is defined
+   in sysdeps/s390/memmove.c and we use a different name here.
+   Otherwise, we have to define memmove here or in
+   sysdeps/s390/memcpy.S depending on the used default implementation.  */
+#  define MEMMOVE MEMMOVE_C
+#  if defined SHARED && IS_IN (libc)
+/* Define the internal symbol.  */
+#   undef libc_hidden_builtin_def
+#   define libc_hidden_builtin_def(name)			\
+  __hidden_ver1 (__memmove_c, __GI_memmove, __memmove_c);
+#  endif
+# endif
+
+# include <string/memmove.c>
+#endif
diff --git a/sysdeps/s390/memmove.c b/sysdeps/s390/memmove.c
new file mode 100644
index 0000000..ac34edf
--- /dev/null
+++ b/sysdeps/s390/memmove.c
@@ -0,0 +1,44 @@
+/* Multiple versions of memmove.
+   Copyright (C) 2016-2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <ifunc-memcpy.h>
+
+#if HAVE_MEMMOVE_IFUNC
+/* If we don't use ifunc, an alias is defined for memmove
+   in sysdeps/s390/memmove-c.c or sysdeps/s390/memcpy.S
+   depending on the used default implementation.  */
+# undef memmove
+# define memmove __redirect_memmove
+# include <string.h>
+# include <ifunc-resolve.h>
+# undef memmove
+
+# if HAVE_MEMMOVE_C
+extern __typeof (__redirect_memmove) MEMMOVE_C attribute_hidden;
+# endif
+
+# if HAVE_MEMMOVE_Z13
+extern __typeof (__redirect_memmove) MEMMOVE_Z13 attribute_hidden;
+# endif
+
+s390_libc_ifunc_expr (__redirect_memmove, memmove,
+		      (HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX))
+		      ? MEMMOVE_Z13
+		      : MEMMOVE_DEFAULT
+		      )
+#endif
diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c
index 6969c48..c05c63e 100644
--- a/sysdeps/s390/multiarch/ifunc-impl-list.c
+++ b/sysdeps/s390/multiarch/ifunc-impl-list.c
@@ -126,6 +126,18 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 	      )
 #endif /* HAVE_MEMCPY_IFUNC  */
 
+#if HAVE_MEMMOVE_IFUNC
+    IFUNC_IMPL (i, name, memmove,
+# if HAVE_MEMMOVE_Z13
+		IFUNC_IMPL_ADD (array, i, memmove,
+				dl_hwcap & HWCAP_S390_VX, MEMMOVE_Z13)
+# endif
+# if HAVE_MEMMOVE_C
+		IFUNC_IMPL_ADD (array, i, memmove, 1, MEMMOVE_C)
+# endif
+		)
+#endif /* HAVE_MEMMOVE_IFUNC  */
+
 #ifdef HAVE_S390_VX_ASM_SUPPORT
 
 # define IFUNC_VX_IMPL(FUNC)						\