aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNoah Goldstein <goldstein.w.n@gmail.com>2022-11-08 17:38:41 -0800
committerNoah Goldstein <goldstein.w.n@gmail.com>2022-11-08 19:22:33 -0800
commit52cf11004eb10f8ebbc193fbdf4094cfecb3dbff (patch)
treedc3c5cf41a53bd42de548c3e0d04f37dac95b72b
parent64b8b6516b3cba19dba4c8f4f9b97daa0556fd98 (diff)
downloadglibc-52cf11004eb10f8ebbc193fbdf4094cfecb3dbff.zip
glibc-52cf11004eb10f8ebbc193fbdf4094cfecb3dbff.tar.gz
glibc-52cf11004eb10f8ebbc193fbdf4094cfecb3dbff.tar.bz2
x86: Add avx2 optimized functions for the wchar_t strcpy family
Implemented: wcscat-avx2 (+ 744 bytes wcscpy-avx2 (+ 539 bytes) wcpcpy-avx2 (+ 577 bytes) wcsncpy-avx2 (+1108 bytes) wcpncpy-avx2 (+1214 bytes) wcsncat-avx2 (+1085 bytes) Performance Changes: Times are from N = 10 runs of the benchmark suite and are reported as geometric mean of all ratios of New Implementation / Best Old Implementation. Best Old Implementation was determined with the highest ISA implementation. wcscat-avx2 -> 0.975 wcscpy-avx2 -> 0.591 wcpcpy-avx2 -> 0.698 wcsncpy-avx2 -> 0.730 wcpncpy-avx2 -> 0.711 wcsncat-avx2 -> 0.954 Code Size Changes: This change increase the size of libc.so by ~5.5kb bytes. For reference the patch optimizing the normal strcpy family functions decreases libc.so by ~5.2kb. Full check passes on x86-64 and build succeeds for all ISA levels w/ and w/o multiarch.
-rw-r--r--sysdeps/x86_64/multiarch/Makefile6
-rw-r--r--sysdeps/x86_64/multiarch/ifunc-impl-list.c28
-rw-r--r--sysdeps/x86_64/multiarch/ifunc-wcs.h7
-rw-r--r--sysdeps/x86_64/multiarch/wcpcpy-avx2.S8
-rw-r--r--sysdeps/x86_64/multiarch/wcpcpy-generic.c2
-rw-r--r--sysdeps/x86_64/multiarch/wcpncpy-avx2.S8
-rw-r--r--sysdeps/x86_64/multiarch/wcpncpy-generic.c2
-rw-r--r--sysdeps/x86_64/multiarch/wcscat-avx2.S10
-rw-r--r--sysdeps/x86_64/multiarch/wcscat-generic.c2
-rw-r--r--sysdeps/x86_64/multiarch/wcscpy-avx2.S7
-rw-r--r--sysdeps/x86_64/multiarch/wcscpy-generic.c2
-rw-r--r--sysdeps/x86_64/multiarch/wcscpy.c5
-rw-r--r--sysdeps/x86_64/multiarch/wcsncat-avx2.S9
-rw-r--r--sysdeps/x86_64/multiarch/wcsncat-generic.c2
-rw-r--r--sysdeps/x86_64/multiarch/wcsncpy-avx2.S7
-rw-r--r--sysdeps/x86_64/multiarch/wcsncpy-generic.c2
-rw-r--r--sysdeps/x86_64/wcpcpy-generic.c2
-rw-r--r--sysdeps/x86_64/wcpcpy.S3
-rw-r--r--sysdeps/x86_64/wcpncpy-generic.c2
-rw-r--r--sysdeps/x86_64/wcpncpy.S3
-rw-r--r--sysdeps/x86_64/wcscat-generic.c2
-rw-r--r--sysdeps/x86_64/wcscat.S3
-rw-r--r--sysdeps/x86_64/wcscpy.S1
-rw-r--r--sysdeps/x86_64/wcsncat-generic.c2
-rw-r--r--sysdeps/x86_64/wcsncat.S3
-rw-r--r--sysdeps/x86_64/wcsncpy-generic.c2
-rw-r--r--sysdeps/x86_64/wcsncpy.S3
27 files changed, 115 insertions, 18 deletions
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index d6e0194..e1e894c 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -131,10 +131,13 @@ endif
ifeq ($(subdir),wcsmbs)
sysdep_routines += \
+ wcpcpy-avx2 \
wcpcpy-evex \
wcpcpy-generic \
+ wcpncpy-avx2 \
wcpncpy-evex \
wcpncpy-generic \
+ wcscat-avx2 \
wcscat-evex \
wcscat-generic \
wcschr-avx2 \
@@ -146,6 +149,7 @@ sysdep_routines += \
wcscmp-avx2-rtm \
wcscmp-evex \
wcscmp-sse2 \
+ wcscpy-avx2 \
wcscpy-evex \
wcscpy-generic \
wcscpy-ssse3 \
@@ -155,11 +159,13 @@ sysdep_routines += \
wcslen-evex512 \
wcslen-sse2 \
wcslen-sse4_1 \
+ wcsncat-avx2 \
wcsncat-evex \
wcsncat-generic \
wcsncmp-avx2 \
wcsncmp-avx2-rtm \
wcsncmp-evex \
+ wcsncpy-avx2 \
wcsncpy-evex \
wcsncpy-generic \
wcsnlen-avx2 \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index c908d6c..0c15dfe 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -907,6 +907,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (BMI2)),
__wcscpy_evex)
X86_IFUNC_IMPL_ADD_V3 (array, i, wcscpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcscpy_avx2)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, wcscpy,
CPU_FEATURE_USABLE (SSSE3),
__wcscpy_ssse3)
X86_IFUNC_IMPL_ADD_V1 (array, i, wcscpy,
@@ -920,7 +924,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__wcsncpy_evex)
- X86_IFUNC_IMPL_ADD_V3 (array, i, wcpncpy,
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcsncpy_avx2)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, wcpncpy,
1,
__wcsncpy_generic))
@@ -932,6 +940,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (BMI2)),
__wcpcpy_evex)
X86_IFUNC_IMPL_ADD_V3 (array, i, wcpcpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcpcpy_avx2)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, wcpcpy,
1,
__wcpcpy_generic))
@@ -942,7 +954,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (BMI2)),
__wcpncpy_evex)
- X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncpy,
+ X86_IFUNC_IMPL_ADD_V3 (array, i, wcpncpy,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcpncpy_avx2)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, wcsncpy,
1,
__wcpncpy_generic))
@@ -954,6 +970,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (BMI2)),
__wcscat_evex)
X86_IFUNC_IMPL_ADD_V3 (array, i, wcscat,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcscat_avx2)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, wcscat,
1,
__wcscat_generic))
@@ -965,6 +985,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& CPU_FEATURE_USABLE (BMI2)),
__wcsncat_evex)
X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncat,
+ (CPU_FEATURE_USABLE (AVX2)
+ && CPU_FEATURE_USABLE (BMI2)),
+ __wcsncat_avx2)
+ X86_IFUNC_IMPL_ADD_V2 (array, i, wcsncat,
1,
__wcsncat_generic))
diff --git a/sysdeps/x86_64/multiarch/ifunc-wcs.h b/sysdeps/x86_64/multiarch/ifunc-wcs.h
index 1d2a634..51194e6 100644
--- a/sysdeps/x86_64/multiarch/ifunc-wcs.h
+++ b/sysdeps/x86_64/multiarch/ifunc-wcs.h
@@ -27,6 +27,8 @@
extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+
extern __typeof (REDIRECT_NAME) OPTIMIZE (GENERIC) attribute_hidden;
static inline void *
@@ -42,6 +44,11 @@ IFUNC_SELECTOR (void)
if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
&& X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
return OPTIMIZE (evex);
+
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+ Prefer_No_VZEROUPPER, !))
+ return OPTIMIZE (avx2);
+
}
return OPTIMIZE (GENERIC);
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-avx2.S b/sysdeps/x86_64/multiarch/wcpcpy-avx2.S
new file mode 100644
index 0000000..0fffd91
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-avx2.S
@@ -0,0 +1,8 @@
+#ifndef WCPCPY
+# define WCPCPY __wcpcpy_avx2
+#endif
+
+#define USE_AS_STPCPY
+#define USE_AS_WCSCPY
+#define STRCPY WCPCPY
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-generic.c b/sysdeps/x86_64/multiarch/wcpcpy-generic.c
index 6039196..0ba29b0 100644
--- a/sysdeps/x86_64/multiarch/wcpcpy-generic.c
+++ b/sysdeps/x86_64/multiarch/wcpcpy-generic.c
@@ -19,7 +19,7 @@
/* We always need to build this implementation as strspn-sse4 needs to
be able to fallback to it. */
#include <isa-level.h>
-#if ISA_SHOULD_BUILD (3)
+#if ISA_SHOULD_BUILD (2)
# define WCPCPY __wcpcpy_generic
# include <wcsmbs/wcpcpy.c>
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-avx2.S b/sysdeps/x86_64/multiarch/wcpncpy-avx2.S
new file mode 100644
index 0000000..b7e594f
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-avx2.S
@@ -0,0 +1,8 @@
+#ifndef WCPNCPY
+# define WCPNCPY __wcpncpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STPCPY
+#define STRNCPY WCPNCPY
+#include "strncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-generic.c b/sysdeps/x86_64/multiarch/wcpncpy-generic.c
index de8d343..4aab4ec 100644
--- a/sysdeps/x86_64/multiarch/wcpncpy-generic.c
+++ b/sysdeps/x86_64/multiarch/wcpncpy-generic.c
@@ -19,7 +19,7 @@
/* We always need to build this implementation as strspn-sse4 needs to
be able to fallback to it. */
#include <isa-level.h>
-#if ISA_SHOULD_BUILD (3)
+#if ISA_SHOULD_BUILD (2)
# define WCPNCPY __wcpncpy_generic
# include <wcsmbs/wcpncpy.c>
diff --git a/sysdeps/x86_64/multiarch/wcscat-avx2.S b/sysdeps/x86_64/multiarch/wcscat-avx2.S
new file mode 100644
index 0000000..a20f23c
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-avx2.S
@@ -0,0 +1,10 @@
+#ifndef WCSCAT
+# define WCSCAT __wcscat_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRCPY WCSCAT
+
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscat-generic.c b/sysdeps/x86_64/multiarch/wcscat-generic.c
index d86b4d5..6476f85 100644
--- a/sysdeps/x86_64/multiarch/wcscat-generic.c
+++ b/sysdeps/x86_64/multiarch/wcscat-generic.c
@@ -19,7 +19,7 @@
/* We always need to build this implementation as strspn-sse4 needs to
be able to fallback to it. */
#include <isa-level.h>
-#if ISA_SHOULD_BUILD (3)
+#if ISA_SHOULD_BUILD (2)
# define WCSCAT __wcscat_generic
# include <wcsmbs/wcscat.c>
diff --git a/sysdeps/x86_64/multiarch/wcscpy-avx2.S b/sysdeps/x86_64/multiarch/wcscpy-avx2.S
new file mode 100644
index 0000000..6bc509d
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscpy-avx2.S
@@ -0,0 +1,7 @@
+#ifndef WCSCPY
+# define WCSCPY __wcscpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define STRCPY WCSCPY
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcscpy-generic.c
index 4a1fffa..600d606 100644
--- a/sysdeps/x86_64/multiarch/wcscpy-generic.c
+++ b/sysdeps/x86_64/multiarch/wcscpy-generic.c
@@ -18,7 +18,7 @@
#include <isa-level.h>
-#if ISA_SHOULD_BUILD (3)
+#if ISA_SHOULD_BUILD (2)
# define WCSCPY __wcscpy_generic
# include <wcsmbs/wcscpy.c>
diff --git a/sysdeps/x86_64/multiarch/wcscpy.c b/sysdeps/x86_64/multiarch/wcscpy.c
index 9ad77da..e204059 100644
--- a/sysdeps/x86_64/multiarch/wcscpy.c
+++ b/sysdeps/x86_64/multiarch/wcscpy.c
@@ -28,6 +28,8 @@
extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE (generic) attribute_hidden;
@@ -44,6 +46,9 @@ IFUNC_SELECTOR (void)
if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
&& X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
return OPTIMIZE (evex);
+
+ if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER, !))
+ return OPTIMIZE (avx2);
}
if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3))
diff --git a/sysdeps/x86_64/multiarch/wcsncat-avx2.S b/sysdeps/x86_64/multiarch/wcsncat-avx2.S
new file mode 100644
index 0000000..a72105b
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-avx2.S
@@ -0,0 +1,9 @@
+#ifndef WCSNCAT
+# define WCSNCAT __wcsncat_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRNCAT WCSNCAT
+#include "strncat-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncat-generic.c b/sysdeps/x86_64/multiarch/wcsncat-generic.c
index 4b55cb4..9ced02b 100644
--- a/sysdeps/x86_64/multiarch/wcsncat-generic.c
+++ b/sysdeps/x86_64/multiarch/wcsncat-generic.c
@@ -19,7 +19,7 @@
/* We always need to build this implementation as strspn-sse4 needs to
be able to fallback to it. */
#include <isa-level.h>
-#if ISA_SHOULD_BUILD (3)
+#if ISA_SHOULD_BUILD (2)
# define WCSNCAT __wcsncat_generic
# include <wcsmbs/wcsncat.c>
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-avx2.S b/sysdeps/x86_64/multiarch/wcsncpy-avx2.S
new file mode 100644
index 0000000..3a1a8a3
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-avx2.S
@@ -0,0 +1,7 @@
+#ifndef WCSNCPY
+# define WCSNCPY __wcsncpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define STRNCPY WCSNCPY
+#include "strncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-generic.c b/sysdeps/x86_64/multiarch/wcsncpy-generic.c
index d0e8a86..6935217 100644
--- a/sysdeps/x86_64/multiarch/wcsncpy-generic.c
+++ b/sysdeps/x86_64/multiarch/wcsncpy-generic.c
@@ -19,7 +19,7 @@
/* We always need to build this implementation as strspn-sse4 needs to
be able to fallback to it. */
#include <isa-level.h>
-#if ISA_SHOULD_BUILD (3)
+#if ISA_SHOULD_BUILD (2)
# define WCSNCPY __wcsncpy_generic
# include <wcsmbs/wcsncpy.c>
diff --git a/sysdeps/x86_64/wcpcpy-generic.c b/sysdeps/x86_64/wcpcpy-generic.c
index 3ddc988..4ab6182 100644
--- a/sysdeps/x86_64/wcpcpy-generic.c
+++ b/sysdeps/x86_64/wcpcpy-generic.c
@@ -24,7 +24,7 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL <= 3
+#if MINIMUM_X86_ISA_LEVEL <= 2
# include <wcsmbs/wcpcpy.c>
diff --git a/sysdeps/x86_64/wcpcpy.S b/sysdeps/x86_64/wcpcpy.S
index 4e4fca7..e64af69 100644
--- a/sysdeps/x86_64/wcpcpy.S
+++ b/sysdeps/x86_64/wcpcpy.S
@@ -24,11 +24,12 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL >= 4
+#if MINIMUM_X86_ISA_LEVEL >= 3
# define WCPCPY __wcpcpy
# define DEFAULT_IMPL_V4 "multiarch/wcpcpy-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcpcpy-avx2.S"
/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
should never be used from here. */
# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
diff --git a/sysdeps/x86_64/wcpncpy-generic.c b/sysdeps/x86_64/wcpncpy-generic.c
index 0c76e56..18c0377 100644
--- a/sysdeps/x86_64/wcpncpy-generic.c
+++ b/sysdeps/x86_64/wcpncpy-generic.c
@@ -24,7 +24,7 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL <= 3
+#if MINIMUM_X86_ISA_LEVEL <= 2
# include <wcsmbs/wcpncpy.c>
diff --git a/sysdeps/x86_64/wcpncpy.S b/sysdeps/x86_64/wcpncpy.S
index b4e5314..0e0f432 100644
--- a/sysdeps/x86_64/wcpncpy.S
+++ b/sysdeps/x86_64/wcpncpy.S
@@ -24,11 +24,12 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL >= 4
+#if MINIMUM_X86_ISA_LEVEL >= 3
# define WCPNCPY __wcpncpy
# define DEFAULT_IMPL_V4 "multiarch/wcpncpy-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcpncpy-avx2.S"
/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
should never be used from here. */
# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
diff --git a/sysdeps/x86_64/wcscat-generic.c b/sysdeps/x86_64/wcscat-generic.c
index 512d0e4..639ceac 100644
--- a/sysdeps/x86_64/wcscat-generic.c
+++ b/sysdeps/x86_64/wcscat-generic.c
@@ -24,7 +24,7 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL <= 3
+#if MINIMUM_X86_ISA_LEVEL <= 2
# include <wcsmbs/wcscat.c>
diff --git a/sysdeps/x86_64/wcscat.S b/sysdeps/x86_64/wcscat.S
index ee8360b..06130f5 100644
--- a/sysdeps/x86_64/wcscat.S
+++ b/sysdeps/x86_64/wcscat.S
@@ -24,11 +24,12 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL >= 4
+#if MINIMUM_X86_ISA_LEVEL >= 3
# define WCSCAT __wcscat
# define DEFAULT_IMPL_V4 "multiarch/wcscat-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcscat-avx2.S"
/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
should never be used from here. */
# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcscpy.S
index e403579..4a85958 100644
--- a/sysdeps/x86_64/wcscpy.S
+++ b/sysdeps/x86_64/wcscpy.S
@@ -29,6 +29,7 @@
# define WCSCPY __wcscpy
# define DEFAULT_IMPL_V4 "multiarch/wcscpy-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcscpy-avx2.S"
# define DEFAULT_IMPL_V2 "multiarch/wcscpy-ssse3.S"
/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
should never be used from here. */
diff --git a/sysdeps/x86_64/wcsncat-generic.c b/sysdeps/x86_64/wcsncat-generic.c
index 86e20d9..57bdd9b 100644
--- a/sysdeps/x86_64/wcsncat-generic.c
+++ b/sysdeps/x86_64/wcsncat-generic.c
@@ -24,7 +24,7 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL <= 3
+#if MINIMUM_X86_ISA_LEVEL <= 2
# include <wcsmbs/wcsncat.c>
diff --git a/sysdeps/x86_64/wcsncat.S b/sysdeps/x86_64/wcsncat.S
index 090055a..e1d8609 100644
--- a/sysdeps/x86_64/wcsncat.S
+++ b/sysdeps/x86_64/wcsncat.S
@@ -24,11 +24,12 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL >= 4
+#if MINIMUM_X86_ISA_LEVEL >= 3
# define WCSNCAT wcsncat
# define DEFAULT_IMPL_V4 "multiarch/wcsncat-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcsncat-avx2.S"
/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
should never be used from here. */
# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"
diff --git a/sysdeps/x86_64/wcsncpy-generic.c b/sysdeps/x86_64/wcsncpy-generic.c
index 0f0ee65..4dcbd8a 100644
--- a/sysdeps/x86_64/wcsncpy-generic.c
+++ b/sysdeps/x86_64/wcsncpy-generic.c
@@ -24,7 +24,7 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL <= 3
+#if MINIMUM_X86_ISA_LEVEL <= 2
# include <wcsmbs/wcsncpy.c>
diff --git a/sysdeps/x86_64/wcsncpy.S b/sysdeps/x86_64/wcsncpy.S
index 32eaf11..f305b5e 100644
--- a/sysdeps/x86_64/wcsncpy.S
+++ b/sysdeps/x86_64/wcsncpy.S
@@ -24,11 +24,12 @@
#include <isa-level.h>
-#if MINIMUM_X86_ISA_LEVEL >= 4
+#if MINIMUM_X86_ISA_LEVEL >= 3
# define WCSNCPY __wcsncpy
# define DEFAULT_IMPL_V4 "multiarch/wcsncpy-evex.S"
+# define DEFAULT_IMPL_V3 "multiarch/wcsncpy-avx2.S"
/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
should never be used from here. */
# define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL"