libatomic: Enable LSE128 128-bit atomics for Armv9.4-a

The armv9.4-a architectural revision adds three new atomic operations
associated with the LSE128 feature:

  * LDCLRP - Atomic AND NOT (bitclear) of a location with 128-bit
  value held in a pair of registers, with original data loaded into
  the same 2 registers.
  * LDSETP - Atomic OR (bitset) of a location with 128-bit value held
  in a pair of registers, with original data loaded into the same 2
  registers.
  * SWPP - Atomic swap of one 128-bit value with 128-bit value held
  in a pair of registers.

It is worth noting that in keeping with existing 128-bit atomic
operations in `atomic_16.S', we have chosen to merge certain
less-restrictive orderings into more restrictive ones.  This is done
to minimize the number of branches in the atomic functions, minimizing
both the likelihood of branch mispredictions and, in keeping code
small, limit the need for extra fetch cycles.

Past benchmarking has revealed that acquire is typically slightly
faster than release (5-10%), such that for the most frequently used
atomics (CAS and SWP) it makes sense to add support for acquire, as
well as release.

Likewise, it was identified that combining acquire and release typically
results in little to no penalty, such that it is of negligible benefit
to distinguish between release and acquire-release, making the
combining release/acq_rel/seq_cst a worthwhile design choice.

This patch adds the logic required to make use of these when the
architectural feature is present and a suitable assembler available.

In order to do this, the following changes are made:

  1. Add a configure-time check to check for LSE128 support in the
  assembler.
  2. Edit host-config.h so that when N == 16, nifunc = 2.
  3. Where available due to LSE128, implement the second ifunc, making
  use of the novel instructions.
  4. For atomic functions unable to make use of these new
  instructions, define a new alias which causes the _i1 function
  variant to point ahead to the corresponding _i2 implementation.

libatomic/ChangeLog:

	* Makefile.am (AM_CPPFLAGS): add conditional setting of
	-DHAVE_FEAT_LSE128.
	* acinclude.m4 (LIBAT_TEST_FEAT_AARCH64_LSE128): New.
	* config/linux/aarch64/atomic_16.S (LSE128): New macro
	definition.
	(libat_exchange_16): New LSE128 variant.
	(libat_fetch_or_16): Likewise.
	(libat_or_fetch_16): Likewise.
	(libat_fetch_and_16): Likewise.
	(libat_and_fetch_16): Likewise.
	* config/linux/aarch64/host-config.h (IFUNC_COND_2): New.
	(IFUNC_NCOND): Add operand size checking.
	(has_lse2): Renamed from `ifunc1`.
	(has_lse128): New.
	(HWCAP2_LSE128): Likewise.
	* configure.ac: Add call to
	LIBAT_TEST_FEAT_AARCH64_LSE128.
	* configure (ac_subst_vars): Regenerated via autoreconf.
	* Makefile.in: Likewise.
	* auto-config.h.in: Likewise.

2 files changed, 205 insertions, 7 deletions

diff --git a/libatomic/config/linux/aarch64/atomic_16.S b/libatomic/config/linux/aarch64/atomic_16.S
index 33f2f56..d4a360a 100644
--- a/libatomic/config/linux/aarch64/atomic_16.S
+++ b/libatomic/config/linux/aarch64/atomic_16.S
@@ -35,12 +35,17 @@
    writes, this will be true when using atomics in actual code.
 
    The libat_<op>_16 entry points are ARMv8.0.
-   The libat_<op>_16_i1 entry points are used when LSE2 is available.  */
-
+   The libat_<op>_16_i1 entry points are used when LSE128 is available.
+   The libat_<op>_16_i2 entry points are used when LSE2 is available.  */
 
+#if HAVE_FEAT_LSE128
+	.arch	armv9-a+lse128
+#else
 	.arch	armv8-a+lse
+#endif
 
-#define LSE2(NAME) 	NAME##_i1
+#define LSE128(NAME) 	NAME##_i1
+#define LSE2(NAME) 	NAME##_i2
 #define CORE(NAME) 	NAME
 
 #define ENTRY_FEAT(NAME, FEAT)  \
@@ -202,6 +207,31 @@ ENTRY (libat_exchange_16)
 END (libat_exchange_16)
 
 
+#if HAVE_FEAT_LSE128
+ENTRY_FEAT (libat_exchange_16, LSE128)
+	mov	tmp0, x0
+	mov	res0, in0
+	mov	res1, in1
+	cbnz	w4, 1f
+
+	/* RELAXED.  */
+	swpp	res0, res1, [tmp0]
+	ret
+1:
+	cmp	w4, ACQUIRE
+	b.hi	2f
+
+	/* ACQUIRE/CONSUME.  */
+	swppa	res0, res1, [tmp0]
+	ret
+
+	/* RELEASE/ACQ_REL/SEQ_CST.  */
+2:	swppal	res0, res1, [tmp0]
+	ret
+END_FEAT (libat_exchange_16, LSE128)
+#endif
+
+
 ENTRY (libat_compare_exchange_16)
 	ldp	exp0, exp1, [x1]
 	cbz	w4, 3f
@@ -395,6 +425,31 @@ ENTRY (libat_fetch_or_16)
 END (libat_fetch_or_16)
 
 
+#if HAVE_FEAT_LSE128
+ENTRY_FEAT (libat_fetch_or_16, LSE128)
+	mov	tmp0, x0
+	mov	res0, in0
+	mov	res1, in1
+	cbnz	w4, 1f
+
+	/* RELAXED.  */
+	ldsetp	res0, res1, [tmp0]
+	ret
+1:
+	cmp	w4, ACQUIRE
+	b.hi	2f
+
+	/* ACQUIRE/CONSUME.  */
+	ldsetpa	res0, res1, [tmp0]
+	ret
+
+	/* RELEASE/ACQ_REL/SEQ_CST.  */
+2:	ldsetpal	res0, res1, [tmp0]
+	ret
+END_FEAT (libat_fetch_or_16, LSE128)
+#endif
+
+
 ENTRY (libat_or_fetch_16)
 	mov	x5, x0
 	cbnz	w4, 2f
@@ -417,6 +472,36 @@ ENTRY (libat_or_fetch_16)
 END (libat_or_fetch_16)
 
 
+#if HAVE_FEAT_LSE128
+ENTRY_FEAT (libat_or_fetch_16, LSE128)
+	cbnz	w4, 1f
+	mov	tmp0, in0
+	mov	tmp1, in1
+
+	/* RELAXED.  */
+	ldsetp	in0, in1, [x0]
+	orr	res0, in0, tmp0
+	orr	res1, in1, tmp1
+	ret
+1:
+	cmp	w4, ACQUIRE
+	b.hi	2f
+
+	/* ACQUIRE/CONSUME.  */
+	ldsetpa	in0, in1, [x0]
+	orr	res0, in0, tmp0
+	orr	res1, in1, tmp1
+	ret
+
+	/* RELEASE/ACQ_REL/SEQ_CST.  */
+2:	ldsetpal	in0, in1, [x0]
+	orr	res0, in0, tmp0
+	orr	res1, in1, tmp1
+	ret
+END_FEAT (libat_or_fetch_16, LSE128)
+#endif
+
+
 ENTRY (libat_fetch_and_16)
 	mov	x5, x0
 	cbnz	w4, 2f
@@ -439,6 +524,32 @@ ENTRY (libat_fetch_and_16)
 END (libat_fetch_and_16)
 
 
+#if HAVE_FEAT_LSE128
+ENTRY_FEAT (libat_fetch_and_16, LSE128)
+	mov	tmp0, x0
+	mvn	res0, in0
+	mvn	res1, in1
+	cbnz	w4, 1f
+
+	/* RELAXED.  */
+	ldclrp	res0, res1, [tmp0]
+	ret
+
+1:
+	cmp	w4, ACQUIRE
+	b.hi	2f
+
+	/* ACQUIRE/CONSUME.  */
+	ldclrpa res0, res1, [tmp0]
+	ret
+
+	/* RELEASE/ACQ_REL/SEQ_CST.  */
+2:	ldclrpal	res0, res1, [tmp0]
+	ret
+END_FEAT (libat_fetch_and_16, LSE128)
+#endif
+
+
 ENTRY (libat_and_fetch_16)
 	mov	x5, x0
 	cbnz	w4, 2f
@@ -461,6 +572,37 @@ ENTRY (libat_and_fetch_16)
 END (libat_and_fetch_16)
 
 
+#if HAVE_FEAT_LSE128
+ENTRY_FEAT (libat_and_fetch_16, LSE128)
+	mvn	tmp0, in0
+	mvn	tmp0, in1
+	cbnz	w4, 1f
+
+	/* RELAXED.  */
+	ldclrp	tmp0, tmp1, [x0]
+	and	res0, tmp0, in0
+	and	res1, tmp1, in1
+	ret
+
+1:
+	cmp	w4, ACQUIRE
+	b.hi	2f
+
+	/* ACQUIRE/CONSUME.  */
+	ldclrpa tmp0, tmp1, [x0]
+	and	res0, tmp0, in0
+	and	res1, tmp1, in1
+	ret
+
+	/* RELEASE/ACQ_REL/SEQ_CST.  */
+2:	ldclrpal	tmp0, tmp1, [x5]
+	and	res0, tmp0, in0
+	and	res1, tmp1, in1
+	ret
+END_FEAT (libat_and_fetch_16, LSE128)
+#endif
+
+
 ENTRY (libat_fetch_xor_16)
 	mov	x5, x0
 	cbnz	w4, 2f
@@ -566,6 +708,28 @@ ENTRY (libat_test_and_set_16)
 END (libat_test_and_set_16)
 
 
+/* Alias entry points which are the same in LSE2 and LSE128.  */
+
+#if !HAVE_FEAT_LSE128
+ALIAS (libat_exchange_16, LSE128, LSE2)
+ALIAS (libat_fetch_or_16, LSE128, LSE2)
+ALIAS (libat_fetch_and_16, LSE128, LSE2)
+ALIAS (libat_or_fetch_16, LSE128, LSE2)
+ALIAS (libat_and_fetch_16, LSE128, LSE2)
+#endif
+ALIAS (libat_load_16, LSE128, LSE2)
+ALIAS (libat_store_16, LSE128, LSE2)
+ALIAS (libat_compare_exchange_16, LSE128, LSE2)
+ALIAS (libat_fetch_add_16, LSE128, LSE2)
+ALIAS (libat_add_fetch_16, LSE128, LSE2)
+ALIAS (libat_fetch_sub_16, LSE128, LSE2)
+ALIAS (libat_sub_fetch_16, LSE128, LSE2)
+ALIAS (libat_fetch_xor_16, LSE128, LSE2)
+ALIAS (libat_xor_fetch_16, LSE128, LSE2)
+ALIAS (libat_fetch_nand_16, LSE128, LSE2)
+ALIAS (libat_nand_fetch_16, LSE128, LSE2)
+ALIAS (libat_test_and_set_16, LSE128, LSE2)
+
 /* Alias entry points which are the same in baseline and LSE2.  */
 
 ALIAS (libat_exchange_16, LSE2, CORE)
diff --git a/libatomic/config/linux/aarch64/host-config.h b/libatomic/config/linux/aarch64/host-config.h
index 8fd4fe3..1bc7d83 100644
--- a/libatomic/config/linux/aarch64/host-config.h
+++ b/libatomic/config/linux/aarch64/host-config.h
@@ -37,14 +37,17 @@ typedef struct __ifunc_arg_t {
 
 #ifdef HWCAP_USCAT
 # if N == 16
-#  define IFUNC_COND_1	ifunc1 (hwcap, features)
+#  define IFUNC_COND_1		(has_lse128 (hwcap, features))
+#  define IFUNC_COND_2		(has_lse2 (hwcap, features))
+#  define IFUNC_NCOND(N)	2
 # else
-#  define IFUNC_COND_1	(hwcap & HWCAP_ATOMICS)
+#  define IFUNC_COND_1		(hwcap & HWCAP_ATOMICS)
+#  define IFUNC_NCOND(N)	1
 # endif
 #else
 #  define IFUNC_COND_1	(false)
+#  define IFUNC_NCOND(N)	1
 #endif
-#define IFUNC_NCOND(N)	(1)
 
 #endif /* HAVE_IFUNC */
 
@@ -59,7 +62,7 @@ typedef struct __ifunc_arg_t {
 #define MIDR_PARTNUM(midr)	(((midr) >> 4) & 0xfff)
 
 static inline bool
-ifunc1 (unsigned long hwcap, const __ifunc_arg_t *features)
+has_lse2 (unsigned long hwcap, const __ifunc_arg_t *features)
 {
   if (hwcap & HWCAP_USCAT)
     return true;
@@ -75,6 +78,37 @@ ifunc1 (unsigned long hwcap, const __ifunc_arg_t *features)
 
   return false;
 }
+
+/* LSE128 atomic support encoded in ID_AA64ISAR0_EL1.Atomic,
+   bits[23:20].  The expected value is 0b0011.  Check that.  */
+
+#define AT_FEAT_FIELD(isar0)	(((isar0) >> 20) & 15)
+
+/* Ensure backwards compatibility with glibc <= 2.38.  */
+#ifndef HWCAP2_LSE128
+#define HWCAP2_LSE128		(1UL << 47)
+#endif
+
+static inline bool
+has_lse128 (unsigned long hwcap, const __ifunc_arg_t *features)
+{
+  if (hwcap & _IFUNC_ARG_HWCAP
+      && features->_hwcap2 & HWCAP2_LSE128)
+    return true;
+  /* A 0 HWCAP2_LSE128 bit may be just as much a sign of missing HWCAP2 bit
+     support in older kernels as it is of CPU feature absence.  Try fallback
+     method to guarantee LSE128 is not implemented.
+
+     In the absence of HWCAP_CPUID, we are unable to check for LSE128.  */
+  if (!(hwcap & HWCAP_CPUID))
+    return false;
+  unsigned long isar0;
+  asm volatile ("mrs %0, ID_AA64ISAR0_EL1" : "=r" (isar0));
+  if (AT_FEAT_FIELD (isar0) >= 3)
+    return true;
+  return false;
+}
+
 #endif
 
 #include_next <host-config.h>