x86_64: Use __seg_fs qualifiers in NPTL accessors

Use __seg_fs named address space qualifiers to cast NPTL accessors
to %fs: prefixed addresses.  Use volatile access only where
strictly necessary.

Use existing assembly RSEQ_* accessors for x32 to
work around the GCC bug:

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121613

because negative value in __rseq_offset is used
as an offset from %fs.

Co-Authored-By: H.J. Lu <hjl.tools@gmail.com>
Signed-off-by: H.J. Lu <hjl.tools@gmail.com>
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
Cc: Florian Weimer <fweimer@redhat.com>
Cc: Carlos O'Donell <carlos@redhat.com>

4 files changed, 105 insertions, 96 deletions

diff --git a/sysdeps/unix/sysv/linux/x86_64/64/Implies b/sysdeps/unix/sysv/linux/x86_64/64/Implies
new file mode 100644
index 0000000..f379d95
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86_64/64/Implies
@@ -0,0 +1 @@
+x86_64/64/nptl
diff --git a/sysdeps/x86_64/64/nptl/rseq-access.h b/sysdeps/x86_64/64/nptl/rseq-access.h
new file mode 100644
index 0000000..214cb96
--- /dev/null
+++ b/sysdeps/x86_64/64/nptl/rseq-access.h
@@ -0,0 +1,61 @@
+/* RSEQ_* accessors.  x86_64 version.
+   Copyright (C) 2002-2025 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Read member of the RSEQ area directly.  */
+#define RSEQ_GETMEM(member) \
+  ({									      \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1			      \
+		     || sizeof (RSEQ_SELF()->member) == 4		      \
+		     || sizeof (RSEQ_SELF()->member) == 8,		      \
+		     "size of rseq data");				      \
+     (*(__typeof (RSEQ_SELF()->member) __seg_fs *)			      \
+      (__rseq_offset + offsetof (struct rseq_area, member)));		      \
+  })
+
+/* Read member of the RSEQ area directly, with single-copy atomicity semantics.  */
+#define RSEQ_GETMEM_ONCE(member) \
+  ({									      \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1			      \
+		     || sizeof (RSEQ_SELF()->member) == 4		      \
+		     || sizeof (RSEQ_SELF()->member) == 8,		      \
+		     "size of rseq data");				      \
+     (*(volatile __typeof (RSEQ_SELF()->member) __seg_fs *)		      \
+      (__rseq_offset + offsetof (struct rseq_area, member)));		      \
+  })
+
+/* Set member of the RSEQ area directly.  */
+#define RSEQ_SETMEM(member, value) \
+  ({									      \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1			      \
+		     || sizeof (RSEQ_SELF()->member) == 4		      \
+		     || sizeof (RSEQ_SELF()->member) == 8,		      \
+		     "size of rseq data");				      \
+     (*(__typeof (RSEQ_SELF()->member) __seg_fs *)			      \
+      (__rseq_offset + offsetof (struct rseq_area, member)) = (value));	      \
+  })
+
+/* Set member of the RSEQ area directly, with single-copy atomicity semantics.  */
+#define RSEQ_SETMEM_ONCE(member, value) \
+  ({									      \
+     _Static_assert (sizeof (RSEQ_SELF()->member) == 1			      \
+		     || sizeof (RSEQ_SELF()->member) == 4		      \
+		     || sizeof (RSEQ_SELF()->member) == 8,		      \
+		     "size of rseq data");				      \
+     (*(volatile __typeof (RSEQ_SELF()->member) __seg_fs *)		      \
+      (__rseq_offset + offsetof (struct rseq_area, member)) = (value));	      \
+  })
diff --git a/sysdeps/x86_64/nptl/tcb-access.h b/sysdeps/x86_64/nptl/tcb-access.h
index defd76f..445f328 100644
--- a/sysdeps/x86_64/nptl/tcb-access.h
+++ b/sysdeps/x86_64/nptl/tcb-access.h
@@ -17,116 +17,55 @@
    <https://www.gnu.org/licenses/>.  */
 
 /* Read member of the thread descriptor directly.  */
-# define THREAD_GETMEM(descr, member) \
-  ({ __typeof (descr->member) __value;					      \
-     _Static_assert (sizeof (__value) == 1				      \
-		     || sizeof (__value) == 4				      \
-		     || sizeof (__value) == 8,				      \
+#define THREAD_GETMEM(descr, member) \
+  ({									      \
+     _Static_assert (sizeof (descr->member) == 1       			      \
+		     || sizeof (descr->member) == 4			      \
+		     || sizeof (descr->member) == 8,			      \
 		     "size of per-thread data");			      \
-     if (sizeof (__value) == 1)						      \
-       asm volatile ("movb %%fs:%P2,%b0"				      \
-		     : "=q" (__value)					      \
-		     : "0" (0), "i" (offsetof (struct pthread, member)));     \
-     else if (sizeof (__value) == 4)					      \
-       asm volatile ("movl %%fs:%P1,%0"					      \
-		     : "=r" (__value)					      \
-		     : "i" (offsetof (struct pthread, member)));	      \
-     else /* 8 */								      \
-       {								      \
-	 asm volatile ("movq %%fs:%P1,%q0"				      \
-		       : "=r" (__value)					      \
-		       : "i" (offsetof (struct pthread, member)));	      \
-       }								      \
-     __value; })
+     (*(__typeof (descr->member) __seg_fs *)				      \
+      offsetof (struct pthread, member));				      \
+  })
 
-/* THREAD_GETMEM already forces a read.  */
-#define THREAD_GETMEM_VOLATILE(descr, member) THREAD_GETMEM (descr, member)
+#define THREAD_GETMEM_VOLATILE(descr, member) \
+  ({									      \
+     _Static_assert (sizeof (descr->member) == 1			      \
+		     || sizeof (descr->member) == 4			      \
+		     || sizeof (descr->member) == 8,			      \
+		     "size of per-thread data");			      \
+     (*(volatile __typeof (descr->member) __seg_fs *)			      \
+      offsetof (struct pthread, member));				      \
+  })
 
 /* Same as THREAD_GETMEM, but the member offset can be non-constant.  */
-# define THREAD_GETMEM_NC(descr, member, idx) \
-  ({ __typeof (descr->member[0]) __value;				      \
-     _Static_assert (sizeof (__value) == 1				      \
-		     || sizeof (__value) == 4				      \
-		     || sizeof (__value) == 8,				      \
+#define THREAD_GETMEM_NC(descr, member, idx) \
+  ({									      \
+     _Static_assert (sizeof (descr->member[0]) == 1			      \
+		     || sizeof (descr->member[0]) == 4			      \
+		     || sizeof (descr->member[0]) == 8,			      \
 		     "size of per-thread data");			      \
-     if (sizeof (__value) == 1)						      \
-       asm volatile ("movb %%fs:%P2(%q3),%b0"				      \
-		     : "=q" (__value)					      \
-		     : "0" (0), "i" (offsetof (struct pthread, member[0])),   \
-		       "r" (idx));					      \
-     else if (sizeof (__value) == 4)					      \
-       asm volatile ("movl %%fs:%P1(,%q2,4),%0"				      \
-		     : "=r" (__value)					      \
-		     : "i" (offsetof (struct pthread, member[0])), "r" (idx));\
-     else /* 8 */							      \
-       {								      \
-	 asm volatile ("movq %%fs:%P1(,%q2,8),%q0"			      \
-		       : "=r" (__value)					      \
-		       : "i" (offsetof (struct pthread, member[0])),	      \
-			 "r" (idx));					      \
-       }								      \
-     __value; })
-
-
-/* Loading addresses of objects on x86-64 needs to be treated special
-   when generating PIC code.  */
-#ifdef __pic__
-# define IMM_MODE "nr"
-#else
-# define IMM_MODE "ir"
-#endif
-
+     (*(__typeof (descr->member[0]) __seg_fs *)				      \
+      offsetof (struct pthread, member[idx]));				      \
+  })
 
 /* Set member of the thread descriptor directly.  */
-# define THREAD_SETMEM(descr, member, value) \
+#define THREAD_SETMEM(descr, member, value) \
   ({									      \
      _Static_assert (sizeof (descr->member) == 1			      \
 		     || sizeof (descr->member) == 4			      \
 		     || sizeof (descr->member) == 8,			      \
 		     "size of per-thread data");			      \
-     if (sizeof (descr->member) == 1)					      \
-       asm volatile ("movb %b0,%%fs:%P1" :				      \
-		     : "iq" (value),					      \
-		       "i" (offsetof (struct pthread, member)));	      \
-     else if (sizeof (descr->member) == 4)				      \
-       asm volatile ("movl %0,%%fs:%P1" :				      \
-		     : IMM_MODE (value),				      \
-		       "i" (offsetof (struct pthread, member)));	      \
-     else /* 8 */							      \
-       {								      \
-	 /* Since movq takes a signed 32-bit immediate or a register source   \
-	    operand, use "er" constraint for 32-bit signed integer constant   \
-	    or register.  */						      \
-	 asm volatile ("movq %q0,%%fs:%P1" :				      \
-		       : "er" ((uint64_t) cast_to_integer (value)),	      \
-			 "i" (offsetof (struct pthread, member)));	      \
-       }})
-
+     (*(__typeof (descr->member) __seg_fs *)				      \
+      offsetof (struct pthread, member) = (value));			      \
+  })
 
 /* Same as THREAD_SETMEM, but the member offset can be non-constant.  */
-# define THREAD_SETMEM_NC(descr, member, idx, value) \
+#define THREAD_SETMEM_NC(descr, member, idx, value) \
   ({									      \
      _Static_assert (sizeof (descr->member[0]) == 1			      \
 		     || sizeof (descr->member[0]) == 4			      \
 		     || sizeof (descr->member[0]) == 8,			      \
 		     "size of per-thread data");			      \
-     if (sizeof (descr->member[0]) == 1)				      \
-       asm volatile ("movb %b0,%%fs:%P1(%q2)" :				      \
-		     : "iq" (value),					      \
-		       "i" (offsetof (struct pthread, member[0])),	      \
-		       "r" (idx));					      \
-     else if (sizeof (descr->member[0]) == 4)				      \
-       asm volatile ("movl %0,%%fs:%P1(,%q2,4)" :			      \
-		     : IMM_MODE (value),				      \
-		       "i" (offsetof (struct pthread, member[0])),	      \
-		       "r" (idx));					      \
-     else /* 8 */							      \
-       {								      \
-	 /* Since movq takes a signed 32-bit immediate or a register source   \
-	    operand, use "er" constraint for 32-bit signed integer constant   \
-	    or register.  */						      \
-	 asm volatile ("movq %q0,%%fs:%P1(,%q2,8)" :			      \
-		       : "er" ((uint64_t) cast_to_integer (value)),	      \
-			 "i" (offsetof (struct pthread, member[0])),	      \
-			 "r" (idx));					      \
-       }})
+     (*(__typeof (descr->member[0]) __seg_fs *)				      \
+      offsetof (struct pthread, member[idx]) = (value));		      \
+  })
diff --git a/sysdeps/x86_64/nptl/rseq-access.h b/sysdeps/x86_64/x32/nptl/rseq-access.h
index bc966b2..8386ebd 100644
--- a/sysdeps/x86_64/nptl/rseq-access.h
+++ b/sysdeps/x86_64/x32/nptl/rseq-access.h
@@ -1,5 +1,5 @@
-/* RSEQ_* accessors.  x86_64 version.
-   Copyright (C) 2002-2025 Free Software Foundation, Inc.
+/* RSEQ_* accessors.  x32 version.
+   Copyright (C) 2025 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -45,6 +45,14 @@
 /* Read member of the RSEQ area directly.  */
 #define RSEQ_GETMEM(member) RSEQ_GETMEM_ONCE(member)
 
+/* Loading addresses of objects on x86-64 needs to be treated special
+   when generating PIC code.  */
+#ifdef __pic__
+# define IMM_MODE "nr"
+#else
+# define IMM_MODE "ir"
+#endif
+
 /* Set member of the RSEQ area directly, with single-copy atomicity semantics.  */
 #define RSEQ_SETMEM_ONCE(member, value) \
   ({									      \