x86/cet: Sync with Linux kernel 6.6 shadow stack interface

Sync with Linux kernel 6.6 shadow stack interface.  Since only x86-64 is
supported, i386 shadow stack codes are unchanged and CET shouldn't be
enabled for i386.

1. When the shadow stack base in TCB is unset, the default shadow stack
is in use.  Use the current shadow stack pointer as the marker for the
default shadow stack. It is used to identify if the current shadow stack
is the same as the target shadow stack when switching ucontexts.  If yes,
INCSSP will be used to unwind shadow stack.  Otherwise, shadow stack
restore token will be used.
2. Allocate shadow stack with the map_shadow_stack syscall.  Since there
is no function to explicitly release ucontext, there is no place to
release shadow stack allocated by map_shadow_stack in ucontext functions.
Such shadow stacks will be leaked.
3. Rename arch_prctl CET commands to ARCH_SHSTK_XXX.
4. Rewrite the CET control functions with the current kernel shadow stack
interface.

Since CET is no longer enabled by kernel, a separate patch will enable
shadow stack during startup.

7 files changed, 111 insertions, 88 deletions

diff --git a/sysdeps/unix/sysv/linux/x86_64/Makefile b/sysdeps/unix/sysv/linux/x86_64/Makefile
index 5e19202..06b8739 100644
--- a/sysdeps/unix/sysv/linux/x86_64/Makefile
+++ b/sysdeps/unix/sysv/linux/x86_64/Makefile
@@ -3,7 +3,7 @@ sysdep_routines += ioperm iopl
 endif
 
 ifeq ($(subdir),stdlib)
-sysdep_routines += __start_context
+sysdep_routines += __start_context allocate-shadow-stack
 endif
 
 ifeq ($(subdir),csu)
diff --git a/sysdeps/unix/sysv/linux/x86_64/__start_context.S b/sysdeps/unix/sysv/linux/x86_64/__start_context.S
index f6436dd..ae04203 100644
--- a/sysdeps/unix/sysv/linux/x86_64/__start_context.S
+++ b/sysdeps/unix/sysv/linux/x86_64/__start_context.S
@@ -24,20 +24,14 @@
 /* Use CALL to push __start_context onto the new stack as well as the new
    shadow stack.  RDI points to ucontext:
    Incoming:
-     __ssp[0]: The original caller's shadow stack pointer.
-     __ssp[1]: The size of the new shadow stack.
-     __ssp[2]: The size of the new shadow stack.
-   Outgoing:
      __ssp[0]: The new shadow stack pointer.
      __ssp[1]: The base address of the new shadow stack.
      __ssp[2]: The size of the new shadow stack.
  */
 
 ENTRY(__push___start_context)
-	/* Save the pointer to ucontext.  */
-	movq	%rdi, %r9
 	/* Get the original shadow stack pointer.  */
-	rdsspq	%r8
+	rdsspq	%rcx
 	/* Save the original stack pointer.  */
 	movq	%rsp, %rdx
 	/* Load the top of the new stack into RSI.  */
@@ -45,24 +39,12 @@ ENTRY(__push___start_context)
 	/* Add 8 bytes to RSI since CALL will push the 8-byte return
 	   address onto stack.  */
 	leaq	8(%rsi), %rsp
-	/* Allocate the new shadow stack.  The size of the new shadow
-	   stack is passed in __ssp[1].  */
-	lea	(oSSP + 8)(%rdi), %RSI_LP
-	movl	$ARCH_CET_ALLOC_SHSTK, %edi
-	movl	$__NR_arch_prctl, %eax
-	/* The new shadow stack base is returned in __ssp[1].  */
-	syscall
-	testq	%rax, %rax
-	jne	L(hlt)		/* This should never happen.  */
-
-	/* Get the size of the new shadow stack.  */
-	movq	8(%rsi), %rdi
-
-	/* Get the base address of the new shadow stack.  */
-	movq	(%rsi), %rsi
-
+	/* The size of the new shadow stack is stored in __ssp[2].  */
+	mov	(oSSP + 16)(%rdi), %RSI_LP
+	/* The new shadow stack base is stored in __ssp[1].  */
+	mov	(oSSP + 8)(%rdi), %RAX_LP
 	/* Use the restore stoken to restore the new shadow stack.  */
-	rstorssp -8(%rsi, %rdi)
+	rstorssp -8(%rax, %rsi)
 
 	/* Save the restore token on the original shadow stack.  */
 	saveprevssp
@@ -73,18 +55,12 @@ ENTRY(__push___start_context)
 	jmp	__start_context
 1:
 
-	/* Get the new shadow stack pointer.  */
-	rdsspq	%rdi
-
 	/* Use the restore stoken to restore the original shadow stack.  */
-	rstorssp -8(%r8)
+	rstorssp -8(%rcx)
 
 	/* Save the restore token on the new shadow stack.  */
 	saveprevssp
 
-	/* Store the new shadow stack pointer in __ssp[0].  */
-	movq	%rdi, oSSP(%r9)
-
 	/* Restore the original stack.  */
 	mov	%rdx, %rsp
 	ret
diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c
new file mode 100644
index 0000000..f2e1d03
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c
@@ -0,0 +1,55 @@
+/* Helper function to allocate shadow stack.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <sysdep.h>
+#include <stdint.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <libc-pointer-arith.h>
+#include <allocate-shadow-stack.h>
+
+/* NB: This can be treated as a syscall by caller.  */
+
+long int
+__allocate_shadow_stack (size_t stack_size,
+			 shadow_stack_size_t *child_stack)
+{
+#ifdef __NR_map_shadow_stack
+  size_t shadow_stack_size
+    = stack_size >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT;
+  /* Align shadow stack to 8 bytes.  */
+  shadow_stack_size = ALIGN_UP (shadow_stack_size, 8);
+  /* Since sigaltstack shares shadow stack with the current context in
+     the thread, add extra 20 stack frames in shadow stack for signal
+     handlers.  */
+  shadow_stack_size += 20 * 8;
+  void *shadow_stack = (void *)INLINE_SYSCALL_CALL
+    (map_shadow_stack, NULL, shadow_stack_size, SHADOW_STACK_SET_TOKEN);
+  /* Report the map_shadow_stack error.  */
+  if (shadow_stack == MAP_FAILED)
+    return -errno;
+
+  /* Save the shadow stack base and size on child stack.  */
+  child_stack[0] = (uintptr_t) shadow_stack;
+  child_stack[1] = shadow_stack_size;
+
+  return 0;
+#else
+  return -ENOSYS;
+#endif
+}
diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h
new file mode 100644
index 0000000..d05aaf1
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h
@@ -0,0 +1,24 @@
+/* Helper function to allocate shadow stack.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <ucontext.h>
+
+typedef __typeof (((ucontext_t *) 0)->__ssp[0]) shadow_stack_size_t;
+
+extern long int __allocate_shadow_stack (size_t, shadow_stack_size_t *)
+  attribute_hidden;
diff --git a/sysdeps/unix/sysv/linux/x86_64/getcontext.S b/sysdeps/unix/sysv/linux/x86_64/getcontext.S
index a00e2f6..71f3802 100644
--- a/sysdeps/unix/sysv/linux/x86_64/getcontext.S
+++ b/sysdeps/unix/sysv/linux/x86_64/getcontext.S
@@ -58,35 +58,15 @@ ENTRY(__getcontext)
 	testl	$X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET
 	jz	L(no_shstk)
 
-	/* Save RDI in RDX which won't be clobbered by syscall.  */
-	movq	%rdi, %rdx
-
 	xorl	%eax, %eax
 	cmpq	%fs:SSP_BASE_OFFSET, %rax
 	jnz	L(shadow_stack_bound_recorded)
 
-	/* Get the base address and size of the default shadow stack
-	   which must be the current shadow stack since nothing has
-	   been recorded yet.  */
-	sub	$24, %RSP_LP
-	mov	%RSP_LP, %RSI_LP
-	movl	$ARCH_CET_STATUS, %edi
-	movl	$__NR_arch_prctl, %eax
-	syscall
-	testq	%rax, %rax
-	jz	L(continue_no_err)
-
-	/* This should never happen.  */
-	hlt
-
-L(continue_no_err):
-	/* Record the base of the current shadow stack.  */
-	movq	8(%rsp), %rax
+	/* When the shadow stack base is unset, the default shadow
+	   stack is in use.  Use the current shadow stack pointer
+	   as the marker for the default shadow stack.  */
+	rdsspq	%rax
 	movq	%rax, %fs:SSP_BASE_OFFSET
-	add	$24, %RSP_LP
-
-	/* Restore RDI.  */
-	movq	%rdx, %rdi
 
 L(shadow_stack_bound_recorded):
 	/* Get the current shadow stack pointer.  */
@@ -94,7 +74,7 @@ L(shadow_stack_bound_recorded):
 	/* NB: Save the caller's shadow stack so that we can jump back
 	   to the caller directly.  */
 	addq	$8, %rax
-	movq	%rax, oSSP(%rdx)
+	movq	%rax, oSSP(%rdi)
 
 	/* Save the current shadow stack base in ucontext.  */
 	movq	%fs:SSP_BASE_OFFSET, %rax
diff --git a/sysdeps/unix/sysv/linux/x86_64/makecontext.c b/sysdeps/unix/sysv/linux/x86_64/makecontext.c
index de9e03e..e4f025b 100644
--- a/sysdeps/unix/sysv/linux/x86_64/makecontext.c
+++ b/sysdeps/unix/sysv/linux/x86_64/makecontext.c
@@ -24,6 +24,7 @@
 # include <pthread.h>
 # include <libc-pointer-arith.h>
 # include <sys/prctl.h>
+# include <allocate-shadow-stack.h>
 #endif
 
 #include "ucontext_i.h"
@@ -88,23 +89,24 @@ __makecontext (ucontext_t *ucp, void (*func) (void), int argc, ...)
   if ((feature_1 & X86_FEATURE_1_SHSTK) != 0)
     {
       /* Shadow stack is enabled.  We need to allocate a new shadow
-         stack.  */
-      unsigned long ssp_size = (((uintptr_t) sp
-				 - (uintptr_t) ucp->uc_stack.ss_sp)
-				>> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT);
-      /* Align shadow stack to 8 bytes.  */
-      ssp_size = ALIGN_UP (ssp_size, 8);
-
-      ucp->__ssp[1] = ssp_size;
-      ucp->__ssp[2] = ssp_size;
-
-      /* Call __push___start_context to allocate a new shadow stack,
-	 push __start_context onto the new stack as well as the new
-	 shadow stack.  NB: After __push___start_context returns,
+         stack.  NB:
 	   ucp->__ssp[0]: The new shadow stack pointer.
 	   ucp->__ssp[1]: The base address of the new shadow stack.
 	   ucp->__ssp[2]: The size of the new shadow stack.
        */
+      long int ret
+	= __allocate_shadow_stack (((uintptr_t) sp
+				    - (uintptr_t) ucp->uc_stack.ss_sp),
+				   &ucp->__ssp[1]);
+      if (ret != 0)
+	{
+	  /* FIXME: What should we do?  */
+	  abort ();
+	}
+
+      ucp->__ssp[0] = ucp->__ssp[1] + ucp->__ssp[2] - 8;
+      /* Call __push___start_context to push __start_context onto the new
+	 stack as well as the new shadow stack.  */
       __push___start_context (ucp);
     }
   else
diff --git a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S
index 5925752..2f2fe98 100644
--- a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S
+++ b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S
@@ -109,25 +109,11 @@ ENTRY(__swapcontext)
 	cmpq	%fs:SSP_BASE_OFFSET, %rax
 	jnz	L(shadow_stack_bound_recorded)
 
-	/* Get the base address and size of the default shadow stack
-	   which must be the current shadow stack since nothing has
-	   been recorded yet.  */
-	sub	$24, %RSP_LP
-	mov	%RSP_LP, %RSI_LP
-	movl	$ARCH_CET_STATUS, %edi
-	movl	$__NR_arch_prctl, %eax
-	syscall
-	testq	%rax, %rax
-	jz	L(continue_no_err)
-
-	/* This should never happen.  */
-	hlt
-
-L(continue_no_err):
-	/* Record the base of the current shadow stack.  */
-	movq	8(%rsp), %rax
+	/* When the shadow stack base is unset, the default shadow
+	   stack is in use.  Use the current shadow stack pointer
+	   as the marker for the default shadow stack.  */
+	rdsspq	%rax
 	movq	%rax, %fs:SSP_BASE_OFFSET
-	add	$24, %RSP_LP
 
 L(shadow_stack_bound_recorded):
         /* If we unwind the stack, we can't undo stack unwinding.  Just