x86/cet: Enable shadow stack during startup

Previously, CET was enabled by kernel before passing control to user
space and the startup code must disable CET if applications or shared
libraries aren't CET enabled.  Since the current kernel only supports
shadow stack and won't enable shadow stack before passing control to
user space, we need to enable shadow stack during startup if the
application and all shared library are shadow stack enabled.  There
is no need to disable shadow stack at startup.  Shadow stack can only
be enabled in a function which will never return.  Otherwise, shadow
stack will underflow at the function return.

1. GL(dl_x86_feature_1) is set to the CET features which are supported
by the processor and are not disabled by the tunable.  Only non-zero
features in GL(dl_x86_feature_1) should be enabled.  After enabling
shadow stack with ARCH_SHSTK_ENABLE, ARCH_SHSTK_STATUS is used to check
if shadow stack is really enabled.
2. Use ARCH_SHSTK_ENABLE in RTLD_START in dynamic executable.  It is
safe since RTLD_START never returns.
3. Call arch_prctl (ARCH_SHSTK_ENABLE) from ARCH_SETUP_TLS in static
executable.  Since the start function using ARCH_SETUP_TLS never returns,
it is safe to enable shadow stack in ARCH_SETUP_TLS.

6 files changed, 93 insertions, 93 deletions

diff --git a/sysdeps/x86/cpu-features-offsets.sym b/sysdeps/x86/cpu-features-offsets.sym
index 6d03cea..5429f60 100644
--- a/sysdeps/x86/cpu-features-offsets.sym
+++ b/sysdeps/x86/cpu-features-offsets.sym
@@ -4,3 +4,4 @@
 
 RTLD_GLOBAL_RO_DL_X86_CPU_FEATURES_OFFSET offsetof (struct rtld_global_ro, _dl_x86_cpu_features)
 XSAVE_STATE_SIZE_OFFSET	offsetof (struct cpu_features, xsave_state_size)
+RTLD_GLOBAL_DL_X86_FEATURE_1_OFFSET offsetof (struct rtld_global, _dl_x86_feature_1)
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index f180f0d..097868c 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -1106,57 +1106,6 @@ no_cpuid:
 	       TUNABLE_CALLBACK (set_x86_ibt));
   TUNABLE_GET (x86_shstk, tunable_val_t *,
 	       TUNABLE_CALLBACK (set_x86_shstk));
-
-  /* Check CET status.  */
-  unsigned int cet_status = get_cet_status ();
-
-  if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT) == 0)
-    CPU_FEATURE_UNSET (cpu_features, IBT)
-  if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK) == 0)
-    CPU_FEATURE_UNSET (cpu_features, SHSTK)
-
-  if (cet_status)
-    {
-      GL(dl_x86_feature_1) = cet_status;
-
-# ifndef SHARED
-      /* Check if IBT and SHSTK are enabled by kernel.  */
-      if ((cet_status
-	   & (GNU_PROPERTY_X86_FEATURE_1_IBT
-	      | GNU_PROPERTY_X86_FEATURE_1_SHSTK)))
-	{
-	  /* Disable IBT and/or SHSTK if they are enabled by kernel, but
-	     disabled by environment variable:
-
-	     GLIBC_TUNABLES=glibc.cpu.hwcaps=-IBT,-SHSTK
-	   */
-	  unsigned int cet_feature = 0;
-	  if (!CPU_FEATURE_USABLE (IBT))
-	    cet_feature |= (cet_status
-			    & GNU_PROPERTY_X86_FEATURE_1_IBT);
-	  if (!CPU_FEATURE_USABLE (SHSTK))
-	    cet_feature |= (cet_status
-			    & GNU_PROPERTY_X86_FEATURE_1_SHSTK);
-
-	  if (cet_feature)
-	    {
-	      int res = dl_cet_disable_cet (cet_feature);
-
-	      /* Clear the disabled bits in dl_x86_feature_1.  */
-	      if (res == 0)
-		GL(dl_x86_feature_1) &= ~cet_feature;
-	    }
-
-	  /* Lock CET if IBT or SHSTK is enabled in executable.  Don't
-	     lock CET if IBT or SHSTK is enabled permissively.  */
-	  if (GL(dl_x86_feature_control).ibt != cet_permissive
-	      && GL(dl_x86_feature_control).shstk != cet_permissive)
-	    dl_cet_lock_cet (GL(dl_x86_feature_1)
-			     & (GNU_PROPERTY_X86_FEATURE_1_IBT
-				| GNU_PROPERTY_X86_FEATURE_1_SHSTK));
-	}
-# endif
-    }
 #endif
 
 #ifndef SHARED
diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c
index 66a7824..82167eb 100644
--- a/sysdeps/x86/dl-cet.c
+++ b/sysdeps/x86/dl-cet.c
@@ -173,40 +173,11 @@ dl_cet_check_startup (struct link_map *m, struct dl_cet_info *info)
     = info->enable_feature_1 ^ info->feature_1_enabled;
   if (disable_feature_1 != 0)
     {
-      /* Disable features in the kernel because of legacy objects or
-	 cet_always_off.  */
-      if (dl_cet_disable_cet (disable_feature_1) != 0)
-	_dl_fatal_printf ("%s: can't disable x86 Features\n",
-			  info->program);
-
       /* Clear the disabled bits.  Sync dl_x86_feature_1 and
          info->feature_1_enabled with info->enable_feature_1.  */
       info->feature_1_enabled = info->enable_feature_1;
       GL(dl_x86_feature_1) = info->enable_feature_1;
     }
-
-  if (HAS_CPU_FEATURE (IBT) || HAS_CPU_FEATURE (SHSTK))
-    {
-      /* Lock CET features only if IBT or SHSTK are enabled and are not
-         enabled permissively.  */
-      unsigned int feature_1_lock = 0;
-
-      if (((info->feature_1_enabled & GNU_PROPERTY_X86_FEATURE_1_IBT)
-	   != 0)
-	  && info->enable_ibt_type != cet_permissive)
-	feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_IBT;
-
-      if (((info->feature_1_enabled & GNU_PROPERTY_X86_FEATURE_1_SHSTK)
-	   != 0)
-	  && info->enable_shstk_type != cet_permissive)
-	feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
-
-      if (feature_1_lock != 0
-	  && dl_cet_lock_cet (feature_1_lock) != 0)
-	_dl_fatal_printf ("%s: can't lock CET\n", info->program);
-    }
-
-  THREAD_SETMEM (THREAD_SELF, header.feature_1, GL(dl_x86_feature_1));
 }
 #endif
 
@@ -298,6 +269,15 @@ dl_cet_check (struct link_map *m, const char *program)
 {
   struct dl_cet_info info;
 
+  /* CET is enabled only if RTLD_START_ENABLE_X86_FEATURES is defined.  */
+#if defined SHARED && defined RTLD_START_ENABLE_X86_FEATURES
+  /* Set dl_x86_feature_1 to features enabled in the executable.  */
+  if (program != NULL)
+    GL(dl_x86_feature_1) = (m->l_x86_feature_1_and
+			    & (X86_FEATURE_1_IBT
+			       | X86_FEATURE_1_SHSTK));
+#endif
+
   /* Check how IBT and SHSTK should be enabled. */
   info.enable_ibt_type = GL(dl_x86_feature_control).ibt;
   info.enable_shstk_type = GL(dl_x86_feature_control).shstk;
@@ -307,17 +287,9 @@ dl_cet_check (struct link_map *m, const char *program)
   /* No legacy object check if IBT and SHSTK are always on.  */
   if (info.enable_ibt_type == cet_always_on
       && info.enable_shstk_type == cet_always_on)
-    {
-#ifdef SHARED
-      /* Set it only during startup.  */
-      if (program != NULL)
-	THREAD_SETMEM (THREAD_SELF, header.feature_1,
-		       info.feature_1_enabled);
-#endif
-      return;
-    }
+    return;
 
-  /* Check if IBT and SHSTK were enabled by kernel.  */
+  /* Check if IBT and SHSTK were enabled.  */
   if (info.feature_1_enabled == 0)
     return;
 
@@ -351,6 +323,32 @@ _dl_cet_open_check (struct link_map *l)
   dl_cet_check (l, NULL);
 }
 
+/* Set GL(dl_x86_feature_1) to the enabled features and clear the
+   active bits of the disabled features.  */
+
+attribute_hidden void
+_dl_cet_setup_features (unsigned int cet_feature)
+{
+  /* NB: cet_feature == GL(dl_x86_feature_1) which is set to features
+     enabled from executable, not necessarily supported by kernel.  */
+  if (cet_feature != 0)
+    {
+      cet_feature = dl_cet_get_cet_status ();
+      if (cet_feature != 0)
+	{
+	  THREAD_SETMEM (THREAD_SELF, header.feature_1, cet_feature);
+
+	  /* Lock CET if IBT or SHSTK is enabled in executable.  Don't
+	     lock CET if IBT or SHSTK is enabled permissively.  */
+	  if (GL(dl_x86_feature_control).ibt != cet_permissive
+	      && (GL(dl_x86_feature_control).shstk != cet_permissive))
+	    dl_cet_lock_cet (cet_feature);
+	}
+      /* Sync GL(dl_x86_feature_1) with kernel.  */
+      GL(dl_x86_feature_1) = cet_feature;
+    }
+}
+
 #ifdef SHARED
 
 # ifndef LINKAGE
diff --git a/sysdeps/x86/get-cpuid-feature-leaf.c b/sysdeps/x86/get-cpuid-feature-leaf.c
index 40a46cc..9317a6b 100644
--- a/sysdeps/x86/get-cpuid-feature-leaf.c
+++ b/sysdeps/x86/get-cpuid-feature-leaf.c
@@ -24,7 +24,7 @@ __x86_get_cpuid_feature_leaf (unsigned int leaf)
   static const struct cpuid_feature feature = {};
   if (leaf < CPUID_INDEX_MAX)
     return ((const struct cpuid_feature *)
-	      &GLRO(dl_x86_cpu_features).features[leaf]);
+	    &GLRO(dl_x86_cpu_features).features[leaf]);
   else
     return &feature;
 }
diff --git a/sysdeps/x86/include/cpu-features.h b/sysdeps/x86/include/cpu-features.h
index 2d7427a..23bd814 100644
--- a/sysdeps/x86/include/cpu-features.h
+++ b/sysdeps/x86/include/cpu-features.h
@@ -990,6 +990,9 @@ extern const struct cpu_features *_dl_x86_get_cpu_features (void)
 # define INIT_ARCH()
 # define _dl_x86_get_cpu_features() (&GLRO(dl_x86_cpu_features))
 extern void _dl_x86_init_cpu_features (void) attribute_hidden;
+
+extern void _dl_cet_setup_features (unsigned int)
+    attribute_hidden;
 #endif
 
 #ifdef __x86_64__
diff --git a/sysdeps/x86/libc-start.h b/sysdeps/x86/libc-start.h
index e93da6e..7a91dd1 100644
--- a/sysdeps/x86/libc-start.h
+++ b/sysdeps/x86/libc-start.h
@@ -19,7 +19,56 @@
 #ifndef SHARED
 # define ARCH_SETUP_IREL() apply_irel ()
 # define ARCH_APPLY_IREL()
-# ifndef ARCH_SETUP_TLS
-#  define ARCH_SETUP_TLS() __libc_setup_tls ()
+# ifdef __CET__
+/* Get CET features enabled in the static executable.  */
+
+static inline unsigned int
+get_cet_feature (void)
+{
+  /* Check if CET is supported and not disabled by tunables.  */
+  const struct cpu_features *cpu_features = __get_cpu_features ();
+  unsigned int cet_feature = 0;
+  if (CPU_FEATURE_USABLE_P (cpu_features, IBT))
+    cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT;
+  if (CPU_FEATURE_USABLE_P (cpu_features, SHSTK))
+    cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
+  if (!cet_feature)
+    return cet_feature;
+
+  struct link_map *main_map = _dl_get_dl_main_map ();
+
+  /* Scan program headers backward to check PT_GNU_PROPERTY early for
+     x86 feature bits on static executable.  */
+  const ElfW(Phdr) *phdr = GL(dl_phdr);
+  const ElfW(Phdr) *ph;
+  for (ph = phdr + GL(dl_phnum); ph != phdr; ph--)
+    if (ph[-1].p_type == PT_GNU_PROPERTY)
+      {
+	_dl_process_pt_gnu_property (main_map, -1, &ph[-1]);
+	/* Enable IBT and SHSTK only if they are enabled on static
+	   executable.  */
+	cet_feature &= (main_map->l_x86_feature_1_and
+			& (GNU_PROPERTY_X86_FEATURE_1_IBT
+			   | GNU_PROPERTY_X86_FEATURE_1_SHSTK));
+	/* Set GL(dl_x86_feature_1) to the enabled CET features.  */
+	GL(dl_x86_feature_1) = cet_feature;
+	break;
+      }
+
+  return cet_feature;
+}
+
+/* The function using this macro to enable shadow stack must not return
+   to avoid shadow stack underflow.  */
+#  define ARCH_SETUP_TLS()						\
+  {									\
+    __libc_setup_tls ();						\
+									\
+    unsigned int cet_feature = get_cet_feature ();			\
+    ENABLE_X86_CET (cet_feature);					\
+    _dl_cet_setup_features (cet_feature);				\
+  }
+# else
+#  define ARCH_SETUP_TLS()	__libc_setup_tls ()
 # endif
 #endif /* !SHARED */