nptl: Move legacy unwinding implementation into libc

It is still used internally.  Since unwinding is now available
unconditionally, avoid indirect calls through function pointers loaded
from the stack by inlining the non-cancellation cleanup code.  This
avoids a regression in security hardening.

The out-of-line  __libc_cleanup_routine implementation is no longer
needed because the inline definition is now static __always_inline.

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>

4 files changed, 66 insertions, 58 deletions

diff --git a/nptl/Versions b/nptl/Versions
index 60202b4..2e5a964 100644
--- a/nptl/Versions
+++ b/nptl/Versions
@@ -87,6 +87,8 @@ libc {
     __futex_abstimed_wait64;
     __futex_abstimed_wait_cancelable64;
     __libc_alloca_cutoff;
+    __libc_cleanup_pop_restore;
+    __libc_cleanup_push_defer;
     __libc_dl_error_tsd;
     __libc_pthread_init;
     __lll_clocklock_elision;
diff --git a/nptl/cleanup_defer_compat.c b/nptl/cleanup_defer_compat.c
index 49ef53e..1957318 100644
--- a/nptl/cleanup_defer_compat.c
+++ b/nptl/cleanup_defer_compat.c
@@ -17,41 +17,15 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include "pthreadP.h"
-
+#include <libc-lock.h>
 
 void
 _pthread_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer,
 			     void (*routine) (void *), void *arg)
 {
-  struct pthread *self = THREAD_SELF;
-
   buffer->__routine = routine;
   buffer->__arg = arg;
-  buffer->__prev = THREAD_GETMEM (self, cleanup);
-
-  int cancelhandling = THREAD_GETMEM (self, cancelhandling);
-
-  /* Disable asynchronous cancellation for now.  */
-  if (__glibc_unlikely (cancelhandling & CANCELTYPE_BITMASK))
-    while (1)
-      {
-	int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
-						cancelhandling
-						& ~CANCELTYPE_BITMASK,
-						cancelhandling);
-	if (__glibc_likely (curval == cancelhandling))
-	  /* Successfully replaced the value.  */
-	  break;
-
-	/* Prepare for the next round.  */
-	cancelhandling = curval;
-      }
-
-  buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
-			  ? PTHREAD_CANCEL_ASYNCHRONOUS
-			  : PTHREAD_CANCEL_DEFERRED);
-
-  THREAD_SETMEM (self, cleanup, buffer);
+  __libc_cleanup_push_defer (buffer);
 }
 strong_alias (_pthread_cleanup_push_defer, __pthread_cleanup_push_defer)
 
@@ -60,31 +34,7 @@ void
 _pthread_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer,
 			      int execute)
 {
-  struct pthread *self = THREAD_SELF;
-
-  THREAD_SETMEM (self, cleanup, buffer->__prev);
-
-  int cancelhandling;
-  if (__builtin_expect (buffer->__canceltype != PTHREAD_CANCEL_DEFERRED, 0)
-      && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
-	  & CANCELTYPE_BITMASK) == 0)
-    {
-      while (1)
-	{
-	  int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
-						  cancelhandling
-						  | CANCELTYPE_BITMASK,
-						  cancelhandling);
-	  if (__glibc_likely (curval == cancelhandling))
-	    /* Successfully replaced the value.  */
-	    break;
-
-	  /* Prepare for the next round.  */
-	  cancelhandling = curval;
-	}
-
-      CANCELLATION_P (self);
-    }
+  __libc_cleanup_pop_restore (buffer);
 
   /* If necessary call the cleanup routine after we removed the
      current cleanup block from the list.  */
diff --git a/nptl/libc-cleanup.c b/nptl/libc-cleanup.c
index 61f97ec..14ccfe9 100644
--- a/nptl/libc-cleanup.c
+++ b/nptl/libc-cleanup.c
@@ -17,11 +17,69 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include "pthreadP.h"
+#include <tls.h>
+#include <libc-lock.h>
 
+void
+__libc_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer)
+{
+  struct pthread *self = THREAD_SELF;
+
+  buffer->__prev = THREAD_GETMEM (self, cleanup);
+
+  int cancelhandling = THREAD_GETMEM (self, cancelhandling);
+
+  /* Disable asynchronous cancellation for now.  */
+  if (__glibc_unlikely (cancelhandling & CANCELTYPE_BITMASK))
+    while (1)
+      {
+	int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						cancelhandling
+						& ~CANCELTYPE_BITMASK,
+						cancelhandling);
+	if (__glibc_likely (curval == cancelhandling))
+	  /* Successfully replaced the value.  */
+	  break;
+
+	/* Prepare for the next round.  */
+	cancelhandling = curval;
+      }
+
+  buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
+			  ? PTHREAD_CANCEL_ASYNCHRONOUS
+			  : PTHREAD_CANCEL_DEFERRED);
+
+  THREAD_SETMEM (self, cleanup, buffer);
+}
+libc_hidden_def (__libc_cleanup_push_defer)
 
 void
-__libc_cleanup_routine (struct __pthread_cleanup_frame *f)
+__libc_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer)
 {
-  if (f->__do_it)
-    f->__cancel_routine (f->__cancel_arg);
+  struct pthread *self = THREAD_SELF;
+
+  THREAD_SETMEM (self, cleanup, buffer->__prev);
+
+  int cancelhandling;
+  if (__builtin_expect (buffer->__canceltype != PTHREAD_CANCEL_DEFERRED, 0)
+      && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
+	  & CANCELTYPE_BITMASK) == 0)
+    {
+      while (1)
+	{
+	  int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						  cancelhandling
+						  | CANCELTYPE_BITMASK,
+						  cancelhandling);
+	  if (__glibc_likely (curval == cancelhandling))
+	    /* Successfully replaced the value.  */
+	    break;
+
+	  /* Prepare for the next round.  */
+	  cancelhandling = curval;
+	}
+
+      CANCELLATION_P (self);
+    }
 }
+libc_hidden_def (__libc_cleanup_pop_restore)
diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c
index 2c7e222..43e2564 100644
--- a/nptl/nptl-init.c
+++ b/nptl/nptl-init.c
@@ -95,8 +95,6 @@ static const struct pthread_functions pthread_functions =
     .ptr___pthread_key_create = __pthread_key_create,
     .ptr___pthread_getspecific = __pthread_getspecific,
     .ptr___pthread_setspecific = __pthread_setspecific,
-    .ptr__pthread_cleanup_push_defer = __pthread_cleanup_push_defer,
-    .ptr__pthread_cleanup_pop_restore = __pthread_cleanup_pop_restore,
     .ptr_nthreads = &__nptl_nthreads,
     .ptr___pthread_unwind = &__pthread_unwind,
     .ptr__nptl_deallocate_tsd = __nptl_deallocate_tsd,