Update.

2003-04-04  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/pthread/createthread.c (create_thread): Add some more
	comments explaining when to set multiple_threads and when not.

	* pthreadP.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
	THREAD_ATOMIC_BIT_SET if not already defined.
	* sysdeps/i386/tls.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
	THREAD_ATOMIC_BIT_SET:
	* sysdeps/x86_64/tls.h: Likewise.
	* cleanup_defer.c (_pthread_cleanup_push_defer): Rewrite to use
	THREAD_ATOMIC_CMPXCHG_VAL.
	(_pthread_cleanup_pop_restore): Likewise.
	* cancellation.c (__pthread_enable_asynccancel): Likewise.
	(__pthread_enable_asynccancel_2): Likewise.
	(__pthread_disable_asynccancel): Likewise.
	* libc-cancellation.c (__libc_enable_asynccancel): Likewise.
	(__libc_disable_asynccancel): Likewise.
	* init.c (sigcancel_handler): Likewise.
	* pthread_setcancelstate.c (__pthread_setcancelstate): Likewise.
	* pthread_setcanceltype.c (__pthread_setcanceltype): Likewise.

10 files changed, 188 insertions, 56 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index d8467e8..7273ece 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,25 @@
+2003-04-04  Ulrich Drepper  <drepper@redhat.com>
+
+	* sysdeps/pthread/createthread.c (create_thread): Add some more
+	comments explaining when to set multiple_threads and when not.
+
+	* pthreadP.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
+	THREAD_ATOMIC_BIT_SET if not already defined.
+	* sysdeps/i386/tls.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
+	THREAD_ATOMIC_BIT_SET:
+	* sysdeps/x86_64/tls.h: Likewise.
+	* cleanup_defer.c (_pthread_cleanup_push_defer): Rewrite to use
+	THREAD_ATOMIC_CMPXCHG_VAL.
+	(_pthread_cleanup_pop_restore): Likewise.
+	* cancellation.c (__pthread_enable_asynccancel): Likewise.
+	(__pthread_enable_asynccancel_2): Likewise.
+	(__pthread_disable_asynccancel): Likewise.
+	* libc-cancellation.c (__libc_enable_asynccancel): Likewise.
+	(__libc_disable_asynccancel): Likewise.
+	* init.c (sigcancel_handler): Likewise.
+	* pthread_setcancelstate.c (__pthread_setcancelstate): Likewise.
+	* pthread_setcanceltype.c (__pthread_setcanceltype): Likewise.
+
 2003-04-03  Ulrich Drepper  <drepper@redhat.com>
 
 	* init.c (sigcancel_handler): Don't set EXITING_BIT here.
diff --git a/nptl/cancellation.c b/nptl/cancellation.c
index d88cae3..c7477d8 100644
--- a/nptl/cancellation.c
+++ b/nptl/cancellation.c
@@ -20,7 +20,6 @@
 #include <setjmp.h>
 #include <stdlib.h>
 #include "pthreadP.h"
-#include "atomic.h"
 
 
 /* The next two functions are similar to pthread_setcanceltype() but
@@ -31,18 +30,18 @@ attribute_hidden
 __pthread_enable_asynccancel (void)
 {
   struct pthread *self = THREAD_SELF;
-  int oldval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -52,6 +51,9 @@ __pthread_enable_asynccancel (void)
 
 	  break;
 	}
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
 
   return oldval;
@@ -63,17 +65,22 @@ internal_function attribute_hidden
 __pthread_enable_asynccancel_2 (int *oldvalp)
 {
   struct pthread *self = THREAD_SELF;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      int oldval = *oldvalp = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      /* We have to store the value before enablying asynchronous
+	 cancellation.  */
+      *oldvalp = oldval;
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -97,17 +104,21 @@ __pthread_disable_asynccancel (int oldtype)
     return;
 
   struct pthread *self = THREAD_SELF;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval & ~CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
 }
diff --git a/nptl/cleanup_defer.c b/nptl/cleanup_defer.c
index 5d6ed03..7945a0d 100644
--- a/nptl/cleanup_defer.c
+++ b/nptl/cleanup_defer.c
@@ -18,7 +18,6 @@
    02111-1307 USA.  */
 
 #include "pthreadP.h"
-#include <atomic.h>
 
 
 void
@@ -37,13 +36,19 @@ _pthread_cleanup_push_defer (buffer, routine, arg)
 
   /* Disable asynchronous cancellation for now.  */
   if (__builtin_expect (cancelhandling & CANCELTYPE_BITMASK, 0))
-    {
-      while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						   cancelhandling
-						   & ~CANCELTYPE_BITMASK,
-						   cancelhandling))
-	cancelhandling = self->cancelhandling;
-    }
+    while (1)
+      {
+	int newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						cancelhandling
+						& ~CANCELTYPE_BITMASK,
+						cancelhandling);
+	if (__builtin_expect (newval == cancelhandling, 1))
+	  /* Successfully replaced the value.  */
+	  break;
+
+	/* Prepare for the next round.  */
+	cancelhandling = newval;
+      }
 
   buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
 			  ? PTHREAD_CANCEL_ASYNCHRONOUS
@@ -53,6 +58,7 @@ _pthread_cleanup_push_defer (buffer, routine, arg)
 }
 strong_alias (_pthread_cleanup_push_defer, __pthread_cleanup_push_defer)
 
+
 void
 _pthread_cleanup_pop_restore (buffer, execute)
      struct _pthread_cleanup_buffer *buffer;
@@ -67,11 +73,19 @@ _pthread_cleanup_pop_restore (buffer, execute)
       && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
 	  & CANCELTYPE_BITMASK) == 0)
     {
-      while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						   cancelhandling
-						   | CANCELTYPE_BITMASK,
-						   cancelhandling))
-	cancelhandling = self->cancelhandling;
+      while (1)
+	{
+	  int newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						  cancelhandling
+						  | CANCELTYPE_BITMASK,
+						  cancelhandling);
+	  if (__builtin_expect (newval == cancelhandling, 1))
+	    /* Successfully replaced the value.  */
+	    break;
+
+	  /* Prepare for the next round.  */
+	  cancelhandling = newval;
+	}
 
       CANCELLATION_P (self);
     }
diff --git a/nptl/init.c b/nptl/init.c
index 9d557cc..7a6883f 100644
--- a/nptl/init.c
+++ b/nptl/init.c
@@ -135,20 +135,21 @@ sigcancel_handler (int sig __attribute ((unused)))
 {
   struct pthread *self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
       /* We are canceled now.  When canceled by another thread this flag
 	 is already set but if the signal is directly send (internally or
 	 from another process) is has to be done here.  */
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELING_BITMASK | CANCELED_BITMASK;
 
       if (oldval == newval || (oldval & EXITING_BITMASK) != 0)
 	/* Already canceled or exiting.  */
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (curval == oldval)
 	{
 	  /* Set the return value.  */
 	  THREAD_SETMEM (self, result, PTHREAD_CANCELED);
@@ -160,6 +161,8 @@ sigcancel_handler (int sig __attribute ((unused)))
 
 	  break;
 	}
+
+      oldval = curval;
     }
 }
 
diff --git a/nptl/libc-cancellation.c b/nptl/libc-cancellation.c
index 344a218..72a6d10 100644
--- a/nptl/libc-cancellation.c
+++ b/nptl/libc-cancellation.c
@@ -33,13 +33,11 @@ attribute_hidden
 __libc_enable_asynccancel (void)
 {
   struct pthread *self = THREAD_SELF;
-  int oldval;
-  int newval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
-  do
+  while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
-      newval = oldval | CANCELTYPE_BITMASK;
+      int newval = oldval | CANCELTYPE_BITMASK;
 
       if (__builtin_expect ((oldval & CANCELED_BITMASK) != 0, 0))
 	{
@@ -47,10 +45,14 @@ __libc_enable_asynccancel (void)
 	  if ((oldval & EXITING_BITMASK) != 0)
 	    break;
 
-	  if (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						    newval, oldval))
-	    /* Somebody else modified the word, try again.  */
-	    continue;
+	  int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						  newval, oldval);
+	  if (__builtin_expect (curval != oldval, 0))
+	    {
+	      /* Somebody else modified the word, try again.  */
+	      oldval = curval;
+	      continue;
+	    }
 
 	  THREAD_SETMEM (self, result, PTHREAD_CANCELED);
 
@@ -58,9 +60,15 @@ __libc_enable_asynccancel (void)
 
 	  /* NOTREACHED */
 	}
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
+	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
-  while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-					       newval, oldval));
 
   return oldval;
 }
@@ -76,19 +84,23 @@ __libc_disable_asynccancel (int oldtype)
     return;
 
   struct pthread *self = THREAD_SELF;
-  int oldval;
-  int newval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
-  do
+  while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
-      newval = oldval & ~CANCELTYPE_BITMASK;
+      int newval = oldval & ~CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
+	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
-  while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling, newval,
-					       oldval));
 }
 
 #endif
diff --git a/nptl/pthread_setcancelstate.c b/nptl/pthread_setcancelstate.c
index a6af063..a452c2e 100644
--- a/nptl/pthread_setcancelstate.c
+++ b/nptl/pthread_setcancelstate.c
@@ -34,9 +34,9 @@ __pthread_setcancelstate (state, oldstate)
 
   self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = (state == PTHREAD_CANCEL_DISABLE
 		    ? oldval | CANCELSTATE_BITMASK
 		    : oldval & ~CANCELSTATE_BITMASK);
@@ -54,14 +54,18 @@ __pthread_setcancelstate (state, oldstate)
 
       /* Update the cancel handling word.  This has to be done
 	 atomically since other bits could be modified as well.  */
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    __do_cancel ();
 
 	  break;
 	}
+
+      /* Prepare for the next round.  */
+      oldval = curval;
     }
 
   return 0;
diff --git a/nptl/pthread_setcanceltype.c b/nptl/pthread_setcanceltype.c
index 5a04635..bbe87ba 100644
--- a/nptl/pthread_setcanceltype.c
+++ b/nptl/pthread_setcanceltype.c
@@ -34,9 +34,9 @@ __pthread_setcanceltype (type, oldtype)
 
   self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = (type == PTHREAD_CANCEL_ASYNCHRONOUS
 		    ? oldval | CANCELTYPE_BITMASK
 		    : oldval & ~CANCELTYPE_BITMASK);
@@ -54,8 +54,9 @@ __pthread_setcanceltype (type, oldtype)
 
       /* Update the cancel handling word.  This has to be done
 	 atomically since other bits could be modified as well.  */
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -65,6 +66,9 @@ __pthread_setcanceltype (type, oldtype)
 
 	  break;
 	}
+
+      /* Prepare for the next round.  */
+      oldval = curval;
     }
 
   return 0;
diff --git a/nptl/sysdeps/i386/tls.h b/nptl/sysdeps/i386/tls.h
index 7fe2222..4f2e818 100644
--- a/nptl/sysdeps/i386/tls.h
+++ b/nptl/sysdeps/i386/tls.h
@@ -169,6 +169,14 @@ union user_desc_init
 # define INIT_SYSINFO
 #endif
 
+#ifndef LOCK
+# ifdef UP
+#  define LOCK  /* nothing */
+# else
+#  define LOCK "lock;"
+# endif
+#endif
+
 /* Code to initially initialize the thread pointer.  This might need
    special attention since 'errno' is not yet available and if the
    operation can cause a failure 'errno' must not be touched.  */
@@ -352,6 +360,32 @@ union user_desc_init
        }})
 
 
+/* Atomic compare and exchange on TLS, returning old value.  */
+#define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \
+  ({ __typeof (descr->member) __ret;					      \
+     __typeof (oldval) __old = (oldval);				      \
+     if (sizeof (descr->member) == 4)					      \
+       asm volatile (LOCK "cmpxchgl %2, %%gs:%P3"			      \
+		     : "=a" (__ret)					      \
+		     : "0" (__old), "r" (newval),			      \
+		       "i" (offsetof (struct pthread, member)));	      \
+     else								      \
+       /* Not necessary for other sizes in the moment.  */		      \
+       abort ();							      \
+     __ret; })
+
+
+/* Atomic set bit.  */
+#define THREAD_ATOMIC_BIT_SET(descr, member, bit) \
+  (void) ({ if (sizeof ((descr)->member) == 4)				      \
+	      asm volatile (LOCK "orl %1, %%gs:%P0"			      \
+			    :: "i" (offsetof (struct pthread, member)),	      \
+			       "ir" (1 << (bit)));			      \
+	    else							      \
+	      /* Not necessary for other sizes in the moment.  */	      \
+	      abort (); })
+
+
 /* Call the user-provided thread function.  */
 #define CALL_THREAD_FCT(descr) \
   ({ void *__res;							      \
diff --git a/nptl/sysdeps/pthread/createthread.c b/nptl/sysdeps/pthread/createthread.c
index fae744f..9d00e4e 100644
--- a/nptl/sysdeps/pthread/createthread.c
+++ b/nptl/sysdeps/pthread/createthread.c
@@ -83,11 +83,11 @@ create_thread (struct pthread *pd, STACK_VARIABLES_PARMS)
 	    /* Failed.  */
 	    return errno;
 
+	  /* We now have for sure more than one thread.  The main
+	     thread might not yet have the flag set.  No need to set
+	     the global variable again if this is what we use.  */
 #ifdef TLS_MULTIPLE_THREADS_IN_TCB
-	  /* We now have for sure more than one thread.  */
-	  pd->header.multiple_threads = 1;
-#else
-	  __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
+	  THREAD_SETMEM (THREAD_SELF, header.multiple_threads, 1);
 #endif
 
 	  /* Now fill in the information about the new thread in
@@ -155,8 +155,10 @@ create_thread (struct pthread *pd, STACK_VARIABLES_PARMS)
     /* Failed.  */
     return errno;
 
+  /* We now have for sure more than one thread.  The main thread might
+     not yet have the flag set.  No need to set the global variable
+     again if this is what we use.  */
 #ifdef TLS_MULTIPLE_THREADS_IN_TCB
-  /* We now have for sure more than one thread.  */
   THREAD_SETMEM (THREAD_SELF, header.multiple_threads, 1);
 #endif
 
diff --git a/nptl/sysdeps/x86_64/tls.h b/nptl/sysdeps/x86_64/tls.h
index f382db2..dec1b5d 100644
--- a/nptl/sysdeps/x86_64/tls.h
+++ b/nptl/sysdeps/x86_64/tls.h
@@ -253,6 +253,32 @@ typedef struct
        }})
 
 
+/* Atomic compare and exchange on TLS, returning old value.  */
+#define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \
+  ({ __typeof (descr->member) __ret;					      \
+     __typeof (oldval) __old = (oldval);				      \
+     if (sizeof (descr->member) == 4)					      \
+       asm volatile (LOCK "cmpxchgl %2, %%fs:%P3"			      \
+		     : "=a" (__ret)					      \
+		     : "0" (__old), "r" (newval),			      \
+		       "i" (offsetof (struct pthread, member)));	      \
+     else								      \
+       /* Not necessary for other sizes in the moment.  */		      \
+       abort ();							      \
+     __ret; })
+
+
+/* Atomic set bit.  */
+#define THREAD_ATOMIC_BIT_SET(descr, member, bit) \
+  (void) ({ if (sizeof ((descr)->member) == 4)				      \
+	      asm volatile (LOCK "orl %1, %%fs:%P0"			      \
+			    :: "i" (offsetof (struct pthread, member)),	      \
+			       "ir" (1 << (bit)));			      \
+	    else							      \
+	      /* Not necessary for other sizes in the moment.  */	      \
+	      abort (); })
+
+
 #define CALL_THREAD_FCT(descr) \
   ({ void *__res;							      \
      asm volatile ("movq %%fs:%P2, %%rdi\n\t"				      \