[BZ #357]

Update.
2004-09-02  Steven Munroe  <sjmunroe@us.ibm.com>

	[BZ #357]
	* stdlib/tst-setcontext.c (test_stack): Added test for stack clobber.
	(main): Call test_stack.
	* sysdeps/unix/sysv/linux/powerpc/powerpc32/getcontext.S
	(__getcontext): Push stack frame then save parms in local frame.
	Improve instruction scheduling.
	* sysdeps/unix/sysv/linux/powerpc/powerpc32/swapcontext.S
	(__swapcontext): Likewise.

16 files changed, 217 insertions, 30 deletions

diff --git a/nptl/sysdeps/pthread/pthread_cond_timedwait.c b/nptl/sysdeps/pthread/pthread_cond_timedwait.c
index 7de2b29..c6606c9 100644
--- a/nptl/sysdeps/pthread/pthread_cond_timedwait.c
+++ b/nptl/sysdeps/pthread/pthread_cond_timedwait.c
@@ -67,6 +67,7 @@ __pthread_cond_timedwait (cond, mutex, abstime)
   /* We have one new user of the condvar.  */
   ++cond->__data.__total_seq;
   ++cond->__data.__futex;
+  cond->__data.__nwaiters += 1 << COND_CLOCK_BITS;
 
   /* Remember the mutex we are using here.  If there is already a
      different address store this is a bad user bug.  Do not store
@@ -98,7 +99,8 @@ __pthread_cond_timedwait (cond, mutex, abstime)
 	INTERNAL_SYSCALL_DECL (err);
 	int ret;
 	ret = INTERNAL_SYSCALL (clock_gettime, err, 2,
-				cond->__data.__clock, &rt);
+				cond->__data.__nwaiters & COND_CLOCK_BITS,
+				&rt);
 # ifndef __ASSUME_POSIX_TIMERS
 	if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (ret, err), 0))
 	  {
@@ -185,6 +187,16 @@ __pthread_cond_timedwait (cond, mutex, abstime)
   ++cond->__data.__woken_seq;
 
  bc_out:
+
+  cond->__data.__nwaiters -= 1 << COND_CLOCK_BITS;
+
+  /* If pthread_cond_destroy was called on this variable already,
+     notify the pthread_cond_destroy caller all waiters have left
+     and it can be successfully destroyed.  */
+  if (cond->__data.__total_seq == -1ULL
+      && cond->__data.__nwaiters < (1 << COND_CLOCK_BITS))
+    lll_futex_wake (&cond->__data.__nwaiters, 1);
+
   /* We are done with the condvar.  */
   lll_mutex_unlock (cond->__data.__lock);
 
diff --git a/nptl/sysdeps/pthread/pthread_cond_wait.c b/nptl/sysdeps/pthread/pthread_cond_wait.c
index 45187b5..8666945 100644
--- a/nptl/sysdeps/pthread/pthread_cond_wait.c
+++ b/nptl/sysdeps/pthread/pthread_cond_wait.c
@@ -42,6 +42,7 @@ __condvar_cleanup (void *arg)
 {
   struct _condvar_cleanup_buffer *cbuffer =
     (struct _condvar_cleanup_buffer *) arg;
+  unsigned int destroying;
 
   /* We are going to modify shared data.  */
   lll_mutex_lock (cbuffer->cond->__data.__lock);
@@ -55,11 +56,25 @@ __condvar_cleanup (void *arg)
       ++cbuffer->cond->__data.__futex;
     }
 
+  cbuffer->cond->__data.__nwaiters -= 1 << COND_CLOCK_BITS;
+
+  /* If pthread_cond_destroy was called on this variable already,
+     notify the pthread_cond_destroy caller all waiters have left
+     and it can be successfully destroyed.  */
+  destroying = 0;
+  if (cbuffer->cond->__data.__total_seq == -1ULL
+      && cbuffer->cond->__data.__nwaiters < (1 << COND_CLOCK_BITS))
+    {
+      lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1);
+      destroying = 1;
+    }
+
   /* We are done.  */
   lll_mutex_unlock (cbuffer->cond->__data.__lock);
 
   /* Wake everybody to make sure no condvar signal gets lost.  */
-  lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX);
+  if (! destroying)
+    lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX);
 
   /* Get the mutex before returning unless asynchronous cancellation
      is in effect.  */
@@ -90,6 +105,7 @@ __pthread_cond_wait (cond, mutex)
   /* We have one new user of the condvar.  */
   ++cond->__data.__total_seq;
   ++cond->__data.__futex;
+  cond->__data.__nwaiters += 1 << COND_CLOCK_BITS;
 
   /* Remember the mutex we are using here.  If there is already a
      different address store this is a bad user bug.  Do not store
@@ -145,6 +161,16 @@ __pthread_cond_wait (cond, mutex)
   ++cond->__data.__woken_seq;
 
  bc_out:
+
+  cond->__data.__nwaiters -= 1 << COND_CLOCK_BITS;
+
+  /* If pthread_cond_destroy was called on this varaible already,
+     notify the pthread_cond_destroy caller all waiters have left
+     and it can be successfully destroyed.  */
+  if (cond->__data.__total_seq == -1ULL
+      && cond->__data.__nwaiters < (1 << COND_CLOCK_BITS))
+    lll_futex_wake (&cond->__data.__nwaiters, 1);
+
   /* We are done with the condvar.  */
   lll_mutex_unlock (cond->__data.__lock);
 
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
index 62c853c..fd20d57 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
@@ -81,7 +81,7 @@ typedef union
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
-    int __clock;
+    unsigned int __nwaiters;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
index 8d7858a..fb62c0d 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
@@ -81,7 +81,7 @@ typedef union
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
-    int __clock;
+    unsigned int __nwaiters;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
index eecec8a..699c2cb 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
@@ -87,6 +87,7 @@ __pthread_cond_timedwait:
 	addl	$1, total_seq(%ebx)
 	adcl	$0, total_seq+4(%ebx)
 	addl	$1, cond_futex(%ebx)
+	addl	$(1 << clock_bits), cond_nwaiters(%ebx)
 
 #define FRAME_SIZE 24
 	subl	$FRAME_SIZE, %esp
@@ -104,8 +105,9 @@ __pthread_cond_timedwait:
 8:	movl	%ebx, %edx
 #ifdef __NR_clock_gettime
 	/* Get the clock number.  */
-	movl	cond_clock(%ebx), %ebx
-	/* Only clocks 0 and 1 are allowed.  Both are handled in the
+	movl	cond_nwaiters(%ebx), %ebx
+	andl	$((1 << clock_bits) - 1), %ebx
+	/* Only clocks 0 and 1 are allowed so far.  Both are handled in the
 	   kernel.  */
 	leal	4(%esp), %ecx
 	movl	$__NR_clock_gettime, %eax
@@ -226,7 +228,25 @@ __pthread_cond_timedwait:
 14:	addl	$1, woken_seq(%ebx)
 	adcl	$0, woken_seq+4(%ebx)
 
-24:	LOCK
+24:	subl	$(1 << clock_bits), cond_nwaiters(%ebx)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	movl	total_seq(%ebx), %eax
+	andl	total_seq+4(%ebx), %eax
+	cmpl	$0xffffffff, %eax
+	jne	25f
+	movl	cond_nwaiters(%ebx), %eax
+	andl	$~((1 << clock_bits) - 1), %eax
+	jne	25f
+
+	addl	$cond_nwaiters, %ebx
+	movl	$SYS_futex, %eax
+	movl	$FUTEX_WAKE, %ecx
+	movl	$1, %edx
+	ENTER_KERNEL
+	subl	$cond_nwaiters, %ebx
+
+25:	LOCK
 #if cond_lock == 0
 	subl	$1, (%ebx)
 #else
@@ -394,7 +414,27 @@ __condvar_tw_cleanup:
 	addl	$1, woken_seq(%ebx)
 	adcl	$0, woken_seq+4(%ebx)
 
-3:	LOCK
+3:	subl	$(1 << clock_bits), cond_nwaiters(%ebx)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	xorl	%edi, %edi
+	movl	total_seq(%ebx), %eax
+	andl	total_seq+4(%ebx), %eax
+	cmpl	$0xffffffff, %eax
+	jne	4f
+	movl	cond_nwaiters(%ebx), %eax
+	andl	$~((1 << clock_bits) - 1), %eax
+	jne	4f
+
+	addl	$cond_nwaiters, %ebx
+	movl	$SYS_futex, %eax
+	movl	$FUTEX_WAKE, %ecx
+	movl	$1, %edx
+	ENTER_KERNEL
+	subl	$cond_nwaiters, %ebx
+	movl	$1, %edi
+
+4:	LOCK
 #if cond_lock == 0
 	subl	$1, (%ebx)
 #else
@@ -410,13 +450,15 @@ __condvar_tw_cleanup:
 	call	__lll_mutex_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
-2:	addl	$cond_futex, %ebx
+2:	testl	%edi, %edi
+	jnz	5f
+	addl	$cond_futex, %ebx
 	movl	$FUTEX_WAKE, %ecx
 	movl	$SYS_futex, %eax
 	movl	$0x7fffffff, %edx
 	ENTER_KERNEL
 
-	movl	24+FRAME_SIZE(%esp), %eax
+5:	movl	24+FRAME_SIZE(%esp), %eax
 	call	__pthread_mutex_cond_lock
 
 	movl	%esi, (%esp)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 3fe7f8c..d282785 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -80,6 +80,7 @@ __pthread_cond_wait:
 	addl	$1, total_seq(%ebx)
 	adcl	$0, total_seq+4(%ebx)
 	addl	$1, cond_futex(%ebx)
+	addl	$(1 << clock_bits), cond_nwaiters(%ebx)
 
 #define FRAME_SIZE 16
 	subl	$FRAME_SIZE, %esp
@@ -156,7 +157,25 @@ __pthread_cond_wait:
 	adcl	$0, woken_seq+4(%ebx)
 
 	/* Unlock */
-16:	LOCK
+16:	subl	$(1 << clock_bits), cond_nwaiters(%ebx)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	movl	total_seq(%ebx), %eax
+	andl	total_seq+4(%ebx), %eax
+	cmpl	$0xffffffff, %eax
+	jne	17f
+	movl	cond_nwaiters(%ebx), %eax
+	andl	$~((1 << clock_bits) - 1), %eax
+	jne	17f
+
+	addl	$cond_nwaiters, %ebx
+	movl	$SYS_futex, %eax
+	movl	$FUTEX_WAKE, %ecx
+	movl	$1, %edx
+	ENTER_KERNEL
+	subl	$cond_nwaiters, %ebx
+
+17:	LOCK
 #if cond_lock == 0
 	subl	$1, (%ebx)
 #else
@@ -286,7 +305,27 @@ __condvar_w_cleanup:
 	addl	$1, woken_seq(%ebx)
 	adcl	$0, woken_seq+4(%ebx)
 
-3:	LOCK
+3:	subl	$(1 << clock_bits), cond_nwaiters(%ebx)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	xorl	%edi, %edi
+	movl	total_seq(%ebx), %eax
+	andl	total_seq+4(%ebx), %eax
+	cmpl	$0xffffffff, %eax
+	jne	4f
+	movl	cond_nwaiters(%ebx), %eax
+	andl	$~((1 << clock_bits) - 1), %eax
+	jne	4f
+
+	addl	$cond_nwaiters, %ebx
+	movl	$SYS_futex, %eax
+	movl	$FUTEX_WAKE, %ecx
+	movl	$1, %edx
+	ENTER_KERNEL
+	subl	$cond_nwaiters, %ebx
+	movl	$1, %edi
+
+4:	LOCK
 #if cond_lock == 0
 	subl	$1, (%ebx)
 #else
@@ -302,13 +341,15 @@ __condvar_w_cleanup:
 	call	__lll_mutex_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
-2:	addl	$cond_futex, %ebx
+2:	testl	%edi, %edi
+	jnz	5f
+	addl	$cond_futex, %ebx
 	movl	$FUTEX_WAKE, %ecx
 	movl	$SYS_futex, %eax
 	movl	$0x7fffffff, %edx
 	ENTER_KERNEL
 
-	movl	20+FRAME_SIZE(%esp), %eax
+5:	movl	20+FRAME_SIZE(%esp), %eax
 	call	__pthread_mutex_cond_lock
 
 	movl	%esi, (%esp)
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
index 958af47..5b442cb 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
@@ -81,7 +81,7 @@ typedef union
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
-    int __clock;
+    unsigned int __nwaiters;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
index 43b6856..1dec19e 100644
--- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
@@ -75,6 +75,12 @@ struct pthread_condattr
 };
 
 
+/* The __NWAITERS field is used as a counter and to house the number
+   of bits which represent the clock.  COND_CLOCK_BITS is the number
+   of bits reserved for the clock.  */
+#define COND_CLOCK_BITS	1
+
+
 /* Read-write lock variable attribute data structure.  */
 struct pthread_rwlockattr
 {
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
index 2e193e6..c5e7978 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
@@ -1,13 +1,16 @@
 #include <stddef.h>
+#include <sched.h>
 #include <bits/pthreadtypes.h>
+#include <internaltypes.h>
 
 --
 
 cond_lock	offsetof (pthread_cond_t, __data.__lock)
 cond_futex	offsetof (pthread_cond_t, __data.__futex)
-cond_clock	offsetof (pthread_cond_t, __data.__clock)
+cond_nwaiters	offsetof (pthread_cond_t, __data.__nwaiters)
 total_seq	offsetof (pthread_cond_t, __data.__total_seq)
 wakeup_seq	offsetof (pthread_cond_t, __data.__wakeup_seq)
 woken_seq	offsetof (pthread_cond_t, __data.__woken_seq)
 dep_mutex	offsetof (pthread_cond_t, __data.__mutex)
 broadcast_seq	offsetof (pthread_cond_t, __data.__broadcast_seq)
+clock_bits	COND_CLOCK_BITS
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
index 5a000d3..a493821 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
@@ -101,7 +101,7 @@ typedef union
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
-    int __clock;
+    unsigned int __nwaiters;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
index 1422864..17cfaa9 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
@@ -100,7 +100,7 @@ typedef union
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
-    int __clock;
+    unsigned int __nwaiters;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h
index b8afbc5..5bf3aff 100644
--- a/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h
@@ -82,7 +82,7 @@ typedef union
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
-    int __clock;
+    unsigned int __nwaiters;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
index e86dd00..34d3df7 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
@@ -101,7 +101,7 @@ typedef union
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
-    int __clock;
+    unsigned int __nwaiters;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
index a7bc581..e5cc605 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
@@ -100,7 +100,7 @@ typedef union
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
-    int __clock;
+    int __nwaiters;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 95f8aab..67bec6c 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -111,6 +111,7 @@ __pthread_cond_timedwait:
 	movq	8(%rsp), %rdi
 	incq	total_seq(%rdi)
 	incl	cond_futex(%rdi)
+	addl	$(1 << clock_bits), cond_nwaiters(%rdi)
 
 	/* Install cancellation handler.  */
 #ifdef PIC
@@ -135,8 +136,9 @@ __pthread_cond_timedwait:
 	/* Get the clock number.  Note that the field in the condvar
 	   structure stores the number minus 1.  */
 	movq	8(%rsp), %rdi
-	movl	cond_clock(%rdi), %edi
-	/* Only clocks 0 and 1 are allowed.  Both are handled in the
+	movl	cond_nwaiters(%rdi), %edi
+	andl	$((1 << clock_bits) - 1), %edi
+	/* Only clocks 0 and 1 are allowed so far.  Both are handled in the
 	   kernel.  */
 	leaq	24(%rsp), %rsi
 	movq	$__NR_clock_gettime, %rax
@@ -244,7 +246,23 @@ __pthread_cond_timedwait:
 9:	xorq	%r14, %r14
 14:	incq	woken_seq(%rdi)
 
-24:	LOCK
+24:	subl	$(1 << clock_bits), cond_nwaiters(%rdi)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	cmpq	$0xffffffffffffffff, total_seq(%rdi)
+	jne	25f
+	movl	cond_nwaiters(%rdi), %eax
+	andl	$~((1 << clock_bits) - 1), %eax
+	jne	25f
+
+	addq	$cond_nwaiters, %rdi
+	movq	$SYS_futex, %rax
+	movq	$FUTEX_WAKE, %rsi
+	movl	$1, %edx
+	syscall
+	subq	$cond_nwaiters, %rdi
+
+25:	LOCK
 #if cond_lock == 0
 	decl	(%rdi)
 #else
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 9e7da30..f5de0a2 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -40,6 +40,8 @@
 	.globl	__condvar_cleanup
 	.hidden	__condvar_cleanup
 __condvar_cleanup:
+	pushq	%r12
+
 	/* Get internal lock.  */
 	movq	%rdi, %r8
 	movq	8(%rdi), %rdi
@@ -66,12 +68,28 @@ __condvar_cleanup:
 	jne	3f
 
 	incq	wakeup_seq(%rdi)
-
 	incq	woken_seq(%rdi)
-
 	incl	cond_futex(%rdi)
 
-3:	LOCK
+3:	subl	$(1 << clock_bits), cond_nwaiters(%rdi)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	xorq	%r12, %r12
+	cmpq	$0xffffffffffffffff, total_seq(%rdi)
+	jne	4f
+	movl	cond_nwaiters(%rdi), %eax
+	andl	$~((1 << clock_bits) - 1), %eax
+	jne	4f
+
+	addq	$cond_nwaiters, %rdi
+	movq	$SYS_futex, %rax
+	movq	$FUTEX_WAKE, %rsi
+	movl	$1, %edx
+	syscall
+	subq	$cond_nwaiters, %rdi
+	movq	$1, %r12
+
+4:	LOCK
 #if cond_lock == 0
 	decl	(%rdi)
 #else
@@ -84,15 +102,19 @@ __condvar_cleanup:
 	callq	__lll_mutex_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
-2:	addq	$cond_futex, %rdi
+2:	testq	%r12, %r12
+	jnz	5f
+	addq	$cond_futex, %rdi
 	movq	$FUTEX_WAKE, %rsi
 	movl	$0x7fffffff, %edx
 	movq	$SYS_futex, %rax
 	syscall
 
-	movq	16(%r8), %rdi
+5:	movq	16(%r8), %rdi
 	callq	__pthread_mutex_cond_lock
 
+	popq	%r12
+
 	retq
 	.size	__condvar_cleanup, .-__condvar_cleanup
 
@@ -157,6 +179,7 @@ __pthread_cond_wait:
 	movq	8(%rsp), %rdi
 	incq	total_seq(%rdi)
 	incl	cond_futex(%rdi)
+	addl	$(1 << clock_bits), cond_nwaiters(%rdi)
 
 	/* Install cancellation handler.  */
 #ifdef PIC
@@ -229,7 +252,23 @@ __pthread_cond_wait:
 	incq	woken_seq(%rdi)
 
 	/* Unlock */
-16:	LOCK
+16:	subl	$(1 << clock_bits), cond_nwaiters(%rdi)
+
+	/* Wake up a thread which wants to destroy the condvar object.  */
+	cmpq	$0xffffffffffffffff, total_seq(%rdi)
+	jne	17f
+	movl	cond_nwaiters(%rdi), %eax
+	andl	$~((1 << clock_bits) - 1), %eax
+	jne	17f
+
+	addq	$cond_nwaiters, %rdi
+	movq	$SYS_futex, %rax
+	movq	$FUTEX_WAKE, %rsi
+	movl	$1, %edx
+	syscall
+	subq	$cond_nwaiters, %rdi
+
+17:	LOCK
 #if cond_lock == 0
 	decl	(%rdi)
 #else