* pthreadP.h (PTHREAD_ROBUST_MUTEX_PSHARED): Define.

	* pthread_mutex_lock.c: Use it instead of PTHREAD_MUTEX_PSHARED when
	dealing with robust mutexes.
	* pthread_mutex_timedlock.c: Likewise.
	* pthread_mutex_trylock.c: Likewise.
	* pthread_mutex_unlock.c: Likewise.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.

2007-08-06  Jakub Jelinek  <jakub@redhat.com>

	* pthreadP.h (PTHREAD_MUTEX_PSHARED_BIT): Define.
	(PTHREAD_MUTEX_TYPE): Mask __kind with 127.
	(PTHREAD_MUTEX_PSHARED): Define.
	* pthread_mutex_init.c (__pthread_mutex_init): Set
	PTHREAD_MUTEX_PSHARED_BIT for pshared or robust
	mutexes.
	* pthread_mutex_lock.c (LLL_MUTEX_LOCK): Take mutex as argument
	instead of its __data.__lock field, pass PTHREAD_MUTEX_PSHARED
	as second argument to lll_lock.
	(LLL_MUTEX_TRYLOCK): Take mutex as argument
	instead of its __data.__lock field.
	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
	to lll_robust_lock.
	(__pthread_mutex_lock): Update LLL_MUTEX_LOCK, LLL_MUTEX_TRYLOCK,
	LLL_ROBUST_MUTEX_LOCK users, use PTHREAD_MUTEX_TYPE (mutex)
	instead of mutex->__data.__kind directly, pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock and lll_futex_wait.
	* pthread_mutex_trylock.c (__pthread_mutex_trylock): Use
	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
	directly, pass PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock.
	(pthread_mutex_timedlock): Pass PTHREAD_MUTEX_PSHARED (mutex)
	to lll_timedlock, lll_robust_timedlock, lll_unlock and
	lll_futex_timed_wait.  Use PTHREAD_MUTEX_TYPE (mutex) instead
	of mutex->__data.__kind directly.
	* pthread_mutex_timedlock.c (pthread_mutex_timedlock): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_timedlock,
	lll_robust_timedlock, lll_unlock and lll_futex_timed_wait.  Use
	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind directly.
	* pthread_mutex_unlock.c (__pthread_mutex_unlock_usercnt): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock, lll_robust_unlock
	and lll_futex_wake.
	* pthread_mutex_setprioceiling.c (pthread_mutex_setprioceiling): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_futex_wait and lll_futex_wake.
	Use PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
	directly.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c (LLL_MUTEX_LOCK):
	Take mutex as argument instead of its __data.__lock field, pass
	PTHREAD_MUTEX_PSHARED as second argument to lll_cond_lock.
	(LLL_MUTEX_TRYLOCK): Take mutex as argument instead of its
	__data.__lock field.
	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
	to lll_robust_cond_lock.
	* pthread_cond_broadcast.c (__pthread_cond_broadcast): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_requeue and
	lll_futex_wake.  Don't use lll_futex_requeue if dependent mutex
	has PTHREAD_MUTEX_PSHARED_BIT bit set in its __data.__kind.
	* pthread_cond_destroy.c (__pthread_cond_destroy): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_wake and
	lll_futex_wait.
	* pthread_cond_signal.c (__pthread_cond_signal): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_wake_unlock and
	lll_futex_wake.
	* pthread_cond_timedwait.c (__pthread_cond_wait): Add
	pshared variable, pass it to lll_lock, lll_unlock,
	lll_futex_timedwait and lll_futex_wake.
	* pthread_cond_wait.c (__condvar_cleanup, __pthread_cond_wait): Add
	pshared variable, pass it to lll_lock, lll_unlock, lll_futex_wait
	and lll_futex_wake.
	* sysdeps/unix/sysv/linux/alpha/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Add private argument, use __lll_private_flag
	macro.
	* sysdeps/unix/sysv/linux/ia64/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_futex_requeue):
	Likewise.
	* sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (lll_futex_requeue):
	Likewise.
	* sysdeps/unix/sysv/linux/s390/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	(lll_futex_wake): Fix a typo.
	* sysdeps/unix/sysv/linux/pthread-pi-defines.sym (PS_BIT): Add.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
	(__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or
	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
	Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
	(__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or
	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
	(__pthread_cond_timedwait): Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:
	(__condvar_cleanup, __pthread_cond_wait): Likewise.

12 files changed, 204 insertions, 72 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
index f3f2919..ab829ad 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
@@ -103,24 +103,24 @@
   while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_CMP_REQUEUE, (nr_wake),	      \
-			      (nr_move), (mutex), (val));		      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
+			      (nr_wake), (nr_move), (mutex), (val));	      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \
+#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_WAKE_OP, (nr_wake),	      \
-			      (nr_wake2), (futexp2),			      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_WAKE_OP, private),    \
+			      (nr_wake), (nr_wake2), (futexp2),		      \
 			      FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);		      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
index 3c28a39..fd30c43 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
@@ -103,18 +103,20 @@ do									\
 while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val)		     \
+#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private)	     \
 ({									     \
-   DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_CMP_REQUEUE,		     \
+   DO_INLINE_SYSCALL(futex, 6, (long) (ftx),				     \
+		     __lll_private_flag (FUTEX_CMP_REQUEUE, private),	     \
 		     (int) (nr_wake), (int) (nr_move), (long) (mutex),	     \
 		     (int) val);					     \
    _r10 == -1;								     \
 })
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2)		     \
+#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2, private)	     \
 ({									     \
-   DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_WAKE_OP,		     \
+   DO_INLINE_SYSCALL(futex, 6, (long) (ftx),				     \
+		     __lll_private_flag (FUTEX_WAKE_OP, private),	     \
 		     (int) (nr_wake), (int) (nr_wake2), (long) (ftx2),	     \
 		     FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);			     \
    _r10 == -1;								     \
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
index 41804d1..87935c1 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
@@ -107,14 +107,14 @@
   while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
 									      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_CMP_REQUEUE, (nr_wake),	      \
-			      (nr_move), (mutex), (val));		      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
+			      (nr_wake), (nr_move), (mutex), (val));	      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
 
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
index a1b6794..d985c6a 100644
--- a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
+++ b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
@@ -4,3 +4,4 @@
 
 MUTEX_KIND	offsetof (pthread_mutex_t, __data.__kind)
 PI_BIT		PTHREAD_MUTEX_PRIO_INHERIT_NP
+PS_BIT		PTHREAD_MUTEX_PSHARED_BIT
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
index 81ecd65..93841c5 100644
--- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
+++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
@@ -1,8 +1,12 @@
 #include <pthreadP.h>
 
-#define LLL_MUTEX_LOCK(mutex) lll_cond_lock (mutex, /* XYZ */ LLL_SHARED)
-#define LLL_MUTEX_TRYLOCK(mutex) lll_cond_trylock (mutex)
-#define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_cond_lock (mutex, id, /* XYZ */ LLL_SHARED)
+#define LLL_MUTEX_LOCK(mutex) \
+  lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
+#define LLL_MUTEX_TRYLOCK(mutex) \
+  lll_cond_trylock ((mutex)->__data.__lock)
+#define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
+  lll_robust_cond_lock ((mutex)->__data.__lock, id, \
+			PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
 #define __pthread_mutex_lock __pthread_mutex_cond_lock
 #define NO_INCR
 
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
index ad4d273..d687e13 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
@@ -93,7 +93,7 @@
   ({									      \
     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
     register unsigned long int __r3 asm ("3")				      \
-      __lll_private_flag (FUTEX_WAKE, private);				      \
+      = __lll_private_flag (FUTEX_WAKE, private);			      \
     register unsigned long int __r4 asm ("4") = (unsigned long int) (nr);     \
     register unsigned long int __result asm ("2");			      \
 									      \
@@ -117,10 +117,11 @@
 
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \
   ({									      \
     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
-    register unsigned long int __r3 asm ("3") = FUTEX_CMP_REQUEUE;	      \
+    register unsigned long int __r3 asm ("3")				      \
+      = __lll_private_flag (FUTEX_CMP_REQUEUE, private);		      \
     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);	      \
     register unsigned long int __r5 asm ("5") = (long int) (nr_move);	      \
     register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex);  \
@@ -137,10 +138,11 @@
 
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2) \
+#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \
   ({									      \
     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
-    register unsigned long int __r3 asm ("3") = FUTEX_WAKE_OP;	      \
+    register unsigned long int __r3 asm ("3")				      \
+      = __lll_private_flag (FUTEX_WAKE_OP, private);			      \
     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);	      \
     register unsigned long int __r5 asm ("5") = (long int) (nr_wake2);	      \
     register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
index 38692bb..24cbbe4 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
@@ -96,14 +96,14 @@
   })
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
 									      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_CMP_REQUEUE, (nr_wake),	      \
-			      (nr_move), (mutex), (val));		      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
+			      (nr_wake), (nr_move), (mutex), (val));	      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
 
@@ -121,14 +121,14 @@
 /* Avoid FUTEX_WAKE_OP if supporting pre-v9 CPUs.  */
 # define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) 1
 #else
-# define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \
+# define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
 									      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_WAKE_OP, (nr_wake),	      \
-			      (nr_wake2), (futexp2),			      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_WAKE_OP, private),    \
+			      (nr_wake), (nr_wake2), (futexp2),		      \
 			      FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);		      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index 192d203..2cd69a1 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -533,7 +533,7 @@ LLL_STUB_UNWIND_INFO_END
   while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \
   ({ int __res;								      \
      register int __nr_move __asm ("r10") = nr_move;			      \
      register void *__mutex __asm ("r8") = mutex;			      \
@@ -541,7 +541,8 @@ LLL_STUB_UNWIND_INFO_END
      __asm __volatile ("syscall"					      \
 		       : "=a" (__res)					      \
 		       : "0" (__NR_futex), "D" ((void *) ftx),		      \
-			 "S" (FUTEX_CMP_REQUEUE), "d" (nr_wake),	      \
+			 "S" (__lll_private_flag (FUTEX_CMP_REQUEUE,	      \
+						  private)), "d" (nr_wake),   \
 			 "r" (__nr_move), "r" (__mutex), "r" (__val)	      \
 		       : "cx", "r11", "cc", "memory");			      \
      __res < 0; })
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index 0c619bf..6b8a29e 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -71,11 +71,18 @@ __pthread_cond_broadcast:
 	je	9f
 
 	/* XXX: The kernel so far doesn't support requeue to PI futex.  */
-	testl	$PI_BIT, MUTEX_KIND(%r8)
+	/* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
+	   type of futex (private resp. shared).  */
+	testl	$(PI_BIT | PS_BIT), MUTEX_KIND(%r8)
 	jne	9f
 
 	/* Wake up all threads.  */
-	movl	$FUTEX_CMP_REQUEUE, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi
+#else
+	movl	%fs:PRIVATE_FUTEX, %esi
+	orl	$FUTEX_CMP_REQUEUE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	movl	$1, %edx
 	movl	$0x7fffffff, %r10d
@@ -104,8 +111,10 @@ __pthread_cond_broadcast:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -114,22 +123,36 @@ __pthread_cond_broadcast:
 
 	/* Unlock in loop requires wakeup.  */
 5:	addq	$cond_lock-cond_futex, %rdi
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	6b
 
 	/* Unlock in loop requires wakeup.  */
 7:	addq	$cond_lock-cond_futex, %rdi
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	subq	$cond_lock-cond_futex, %rdi
 	jmp	8b
 
 9:	/* The futex requeue functionality is not available.  */
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
 	movl	$0x7fffffff, %edx
-	movl	$FUTEX_WAKE, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 	jmp	10b
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 2fc9d1f..8be6d4a 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -55,10 +55,20 @@ __pthread_cond_signal:
 	addl	$1, (%rdi)
 
 	/* Wake up one thread.  */
-	movl	$FUTEX_WAKE_OP, %esi
-	movl	$SYS_futex, %eax
+	cmpq	$-1, dep_mutex(%r8)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE_OP, %eax
+	movl	$(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE_OP, %esi
+#endif
 	movl	$1, %r10d
+	movl	$SYS_futex, %eax
 #if cond_lock != 0
 	addq	$cond_lock, %r8
 #endif
@@ -75,7 +85,9 @@ __pthread_cond_signal:
 	xorl	%eax, %eax
 	retq
 
-7:	movl	$FUTEX_WAKE, %esi
+7:	/* %esi should be either FUTEX_WAKE_OP or
+	   FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall.  */
+	xorl	$(FUTEX_WAKE | FUTEX_WAKE_OP), %esi
 	movl	$SYS_futex, %eax
 	/* %rdx should be 1 already from $FUTEX_WAKE_OP syscall.
 	movl	$1, %edx  */
@@ -98,8 +110,10 @@ __pthread_cond_signal:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -109,8 +123,13 @@ __pthread_cond_signal:
 	/* Unlock in loop requires wakeup.  */
 5:
 	movq	%r8, %rdi
-	/* XYZ */
+#if cond_lock != 0
+	addq	$cond_lock, %rdi
+#endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	6b
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 003069f..415f06f 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -186,12 +186,20 @@ __pthread_cond_timedwait:
 	movl	%eax, (%rsp)
 
 	leaq	24(%rsp), %r10
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	cmpq	$-1, dep_mutex(%rdi)
+	movq	%r12, %rdx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT, %eax
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	movl	$FUTEX_WAIT, %esi
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+# if FUTEX_WAIT != 0
+	orl	$FUTEX_WAIT, %esi
+# endif
 #endif
-	movq	%r12, %rdx
 	addq	$cond_futex, %rdi
 	movl	$SYS_futex, %eax
 	syscall
@@ -251,9 +259,19 @@ __pthread_cond_timedwait:
 	jne	25f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
@@ -292,8 +310,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 	jmp	2b
 
@@ -302,8 +322,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	4b
 
@@ -312,8 +334,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -325,8 +349,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	11b
 
@@ -344,8 +370,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 17:	movq	(%rsp), %rax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 34ef2c7..db2683b 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -49,8 +49,10 @@ __condvar_cleanup:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -81,9 +83,19 @@ __condvar_cleanup:
 	jne	4f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 	movl	$1, %r12d
@@ -98,16 +110,28 @@ __condvar_cleanup:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
 2:	testq	%r12, %r12
 	jnz	5f
 	addq	$cond_futex, %rdi
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -216,12 +240,20 @@ __pthread_cond_wait:
 	xorq	%r10, %r10
 	movq	%r12, %rdx
 	addq	$cond_futex-cond_lock, %rdi
-	movl	$SYS_futex, %eax
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT, %eax
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	movl	$FUTEX_WAIT, %esi
+	movl	$FUTEX_WAIT, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+# if FUTEX_WAIT != 0
+	orl	$FUTEX_WAIT, %esi
+# endif
 #endif
+	movl	$SYS_futex, %eax
 	syscall
 
 	movl	(%rsp), %edi
@@ -267,9 +299,19 @@ __pthread_cond_wait:
 	jne	17f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
@@ -302,8 +344,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 	jmp	2b
 
@@ -312,8 +356,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	4b
 
@@ -322,8 +368,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -335,8 +383,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	11b
 
@@ -354,8 +404,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 13:	movq	%r10, %rax