* pthreadP.h (PTHREAD_ROBUST_MUTEX_PSHARED): Define.

	* pthread_mutex_lock.c: Use it instead of PTHREAD_MUTEX_PSHARED when
	dealing with robust mutexes.
	* pthread_mutex_timedlock.c: Likewise.
	* pthread_mutex_trylock.c: Likewise.
	* pthread_mutex_unlock.c: Likewise.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.

2007-08-06  Jakub Jelinek  <jakub@redhat.com>

	* pthreadP.h (PTHREAD_MUTEX_PSHARED_BIT): Define.
	(PTHREAD_MUTEX_TYPE): Mask __kind with 127.
	(PTHREAD_MUTEX_PSHARED): Define.
	* pthread_mutex_init.c (__pthread_mutex_init): Set
	PTHREAD_MUTEX_PSHARED_BIT for pshared or robust
	mutexes.
	* pthread_mutex_lock.c (LLL_MUTEX_LOCK): Take mutex as argument
	instead of its __data.__lock field, pass PTHREAD_MUTEX_PSHARED
	as second argument to lll_lock.
	(LLL_MUTEX_TRYLOCK): Take mutex as argument
	instead of its __data.__lock field.
	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
	to lll_robust_lock.
	(__pthread_mutex_lock): Update LLL_MUTEX_LOCK, LLL_MUTEX_TRYLOCK,
	LLL_ROBUST_MUTEX_LOCK users, use PTHREAD_MUTEX_TYPE (mutex)
	instead of mutex->__data.__kind directly, pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock and lll_futex_wait.
	* pthread_mutex_trylock.c (__pthread_mutex_trylock): Use
	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
	directly, pass PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock.
	(pthread_mutex_timedlock): Pass PTHREAD_MUTEX_PSHARED (mutex)
	to lll_timedlock, lll_robust_timedlock, lll_unlock and
	lll_futex_timed_wait.  Use PTHREAD_MUTEX_TYPE (mutex) instead
	of mutex->__data.__kind directly.
	* pthread_mutex_timedlock.c (pthread_mutex_timedlock): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_timedlock,
	lll_robust_timedlock, lll_unlock and lll_futex_timed_wait.  Use
	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind directly.
	* pthread_mutex_unlock.c (__pthread_mutex_unlock_usercnt): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock, lll_robust_unlock
	and lll_futex_wake.
	* pthread_mutex_setprioceiling.c (pthread_mutex_setprioceiling): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_futex_wait and lll_futex_wake.
	Use PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
	directly.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c (LLL_MUTEX_LOCK):
	Take mutex as argument instead of its __data.__lock field, pass
	PTHREAD_MUTEX_PSHARED as second argument to lll_cond_lock.
	(LLL_MUTEX_TRYLOCK): Take mutex as argument instead of its
	__data.__lock field.
	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
	to lll_robust_cond_lock.
	* pthread_cond_broadcast.c (__pthread_cond_broadcast): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_requeue and
	lll_futex_wake.  Don't use lll_futex_requeue if dependent mutex
	has PTHREAD_MUTEX_PSHARED_BIT bit set in its __data.__kind.
	* pthread_cond_destroy.c (__pthread_cond_destroy): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_wake and
	lll_futex_wait.
	* pthread_cond_signal.c (__pthread_cond_signal): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_wake_unlock and
	lll_futex_wake.
	* pthread_cond_timedwait.c (__pthread_cond_wait): Add
	pshared variable, pass it to lll_lock, lll_unlock,
	lll_futex_timedwait and lll_futex_wake.
	* pthread_cond_wait.c (__condvar_cleanup, __pthread_cond_wait): Add
	pshared variable, pass it to lll_lock, lll_unlock, lll_futex_wait
	and lll_futex_wake.
	* sysdeps/unix/sysv/linux/alpha/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Add private argument, use __lll_private_flag
	macro.
	* sysdeps/unix/sysv/linux/ia64/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_futex_requeue):
	Likewise.
	* sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (lll_futex_requeue):
	Likewise.
	* sysdeps/unix/sysv/linux/s390/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	(lll_futex_wake): Fix a typo.
	* sysdeps/unix/sysv/linux/pthread-pi-defines.sym (PS_BIT): Add.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
	(__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or
	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
	Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
	(__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or
	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
	(__pthread_cond_timedwait): Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:
	(__condvar_cleanup, __pthread_cond_wait): Likewise.

25 files changed, 421 insertions, 172 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 34fd724d9b..b81c5c5314 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,102 @@
+2007-08-11  Ulrich Drepper  <drepper@redhat.com>
+
+	* pthreadP.h (PTHREAD_ROBUST_MUTEX_PSHARED): Define.
+	* pthread_mutex_lock.c: Use it instead of PTHREAD_MUTEX_PSHARED when
+	dealing with robust mutexes.
+	* pthread_mutex_timedlock.c: Likewise.
+	* pthread_mutex_trylock.c: Likewise.
+	* pthread_mutex_unlock.c: Likewise.
+	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.
+
+2007-08-06  Jakub Jelinek  <jakub@redhat.com>
+
+	* pthreadP.h (PTHREAD_MUTEX_PSHARED_BIT): Define.
+	(PTHREAD_MUTEX_TYPE): Mask __kind with 127.
+	(PTHREAD_MUTEX_PSHARED): Define.
+	* pthread_mutex_init.c (__pthread_mutex_init): Set
+	PTHREAD_MUTEX_PSHARED_BIT for pshared or robust
+	mutexes.
+	* pthread_mutex_lock.c (LLL_MUTEX_LOCK): Take mutex as argument
+	instead of its __data.__lock field, pass PTHREAD_MUTEX_PSHARED
+	as second argument to lll_lock.
+	(LLL_MUTEX_TRYLOCK): Take mutex as argument
+	instead of its __data.__lock field.
+	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
+	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
+	to lll_robust_lock.
+	(__pthread_mutex_lock): Update LLL_MUTEX_LOCK, LLL_MUTEX_TRYLOCK,
+	LLL_ROBUST_MUTEX_LOCK users, use PTHREAD_MUTEX_TYPE (mutex)
+	instead of mutex->__data.__kind directly, pass
+	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock and lll_futex_wait.
+	* pthread_mutex_trylock.c (__pthread_mutex_trylock): Use
+	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
+	directly, pass PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock.
+	(pthread_mutex_timedlock): Pass PTHREAD_MUTEX_PSHARED (mutex)
+	to lll_timedlock, lll_robust_timedlock, lll_unlock and
+	lll_futex_timed_wait.  Use PTHREAD_MUTEX_TYPE (mutex) instead
+	of mutex->__data.__kind directly.
+	* pthread_mutex_timedlock.c (pthread_mutex_timedlock): Pass
+	PTHREAD_MUTEX_PSHARED (mutex) to lll_timedlock,
+	lll_robust_timedlock, lll_unlock and lll_futex_timed_wait.  Use
+	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind directly.
+	* pthread_mutex_unlock.c (__pthread_mutex_unlock_usercnt): Pass
+	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock, lll_robust_unlock
+	and lll_futex_wake.
+	* pthread_mutex_setprioceiling.c (pthread_mutex_setprioceiling): Pass
+	PTHREAD_MUTEX_PSHARED (mutex) to lll_futex_wait and lll_futex_wake.
+	Use PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
+	directly.
+	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c (LLL_MUTEX_LOCK):
+	Take mutex as argument instead of its __data.__lock field, pass
+	PTHREAD_MUTEX_PSHARED as second argument to lll_cond_lock.
+	(LLL_MUTEX_TRYLOCK): Take mutex as argument instead of its
+	__data.__lock field.
+	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
+	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
+	to lll_robust_cond_lock.
+	* pthread_cond_broadcast.c (__pthread_cond_broadcast): Add pshared
+	variable, pass it to lll_lock, lll_unlock, lll_futex_requeue and
+	lll_futex_wake.  Don't use lll_futex_requeue if dependent mutex
+	has PTHREAD_MUTEX_PSHARED_BIT bit set in its __data.__kind.
+	* pthread_cond_destroy.c (__pthread_cond_destroy): Add pshared
+	variable, pass it to lll_lock, lll_unlock, lll_futex_wake and
+	lll_futex_wait.
+	* pthread_cond_signal.c (__pthread_cond_signal): Add pshared
+	variable, pass it to lll_lock, lll_unlock, lll_futex_wake_unlock and
+	lll_futex_wake.
+	* pthread_cond_timedwait.c (__pthread_cond_wait): Add
+	pshared variable, pass it to lll_lock, lll_unlock,
+	lll_futex_timedwait and lll_futex_wake.
+	* pthread_cond_wait.c (__condvar_cleanup, __pthread_cond_wait): Add
+	pshared variable, pass it to lll_lock, lll_unlock, lll_futex_wait
+	and lll_futex_wake.
+	* sysdeps/unix/sysv/linux/alpha/lowlevellock.h (lll_futex_requeue,
+	lll_futex_wake_unlock): Add private argument, use __lll_private_flag
+	macro.
+	* sysdeps/unix/sysv/linux/ia64/lowlevellock.h (lll_futex_requeue,
+	lll_futex_wake_unlock): Likewise.
+	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_futex_requeue):
+	Likewise.
+	* sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_futex_requeue,
+	lll_futex_wake_unlock): Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (lll_futex_requeue):
+	Likewise.
+	* sysdeps/unix/sysv/linux/s390/lowlevellock.h (lll_futex_requeue,
+	lll_futex_wake_unlock): Likewise.
+	(lll_futex_wake): Fix a typo.
+	* sysdeps/unix/sysv/linux/pthread-pi-defines.sym (PS_BIT): Add.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+	(__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or
+	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
+	Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+	(__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or
+	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+	(__pthread_cond_timedwait): Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:
+	(__condvar_cleanup, __pthread_cond_wait): Likewise.
+
 2007-08-05  Jakub Jelinek  <jakub@redhat.com>
 
 	* sysdeps/unix/sysv/linux/powerpc/powerpc32/sysdep-cancel.h (PSEUDO):
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 82c0f1ecf6..17b6492ad8 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -96,9 +96,22 @@ enum
   PTHREAD_MUTEX_PP_ADAPTIVE_NP
   = PTHREAD_MUTEX_PRIO_PROTECT_NP | PTHREAD_MUTEX_ADAPTIVE_NP
 };
+#define PTHREAD_MUTEX_PSHARED_BIT 128
 
 #define PTHREAD_MUTEX_TYPE(m) \
-  ((m)->__data.__kind)
+  ((m)->__data.__kind & 127)
+
+#if LLL_PRIVATE == 0 && LLL_SHARED == 128
+# define PTHREAD_MUTEX_PSHARED(m) \
+  ((m)->__data.__kind & 128)
+#else
+# define PTHREAD_MUTEX_PSHARED(m) \
+  (((m)->__data.__kind & 128) ? LLL_SHARED : LLL_PRIVATE)
+#endif
+
+/* The kernel when waking robust mutexes on exit never uses
+   FUTEX_PRIVATE_FLAG FUTEX_WAKE.  */
+#define PTHREAD_ROBUST_MUTEX_PSHARED(m) LLL_SHARED
 
 /* Ceiling in __data.__lock.  __data.__lock is signed, so don't
    use the MSB bit in there, but in the mask also include that bit,
diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c
index 5c0d76effc..22523c2973 100644
--- a/nptl/pthread_cond_broadcast.c
+++ b/nptl/pthread_cond_broadcast.c
@@ -32,8 +32,10 @@ int
 __pthread_cond_broadcast (cond)
      pthread_cond_t *cond;
 {
+  int pshared = (cond->__data.__mutex == (void *) ~0l)
+		? LLL_SHARED : LLL_PRIVATE;
   /* Make sure we are alone.  */
-  lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_lock (cond->__data.__lock, pshared);
 
   /* Are there any waiters to be woken?  */
   if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
@@ -47,7 +49,7 @@ __pthread_cond_broadcast (cond)
       ++cond->__data.__broadcast_seq;
 
       /* We are done.  */
-      lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_unlock (cond->__data.__lock, pshared);
 
       /* Do not use requeue for pshared condvars.  */
       if (cond->__data.__mutex == (void *) ~0l)
@@ -57,21 +59,22 @@ __pthread_cond_broadcast (cond)
       pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
 
       /* XXX: Kernel so far doesn't support requeue to PI futex.  */
-      if (__builtin_expect (mut->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP,
-			    0))
+      /* XXX: Kernel so far can only requeue to the same type of futex,
+	 in this case private (we don't requeue for pshared condvars).  */
+      if (__builtin_expect (mut->__data.__kind
+			    & (PTHREAD_MUTEX_PRIO_INHERIT_NP
+			       | PTHREAD_MUTEX_PSHARED_BIT), 0))
 	goto wake_all;
 
       /* lll_futex_requeue returns 0 for success and non-zero
 	 for errors.  */
       if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
 					       INT_MAX, &mut->__data.__lock,
-					       futex_val), 0))
+					       futex_val, LLL_PRIVATE), 0))
 	{
 	  /* The requeue functionality is not available.  */
 	wake_all:
-	  lll_futex_wake (&cond->__data.__futex, INT_MAX,
-			  // XYZ check mutex flag
-			  LLL_SHARED);
+	  lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared);
 	}
 
       /* That's all.  */
@@ -79,7 +82,7 @@ __pthread_cond_broadcast (cond)
     }
 
   /* We are done.  */
-  lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_unlock (cond->__data.__lock, pshared);
 
   return 0;
 }
diff --git a/nptl/pthread_cond_destroy.c b/nptl/pthread_cond_destroy.c
index 53b5cd272f..35135a68bc 100644
--- a/nptl/pthread_cond_destroy.c
+++ b/nptl/pthread_cond_destroy.c
@@ -26,14 +26,17 @@ int
 __pthread_cond_destroy (cond)
      pthread_cond_t *cond;
 {
+  int pshared = (cond->__data.__mutex == (void *) ~0l)
+		? LLL_SHARED : LLL_PRIVATE;
+
   /* Make sure we are alone.  */
-  lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_lock (cond->__data.__lock, pshared);
 
   if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
     {
       /* If there are still some waiters which have not been
 	 woken up, this is an application bug.  */
-      lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_unlock (cond->__data.__lock, pshared);
       return EBUSY;
     }
 
@@ -60,19 +63,16 @@ __pthread_cond_destroy (cond)
 	{
 	  pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
 	  lll_futex_wake (&mut->__data.__lock, INT_MAX,
-			  // XYZ check mutex flag
-			  LLL_SHARED);
+			  PTHREAD_MUTEX_PSHARED (mut));
 	}
 
       do
 	{
-	  lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+	  lll_unlock (cond->__data.__lock, pshared);
 
-	  lll_futex_wait (&cond->__data.__nwaiters, nwaiters,
-			  // XYZ check mutex flag
-			  LLL_SHARED);
+	  lll_futex_wait (&cond->__data.__nwaiters, nwaiters, pshared);
 
-	  lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+	  lll_lock (cond->__data.__lock, pshared);
 
 	  nwaiters = cond->__data.__nwaiters;
 	}
diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c
index f2de58fa1d..023bbb5e9b 100644
--- a/nptl/pthread_cond_signal.c
+++ b/nptl/pthread_cond_signal.c
@@ -32,8 +32,11 @@ int
 __pthread_cond_signal (cond)
      pthread_cond_t *cond;
 {
+  int pshared = (cond->__data.__mutex == (void *) ~0l)
+		? LLL_SHARED : LLL_PRIVATE;
+
   /* Make sure we are alone.  */
-  lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_lock (cond->__data.__lock, pshared);
 
   /* Are there any waiters to be woken?  */
   if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
@@ -45,18 +48,14 @@ __pthread_cond_signal (cond)
       /* Wake one.  */
       if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1,
 						     1, &cond->__data.__lock,
-						     // XYZ check mutex flag
-						     LLL_SHARED),
-						     0))
+						     pshared), 0))
 	return 0;
 
-      lll_futex_wake (&cond->__data.__futex, 1,
-		      // XYZ check mutex flag
-		      LLL_SHARED);
+      lll_futex_wake (&cond->__data.__futex, 1, pshared);
     }
 
   /* We are done.  */
-  lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_unlock (cond->__data.__lock, pshared);
 
   return 0;
 }
diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c
index a8d95dc224..54839c015a 100644
--- a/nptl/pthread_cond_timedwait.c
+++ b/nptl/pthread_cond_timedwait.c
@@ -53,14 +53,17 @@ __pthread_cond_timedwait (cond, mutex, abstime)
   if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
     return EINVAL;
 
+  int pshared = (cond->__data.__mutex == (void *) ~0l)
+		? LLL_SHARED : LLL_PRIVATE;
+
   /* Make sure we are along.  */
-  lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_lock (cond->__data.__lock, pshared);
 
   /* Now we can release the mutex.  */
   int err = __pthread_mutex_unlock_usercnt (mutex, 0);
   if (err)
     {
-      lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_unlock (cond->__data.__lock, pshared);
       return err;
     }
 
@@ -146,22 +149,20 @@ __pthread_cond_timedwait (cond, mutex, abstime)
       unsigned int futex_val = cond->__data.__futex;
 
       /* Prepare to wait.  Release the condvar futex.  */
-      lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_unlock (cond->__data.__lock, pshared);
 
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
       /* Wait until woken by signal or broadcast.  */
       err = lll_futex_timed_wait (&cond->__data.__futex,
-				  futex_val, &rt,
-				  // XYZ check mutex flag
-				  LLL_SHARED);
+				  futex_val, &rt, pshared);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
 
       /* We are going to look at shared data again, so get the lock.  */
-      lll_lock(cond->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_lock (cond->__data.__lock, pshared);
 
       /* If a broadcast happened, we are done.  */
       if (cbuffer.bc_seq != cond->__data.__broadcast_seq)
@@ -198,12 +199,10 @@ __pthread_cond_timedwait (cond, mutex, abstime)
      and it can be successfully destroyed.  */
   if (cond->__data.__total_seq == -1ULL
       && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
-    lll_futex_wake (&cond->__data.__nwaiters, 1,
-		    // XYZ check mutex flag
-		    LLL_SHARED);
+    lll_futex_wake (&cond->__data.__nwaiters, 1, pshared);
 
   /* We are done with the condvar.  */
-  lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_unlock (cond->__data.__lock, pshared);
 
   /* The cancellation handling is back to normal, remove the handler.  */
   __pthread_cleanup_pop (&buffer, 0);
diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c
index 679655f8fd..670fba5736 100644
--- a/nptl/pthread_cond_wait.c
+++ b/nptl/pthread_cond_wait.c
@@ -43,9 +43,11 @@ __condvar_cleanup (void *arg)
   struct _condvar_cleanup_buffer *cbuffer =
     (struct _condvar_cleanup_buffer *) arg;
   unsigned int destroying;
+  int pshared = (cbuffer->cond->__data.__mutex == (void *) ~0l)
+  		? LLL_SHARED : LLL_PRIVATE;
 
   /* We are going to modify shared data.  */
-  lll_lock (cbuffer->cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_lock (cbuffer->cond->__data.__lock, pshared);
 
   if (cbuffer->bc_seq == cbuffer->cond->__data.__broadcast_seq)
     {
@@ -71,20 +73,16 @@ __condvar_cleanup (void *arg)
   if (cbuffer->cond->__data.__total_seq == -1ULL
       && cbuffer->cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
     {
-      lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1,
-		      // XYZ check mutex flag
-		      LLL_SHARED);
+      lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1, pshared);
       destroying = 1;
     }
 
   /* We are done.  */
-  lll_unlock (cbuffer->cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_unlock (cbuffer->cond->__data.__lock, pshared);
 
   /* Wake everybody to make sure no condvar signal gets lost.  */
   if (! destroying)
-    lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX,
-		    // XYZ check mutex flag
-		    LLL_SHARED);
+    lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, pshared);
 
   /* Get the mutex before returning unless asynchronous cancellation
      is in effect.  */
@@ -100,15 +98,17 @@ __pthread_cond_wait (cond, mutex)
   struct _pthread_cleanup_buffer buffer;
   struct _condvar_cleanup_buffer cbuffer;
   int err;
+  int pshared = (cond->__data.__mutex == (void *) ~0l)
+  		? LLL_SHARED : LLL_PRIVATE;
 
   /* Make sure we are along.  */
-  lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_lock (cond->__data.__lock, pshared);
 
   /* Now we can release the mutex.  */
   err = __pthread_mutex_unlock_usercnt (mutex, 0);
   if (__builtin_expect (err, 0))
     {
-      lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_unlock (cond->__data.__lock, pshared);
       return err;
     }
 
@@ -144,21 +144,19 @@ __pthread_cond_wait (cond, mutex)
       unsigned int futex_val = cond->__data.__futex;
 
       /* Prepare to wait.  Release the condvar futex.  */
-      lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_unlock (cond->__data.__lock, pshared);
 
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
       /* Wait until woken by signal or broadcast.  */
-      lll_futex_wait (&cond->__data.__futex, futex_val,
-		      // XYZ check mutex flag
-		      LLL_SHARED);
+      lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
 
       /* We are going to look at shared data again, so get the lock.  */
-      lll_lock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_lock (cond->__data.__lock, pshared);
 
       /* If a broadcast happened, we are done.  */
       if (cbuffer.bc_seq != cond->__data.__broadcast_seq)
@@ -181,12 +179,10 @@ __pthread_cond_wait (cond, mutex)
      and it can be successfully destroyed.  */
   if (cond->__data.__total_seq == -1ULL
       && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
-    lll_futex_wake (&cond->__data.__nwaiters, 1,
-		    // XYZ check mutex flag
-		    LLL_SHARED);
+    lll_futex_wake (&cond->__data.__nwaiters, 1, pshared);
 
   /* We are done with the condvar.  */
-  lll_unlock (cond->__data.__lock, /* XYZ */ LLL_SHARED);
+  lll_unlock (cond->__data.__lock, pshared);
 
   /* The cancellation handling is back to normal, remove the handler.  */
   __pthread_cleanup_pop (&buffer, 0);
diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c
index 96f1fb00f8..a013ca83fe 100644
--- a/nptl/pthread_mutex_init.c
+++ b/nptl/pthread_mutex_init.c
@@ -1,4 +1,5 @@
-/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007
+   Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -120,6 +121,12 @@ __pthread_mutex_init (mutex, mutexattr)
       break;
     }
 
+  /* The kernel when waking robust mutexes on exit never uses
+     FUTEX_PRIVATE_FLAG FUTEX_WAKE.  */
+  if ((imutexattr->mutexkind & (PTHREAD_MUTEXATTR_FLAG_PSHARED
+				| PTHREAD_MUTEXATTR_FLAG_ROBUST)) != 0)
+    mutex->__data.__kind |= PTHREAD_MUTEX_PSHARED_BIT;
+
   /* Default values: mutex not used yet.  */
   // mutex->__count = 0;	already done by memset
   // mutex->__owner = 0;	already done by memset
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index a82922e99a..ed98dfc6c3 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -27,9 +27,13 @@
 
 
 #ifndef LLL_MUTEX_LOCK
-# define LLL_MUTEX_LOCK(mutex) lll_lock (mutex, /* XYZ */ LLL_SHARED)
-# define LLL_MUTEX_TRYLOCK(mutex) lll_trylock (mutex)
-# define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_lock (mutex, id, /* XYZ */ LLL_SHARED)
+# define LLL_MUTEX_LOCK(mutex) \
+  lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
+# define LLL_MUTEX_TRYLOCK(mutex) \
+  lll_trylock ((mutex)->__data.__lock)
+# define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
+  lll_robust_lock ((mutex)->__data.__lock, id, \
+		   PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
 #endif
 
 
@@ -62,7 +66,7 @@ __pthread_mutex_lock (mutex)
 	}
 
       /* We have to get the mutex.  */
-      LLL_MUTEX_LOCK (mutex->__data.__lock);
+      LLL_MUTEX_LOCK (mutex);
 
       assert (mutex->__data.__owner == 0);
       mutex->__data.__count = 1;
@@ -79,7 +83,7 @@ __pthread_mutex_lock (mutex)
     case PTHREAD_MUTEX_TIMED_NP:
     simple:
       /* Normal mutex.  */
-      LLL_MUTEX_LOCK (mutex->__data.__lock);
+      LLL_MUTEX_LOCK (mutex);
       assert (mutex->__data.__owner == 0);
       break;
 
@@ -87,7 +91,7 @@ __pthread_mutex_lock (mutex)
       if (! __is_smp)
 	goto simple;
 
-      if (LLL_MUTEX_TRYLOCK (mutex->__data.__lock) != 0)
+      if (LLL_MUTEX_TRYLOCK (mutex) != 0)
 	{
 	  int cnt = 0;
 	  int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
@@ -96,7 +100,7 @@ __pthread_mutex_lock (mutex)
 	    {
 	      if (cnt++ >= max_cnt)
 		{
-		  LLL_MUTEX_LOCK (mutex->__data.__lock);
+		  LLL_MUTEX_LOCK (mutex);
 		  break;
 		}
 
@@ -104,7 +108,7 @@ __pthread_mutex_lock (mutex)
 	      BUSY_WAIT_NOP;
 #endif
 	    }
-	  while (LLL_MUTEX_TRYLOCK (mutex->__data.__lock) != 0);
+	  while (LLL_MUTEX_TRYLOCK (mutex) != 0);
 
 	  mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
 	}
@@ -166,16 +170,15 @@ __pthread_mutex_lock (mutex)
 	  /* Check whether we already hold the mutex.  */
 	  if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 	    {
-	      if (mutex->__data.__kind
-		  == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
+	      int kind = PTHREAD_MUTEX_TYPE (mutex);
+	      if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 		{
 		  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 				 NULL);
 		  return EDEADLK;
 		}
 
-	      if (mutex->__data.__kind
-		  == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
+	      if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 		{
 		  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 				 NULL);
@@ -191,14 +194,15 @@ __pthread_mutex_lock (mutex)
 		}
 	    }
 
-	  oldval = LLL_ROBUST_MUTEX_LOCK (mutex->__data.__lock, id);
+	  oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
 
 	  if (__builtin_expect (mutex->__data.__owner
 				== PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 	    {
 	      /* This mutex is now not recoverable.  */
 	      mutex->__data.__count = 0;
-	      lll_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED);
+	      lll_unlock (mutex->__data.__lock,
+			  PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 	      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 	      return ENOTRECOVERABLE;
 	    }
@@ -410,8 +414,7 @@ __pthread_mutex_lock (mutex)
 
 		if (oldval != ceilval)
 		  lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
-				  // XYZ check mutex flag
-				  LLL_SHARED);
+				  PTHREAD_MUTEX_PSHARED (mutex));
 	      }
 	    while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 							ceilval | 2, ceilval)
diff --git a/nptl/pthread_mutex_setprioceiling.c b/nptl/pthread_mutex_setprioceiling.c
index 301fb63d21..836c9a3e93 100644
--- a/nptl/pthread_mutex_setprioceiling.c
+++ b/nptl/pthread_mutex_setprioceiling.c
@@ -47,12 +47,13 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling)
 
   /* Check whether we already hold the mutex.  */
   bool locked = false;
+  int kind = PTHREAD_MUTEX_TYPE (mutex);
   if (mutex->__data.__owner == THREAD_GETMEM (THREAD_SELF, tid))
     {
-      if (mutex->__data.__kind == PTHREAD_MUTEX_PP_ERRORCHECK_NP)
+      if (kind == PTHREAD_MUTEX_PP_ERRORCHECK_NP)
 	return EDEADLK;
 
-      if (mutex->__data.__kind == PTHREAD_MUTEX_PP_RECURSIVE_NP)
+      if (kind == PTHREAD_MUTEX_PP_RECURSIVE_NP)
 	locked = true;
     }
 
@@ -81,8 +82,7 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling)
 
 	    if (oldval != ceilval)
 	      lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
-			      // XYZ check mutex flag
-			      LLL_SHARED);
+			      PTHREAD_MUTEX_PSHARED (mutex));
 	  }
 	while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 						    ceilval | 2, ceilval)
@@ -113,8 +113,7 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling)
   atomic_full_barrier ();
 
   lll_futex_wake (&mutex->__data.__lock, INT_MAX,
-		  // XYZ check mutex flag
-		  LLL_SHARED);
+		  PTHREAD_MUTEX_PSHARED (mutex));
 
   return 0;
 }
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index 7a0ed57eaa..4bf0efea34 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -57,7 +57,7 @@ pthread_mutex_timedlock (mutex, abstime)
 
       /* We have to get the mutex.  */
       result = lll_timedlock (mutex->__data.__lock, abstime,
-			      /* XYZ */ LLL_SHARED);
+			      PTHREAD_MUTEX_PSHARED (mutex));
 
       if (result != 0)
 	goto out;
@@ -78,7 +78,7 @@ pthread_mutex_timedlock (mutex, abstime)
     simple:
       /* Normal mutex.  */
       result = lll_timedlock (mutex->__data.__lock, abstime,
-			      /* XYZ */ LLL_SHARED);
+			      PTHREAD_MUTEX_PSHARED (mutex));
       break;
 
     case PTHREAD_MUTEX_ADAPTIVE_NP:
@@ -95,7 +95,7 @@ pthread_mutex_timedlock (mutex, abstime)
 	      if (cnt++ >= max_cnt)
 		{
 		  result = lll_timedlock (mutex->__data.__lock, abstime,
-					  /* XYZ */ LLL_SHARED);
+					  PTHREAD_MUTEX_PSHARED (mutex));
 		  break;
 		}
 
@@ -152,16 +152,15 @@ pthread_mutex_timedlock (mutex, abstime)
 	  /* Check whether we already hold the mutex.  */
 	  if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 	    {
-	      if (mutex->__data.__kind
-		  == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
+	      int kind = PTHREAD_MUTEX_TYPE (mutex);
+	      if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 		{
 		  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 				 NULL);
 		  return EDEADLK;
 		}
 
-	      if (mutex->__data.__kind
-		  == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
+	      if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 		{
 		  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 				 NULL);
@@ -178,14 +177,15 @@ pthread_mutex_timedlock (mutex, abstime)
 	    }
 
 	  result = lll_robust_timedlock (mutex->__data.__lock, abstime, id,
-					 /* XYZ */ LLL_SHARED);
+					 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 
 	  if (__builtin_expect (mutex->__data.__owner
 				== PTHREAD_MUTEX_NOTRECOVERABLE, 0))
 	    {
 	      /* This mutex is now not recoverable.  */
 	      mutex->__data.__count = 0;
-	      lll_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED);
+	      lll_unlock (mutex->__data.__lock,
+			  PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 	      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 	      return ENOTRECOVERABLE;
 	    }
@@ -446,8 +446,7 @@ pthread_mutex_timedlock (mutex, abstime)
 
 		    lll_futex_timed_wait (&mutex->__data.__lock,
 					  ceilval | 2, &rt,
-					  // XYZ check mutex flag
-					  LLL_SHARED);
+					  PTHREAD_MUTEX_PSHARED (mutex));
 		  }
 	      }
 	    while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c
index 9478d382c2..f6e24d4138 100644
--- a/nptl/pthread_mutex_trylock.c
+++ b/nptl/pthread_mutex_trylock.c
@@ -115,16 +115,15 @@ __pthread_mutex_trylock (mutex)
 	  /* Check whether we already hold the mutex.  */
 	  if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
 	    {
-	      if (mutex->__data.__kind
-		  == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
+	      int kind = PTHREAD_MUTEX_TYPE (mutex);
+	      if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
 		{
 		  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 				 NULL);
 		  return EDEADLK;
 		}
 
-	      if (mutex->__data.__kind
-		  == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
+	      if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
 		{
 		  THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
 				 NULL);
@@ -154,7 +153,8 @@ __pthread_mutex_trylock (mutex)
 	      /* This mutex is now not recoverable.  */
 	      mutex->__data.__count = 0;
 	      if (oldval == id)
-		lll_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED);
+		lll_unlock (mutex->__data.__lock,
+			    PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 	      THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
 	      return ENOTRECOVERABLE;
 	    }
diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c
index 6226089ebe..d33d0593d8 100644
--- a/nptl/pthread_mutex_unlock.c
+++ b/nptl/pthread_mutex_unlock.c
@@ -61,7 +61,7 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
 	--mutex->__data.__nusers;
 
       /* Unlock.  */
-      lll_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
       break;
 
     case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
@@ -115,7 +115,8 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
 	--mutex->__data.__nusers;
 
       /* Unlock.  */
-      lll_robust_unlock (mutex->__data.__lock, /* XYZ */ LLL_SHARED);
+      lll_robust_unlock (mutex->__data.__lock,
+			 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 
       THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
       break;
@@ -242,8 +243,7 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
 
       if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
 	lll_futex_wake (&mutex->__data.__lock, 1,
-			// XYZ check mutex flag
-			LLL_SHARED);
+			PTHREAD_MUTEX_PSHARED (mutex));
 
       int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
       return __pthread_tpp_change_priority (oldprio, -1);
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
index f3f291979a..ab829ad7af 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
@@ -103,24 +103,24 @@
   while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_CMP_REQUEUE, (nr_wake),	      \
-			      (nr_move), (mutex), (val));		      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
+			      (nr_wake), (nr_move), (mutex), (val));	      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \
+#define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_WAKE_OP, (nr_wake),	      \
-			      (nr_wake2), (futexp2),			      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_WAKE_OP, private),    \
+			      (nr_wake), (nr_wake2), (futexp2),		      \
 			      FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);		      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
index 3c28a397ea..fd30c43103 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
@@ -103,18 +103,20 @@ do									\
 while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val)		     \
+#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private)	     \
 ({									     \
-   DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_CMP_REQUEUE,		     \
+   DO_INLINE_SYSCALL(futex, 6, (long) (ftx),				     \
+		     __lll_private_flag (FUTEX_CMP_REQUEUE, private),	     \
 		     (int) (nr_wake), (int) (nr_move), (long) (mutex),	     \
 		     (int) val);					     \
    _r10 == -1;								     \
 })
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2)		     \
+#define lll_futex_wake_unlock(ftx, nr_wake, nr_wake2, ftx2, private)	     \
 ({									     \
-   DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_WAKE_OP,		     \
+   DO_INLINE_SYSCALL(futex, 6, (long) (ftx),				     \
+		     __lll_private_flag (FUTEX_WAKE_OP, private),	     \
 		     (int) (nr_wake), (int) (nr_wake2), (long) (ftx2),	     \
 		     FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);			     \
    _r10 == -1;								     \
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
index 41804d1372..87935c1a85 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
@@ -107,14 +107,14 @@
   while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
 									      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_CMP_REQUEUE, (nr_wake),	      \
-			      (nr_move), (mutex), (val));		      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
+			      (nr_wake), (nr_move), (mutex), (val));	      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
 
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
index a1b6794260..d985c6a79b 100644
--- a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
+++ b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
@@ -4,3 +4,4 @@
 
 MUTEX_KIND	offsetof (pthread_mutex_t, __data.__kind)
 PI_BIT		PTHREAD_MUTEX_PRIO_INHERIT_NP
+PS_BIT		PTHREAD_MUTEX_PSHARED_BIT
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
index 81ecd6556b..93841c5b3e 100644
--- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
+++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
@@ -1,8 +1,12 @@
 #include <pthreadP.h>
 
-#define LLL_MUTEX_LOCK(mutex) lll_cond_lock (mutex, /* XYZ */ LLL_SHARED)
-#define LLL_MUTEX_TRYLOCK(mutex) lll_cond_trylock (mutex)
-#define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_cond_lock (mutex, id, /* XYZ */ LLL_SHARED)
+#define LLL_MUTEX_LOCK(mutex) \
+  lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
+#define LLL_MUTEX_TRYLOCK(mutex) \
+  lll_cond_trylock ((mutex)->__data.__lock)
+#define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
+  lll_robust_cond_lock ((mutex)->__data.__lock, id, \
+			PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
 #define __pthread_mutex_lock __pthread_mutex_cond_lock
 #define NO_INCR
 
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
index ad4d27300f..d687e13c0b 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
@@ -93,7 +93,7 @@
   ({									      \
     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
     register unsigned long int __r3 asm ("3")				      \
-      __lll_private_flag (FUTEX_WAKE, private);				      \
+      = __lll_private_flag (FUTEX_WAKE, private);			      \
     register unsigned long int __r4 asm ("4") = (unsigned long int) (nr);     \
     register unsigned long int __result asm ("2");			      \
 									      \
@@ -117,10 +117,11 @@
 
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val, private) \
   ({									      \
     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
-    register unsigned long int __r3 asm ("3") = FUTEX_CMP_REQUEUE;	      \
+    register unsigned long int __r3 asm ("3")				      \
+      = __lll_private_flag (FUTEX_CMP_REQUEUE, private);		      \
     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);	      \
     register unsigned long int __r5 asm ("5") = (long int) (nr_move);	      \
     register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex);  \
@@ -137,10 +138,11 @@
 
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2) \
+#define lll_futex_wake_unlock(futex, nr_wake, nr_wake2, futex2, private) \
   ({									      \
     register unsigned long int __r2 asm ("2") = (unsigned long int) (futex);  \
-    register unsigned long int __r3 asm ("3") = FUTEX_WAKE_OP;	      \
+    register unsigned long int __r3 asm ("3")				      \
+      = __lll_private_flag (FUTEX_WAKE_OP, private);			      \
     register unsigned long int __r4 asm ("4") = (long int) (nr_wake);	      \
     register unsigned long int __r5 asm ("5") = (long int) (nr_wake2);	      \
     register unsigned long int __r6 asm ("6") = (unsigned long int) (futex2); \
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
index 38692bbd2d..24cbbe413d 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
@@ -96,14 +96,14 @@
   })
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
 									      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_CMP_REQUEUE, (nr_wake),	      \
-			      (nr_move), (mutex), (val));		      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_CMP_REQUEUE, private),\
+			      (nr_wake), (nr_move), (mutex), (val));	      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
 
@@ -121,14 +121,14 @@
 /* Avoid FUTEX_WAKE_OP if supporting pre-v9 CPUs.  */
 # define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) 1
 #else
-# define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2) \
+# define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
   ({									      \
     INTERNAL_SYSCALL_DECL (__err);					      \
     long int __ret;							      \
 									      \
-    __ret = INTERNAL_SYSCALL (futex, __err, 6,				      \
-			      (futexp), FUTEX_WAKE_OP, (nr_wake),	      \
-			      (nr_wake2), (futexp2),			      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_WAKE_OP, private),    \
+			      (nr_wake), (nr_wake2), (futexp2),		      \
 			      FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);		      \
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index 192d203926..2cd69a14ce 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -533,7 +533,7 @@ LLL_STUB_UNWIND_INFO_END
   while (0)
 
 /* Returns non-zero if error happened, zero if success.  */
-#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \
+#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \
   ({ int __res;								      \
      register int __nr_move __asm ("r10") = nr_move;			      \
      register void *__mutex __asm ("r8") = mutex;			      \
@@ -541,7 +541,8 @@ LLL_STUB_UNWIND_INFO_END
      __asm __volatile ("syscall"					      \
 		       : "=a" (__res)					      \
 		       : "0" (__NR_futex), "D" ((void *) ftx),		      \
-			 "S" (FUTEX_CMP_REQUEUE), "d" (nr_wake),	      \
+			 "S" (__lll_private_flag (FUTEX_CMP_REQUEUE,	      \
+						  private)), "d" (nr_wake),   \
 			 "r" (__nr_move), "r" (__mutex), "r" (__val)	      \
 		       : "cx", "r11", "cc", "memory");			      \
      __res < 0; })
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index 0c619bf271..6b8a29e768 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -71,11 +71,18 @@ __pthread_cond_broadcast:
 	je	9f
 
 	/* XXX: The kernel so far doesn't support requeue to PI futex.  */
-	testl	$PI_BIT, MUTEX_KIND(%r8)
+	/* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
+	   type of futex (private resp. shared).  */
+	testl	$(PI_BIT | PS_BIT), MUTEX_KIND(%r8)
 	jne	9f
 
 	/* Wake up all threads.  */
-	movl	$FUTEX_CMP_REQUEUE, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi
+#else
+	movl	%fs:PRIVATE_FUTEX, %esi
+	orl	$FUTEX_CMP_REQUEUE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	movl	$1, %edx
 	movl	$0x7fffffff, %r10d
@@ -104,8 +111,10 @@ __pthread_cond_broadcast:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -114,22 +123,36 @@ __pthread_cond_broadcast:
 
 	/* Unlock in loop requires wakeup.  */
 5:	addq	$cond_lock-cond_futex, %rdi
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	6b
 
 	/* Unlock in loop requires wakeup.  */
 7:	addq	$cond_lock-cond_futex, %rdi
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	subq	$cond_lock-cond_futex, %rdi
 	jmp	8b
 
 9:	/* The futex requeue functionality is not available.  */
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
 	movl	$0x7fffffff, %edx
-	movl	$FUTEX_WAKE, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 	jmp	10b
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 2fc9d1fad7..8be6d4a21b 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -55,10 +55,20 @@ __pthread_cond_signal:
 	addl	$1, (%rdi)
 
 	/* Wake up one thread.  */
-	movl	$FUTEX_WAKE_OP, %esi
-	movl	$SYS_futex, %eax
+	cmpq	$-1, dep_mutex(%r8)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE_OP, %eax
+	movl	$(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE_OP, %esi
+#endif
 	movl	$1, %r10d
+	movl	$SYS_futex, %eax
 #if cond_lock != 0
 	addq	$cond_lock, %r8
 #endif
@@ -75,7 +85,9 @@ __pthread_cond_signal:
 	xorl	%eax, %eax
 	retq
 
-7:	movl	$FUTEX_WAKE, %esi
+7:	/* %esi should be either FUTEX_WAKE_OP or
+	   FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall.  */
+	xorl	$(FUTEX_WAKE | FUTEX_WAKE_OP), %esi
 	movl	$SYS_futex, %eax
 	/* %rdx should be 1 already from $FUTEX_WAKE_OP syscall.
 	movl	$1, %edx  */
@@ -98,8 +110,10 @@ __pthread_cond_signal:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -109,8 +123,13 @@ __pthread_cond_signal:
 	/* Unlock in loop requires wakeup.  */
 5:
 	movq	%r8, %rdi
-	/* XYZ */
+#if cond_lock != 0
+	addq	$cond_lock, %rdi
+#endif
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	6b
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 003069fb6b..415f06f467 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -186,12 +186,20 @@ __pthread_cond_timedwait:
 	movl	%eax, (%rsp)
 
 	leaq	24(%rsp), %r10
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	cmpq	$-1, dep_mutex(%rdi)
+	movq	%r12, %rdx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT, %eax
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	movl	$FUTEX_WAIT, %esi
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+# if FUTEX_WAIT != 0
+	orl	$FUTEX_WAIT, %esi
+# endif
 #endif
-	movq	%r12, %rdx
 	addq	$cond_futex, %rdi
 	movl	$SYS_futex, %eax
 	syscall
@@ -251,9 +259,19 @@ __pthread_cond_timedwait:
 	jne	25f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
@@ -292,8 +310,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 	jmp	2b
 
@@ -302,8 +322,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	4b
 
@@ -312,8 +334,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -325,8 +349,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	11b
 
@@ -344,8 +370,10 @@ __pthread_cond_timedwait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 17:	movq	(%rsp), %rax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 34ef2c7b77..db2683b87b 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -49,8 +49,10 @@ __condvar_cleanup:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -81,9 +83,19 @@ __condvar_cleanup:
 	jne	4f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 	movl	$1, %r12d
@@ -98,16 +110,28 @@ __condvar_cleanup:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
 2:	testq	%r12, %r12
 	jnz	5f
 	addq	$cond_futex, %rdi
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -216,12 +240,20 @@ __pthread_cond_wait:
 	xorq	%r10, %r10
 	movq	%r12, %rdx
 	addq	$cond_futex-cond_lock, %rdi
-	movl	$SYS_futex, %eax
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT, %eax
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	movl	$FUTEX_WAIT, %esi
+	movl	$FUTEX_WAIT, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+# if FUTEX_WAIT != 0
+	orl	$FUTEX_WAIT, %esi
+# endif
 #endif
+	movl	$SYS_futex, %eax
 	syscall
 
 	movl	(%rsp), %edi
@@ -267,9 +299,19 @@ __pthread_cond_wait:
 	jne	17f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
@@ -302,8 +344,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 	jmp	2b
 
@@ -312,8 +356,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	4b
 
@@ -322,8 +368,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -335,8 +383,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	11b
 
@@ -354,8 +404,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 13:	movq	%r10, %rax