Update.

2003-05-25  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/unix/sysv/linux/kernel-features.h: Define
	__ASSUME_FUTEX_REQUEUE for >= 2.5.70.

	* math/test-fenv.c (feexcp_nomask_test): Fix comment.

25 files changed, 366 insertions, 68 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 0f4d0855a2..7a0af3bfde 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,52 @@
+2003-05-25  Ulrich Drepper  <drepper@redhat.com>
+
+	* sysdeps/pthread/pthread_cond_broadcast.c: Try using FUTEX_REQUEUE
+	instead of FUTEX_WAIT.
+	* sysdeps/pthread/pthread_cond_signal.c: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.
+	* sysdeps/pthread/pthread_cond_timedwait.c: Remember mutex which was
+	used in condvar structure.  Call __pthread_mutex_cond_lock instead
+	of __pthread_mutex_lock_internal.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise.
+	* sysdeps/pthread/pthread_cond_wait.c: Likewise.
+	(__condvar_cleanup): Always call __pthread_mutex_cond_lock.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
+	* sysdeps/unix/sysv/linux/Makefile (libpthread-sysdep_routines):
+	Add pthread_mutex_cond_lock.
+	* sysdeps/unix/sysv/linux/lowlevelcond.sym: Add dep_mutex.
+	* sysdeps/unix/sysv/linux/pthread_cond_mutex_lock.c: New file.
+	* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Define
+	lll_mutex_cond_lock.
+	* sysdeps/unix/sysv/linux/ia64/lowlevellock.h: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
+	* sysdeps/unix/sysv/linux/i386/bits/pthread_types.h (pthread_cond_t):
+	Add __mutex field.
+	* sysdeps/unix/sysv/linux/ia64/bits/pthread_types.h: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h: Likewise.
+
+	* sysdeps/i386/tcb-offsets.sym: Define MUTEX_FUTEX.
+	* sysdeps/x86_64/tcb-offsets.sym: Likewise.
+
+	* pthreadP.h: Declare __pthread_mutex_cond_lock.
+	* pthread_mutex_lock.c: Define LLL_MUTEX_LOCK if not already defined.
+	Use it instead of lll_mutex_lock.  If __pthread_mutex_lock is a
+	macro don't define aliases.
+
+	* cancellation.c: Remove __pthread_enable_asynccancel_2.
+	* pthreadP.h: Remove declaration of __pthread_enable_asynccancel_2.
+	* sysdeps/pthread/pthread_cond_timedwait.c: Use
+	__pthread_enable_asynccancel instead of __pthread_enable_asynccancel_2.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise.
+	* sysdeps/pthread/pthread_cond_wait.c: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
+
 2003-05-17  Ulrich Drepper  <drepper@redhat.com>
 
 	* sem_open.c: Fix one endless loop.  Implement correct semantics
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 71f1a7fede..fe982af87b 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -241,6 +241,8 @@ extern int __pthread_mutex_destroy_internal (pthread_mutex_t *__mutex);
 extern int __pthread_mutex_trylock (pthread_mutex_t *_mutex);
 extern int __pthread_mutex_lock (pthread_mutex_t *__mutex);
 extern int __pthread_mutex_lock_internal (pthread_mutex_t *__mutex);
+extern int __pthread_mutex_cond_lock (pthread_mutex_t *__mutex)
+     attribute_hidden;
 extern int __pthread_mutex_unlock (pthread_mutex_t *__mutex);
 extern int __pthread_mutex_unlock_internal (pthread_mutex_t *__mutex);
 extern int __pthread_mutexattr_init (pthread_mutexattr_t *attr);
@@ -320,8 +322,6 @@ extern int __pthread_kill (pthread_t threadid, int signo);
 extern void __pthread_exit (void *value);
 extern int __pthread_setcanceltype (int type, int *oldtype);
 extern int __pthread_enable_asynccancel (void) attribute_hidden;
-extern void __pthread_enable_asynccancel_2 (int *oldvalp)
-     internal_function attribute_hidden;
 extern void __pthread_disable_asynccancel (int oldtype)
      internal_function attribute_hidden;
 
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index f70445acfc..4fdb13718c 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -22,6 +22,11 @@
 #include <lowlevellock.h>
 
 
+#ifndef LLL_MUTEX_LOCK
+# define LLL_MUTEX_LOCK(mutex) lll_mutex_lock (mutex)
+#endif
+
+
 int
 __pthread_mutex_lock (mutex)
      pthread_mutex_t *mutex;
@@ -45,7 +50,7 @@ __pthread_mutex_lock (mutex)
       else
 	{
 	  /* We have to get the mutex.  */
-	  lll_mutex_lock (mutex->__data.__lock);
+	  LLL_MUTEX_LOCK (mutex->__data.__lock);
 
 	  /* Record the ownership.  */
 	  mutex->__data.__owner = id;
@@ -66,7 +71,7 @@ __pthread_mutex_lock (mutex)
     case PTHREAD_MUTEX_TIMED_NP:
     case PTHREAD_MUTEX_ADAPTIVE_NP:
       /* Normal mutex.  */
-      lll_mutex_lock (mutex->__data.__lock);
+      LLL_MUTEX_LOCK (mutex->__data.__lock);
       /* Record the ownership.  */
       mutex->__data.__owner = id;
       break;
@@ -74,5 +79,7 @@ __pthread_mutex_lock (mutex)
 
   return 0;
 }
+#ifndef __pthread_mutex_lock
 strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
 strong_alias (__pthread_mutex_lock, __pthread_mutex_lock_internal)
+#endif
diff --git a/nptl/sysdeps/i386/tcb-offsets.sym b/nptl/sysdeps/i386/tcb-offsets.sym
index 6a5c16c0ee..562ac70d4b 100644
--- a/nptl/sysdeps/i386/tcb-offsets.sym
+++ b/nptl/sysdeps/i386/tcb-offsets.sym
@@ -6,3 +6,4 @@ MULTIPLE_THREADS_OFFSET	offsetof (tcbhead_t, multiple_threads)
 SYSINFO_OFFSET		offsetof (tcbhead_t, sysinfo)
 CLEANUP			offsetof (struct pthread, cleanup)
 CLEANUP_PREV		offsetof (struct _pthread_cleanup_buffer, __prev)
+MUTEX_FUTEX		offsetof (pthread_mutex_t, __data.__lock)
diff --git a/nptl/sysdeps/pthread/pthread_cond_broadcast.c b/nptl/sysdeps/pthread/pthread_cond_broadcast.c
index 1076fe3c50..f34f58cc99 100644
--- a/nptl/sysdeps/pthread/pthread_cond_broadcast.c
+++ b/nptl/sysdeps/pthread/pthread_cond_broadcast.c
@@ -25,6 +25,7 @@
 #include <pthreadP.h>
 
 #include <shlib-compat.h>
+#include <kernel-features.h>
 
 
 int
@@ -54,7 +55,16 @@ __pthread_cond_broadcast (cond)
 #endif
 
       /* Wake everybody.  */
-      lll_futex_wake (futex, INT_MAX);
+      pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
+      if (__builtin_expect (lll_futex_requeue (futex, 1, MAX_INT,
+					       &mut->__data.__lock) == -EINVAL,
+			    0))
+	{
+	  /* The requeue functionality is not available.  */
+#ifndef __ASSUME_FUTEX_REQUEUE
+	  lll_futex_wake (futex, MAX_INT);
+#endif
+	}
 
       /* That's all.  */
       return 0;
diff --git a/nptl/sysdeps/pthread/pthread_cond_signal.c b/nptl/sysdeps/pthread/pthread_cond_signal.c
index 1a035fe05f..b9d8af07b2 100644
--- a/nptl/sysdeps/pthread/pthread_cond_signal.c
+++ b/nptl/sysdeps/pthread/pthread_cond_signal.c
@@ -25,13 +25,15 @@
 #include <pthreadP.h>
 
 #include <shlib-compat.h>
+#include <kernel-features.h>
+
 
 int
 __pthread_cond_signal (cond)
      pthread_cond_t *cond;
 {
   /* Make sure we are alone.  */
-  lll_mutex_lock(cond->__data.__lock);
+  lll_mutex_lock (cond->__data.__lock);
 
   /* Are there any waiters to be woken?  */
   if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
@@ -50,7 +52,22 @@ __pthread_cond_signal (cond)
 #endif
 
       /* Wake one.  */
-      lll_futex_wake (futex, 1);
+      int r = lll_futex_requeue (futex, 0, 1, &cond->__data.__lock);
+      if (__builtin_expect (r == -EINVAL, 0))
+	{
+	  /* The requeue functionality is not available.  */
+#ifndef __ASSUME_FUTEX_REQUEUE
+	  lll_futex_wake (futex, 1);
+#endif
+	}
+      else if (r != 0)
+	{
+	  /* We always have to make the syscall if requeue actually
+	     moved a thread.  */
+	  lll_mutex_unlock_force (cond->__data.__lock);
+
+	  return 0;
+	}
     }
 
   /* We are done.  */
diff --git a/nptl/sysdeps/pthread/pthread_cond_timedwait.c b/nptl/sysdeps/pthread/pthread_cond_timedwait.c
index 23cf0acb99..4dd6f2e02a 100644
--- a/nptl/sysdeps/pthread/pthread_cond_timedwait.c
+++ b/nptl/sysdeps/pthread/pthread_cond_timedwait.c
@@ -66,6 +66,10 @@ __pthread_cond_timedwait (cond, mutex, abstime)
   /* We have one new user of the condvar.  */
   ++cond->__data.__total_seq;
 
+  /* Remember the mutex we are using here.  If there is already a
+     different address store this is a bad user bug.  */
+  cond->__data.__mutex = mutex;
+
   /* Prepare structure passed to cancellation handler.  */
   cbuffer.cond = cond;
   cbuffer.mutex = mutex;
@@ -145,7 +149,7 @@ __pthread_cond_timedwait (cond, mutex, abstime)
       lll_mutex_unlock (cond->__data.__lock);
 
       /* Enable asynchronous cancellation.  Required by the standard.  */
-      __pthread_enable_asynccancel_2 (&cbuffer.oldtype);
+      cbuffer.oldtype = __pthread_enable_asynccancel ();
 
       /* Wait until woken by signal or broadcast.  Note that we
 	 truncate the 'val' value to 32 bits.  */
@@ -184,7 +188,7 @@ __pthread_cond_timedwait (cond, mutex, abstime)
   __pthread_cleanup_pop (&buffer, 0);
 
   /* Get the mutex before returning.  */
-  err = __pthread_mutex_lock_internal (mutex);
+  err = __pthread_mutex_cond_lock (mutex);
 
   return err ?: result;
 }
diff --git a/nptl/sysdeps/pthread/pthread_cond_wait.c b/nptl/sysdeps/pthread/pthread_cond_wait.c
index 708566be03..da94cc2d6b 100644
--- a/nptl/sysdeps/pthread/pthread_cond_wait.c
+++ b/nptl/sysdeps/pthread/pthread_cond_wait.c
@@ -65,8 +65,7 @@ __condvar_cleanup (void *arg)
 
   /* Get the mutex before returning unless asynchronous cancellation
      is in effect.  */
-  if (!(cbuffer->oldtype & CANCELTYPE_BITMASK))
-    __pthread_mutex_lock_internal (cbuffer->mutex);
+  __pthread_mutex_cond_lock (cbuffer->mutex);
 }
 
 
@@ -93,6 +92,10 @@ __pthread_cond_wait (cond, mutex)
   /* We have one new user of the condvar.  */
   ++cond->__data.__total_seq;
 
+  /* Remember the mutex we are using here.  If there is already a
+     different address store this is a bad user bug.  */
+  cond->__data.__mutex = mutex;
+
   /* Prepare structure passed to cancellation handler.  */
   cbuffer.cond = cond;
   cbuffer.mutex = mutex;
@@ -123,7 +126,7 @@ __pthread_cond_wait (cond, mutex)
       lll_mutex_unlock (cond->__data.__lock);
 
       /* Enable asynchronous cancellation.  Required by the standard.  */
-      __pthread_enable_asynccancel_2 (&cbuffer.oldtype);
+      cbuffer.oldtype = __pthread_enable_asynccancel ();
 
       /* Wait until woken by signal or broadcast.  Note that we
 	 truncate the 'val' value to 32 bits.  */
@@ -150,7 +153,7 @@ __pthread_cond_wait (cond, mutex)
   __pthread_cleanup_pop (&buffer, 0);
 
   /* Get the mutex before returning.  */
-  return __pthread_mutex_lock_internal (mutex);
+  return __pthread_mutex_cond_lock (mutex);
 }
 
 versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
diff --git a/nptl/sysdeps/unix/sysv/linux/Makefile b/nptl/sysdeps/unix/sysv/linux/Makefile
index 5701887c06..510232c9c6 100644
--- a/nptl/sysdeps/unix/sysv/linux/Makefile
+++ b/nptl/sysdeps/unix/sysv/linux/Makefile
@@ -21,7 +21,7 @@ ifeq ($(subdir),nptl)
 sysdep_routines += register-atfork unregister-atfork libc_pthread_init \
 		   libc_multiple_threads
 
-libpthread-sysdep_routines += pt-fork
+libpthread-sysdep_routines += pt-fork pthread_mutex_cond_lock
 
 gen-as-const-headers += lowlevelcond.sym lowlevelrwlock.sym lowlevelbarrier.sym
 endif
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
index 0834894c25..4b0f11adc5 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
@@ -78,6 +78,7 @@ typedef union
     unsigned long long int __total_seq;
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
+    void *__mutex;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
   long long int __align;
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 0fa402a95e..06821ad376 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		240
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -41,8 +45,10 @@
 __pthread_cond_broadcast:
 
 	pushl	%ebx
+	pushl	%esi
+	pushl	%edi
 
-	movl	8(%esp), %ebx
+	movl	16(%esp), %ebx
 
 	/* Get internal lock.  */
 	movl	$1, %eax
@@ -69,18 +75,34 @@ __pthread_cond_broadcast:
 3:	movl	%ecx, (%ebx)
 	movl	%eax, 4(%ebx)
 
+	/* Get the address of the mutex used.  */
+	movl	dep_mutex-wakeup_seq(%ebx), %edi
+
 	/* Unlock.  */
 	LOCK
 	subl	$1, cond_lock-wakeup_seq(%ebx)
 	jne	7f
 
 	/* Wake up all threads.  */
-8:	movl	$FUTEX_WAKE, %ecx
+8:	movl	$FUTEX_REQUEUE, %ecx
 	movl	$SYS_futex, %eax
-	movl	$0x7fffffff, %edx
+	movl	$0x7fffffff, %esi
+	movl	$1, %edx
+	/* Get the address of the futex involved.  */
+# if MUTEX_FUTEX != 0
+	addl	$MUTEX_FUTEX, %edi
+# endif
 	ENTER_KERNEL
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpl	$-EINVAL, %eax
+	je	9f
+10:
+#endif
+
 	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -91,6 +113,8 @@ __pthread_cond_broadcast:
 	jne	5f
 
 6:	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -113,6 +137,15 @@ __pthread_cond_broadcast:
 7:	leal	cond_lock-wakeup_seq(%ebx), %eax
 	call	__lll_mutex_unlock_wake
 	jmp	8b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+9:	/* The futex requeue functionality is not available.  */
+	movl	$0x7fffffff, %edx
+	movl	$FUTEX_WAKE, %ecx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	jmp	10b
+#endif
 	.size	__pthread_cond_broadcast, .-__pthread_cond_broadcast
 versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index 5465d7b2a8..ed25c554d2 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		240
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -41,23 +45,25 @@
 __pthread_cond_signal:
 
 	pushl	%ebx
+	pushl	%esi
+	pushl	%edi
 
-	movl	8(%esp), %ebx
+	movl	16(%esp), %edi
 
 	/* Get internal lock.  */
 	movl	$1, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	xaddl	%eax, (%edi)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	xaddl	%eax, cond_lock(%edi)
 #endif
 	testl	%eax, %eax
 	jne	1f
 
-2:	addl	$wakeup_seq, %ebx
-	movl	total_seq+4-wakeup_seq(%ebx), %eax
-	movl	total_seq-wakeup_seq(%ebx), %ecx
+2:	leal	wakeup_seq(%edi), %ebx
+	movl	total_seq+4(%edi), %eax
+	movl	total_seq(%edi), %ecx
 	cmpl	4(%ebx), %eax
 	ja	3f
 	jb	4f
@@ -68,18 +74,30 @@ __pthread_cond_signal:
 3:	addl	$1, (%ebx)
 	adcl	$0, 4(%ebx)
 
-	/* Wake up one thread.  */
-	movl	$FUTEX_WAKE, %ecx
+	/* Wake up one thread by moving it to the internal lock futex.  */
+	movl	$FUTEX_REQUEUE, %ecx
 	movl	$SYS_futex, %eax
-	movl	%ecx, %edx	/* movl $1, %edx */
+	xorl	%edx, %edx
+	movl	$1, %esi
 	ENTER_KERNEL
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpl	$-EINVAL, %eax
+	je	7f
+#endif
+
+	/* If we moved a thread we in any case have to make the syscall.  */
+	testl	%eax, %eax
+	jne	5f
+
 	/* Unlock.  */
 4:	LOCK
-	subl	$1, cond_lock-wakeup_seq(%ebx)
+	subl	$1, (%edi)
 	jne	5f
 
 6:	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -93,11 +111,24 @@ __pthread_cond_signal:
 	call	__lll_mutex_lock_wait
 	jmp	2b
 
-	/* Unlock in loop requires waekup.  */
+	/* Unlock in loop requires wakeup.  */
 5:
-	leal	cond_lock-wakeup_seq(%ebx), %eax
+#if cond_lock == 0
+	movl	%edi, %eax
+#else
+	leal	cond_lock(%edi), %eax
+#endif
 	call	__lll_mutex_unlock_wake
 	jmp	6b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+7:	/* The futex requeue functionality is not available.  */
+	movl	$1, %edx
+	movl	$FUTEX_WAKE, %ecx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	jmp	4b
+#endif
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
 versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
index d9bffe5139..5eec268b29 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
@@ -66,8 +66,13 @@ __pthread_cond_timedwait:
 	testl	%eax, %eax
 	jne	1f
 
+	/* Store the reference to the mutex.  If there is already a
+	   different value in there this is a bad user bug.  */
+2:	movl	24(%esp), %eax
+	movl	%eax, dep_mutex(%ebx)
+
 	/* Unlock the mutex.  */
-2:	pushl	24(%esp)
+	pushl	%eax
 .Lpush4:
 	call	__pthread_mutex_unlock_internal
 
@@ -113,8 +118,8 @@ __pthread_cond_timedwait:
 #endif
 	jne	3f
 
-4:	leal	8(%esp), %eax
-	call	__pthread_enable_asynccancel_2
+4:	call	__pthread_enable_asynccancel
+	movl	%eax, 8(%esp)
 
 	/* Get the current time.  */
 	movl	%ebx, %edx
@@ -230,7 +235,7 @@ __pthread_cond_timedwait:
 	movl	%edx, %gs:CLEANUP
 
 	/* Trick ahead:	 (%esp) contains the address of the mutex.  */
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	addl	$44, %esp
 .Laddl:
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 1ac6c1a7d6..61d3d8d225 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -83,15 +83,11 @@ __condvar_cleanup:
 	movl	$0x7fffffff, %edx
 	ENTER_KERNEL
 
-	/* Lock the mutex unless asynchronous cancellation is in effect.  */
-	testl	$2, 8(%esi)
-	jne	3f
-
 	pushl	(%esi)
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	popl	%eax
 
-3:	popl	%esi
+	popl	%esi
 	popl	%ebx
 	ret
 	.size	__condvar_cleanup, .-__condvar_cleanup
@@ -125,8 +121,13 @@ __pthread_cond_wait:
 	testl	%eax, %eax
 	jne	1f
 
+	/* Store the reference to the mutex.  If there is already a
+	   different value in there this is a bad user bug.  */
+2:	movl	20(%esp), %eax
+	movl	%eax, dep_mutex(%ebx)
+
 	/* Unlock the mutex.  */
-2:	pushl	20(%esp)
+	pushl	%eax
 .Lpush4:
 	call	__pthread_mutex_unlock_internal
 
@@ -171,8 +172,8 @@ __pthread_cond_wait:
 #endif
 	jne	3f
 
-4:	leal	8(%esp), %eax
-	call	__pthread_enable_asynccancel_2
+4:	call	__pthread_enable_asynccancel
+	movl	%eax, 8(%esp)
 
 	movl	%esi, %ecx	/* movl $FUTEX_WAIT, %ecx */
 	movl	%edi, %edx
@@ -229,7 +230,7 @@ __pthread_cond_wait:
 	movl	%edx, %gs:CLEANUP
 
 	/* Trick ahead:	 (%esp) contains the address of the mutex.  */
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	addl	$36, %esp
 .Laddl:
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index 5fd50b97db..8923afbfed 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -126,6 +126,24 @@ extern int __lll_mutex_unlock_wait (int *__futex)
 			      : "memory"); })
 
 
+/* Special version of lll_mutex_lock which causes the unlock function to
+   always wakeup waiters.  */
+#define lll_mutex_cond_lock(futex) \
+  (void) ({ int ignore1, ignore2;					      \
+	    __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"		      \
+			      "testl %0, %0\n\t"			      \
+			      "jne 1f\n\t"				      \
+			      ".subsection 1\n"				      \
+			      "1:\tleal %2, %%ecx\n\t"			      \
+			      "call __lll_mutex_lock_wait\n\t"		      \
+			      "jmp 2f\n\t"				      \
+			      ".previous\n"				      \
+			      "2:"					      \
+			      : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \
+			      : "0" (2), "2" (futex)			      \
+			      : "memory"); })
+
+
 #define lll_mutex_timedlock(futex, timeout) \
   ({ int result, ignore1, ignore2;					      \
      __asm __volatile (LOCK_INSTR "xaddl %0, %3\n\t"			      \
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
index 92c0b5c524..b1941e07c6 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
@@ -29,6 +29,7 @@
 #define SYS_futex		1230
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
 
 /* Initializer for compatibility lock.	*/
 #define LLL_MUTEX_LOCK_INITIALIZER (0)
@@ -64,7 +65,7 @@
 			 "=r" (__o0), "=r" (__o1), "=r" (__o2), "=r" (__o3)   \
 		       : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
 		 	 "5" (__o2), "6" (__o3)				      \
-		       : lll_futex_clobbers);				      \
+		       : "out4", lll_futex_clobbers);			      \
      __r10 == -1 ? -__r8 : __r8;					      \
   })
 
@@ -83,10 +84,33 @@
 			 "=r" (__o0), "=r" (__o1), "=r" (__o2)		      \
 		       : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
 		 	 "5" (__o2)					      \
-		       : "out3", lll_futex_clobbers);			      \
+		       : "out3", "out4", lll_futex_clobbers);		      \
      __r10 == -1 ? -__r8 : __r8;					      \
   })
 
+
+#define lll_futex_requeue(futex, nr_wake, nr_move, mutex) \
+  ({									      \
+     register long int __o0 asm ("out0") = (long int) (futex);		      \
+     register long int __o1 asm ("out1") = FUTEX_REQUEUE;		      \
+     register long int __o2 asm ("out2") = (long int) (nr_wake);	      \
+     register long int __o3 asm ("out3") = (long int) (nr_move);	      \
+     register long int __o4 asm ("out4") = (long int) (mutex);		      \
+     register long int __r8 asm ("r8");					      \
+     register long int __r10 asm ("r10");				      \
+     register long int __r15 asm ("r15") = SYS_futex;			      \
+									      \
+     __asm __volatile ("break %7;;"					      \
+		       : "=r" (__r8), "=r" (__r10), "=r" (__r15),	      \
+			 "=r" (__o0), "=r" (__o1), "=r" (__o2), "r" (__o3),   \
+			 "=r" (__o4)					      \
+		       : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
+			 "5" (__o2), "6" (__o3), "7" (__o4)		      \
+		       : lll_futex_clobbers);				      \
+     __r8;								      \
+  })
+
+
 static inline int
 __attribute__ ((always_inline))
 __lll_mutex_trylock (int *futex)
@@ -111,6 +135,18 @@ __lll_mutex_lock (int *futex)
 #define lll_mutex_lock(futex) __lll_mutex_lock (&(futex))
 
 
+static inline void
+__attribute__ ((always_inline))
+__lll_mutex_cond_lock (int *futex)
+{
+  int val = atomic_exchange_and_add (futex, 2);
+
+  if (__builtin_expect (val != 0, 0))
+    __lll_lock_wait (futex, val);
+}
+#define lll_mutex_cond_lock(futex) __lll_mutex_cond_lock (&(futex))
+
+
 extern int __lll_timedlock_wait (int *futex, int val, const struct timespec *)
      attribute_hidden;
 
@@ -140,7 +176,11 @@ __lll_mutex_unlock (int *futex)
   if (__builtin_expect (val > 1, 0))
     lll_futex_wake (futex, 1);
 }
-#define lll_mutex_unlock(futex) __lll_mutex_unlock(&(futex))
+#define lll_mutex_unlock(futex) \
+  __lll_mutex_unlock(&(futex))
+
+#define lll_mutex_unlock_force(futex) \
+  lll_futex_wake (&(futex), 1)
 
 #define lll_mutex_islocked(futex) \
   (futex != 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
index 5eb535e157..1463e0810e 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
@@ -8,3 +8,4 @@ cond_clock	offsetof (pthread_cond_t, __data.__clock)
 total_seq	offsetof (pthread_cond_t, __data.__total_seq)
 wakeup_seq	offsetof (pthread_cond_t, __data.__wakeup_seq)
 woken_seq	offsetof (pthread_cond_t, __data.__woken_seq)
+dep_mutex	offsetof (pthread_cond_t, __data.__mutex)
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
new file mode 100644
index 0000000000..893a5e932d
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
@@ -0,0 +1,6 @@
+#include <pthreadP.h>
+
+#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock(mutex)
+#define __pthread_mutex_lock __pthread_mutex_cond_lock
+
+#include <nptl/pthread_mutex_lock.c>
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
index 47d856cae4..7c12db6fc6 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
@@ -78,6 +78,7 @@ typedef union
     unsigned long long int __total_seq;
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
+    void *__mutex;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
   long int __align;
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index 8e1742b671..136dc574e0 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -102,6 +102,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
 			      : "0" (1), "2" (futex)			      \
 			      : "cx", "r11", "cc", "memory"); })
 
+
+#define lll_mutex_cond_lock(futex) \
+  (void) ({ int ignore1, ignore2;					      \
+	    __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"		      \
+			      "testl %0, %0\n\t"			      \
+			      "jne 1f\n\t"				      \
+			      ".subsection 1\n"				      \
+			      "1:\tleaq %2, %%rdi\n\t"			      \
+			      "subq $128, %%rsp\n\t"			      \
+			      "callq __lll_mutex_lock_wait\n\t"		      \
+			      "addq $128, %%rsp\n\t"			      \
+			      "jmp 2f\n\t"				      \
+			      ".previous\n"				      \
+			      "2:"					      \
+			      : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \
+			      : "0" (2), "2" (futex)			      \
+			      : "cx", "r11", "cc", "memory"); })
+
+
 #define lll_mutex_timedlock(futex, timeout) \
   ({ int result, ignore1, ignore2, ignore3;				      \
      __asm __volatile (LOCK_INSTR "xaddl %0, %4\n\t"			      \
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index cc12f5407c..66edb9ac85 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		202
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -60,17 +64,27 @@ __pthread_cond_broadcast:
 	   woken up.  */
 	movq	%rcx, (%rdi)
 
+	/* Get the address of the mutex used.  */
+	movq	dep_mutex-wakeup_seq(%rdi), %r8
+
 	/* Unlock.  */
 	LOCK
 	decl	cond_lock-wakeup_seq(%rdi)
 	jne	7f
 
 	/* Wake up all threads.  */
-8:	movq	$FUTEX_WAKE, %rsi
+8:	movq	$FUTEX_REQUEUE, %rsi
 	movq	$SYS_futex, %rax
-	movl	$0x7fffffff, %edx
+	movl	$1, %edx
+	movq	$0x7fffffff, %r10
 	syscall
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpq	$-EINVAL, %eax
+	je	9f
+10:
+#endif
+
 	xorl	%eax, %eax
 	retq
 
@@ -104,6 +118,15 @@ __pthread_cond_broadcast:
 	callq	__lll_mutex_unlock_wake
 	subq	$cond_lock-wakeup_seq, %rdi
 	jmp	8b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+9:	/* The futex requeue functionality is not available.  */
+	movq	$0x7fffffff, %rdx
+	movq	$FUTEX_WAKE, %rsi
+	movq	$SYS_futex, %rax
+	syscall
+	jmp	10b
+#endif
 	.size	__pthread_cond_broadcast, .-__pthread_cond_broadcast
 versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 11635baa22..709fcf4ca8 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		202
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -44,15 +48,15 @@ __pthread_cond_signal:
 	movl	$1, %esi
 	LOCK
 #if cond_lock == 0
-	xaddl	%esi, (%rdi)
+	xaddl	%esi, (%r8)
 #else
-	xaddl	%esi, cond_lock(%rdi)
+	xaddl	%esi, cond_lock(%r8)
 #endif
 	testl	%esi, %esi
 	jne	1f
 
-2:	addq	$wakeup_seq, %rdi
-	movq	total_seq-wakeup_seq(%rdi), %rcx
+2:	leaq	wakeup_seq(%r8), %rdi
+	movq	total_seq(%r8), %rcx
 	cmpq	(%rdi), %rcx
 	jbe	4f
 
@@ -62,12 +66,22 @@ __pthread_cond_signal:
 	/* Wake up one thread.  */
 	movq	$FUTEX_WAKE, %rsi
 	movq	$SYS_futex, %rax
-	movq	%rsi, %rdx	/* movl $1, %edx */
+	xorq	%rdx, %rdx
+	movq	$1, %r10
 	syscall
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpq	$-EINVAL, %rax
+	je	7f
+#endif
+
+	/* If we moved a thread we in any case have to make the syscall.  */
+	testq	%rax, %rax
+	jne	5f
+
 	/* Unlock.  */
 4:	LOCK
-	decl	cond_lock-wakeup_seq(%rdi)
+	decl	cond_lock(%r8)
 	jne	5f
 
 6:	xorl	%eax, %eax
@@ -85,9 +99,22 @@ __pthread_cond_signal:
 	jmp	2b
 
 	/* Unlock in loop requires waekup.  */
-5:	addq	$cond_lock-wakeup_seq, %rdi
+5:
+#if cond_lock != 0
+	addq	$cond_lock-wakeup_seq, %rdi
+#else
+	movq	%r8, %rdi
+#endif
 	callq	__lll_mutex_unlock_wake
 	jmp	6b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+7:	/* The futex requeue functionality is not available.  */
+	movq	$1, %rdx
+	movq	$FUTEX_WAKE, %esi
+	movq	$SYS_futex, %rax
+	syscall
+	jmp	4b
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
 versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 151395048a..c8fd4ea129 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -88,7 +88,9 @@ __pthread_cond_timedwait:
 	jne	1f
 
 	/* Unlock the mutex.  */
-2:	movq	16(%rsp), %rdi
+2:	movq	%rdi, %rax
+	movq	16(%rsp), %rdi
+	movq	%rdi, dep_mutex(%rax)
 	callq	__pthread_mutex_unlock_internal
 
 	testl	%eax, %eax
@@ -121,8 +123,8 @@ __pthread_cond_timedwait:
 #endif
 	jne	3f
 
-4:	movq	%rsp, %rdi
-	callq	__pthread_enable_asynccancel_2
+4:	callq	__pthread_enable_asynccancel
+	movq	%rax, (%rsp)
 
 	/* Get the current time.  */
 #ifdef __NR_clock_gettime
@@ -227,7 +229,7 @@ __pthread_cond_timedwait:
 	movq	%rdx, %fs:CLEANUP
 
 	movq	16(%rsp), %rdi
-	callq	__pthread_mutex_lock_internal
+	callq	__pthread_mutex_cond_lock
 
 	testq	%rax, %rax
 	cmoveq	%r14, %rax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 6cad2812f0..5189972fc7 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -84,12 +84,8 @@ __condvar_cleanup:
 	movq	$SYS_futex, %rax
 	syscall
 
-	/* Lock the mutex unless asynchronous cancellation is in effect.  */
-	testq	$2, (%r8)
-	jne	3f
-
 	movq	16(%r8), %rdi
-	callq	__pthread_mutex_lock_internal
+	callq	__pthread_mutex_cond_lock
 
 3:	retq
 	.size	__condvar_cleanup, .-__condvar_cleanup
@@ -137,7 +133,9 @@ __pthread_cond_wait:
 	jne	1f
 
 	/* Unlock the mutex.  */
-2:	movq	16(%rsp), %rdi
+2:	movq	%rdi, %rax
+	movq	16(%rsp), %rdi
+	movq	%rdi, dep_mutex(%rax)
 	callq	__pthread_mutex_unlock_internal
 
 	testl	%eax, %eax
@@ -170,8 +168,8 @@ __pthread_cond_wait:
 #endif
 	jne	3f
 
-4:	movq	%rsp, %rdi
-	callq	__pthread_enable_asynccancel_2
+4:	callq	__pthread_enable_asynccancel
+	movq	%rax, (%rsp)
 
 	movq	8(%rsp), %rdi
 	xorq	%r10, %r10
@@ -221,7 +219,7 @@ __pthread_cond_wait:
 	movq	%rdx, %fs:CLEANUP
 
 	movq	16(%rsp), %rdi
-	callq	__pthread_mutex_lock_internal
+	callq	__pthread_mutex_cond_lock
 14:	addq	$64, %rsp
 .Laddq:
 
diff --git a/nptl/sysdeps/x86_64/tcb-offsets.sym b/nptl/sysdeps/x86_64/tcb-offsets.sym
index 046ad5bee8..e2abc02647 100644
--- a/nptl/sysdeps/x86_64/tcb-offsets.sym
+++ b/nptl/sysdeps/x86_64/tcb-offsets.sym
@@ -4,3 +4,4 @@
 SELF			offsetof (tcbhead_t, self)
 CLEANUP			offsetof (struct pthread, cleanup)
 CLEANUP_PREV		offsetof (struct _pthread_cleanup_buffer, __prev)
+MUTEX_FUTEX		offsetof (pthread_mutex_t, __data.__lock)