Update.

2003-05-25  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/unix/sysv/linux/kernel-features.h: Define
	__ASSUME_FUTEX_REQUEUE for >= 2.5.70.

	* math/test-fenv.c (feexcp_nomask_test): Fix comment.

16 files changed, 261 insertions, 55 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/Makefile b/nptl/sysdeps/unix/sysv/linux/Makefile
index 5701887c06..510232c9c6 100644
--- a/nptl/sysdeps/unix/sysv/linux/Makefile
+++ b/nptl/sysdeps/unix/sysv/linux/Makefile
@@ -21,7 +21,7 @@ ifeq ($(subdir),nptl)
 sysdep_routines += register-atfork unregister-atfork libc_pthread_init \
 		   libc_multiple_threads
 
-libpthread-sysdep_routines += pt-fork
+libpthread-sysdep_routines += pt-fork pthread_mutex_cond_lock
 
 gen-as-const-headers += lowlevelcond.sym lowlevelrwlock.sym lowlevelbarrier.sym
 endif
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
index 0834894c25..4b0f11adc5 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
@@ -78,6 +78,7 @@ typedef union
     unsigned long long int __total_seq;
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
+    void *__mutex;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
   long long int __align;
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 0fa402a95e..06821ad376 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		240
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -41,8 +45,10 @@
 __pthread_cond_broadcast:
 
 	pushl	%ebx
+	pushl	%esi
+	pushl	%edi
 
-	movl	8(%esp), %ebx
+	movl	16(%esp), %ebx
 
 	/* Get internal lock.  */
 	movl	$1, %eax
@@ -69,18 +75,34 @@ __pthread_cond_broadcast:
 3:	movl	%ecx, (%ebx)
 	movl	%eax, 4(%ebx)
 
+	/* Get the address of the mutex used.  */
+	movl	dep_mutex-wakeup_seq(%ebx), %edi
+
 	/* Unlock.  */
 	LOCK
 	subl	$1, cond_lock-wakeup_seq(%ebx)
 	jne	7f
 
 	/* Wake up all threads.  */
-8:	movl	$FUTEX_WAKE, %ecx
+8:	movl	$FUTEX_REQUEUE, %ecx
 	movl	$SYS_futex, %eax
-	movl	$0x7fffffff, %edx
+	movl	$0x7fffffff, %esi
+	movl	$1, %edx
+	/* Get the address of the futex involved.  */
+# if MUTEX_FUTEX != 0
+	addl	$MUTEX_FUTEX, %edi
+# endif
 	ENTER_KERNEL
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpl	$-EINVAL, %eax
+	je	9f
+10:
+#endif
+
 	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -91,6 +113,8 @@ __pthread_cond_broadcast:
 	jne	5f
 
 6:	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -113,6 +137,15 @@ __pthread_cond_broadcast:
 7:	leal	cond_lock-wakeup_seq(%ebx), %eax
 	call	__lll_mutex_unlock_wake
 	jmp	8b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+9:	/* The futex requeue functionality is not available.  */
+	movl	$0x7fffffff, %edx
+	movl	$FUTEX_WAKE, %ecx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	jmp	10b
+#endif
 	.size	__pthread_cond_broadcast, .-__pthread_cond_broadcast
 versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index 5465d7b2a8..ed25c554d2 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		240
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -41,23 +45,25 @@
 __pthread_cond_signal:
 
 	pushl	%ebx
+	pushl	%esi
+	pushl	%edi
 
-	movl	8(%esp), %ebx
+	movl	16(%esp), %edi
 
 	/* Get internal lock.  */
 	movl	$1, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	xaddl	%eax, (%edi)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	xaddl	%eax, cond_lock(%edi)
 #endif
 	testl	%eax, %eax
 	jne	1f
 
-2:	addl	$wakeup_seq, %ebx
-	movl	total_seq+4-wakeup_seq(%ebx), %eax
-	movl	total_seq-wakeup_seq(%ebx), %ecx
+2:	leal	wakeup_seq(%edi), %ebx
+	movl	total_seq+4(%edi), %eax
+	movl	total_seq(%edi), %ecx
 	cmpl	4(%ebx), %eax
 	ja	3f
 	jb	4f
@@ -68,18 +74,30 @@ __pthread_cond_signal:
 3:	addl	$1, (%ebx)
 	adcl	$0, 4(%ebx)
 
-	/* Wake up one thread.  */
-	movl	$FUTEX_WAKE, %ecx
+	/* Wake up one thread by moving it to the internal lock futex.  */
+	movl	$FUTEX_REQUEUE, %ecx
 	movl	$SYS_futex, %eax
-	movl	%ecx, %edx	/* movl $1, %edx */
+	xorl	%edx, %edx
+	movl	$1, %esi
 	ENTER_KERNEL
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpl	$-EINVAL, %eax
+	je	7f
+#endif
+
+	/* If we moved a thread we in any case have to make the syscall.  */
+	testl	%eax, %eax
+	jne	5f
+
 	/* Unlock.  */
 4:	LOCK
-	subl	$1, cond_lock-wakeup_seq(%ebx)
+	subl	$1, (%edi)
 	jne	5f
 
 6:	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -93,11 +111,24 @@ __pthread_cond_signal:
 	call	__lll_mutex_lock_wait
 	jmp	2b
 
-	/* Unlock in loop requires waekup.  */
+	/* Unlock in loop requires wakeup.  */
 5:
-	leal	cond_lock-wakeup_seq(%ebx), %eax
+#if cond_lock == 0
+	movl	%edi, %eax
+#else
+	leal	cond_lock(%edi), %eax
+#endif
 	call	__lll_mutex_unlock_wake
 	jmp	6b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+7:	/* The futex requeue functionality is not available.  */
+	movl	$1, %edx
+	movl	$FUTEX_WAKE, %ecx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	jmp	4b
+#endif
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
 versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
index d9bffe5139..5eec268b29 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
@@ -66,8 +66,13 @@ __pthread_cond_timedwait:
 	testl	%eax, %eax
 	jne	1f
 
+	/* Store the reference to the mutex.  If there is already a
+	   different value in there this is a bad user bug.  */
+2:	movl	24(%esp), %eax
+	movl	%eax, dep_mutex(%ebx)
+
 	/* Unlock the mutex.  */
-2:	pushl	24(%esp)
+	pushl	%eax
 .Lpush4:
 	call	__pthread_mutex_unlock_internal
 
@@ -113,8 +118,8 @@ __pthread_cond_timedwait:
 #endif
 	jne	3f
 
-4:	leal	8(%esp), %eax
-	call	__pthread_enable_asynccancel_2
+4:	call	__pthread_enable_asynccancel
+	movl	%eax, 8(%esp)
 
 	/* Get the current time.  */
 	movl	%ebx, %edx
@@ -230,7 +235,7 @@ __pthread_cond_timedwait:
 	movl	%edx, %gs:CLEANUP
 
 	/* Trick ahead:	 (%esp) contains the address of the mutex.  */
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	addl	$44, %esp
 .Laddl:
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 1ac6c1a7d6..61d3d8d225 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -83,15 +83,11 @@ __condvar_cleanup:
 	movl	$0x7fffffff, %edx
 	ENTER_KERNEL
 
-	/* Lock the mutex unless asynchronous cancellation is in effect.  */
-	testl	$2, 8(%esi)
-	jne	3f
-
 	pushl	(%esi)
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	popl	%eax
 
-3:	popl	%esi
+	popl	%esi
 	popl	%ebx
 	ret
 	.size	__condvar_cleanup, .-__condvar_cleanup
@@ -125,8 +121,13 @@ __pthread_cond_wait:
 	testl	%eax, %eax
 	jne	1f
 
+	/* Store the reference to the mutex.  If there is already a
+	   different value in there this is a bad user bug.  */
+2:	movl	20(%esp), %eax
+	movl	%eax, dep_mutex(%ebx)
+
 	/* Unlock the mutex.  */
-2:	pushl	20(%esp)
+	pushl	%eax
 .Lpush4:
 	call	__pthread_mutex_unlock_internal
 
@@ -171,8 +172,8 @@ __pthread_cond_wait:
 #endif
 	jne	3f
 
-4:	leal	8(%esp), %eax
-	call	__pthread_enable_asynccancel_2
+4:	call	__pthread_enable_asynccancel
+	movl	%eax, 8(%esp)
 
 	movl	%esi, %ecx	/* movl $FUTEX_WAIT, %ecx */
 	movl	%edi, %edx
@@ -229,7 +230,7 @@ __pthread_cond_wait:
 	movl	%edx, %gs:CLEANUP
 
 	/* Trick ahead:	 (%esp) contains the address of the mutex.  */
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	addl	$36, %esp
 .Laddl:
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index 5fd50b97db..8923afbfed 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -126,6 +126,24 @@ extern int __lll_mutex_unlock_wait (int *__futex)
 			      : "memory"); })
 
 
+/* Special version of lll_mutex_lock which causes the unlock function to
+   always wakeup waiters.  */
+#define lll_mutex_cond_lock(futex) \
+  (void) ({ int ignore1, ignore2;					      \
+	    __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"		      \
+			      "testl %0, %0\n\t"			      \
+			      "jne 1f\n\t"				      \
+			      ".subsection 1\n"				      \
+			      "1:\tleal %2, %%ecx\n\t"			      \
+			      "call __lll_mutex_lock_wait\n\t"		      \
+			      "jmp 2f\n\t"				      \
+			      ".previous\n"				      \
+			      "2:"					      \
+			      : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \
+			      : "0" (2), "2" (futex)			      \
+			      : "memory"); })
+
+
 #define lll_mutex_timedlock(futex, timeout) \
   ({ int result, ignore1, ignore2;					      \
      __asm __volatile (LOCK_INSTR "xaddl %0, %3\n\t"			      \
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
index 92c0b5c524..b1941e07c6 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
@@ -29,6 +29,7 @@
 #define SYS_futex		1230
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
 
 /* Initializer for compatibility lock.	*/
 #define LLL_MUTEX_LOCK_INITIALIZER (0)
@@ -64,7 +65,7 @@
 			 "=r" (__o0), "=r" (__o1), "=r" (__o2), "=r" (__o3)   \
 		       : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
 		 	 "5" (__o2), "6" (__o3)				      \
-		       : lll_futex_clobbers);				      \
+		       : "out4", lll_futex_clobbers);			      \
      __r10 == -1 ? -__r8 : __r8;					      \
   })
 
@@ -83,10 +84,33 @@
 			 "=r" (__o0), "=r" (__o1), "=r" (__o2)		      \
 		       : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
 		 	 "5" (__o2)					      \
-		       : "out3", lll_futex_clobbers);			      \
+		       : "out3", "out4", lll_futex_clobbers);		      \
      __r10 == -1 ? -__r8 : __r8;					      \
   })
 
+
+#define lll_futex_requeue(futex, nr_wake, nr_move, mutex) \
+  ({									      \
+     register long int __o0 asm ("out0") = (long int) (futex);		      \
+     register long int __o1 asm ("out1") = FUTEX_REQUEUE;		      \
+     register long int __o2 asm ("out2") = (long int) (nr_wake);	      \
+     register long int __o3 asm ("out3") = (long int) (nr_move);	      \
+     register long int __o4 asm ("out4") = (long int) (mutex);		      \
+     register long int __r8 asm ("r8");					      \
+     register long int __r10 asm ("r10");				      \
+     register long int __r15 asm ("r15") = SYS_futex;			      \
+									      \
+     __asm __volatile ("break %7;;"					      \
+		       : "=r" (__r8), "=r" (__r10), "=r" (__r15),	      \
+			 "=r" (__o0), "=r" (__o1), "=r" (__o2), "r" (__o3),   \
+			 "=r" (__o4)					      \
+		       : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \
+			 "5" (__o2), "6" (__o3), "7" (__o4)		      \
+		       : lll_futex_clobbers);				      \
+     __r8;								      \
+  })
+
+
 static inline int
 __attribute__ ((always_inline))
 __lll_mutex_trylock (int *futex)
@@ -111,6 +135,18 @@ __lll_mutex_lock (int *futex)
 #define lll_mutex_lock(futex) __lll_mutex_lock (&(futex))
 
 
+static inline void
+__attribute__ ((always_inline))
+__lll_mutex_cond_lock (int *futex)
+{
+  int val = atomic_exchange_and_add (futex, 2);
+
+  if (__builtin_expect (val != 0, 0))
+    __lll_lock_wait (futex, val);
+}
+#define lll_mutex_cond_lock(futex) __lll_mutex_cond_lock (&(futex))
+
+
 extern int __lll_timedlock_wait (int *futex, int val, const struct timespec *)
      attribute_hidden;
 
@@ -140,7 +176,11 @@ __lll_mutex_unlock (int *futex)
   if (__builtin_expect (val > 1, 0))
     lll_futex_wake (futex, 1);
 }
-#define lll_mutex_unlock(futex) __lll_mutex_unlock(&(futex))
+#define lll_mutex_unlock(futex) \
+  __lll_mutex_unlock(&(futex))
+
+#define lll_mutex_unlock_force(futex) \
+  lll_futex_wake (&(futex), 1)
 
 #define lll_mutex_islocked(futex) \
   (futex != 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
index 5eb535e157..1463e0810e 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
@@ -8,3 +8,4 @@ cond_clock	offsetof (pthread_cond_t, __data.__clock)
 total_seq	offsetof (pthread_cond_t, __data.__total_seq)
 wakeup_seq	offsetof (pthread_cond_t, __data.__wakeup_seq)
 woken_seq	offsetof (pthread_cond_t, __data.__woken_seq)
+dep_mutex	offsetof (pthread_cond_t, __data.__mutex)
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
new file mode 100644
index 0000000000..893a5e932d
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
@@ -0,0 +1,6 @@
+#include <pthreadP.h>
+
+#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock(mutex)
+#define __pthread_mutex_lock __pthread_mutex_cond_lock
+
+#include <nptl/pthread_mutex_lock.c>
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
index 47d856cae4..7c12db6fc6 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
@@ -78,6 +78,7 @@ typedef union
     unsigned long long int __total_seq;
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
+    void *__mutex;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
   long int __align;
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index 8e1742b671..136dc574e0 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -102,6 +102,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
 			      : "0" (1), "2" (futex)			      \
 			      : "cx", "r11", "cc", "memory"); })
 
+
+#define lll_mutex_cond_lock(futex) \
+  (void) ({ int ignore1, ignore2;					      \
+	    __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"		      \
+			      "testl %0, %0\n\t"			      \
+			      "jne 1f\n\t"				      \
+			      ".subsection 1\n"				      \
+			      "1:\tleaq %2, %%rdi\n\t"			      \
+			      "subq $128, %%rsp\n\t"			      \
+			      "callq __lll_mutex_lock_wait\n\t"		      \
+			      "addq $128, %%rsp\n\t"			      \
+			      "jmp 2f\n\t"				      \
+			      ".previous\n"				      \
+			      "2:"					      \
+			      : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \
+			      : "0" (2), "2" (futex)			      \
+			      : "cx", "r11", "cc", "memory"); })
+
+
 #define lll_mutex_timedlock(futex, timeout) \
   ({ int result, ignore1, ignore2, ignore3;				      \
      __asm __volatile (LOCK_INSTR "xaddl %0, %4\n\t"			      \
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index cc12f5407c..66edb9ac85 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		202
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -60,17 +64,27 @@ __pthread_cond_broadcast:
 	   woken up.  */
 	movq	%rcx, (%rdi)
 
+	/* Get the address of the mutex used.  */
+	movq	dep_mutex-wakeup_seq(%rdi), %r8
+
 	/* Unlock.  */
 	LOCK
 	decl	cond_lock-wakeup_seq(%rdi)
 	jne	7f
 
 	/* Wake up all threads.  */
-8:	movq	$FUTEX_WAKE, %rsi
+8:	movq	$FUTEX_REQUEUE, %rsi
 	movq	$SYS_futex, %rax
-	movl	$0x7fffffff, %edx
+	movl	$1, %edx
+	movq	$0x7fffffff, %r10
 	syscall
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpq	$-EINVAL, %eax
+	je	9f
+10:
+#endif
+
 	xorl	%eax, %eax
 	retq
 
@@ -104,6 +118,15 @@ __pthread_cond_broadcast:
 	callq	__lll_mutex_unlock_wake
 	subq	$cond_lock-wakeup_seq, %rdi
 	jmp	8b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+9:	/* The futex requeue functionality is not available.  */
+	movq	$0x7fffffff, %rdx
+	movq	$FUTEX_WAKE, %rsi
+	movq	$SYS_futex, %rax
+	syscall
+	jmp	10b
+#endif
 	.size	__pthread_cond_broadcast, .-__pthread_cond_broadcast
 versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 11635baa22..709fcf4ca8 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		202
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -44,15 +48,15 @@ __pthread_cond_signal:
 	movl	$1, %esi
 	LOCK
 #if cond_lock == 0
-	xaddl	%esi, (%rdi)
+	xaddl	%esi, (%r8)
 #else
-	xaddl	%esi, cond_lock(%rdi)
+	xaddl	%esi, cond_lock(%r8)
 #endif
 	testl	%esi, %esi
 	jne	1f
 
-2:	addq	$wakeup_seq, %rdi
-	movq	total_seq-wakeup_seq(%rdi), %rcx
+2:	leaq	wakeup_seq(%r8), %rdi
+	movq	total_seq(%r8), %rcx
 	cmpq	(%rdi), %rcx
 	jbe	4f
 
@@ -62,12 +66,22 @@ __pthread_cond_signal:
 	/* Wake up one thread.  */
 	movq	$FUTEX_WAKE, %rsi
 	movq	$SYS_futex, %rax
-	movq	%rsi, %rdx	/* movl $1, %edx */
+	xorq	%rdx, %rdx
+	movq	$1, %r10
 	syscall
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpq	$-EINVAL, %rax
+	je	7f
+#endif
+
+	/* If we moved a thread we in any case have to make the syscall.  */
+	testq	%rax, %rax
+	jne	5f
+
 	/* Unlock.  */
 4:	LOCK
-	decl	cond_lock-wakeup_seq(%rdi)
+	decl	cond_lock(%r8)
 	jne	5f
 
 6:	xorl	%eax, %eax
@@ -85,9 +99,22 @@ __pthread_cond_signal:
 	jmp	2b
 
 	/* Unlock in loop requires waekup.  */
-5:	addq	$cond_lock-wakeup_seq, %rdi
+5:
+#if cond_lock != 0
+	addq	$cond_lock-wakeup_seq, %rdi
+#else
+	movq	%r8, %rdi
+#endif
 	callq	__lll_mutex_unlock_wake
 	jmp	6b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+7:	/* The futex requeue functionality is not available.  */
+	movq	$1, %rdx
+	movq	$FUTEX_WAKE, %esi
+	movq	$SYS_futex, %rax
+	syscall
+	jmp	4b
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
 versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 151395048a..c8fd4ea129 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -88,7 +88,9 @@ __pthread_cond_timedwait:
 	jne	1f
 
 	/* Unlock the mutex.  */
-2:	movq	16(%rsp), %rdi
+2:	movq	%rdi, %rax
+	movq	16(%rsp), %rdi
+	movq	%rdi, dep_mutex(%rax)
 	callq	__pthread_mutex_unlock_internal
 
 	testl	%eax, %eax
@@ -121,8 +123,8 @@ __pthread_cond_timedwait:
 #endif
 	jne	3f
 
-4:	movq	%rsp, %rdi
-	callq	__pthread_enable_asynccancel_2
+4:	callq	__pthread_enable_asynccancel
+	movq	%rax, (%rsp)
 
 	/* Get the current time.  */
 #ifdef __NR_clock_gettime
@@ -227,7 +229,7 @@ __pthread_cond_timedwait:
 	movq	%rdx, %fs:CLEANUP
 
 	movq	16(%rsp), %rdi
-	callq	__pthread_mutex_lock_internal
+	callq	__pthread_mutex_cond_lock
 
 	testq	%rax, %rax
 	cmoveq	%r14, %rax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 6cad2812f0..5189972fc7 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -84,12 +84,8 @@ __condvar_cleanup:
 	movq	$SYS_futex, %rax
 	syscall
 
-	/* Lock the mutex unless asynchronous cancellation is in effect.  */
-	testq	$2, (%r8)
-	jne	3f
-
 	movq	16(%r8), %rdi
-	callq	__pthread_mutex_lock_internal
+	callq	__pthread_mutex_cond_lock
 
 3:	retq
 	.size	__condvar_cleanup, .-__condvar_cleanup
@@ -137,7 +133,9 @@ __pthread_cond_wait:
 	jne	1f
 
 	/* Unlock the mutex.  */
-2:	movq	16(%rsp), %rdi
+2:	movq	%rdi, %rax
+	movq	16(%rsp), %rdi
+	movq	%rdi, dep_mutex(%rax)
 	callq	__pthread_mutex_unlock_internal
 
 	testl	%eax, %eax
@@ -170,8 +168,8 @@ __pthread_cond_wait:
 #endif
 	jne	3f
 
-4:	movq	%rsp, %rdi
-	callq	__pthread_enable_asynccancel_2
+4:	callq	__pthread_enable_asynccancel
+	movq	%rax, (%rsp)
 
 	movq	8(%rsp), %rdi
 	xorq	%r10, %r10
@@ -221,7 +219,7 @@ __pthread_cond_wait:
 	movq	%rdx, %fs:CLEANUP
 
 	movq	16(%rsp), %rdi
-	callq	__pthread_mutex_lock_internal
+	callq	__pthread_mutex_cond_lock
 14:	addq	$64, %rsp
 .Laddq: