Update.

2003-05-25  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/unix/sysv/linux/kernel-features.h: Define
	__ASSUME_FUTEX_REQUEUE for >= 2.5.70.

	* math/test-fenv.c (feexcp_nomask_test): Fix comment.

6 files changed, 118 insertions, 29 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
index 0834894c25..4b0f11adc5 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
@@ -78,6 +78,7 @@ typedef union
     unsigned long long int __total_seq;
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
+    void *__mutex;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
   long long int __align;
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 0fa402a95e..06821ad376 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		240
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -41,8 +45,10 @@
 __pthread_cond_broadcast:
 
 	pushl	%ebx
+	pushl	%esi
+	pushl	%edi
 
-	movl	8(%esp), %ebx
+	movl	16(%esp), %ebx
 
 	/* Get internal lock.  */
 	movl	$1, %eax
@@ -69,18 +75,34 @@ __pthread_cond_broadcast:
 3:	movl	%ecx, (%ebx)
 	movl	%eax, 4(%ebx)
 
+	/* Get the address of the mutex used.  */
+	movl	dep_mutex-wakeup_seq(%ebx), %edi
+
 	/* Unlock.  */
 	LOCK
 	subl	$1, cond_lock-wakeup_seq(%ebx)
 	jne	7f
 
 	/* Wake up all threads.  */
-8:	movl	$FUTEX_WAKE, %ecx
+8:	movl	$FUTEX_REQUEUE, %ecx
 	movl	$SYS_futex, %eax
-	movl	$0x7fffffff, %edx
+	movl	$0x7fffffff, %esi
+	movl	$1, %edx
+	/* Get the address of the futex involved.  */
+# if MUTEX_FUTEX != 0
+	addl	$MUTEX_FUTEX, %edi
+# endif
 	ENTER_KERNEL
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpl	$-EINVAL, %eax
+	je	9f
+10:
+#endif
+
 	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -91,6 +113,8 @@ __pthread_cond_broadcast:
 	jne	5f
 
 6:	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -113,6 +137,15 @@ __pthread_cond_broadcast:
 7:	leal	cond_lock-wakeup_seq(%ebx), %eax
 	call	__lll_mutex_unlock_wake
 	jmp	8b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+9:	/* The futex requeue functionality is not available.  */
+	movl	$0x7fffffff, %edx
+	movl	$FUTEX_WAKE, %ecx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	jmp	10b
+#endif
 	.size	__pthread_cond_broadcast, .-__pthread_cond_broadcast
 versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index 5465d7b2a8..ed25c554d2 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -20,6 +20,7 @@
 #include <sysdep.h>
 #include <shlib-compat.h>
 #include <lowlevelcond.h>
+#include <kernel-features.h>
 
 #ifdef UP
 # define LOCK
@@ -30,6 +31,9 @@
 #define SYS_futex		240
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_REQUEUE		3
+
+#define EINVAL			22
 
 
 	.text
@@ -41,23 +45,25 @@
 __pthread_cond_signal:
 
 	pushl	%ebx
+	pushl	%esi
+	pushl	%edi
 
-	movl	8(%esp), %ebx
+	movl	16(%esp), %edi
 
 	/* Get internal lock.  */
 	movl	$1, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	xaddl	%eax, (%edi)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	xaddl	%eax, cond_lock(%edi)
 #endif
 	testl	%eax, %eax
 	jne	1f
 
-2:	addl	$wakeup_seq, %ebx
-	movl	total_seq+4-wakeup_seq(%ebx), %eax
-	movl	total_seq-wakeup_seq(%ebx), %ecx
+2:	leal	wakeup_seq(%edi), %ebx
+	movl	total_seq+4(%edi), %eax
+	movl	total_seq(%edi), %ecx
 	cmpl	4(%ebx), %eax
 	ja	3f
 	jb	4f
@@ -68,18 +74,30 @@ __pthread_cond_signal:
 3:	addl	$1, (%ebx)
 	adcl	$0, 4(%ebx)
 
-	/* Wake up one thread.  */
-	movl	$FUTEX_WAKE, %ecx
+	/* Wake up one thread by moving it to the internal lock futex.  */
+	movl	$FUTEX_REQUEUE, %ecx
 	movl	$SYS_futex, %eax
-	movl	%ecx, %edx	/* movl $1, %edx */
+	xorl	%edx, %edx
+	movl	$1, %esi
 	ENTER_KERNEL
 
+#ifndef __ASSUME_FUTEX_REQUEUE
+	cmpl	$-EINVAL, %eax
+	je	7f
+#endif
+
+	/* If we moved a thread we in any case have to make the syscall.  */
+	testl	%eax, %eax
+	jne	5f
+
 	/* Unlock.  */
 4:	LOCK
-	subl	$1, cond_lock-wakeup_seq(%ebx)
+	subl	$1, (%edi)
 	jne	5f
 
 6:	xorl	%eax, %eax
+	popl	%edi
+	popl	%esi
 	popl	%ebx
 	ret
 
@@ -93,11 +111,24 @@ __pthread_cond_signal:
 	call	__lll_mutex_lock_wait
 	jmp	2b
 
-	/* Unlock in loop requires waekup.  */
+	/* Unlock in loop requires wakeup.  */
 5:
-	leal	cond_lock-wakeup_seq(%ebx), %eax
+#if cond_lock == 0
+	movl	%edi, %eax
+#else
+	leal	cond_lock(%edi), %eax
+#endif
 	call	__lll_mutex_unlock_wake
 	jmp	6b
+
+#ifndef __ASSUME_FUTEX_REQUEUE
+7:	/* The futex requeue functionality is not available.  */
+	movl	$1, %edx
+	movl	$FUTEX_WAKE, %ecx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	jmp	4b
+#endif
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
 versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
index d9bffe5139..5eec268b29 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
@@ -66,8 +66,13 @@ __pthread_cond_timedwait:
 	testl	%eax, %eax
 	jne	1f
 
+	/* Store the reference to the mutex.  If there is already a
+	   different value in there this is a bad user bug.  */
+2:	movl	24(%esp), %eax
+	movl	%eax, dep_mutex(%ebx)
+
 	/* Unlock the mutex.  */
-2:	pushl	24(%esp)
+	pushl	%eax
 .Lpush4:
 	call	__pthread_mutex_unlock_internal
 
@@ -113,8 +118,8 @@ __pthread_cond_timedwait:
 #endif
 	jne	3f
 
-4:	leal	8(%esp), %eax
-	call	__pthread_enable_asynccancel_2
+4:	call	__pthread_enable_asynccancel
+	movl	%eax, 8(%esp)
 
 	/* Get the current time.  */
 	movl	%ebx, %edx
@@ -230,7 +235,7 @@ __pthread_cond_timedwait:
 	movl	%edx, %gs:CLEANUP
 
 	/* Trick ahead:	 (%esp) contains the address of the mutex.  */
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	addl	$44, %esp
 .Laddl:
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 1ac6c1a7d6..61d3d8d225 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -83,15 +83,11 @@ __condvar_cleanup:
 	movl	$0x7fffffff, %edx
 	ENTER_KERNEL
 
-	/* Lock the mutex unless asynchronous cancellation is in effect.  */
-	testl	$2, 8(%esi)
-	jne	3f
-
 	pushl	(%esi)
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	popl	%eax
 
-3:	popl	%esi
+	popl	%esi
 	popl	%ebx
 	ret
 	.size	__condvar_cleanup, .-__condvar_cleanup
@@ -125,8 +121,13 @@ __pthread_cond_wait:
 	testl	%eax, %eax
 	jne	1f
 
+	/* Store the reference to the mutex.  If there is already a
+	   different value in there this is a bad user bug.  */
+2:	movl	20(%esp), %eax
+	movl	%eax, dep_mutex(%ebx)
+
 	/* Unlock the mutex.  */
-2:	pushl	20(%esp)
+	pushl	%eax
 .Lpush4:
 	call	__pthread_mutex_unlock_internal
 
@@ -171,8 +172,8 @@ __pthread_cond_wait:
 #endif
 	jne	3f
 
-4:	leal	8(%esp), %eax
-	call	__pthread_enable_asynccancel_2
+4:	call	__pthread_enable_asynccancel
+	movl	%eax, 8(%esp)
 
 	movl	%esi, %ecx	/* movl $FUTEX_WAIT, %ecx */
 	movl	%edi, %edx
@@ -229,7 +230,7 @@ __pthread_cond_wait:
 	movl	%edx, %gs:CLEANUP
 
 	/* Trick ahead:	 (%esp) contains the address of the mutex.  */
-	call	__pthread_mutex_lock_internal
+	call	__pthread_mutex_cond_lock
 	addl	$36, %esp
 .Laddl:
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index 5fd50b97db..8923afbfed 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -126,6 +126,24 @@ extern int __lll_mutex_unlock_wait (int *__futex)
 			      : "memory"); })
 
 
+/* Special version of lll_mutex_lock which causes the unlock function to
+   always wakeup waiters.  */
+#define lll_mutex_cond_lock(futex) \
+  (void) ({ int ignore1, ignore2;					      \
+	    __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"		      \
+			      "testl %0, %0\n\t"			      \
+			      "jne 1f\n\t"				      \
+			      ".subsection 1\n"				      \
+			      "1:\tleal %2, %%ecx\n\t"			      \
+			      "call __lll_mutex_lock_wait\n\t"		      \
+			      "jmp 2f\n\t"				      \
+			      ".previous\n"				      \
+			      "2:"					      \
+			      : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \
+			      : "0" (2), "2" (futex)			      \
+			      : "memory"); })
+
+
 #define lll_mutex_timedlock(futex, timeout) \
   ({ int result, ignore1, ignore2;					      \
      __asm __volatile (LOCK_INSTR "xaddl %0, %3\n\t"			      \