Update.

2003-09-21  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Completely revamp the
	locking macros.  No distinction between normal and mutex locking
	anymore.
	* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Rewrite mutex
	locking.  Merge bits from lowlevelmutex.S we still need.
	* sysdeps/unix/sysv/linux/i386/i486/lowlevelmutex.S: Removed.
	* Makefile (routines): Remove libc-lowlevelmutex.
	(libpthread-rountines): Remove lowlevelmutex.
	* pthread_barrier_wait.S: Adjust for new mutex implementation.
	* pthread_cond_broadcast.S: Likewise.
	* pthread_cond_timedwait.S: Likewise.
	* pthread_cond_wait.S: Likewise.
	* pthread_rwlock_rdlock.S: Likewise.
	* pthread_rwlock_timedrdlock.S: Likewise.
	* pthread_rwlock_timedwrlock.S: Likewise.
	* pthread_rwlock_unlock.S: Likewise.
	* pthread_rwlock_wrlock.S: Likewise.
	* pthread_cond_signal.S: Likewise.  Don't use requeue.

13 files changed, 266 insertions, 373 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
index f27fe2bc1f..6e4b077295 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
@@ -36,11 +36,11 @@
 #define FUTEX_WAKE		1
 
 
-	.globl	__lll_lock_wait
-	.type	__lll_lock_wait,@function
-	.hidden	__lll_lock_wait
+	.globl	__lll_mutex_lock_wait
+	.type	__lll_mutex_lock_wait,@function
+	.hidden	__lll_mutex_lock_wait
 	.align	16
-__lll_lock_wait:
+__lll_mutex_lock_wait:
 	pushl	%esi
 	pushl	%ebx
 	pushl	%edx
@@ -48,23 +48,124 @@ __lll_lock_wait:
 	movl	%ecx, %ebx
 	xorl	%esi, %esi	/* No timeout.  */
 	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+	movl	$2, %edx
+
 1:
-	leal	-1(%eax), %edx	/* account for the preceeded xadd.  */
+	movl	$1, %eax
+	LOCK
+	cmpxchgl %edx, (%ebx)
+
+	testl	%eax, %eax
+	je	2f
+
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 
-	orl	$-1, %eax	/* Load -1.  */
-	LOCK
-	xaddl	%eax, (%ebx)
-	jne,pn	1b
+	xorl	%eax, %eax
+2:	LOCK
+	cmpxchgl %edx, (%ebx)
 
-	movl	$-1, (%ebx)
+	testl	%eax, %eax
+	jne,pn	1b
 
 	popl	%edx
 	popl	%ebx
 	popl	%esi
 	ret
-	.size	__lll_lock_wait,.-__lll_lock_wait
+	.size	__lll_mutex_lock_wait,.-__lll_mutex_lock_wait
+
+
+#ifdef NOT_IN_libc
+	.globl	__lll_mutex_timedlock_wait
+	.type	__lll_mutex_timedlock_wait,@function
+	.hidden	__lll_mutex_timedlock_wait
+	.align	16
+__lll_mutex_timedlock_wait:
+	/* Check for a valid timeout value.  */
+	cmpl	$1000000000, 4(%edx)
+	jae	3f
+
+	pushl	%edi
+	pushl	%esi
+	pushl	%ebx
+	pushl	%ebp
+
+	/* Stack frame for the timespec and timeval structs.  */
+	subl	$8, %esp
+
+	movl	%ecx, %ebp
+	movl	%edx, %edi
+
+1:
+	/* Get current time.  */
+	movl	%esp, %ebx
+	xorl	%ecx, %ecx
+	movl	$SYS_gettimeofday, %eax
+	ENTER_KERNEL
+
+	/* Compute relative timeout.  */
+	movl	4(%esp), %eax
+	movl	$1000, %edx
+	mul	%edx		/* Milli seconds to nano seconds.  */
+	movl	(%edi), %ecx
+	movl	4(%edi), %edx
+	subl	(%esp), %ecx
+	subl	%eax, %edx
+	jns	4f
+	addl	$1000000000, %edx
+	subl	$1, %ecx
+4:	testl	%ecx, %ecx
+	js	5f		/* Time is already up.  */
+
+	/* Store relative timeout.  */
+	movl	%ecx, (%esp)
+	movl	%edx, 4(%esp)
+
+	movl	%ebp, %ebx
+
+	movl	$1, %eax
+	movl	$2, %edx
+	LOCK
+	cmpxchgl %edx, (%ebx)
+
+	testl	%eax, %eax
+	je	8f
+
+	/* Futex call.  */
+	movl	%esp, %esi
+	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	movl	%eax, %ecx
+
+8:
+	xorl	%eax, %eax
+	movl	$2, %edx
+	LOCK
+	cmpxchgl %edx, (%ebx)
+
+	testl	%eax, %eax
+	jne	7f
+
+6:	addl	$8, %esp
+	popl	%ebp
+	popl	%ebx
+	popl	%esi
+	popl	%edi
+	ret
+
+	/* Check whether the time expired.  */
+7:	cmpl	$-ETIMEDOUT, %ecx
+	je	5f
+	jmp	1b
+
+3:	movl	$EINVAL, %eax
+	ret
+
+5:	movl	$ETIMEDOUT, %eax
+	jmp	6b
+	.size	__lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait
+#endif
 
 
 #ifdef NOT_IN_libc
@@ -79,10 +180,16 @@ lll_unlock_wake_cb:
 
 	movl	20(%esp), %ebx
 	LOCK
-	addl	$1, (%ebx)
-	jng	1f
+	subl	$1, (%ebx)
+	je	1f
 
-	popl	%edx
+	movl	$FUTEX_WAKE, %ecx
+	movl	$1, %edx	/* Wake one thread.  */
+	movl	$SYS_futex, %eax
+	movl	$0, (%ebx)
+	ENTER_KERNEL
+
+1:	popl	%edx
 	popl	%ecx
 	popl	%ebx
 	ret
@@ -90,27 +197,27 @@ lll_unlock_wake_cb:
 #endif
 
 
-	.globl	__lll_unlock_wake
-	.type	__lll_unlock_wake,@function
-	.hidden	__lll_unlock_wake
+	.globl	__lll_mutex_unlock_wake
+	.type	__lll_mutex_unlock_wake,@function
+	.hidden	__lll_mutex_unlock_wake
 	.align	16
-__lll_unlock_wake:
+__lll_mutex_unlock_wake:
 	pushl	%ebx
 	pushl	%ecx
 	pushl	%edx
 
 	movl	%eax, %ebx
-1:	movl	$FUTEX_WAKE, %ecx
+	movl	$0, (%eax)
+	movl	$FUTEX_WAKE, %ecx
 	movl	$1, %edx	/* Wake one thread.  */
 	movl	$SYS_futex, %eax
-	movl	%edx, (%ebx)	/* Stores '$1'.  */
 	ENTER_KERNEL
 
 	popl	%edx
 	popl	%ecx
 	popl	%ebx
 	ret
-	.size	__lll_unlock_wake,.-__lll_unlock_wake
+	.size	__lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake
 
 
 #ifdef NOT_IN_libc
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelmutex.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelmutex.S
deleted file mode 100644
index a54e48c9e6..0000000000
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelmutex.S
+++ /dev/null
@@ -1,180 +0,0 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, write to the Free
-   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-   02111-1307 USA.  */
-
-#include <sysdep.h>
-#include <pthread-errnos.h>
-
-	.text
-
-#ifndef LOCK
-# ifdef UP
-#  define LOCK
-# else
-#  define LOCK lock
-# endif
-#endif
-
-#define SYS_gettimeofday	__NR_gettimeofday
-#define SYS_futex		240
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-
-
-	.globl	__lll_mutex_lock_wait
-	.type	__lll_mutex_lock_wait,@function
-	.hidden	__lll_mutex_lock_wait
-	.align	16
-__lll_mutex_lock_wait:
-	pushl	%esi
-	pushl	%ebx
-	pushl	%edx
-
-	/* In the loop we are going to add 2 instead of 1 which is what
-	   the caller did.  Account for that.  */
-	subl	$1, %eax
-
-	movl	%ecx, %ebx
-	xorl	%esi, %esi	/* No timeout.  */
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
-1:
-	leal	2(%eax), %edx	/* account for the preceeded xadd.  */
-	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-
-	movl	$2, %eax
-	LOCK
-	xaddl	%eax, (%ebx)
-	testl	%eax, %eax
-	jne,pn	1b
-
-	popl	%edx
-	popl	%ebx
-	popl	%esi
-	ret
-	.size	__lll_mutex_lock_wait,.-__lll_mutex_lock_wait
-
-
-#ifdef NOT_IN_libc
-	.globl	__lll_mutex_timedlock_wait
-	.type	__lll_mutex_timedlock_wait,@function
-	.hidden	__lll_mutex_timedlock_wait
-	.align	16
-__lll_mutex_timedlock_wait:
-	/* Check for a valid timeout value.  */
-	cmpl	$1000000000, 4(%edx)
-	jae	3f
-
-	pushl	%edi
-	pushl	%esi
-	pushl	%ebx
-	pushl	%ebp
-
-	/* In the loop we are going to add 2 instead of 1 which is what
-	   the caller did.  Account for that.  */
-	subl	$1, %eax
-
-	/* Stack frame for the timespec and timeval structs.  */
-	subl	$8, %esp
-
-	movl	%ecx, %ebp
-	movl	%edx, %edi
-
-1:	leal	2(%eax), %esi
-
-	/* Get current time.  */
-	movl	%esp, %ebx
-	xorl	%ecx, %ecx
-	movl	$SYS_gettimeofday, %eax
-	ENTER_KERNEL
-
-	/* Compute relative timeout.  */
-	movl	4(%esp), %eax
-	movl	$1000, %edx
-	mul	%edx		/* Milli seconds to nano seconds.  */
-	movl	(%edi), %ecx
-	movl	4(%edi), %edx
-	subl	(%esp), %ecx
-	subl	%eax, %edx
-	jns	4f
-	addl	$1000000000, %edx
-	subl	$1, %ecx
-4:	testl	%ecx, %ecx
-	js	5f		/* Time is already up.  */
-
-	/* Futex call.  */
-	movl	%ecx, (%esp)	/* Store relative timeout.  */
-	movl	%edx, 4(%esp)
-	movl	%esi, %edx
-	movl	%esp, %esi
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
-	movl	%ebp, %ebx
-	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-	movl	%eax, %ecx
-
-	movl	$2, %eax
-	LOCK
-	xaddl	%eax, (%ebx)
-	testl	%eax, %eax
-	jne	7f
-
-	xorl	%eax, %eax
-
-6:	addl	$8, %esp
-	popl	%ebp
-	popl	%ebx
-	popl	%esi
-	popl	%edi
-	ret
-
-	/* Check whether the time expired.  */
-7:	cmpl	$-ETIMEDOUT, %ecx
-	je	5f
-	jmp	1b
-
-3:	movl	$EINVAL, %eax
-	ret
-
-5:	movl	$ETIMEDOUT, %eax
-	jmp	6b
-	.size	__lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait
-#endif
-
-
-	.globl	__lll_mutex_unlock_wake
-	.type	__lll_mutex_unlock_wake,@function
-	.hidden	__lll_mutex_unlock_wake
-	.align	16
-__lll_mutex_unlock_wake:
-	pushl	%ebx
-	pushl	%ecx
-	pushl	%edx
-
-	movl	%eax, %ebx
-	movl	$0, (%eax)
-	movl	$FUTEX_WAKE, %ecx
-	movl	$1, %edx	/* Wake one thread.  */
-	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-
-	popl	%edx
-	popl	%ecx
-	popl	%ebx
-	ret
-	.size	__lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
index c9b3d8b7bc..08c6e915b9 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
@@ -42,9 +42,11 @@ pthread_barrier_wait:
 	movl	8(%esp), %ebx
 
 	/* Get the mutex.  */
-	orl	$-1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
-	xaddl	%eax, MUTEX(%ebx)
+	cmpxchgl %edx, MUTEX(%ebx)
+	testl	%eax, %eax
 	jne	1f
 
 	/* One less waiter.  If this was the last one needed wake
@@ -63,8 +65,8 @@ pthread_barrier_wait:
 
 	/* Release the mutex.  */
 	LOCK
-	addl	$1, MUTEX(%ebx)
-	jng	6f
+	subl	$1, MUTEX(%ebx)
+	jne	6f
 
 	/* Wait for the remaining threads.  The call will return immediately
 	   if the CURR_EVENT memory has meanwhile been changed.  */
@@ -110,8 +112,8 @@ pthread_barrier_wait:
 	   waking the waiting threads since otherwise a new thread might
 	   arrive and gets waken up, too.  */
 	LOCK
-	addl	$1, MUTEX(%ebx)
-	jng	4f
+	subl	$1, MUTEX(%ebx)
+	jne	4f
 
 5:	orl	$-1, %eax		/* == PTHREAD_BARRIER_SERIAL_THREAD */
 
@@ -119,14 +121,14 @@ pthread_barrier_wait:
 	ret
 
 1:	leal	MUTEX(%ebx), %ecx
-	call	__lll_lock_wait
+	call	__lll_mutex_lock_wait
 	jmp	2b
 
 4:	leal	MUTEX(%ebx), %eax
-	call	__lll_unlock_wake
+	call	__lll_mutex_unlock_wake
 	jmp	5b
 
 6:	leal	MUTEX(%ebx), %eax
-	call	__lll_unlock_wake
+	call	__lll_mutex_unlock_wake
 	jmp	7b
 	.size	pthread_barrier_wait,.-pthread_barrier_wait
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 06821ad376..1049e0f4dd 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -51,12 +51,13 @@ __pthread_cond_broadcast:
 	movl	16(%esp), %ebx
 
 	/* Get internal lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	cmpxchgl %edx, cond_lock(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	1f
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index 95f3aad1d8..d86ee054a9 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -45,18 +45,18 @@
 __pthread_cond_signal:
 
 	pushl	%ebx
-	pushl	%esi
 	pushl	%edi
 
-	movl	16(%esp), %edi
+	movl	12(%esp), %edi
 
 	/* Get internal lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%edi)
+	cmpxchgl %edx, (%edi)
 #else
-	xaddl	%eax, cond_lock(%edi)
+	cmpxchgl %edx, cond_lock(%edi)
 #endif
 	testl	%eax, %eax
 	jne	1f
@@ -78,21 +78,11 @@ __pthread_cond_signal:
 	adcl	$0, 4(%ebx)
 
 	/* Wake up one thread by moving it to the internal lock futex.  */
-	movl	$FUTEX_REQUEUE, %ecx
+	movl	$FUTEX_WAKE, %ecx
 	movl	$SYS_futex, %eax
-	xorl	%edx, %edx
-	movl	$1, %esi
+	movl	$1, %edx
 	ENTER_KERNEL
 
-#ifndef __ASSUME_FUTEX_REQUEUE
-	cmpl	$-EINVAL, %eax
-	je	7f
-#endif
-
-	/* If we moved a thread we in any case have to make the syscall.  */
-	testl	%eax, %eax
-	jne	5f
-
 	/* Unlock.  Note that at this point %edi always points to
 	   cond_lock.  */
 4:	LOCK
@@ -101,7 +91,6 @@ __pthread_cond_signal:
 
 6:	xorl	%eax, %eax
 	popl	%edi
-	popl	%esi
 	popl	%ebx
 	ret
 
@@ -119,15 +108,6 @@ __pthread_cond_signal:
 5:	movl	%edi, %eax
 	call	__lll_mutex_unlock_wake
 	jmp	6b
-
-#ifndef __ASSUME_FUTEX_REQUEUE
-7:	/* The futex requeue functionality is not available.  */
-	movl	$1, %edx
-	movl	$FUTEX_WAKE, %ecx
-	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-	jmp	4b
-#endif
 	.size	__pthread_cond_signal, .-__pthread_cond_signal
 versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
 		  GLIBC_2_3_2)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
index f88d2d33bb..5799d8f862 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
@@ -56,12 +56,13 @@ __pthread_cond_timedwait:
 	movl	28(%esp), %ebp
 
 	/* Get internal lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	cmpxchgl %edx, cond_lock(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	1f
@@ -170,12 +171,13 @@ __pthread_cond_timedwait:
 .LcleanupEND:
 
 	/* Lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	cmpxchgl %edx, cond_lock(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	5f
@@ -353,12 +355,13 @@ __condvar_tw_cleanup:
 	movl	%eax, %esi
 
 	/* Get internal lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	cmpxchgl %edx, cond_lock(%ebx)
 #endif
 	testl	%eax, %eax
 	je	1f
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 1680b3d859..db0428f527 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -53,12 +53,13 @@ __pthread_cond_wait:
 	movl	16(%esp), %ebx
 
 	/* Get internal lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	cmpxchgl %edx, cond_lock(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	1f
@@ -114,12 +115,13 @@ __pthread_cond_wait:
 .LcleanupEND:
 
 	/* Lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	cmpxchgl %edx, cond_lock(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	5f
@@ -246,12 +248,13 @@ __condvar_w_cleanup:
 	movl	%eax, %esi
 
 	/* Get internal lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if cond_lock == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, cond_lock(%ebx)
+	cmpxchgl %edx, cond_lock(%ebx)
 #endif
 	testl	%eax, %eax
 	je	1f
@@ -270,7 +273,11 @@ __condvar_w_cleanup:
 	adcl	$0, woken_seq+4(%ebx)
 
 	LOCK
+#if cond_lock == 0
+	subl	$1, (%ebx)
+#else
 	subl	$1, cond_lock(%ebx)
+#endif
 	je	2f
 
 #if cond_lock == 0
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
index 81d9766056..1c149433f0 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
@@ -46,12 +46,13 @@ __pthread_rwlock_rdlock:
 	movl	12(%esp), %ebx
 
 	/* Get the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, MUTEX(%ebx)
+	cmpxchgl %edx, MUTEX(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	1f
@@ -85,12 +86,13 @@ __pthread_rwlock_rdlock:
 	subl	$READERS_WAKEUP, %ebx
 
 	/* Reget the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, MUTEX(%ebx)
+	cmpxchgl %edx, MUTEX(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	12f
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
index 7d9aa93ebb..409c046b7b 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
@@ -50,12 +50,13 @@ pthread_rwlock_timedrdlock:
 	movl	32(%esp), %edi
 
 	/* Get the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%ebp)
+	cmpxchgl %edx, (%ebp)
 #else
-	xaddl	%eax, MUTEX(%ebp)
+	cmpxchgl %edx, MUTEX(%ebp)
 #endif
 	testl	%eax, %eax
 	jne	1f
@@ -114,22 +115,23 @@ pthread_rwlock_timedrdlock:
 	leal	READERS_WAKEUP(%ebp), %ebx
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
-	movl	%eax, %edx
+	movl	%eax, %ecx
 17:
 
 	/* Reget the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%ebp)
+	cmpxchgl %edx, (%ebp)
 #else
-	xaddl	%eax, MUTEX(%ebp)
+	cmpxchgl %edx, MUTEX(%ebp)
 #endif
 	testl	%eax, %eax
 	jne	12f
 
 13:	subl	$1, READERS_QUEUED(%ebp)
-	cmpl	$-ETIMEDOUT, %edx
+	cmpl	$-ETIMEDOUT, %ecx
 	jne	2b
 
 18:	movl	$ETIMEDOUT, %ecx
@@ -207,7 +209,7 @@ pthread_rwlock_timedrdlock:
 	call	__lll_mutex_lock_wait
 	jmp	13b
 
-16:	movl	$-ETIMEDOUT, %edx
+16:	movl	$-ETIMEDOUT, %ecx
 	jmp	17b
 
 19:	movl	$EINVAL, %ecx
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
index 4a144453fa..00d9906e59 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
@@ -50,12 +50,13 @@ pthread_rwlock_timedwrlock:
 	movl	32(%esp), %edi
 
 	/* Get the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%ebp)
+	cmpxchgl %edx, (%ebp)
 #else
-	xaddl	%eax, MUTEX(%ebp)
+	cmpxchgl %edx, MUTEX(%ebp)
 #endif
 	testl	%eax, %eax
 	jne	1f
@@ -112,22 +113,23 @@ pthread_rwlock_timedwrlock:
 	leal	WRITERS_WAKEUP(%ebp), %ebx
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
-	movl	%eax, %edx
+	movl	%eax, %ecx
 17:
 
 	/* Reget the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%ebp)
+	cmpxchgl %edx, (%ebp)
 #else
-	xaddl	%eax, MUTEX(%ebp)
+	cmpxchgl %edx, MUTEX(%ebp)
 #endif
 	testl	%eax, %eax
 	jne	12f
 
 13:	subl	$1, WRITERS_QUEUED(%ebp)
-	cmpl	$-ETIMEDOUT, %edx
+	cmpl	$-ETIMEDOUT, %ecx
 	jne	2b
 
 18:	movl	$ETIMEDOUT, %ecx
@@ -200,7 +202,7 @@ pthread_rwlock_timedwrlock:
 	call	__lll_mutex_lock_wait
 	jmp	13b
 
-16:	movl	$-ETIMEDOUT, %edx
+16:	movl	$-ETIMEDOUT, %ecx
 	jmp	17b
 
 19:	movl	$EINVAL, %ecx
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
index 35f61bf0f9..839a5eb336 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
@@ -44,12 +44,13 @@ __pthread_rwlock_unlock:
 	movl	12(%esp), %edi
 
 	/* Get the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%edi)
+	cmpxchgl %edx, (%edi)
 #else
-	xaddl	%eax, MUTEX(%edi)
+	cmpxchgl %edx, MUTEX(%edi)
 #endif
 	testl	%eax, %eax
 	jne	1f
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
index 28c24bad65..89ad6a3774 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
@@ -46,12 +46,13 @@ __pthread_rwlock_wrlock:
 	movl	12(%esp), %ebx
 
 	/* Get the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, MUTEX(%ebx)
+	cmpxchgl %edx, MUTEX(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	1f
@@ -83,12 +84,13 @@ __pthread_rwlock_wrlock:
 	subl	$WRITERS_WAKEUP, %ebx
 
 	/* Reget the lock.  */
-	movl	$1, %eax
+	movl	$1, %edx
+	xorl	%eax, %eax
 	LOCK
 #if MUTEX == 0
-	xaddl	%eax, (%ebx)
+	cmpxchgl %edx, (%ebx)
 #else
-	xaddl	%eax, MUTEX(%ebx)
+	cmpxchgl %edx, MUTEX(%ebx)
 #endif
 	testl	%eax, %eax
 	jne	12f
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index ebda2f8420..ba32591623 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -38,7 +38,8 @@
 
 
 /* Initializer for compatibility lock.  */
-#define LLL_MUTEX_LOCK_INITIALIZER (0)
+#define LLL_MUTEX_LOCK_INITIALIZER		(0)
+#define LLL_MUTEX_LOCK_INITIALIZER_LOCKED	(1)
 
 
 #ifdef PIC
@@ -97,7 +98,7 @@ extern int __lll_mutex_timedlock_wait (int val, int *__futex,
 				       const struct timespec *abstime)
      __attribute ((regparm (3))) attribute_hidden;
 /* Preserves all registers but %eax.  */
-extern int __lll_mutex_unlock_wait (int *__futex)
+extern int __lll_mutex_unlock_wake (int *__futex)
      __attribute ((regparm (1))) attribute_hidden;
 
 
@@ -105,14 +106,15 @@ extern int __lll_mutex_unlock_wait (int *__futex)
   ({ unsigned char ret;							      \
      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1; setne %0"		      \
 		       : "=a" (ret), "=m" (futex)			      \
-		       : "r" (1), "m" (futex), "0" (0)			      \
+		       : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
+			 "0" (LLL_MUTEX_LOCK_INITIALIZER)		      \
 		       : "memory");					      \
      ret; })
 
 
 #define lll_mutex_lock(futex) \
   (void) ({ int ignore1, ignore2;					      \
-	    __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"		      \
+	    __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
 			      "testl %0, %0\n\t"			      \
 			      "jne _L_mutex_lock_%=\n\t"		      \
 			      ".subsection 1\n\t"			      \
@@ -124,8 +126,8 @@ extern int __lll_mutex_unlock_wait (int *__futex)
 			      ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n"   \
 			      ".previous\n"				      \
 			      "1:"					      \
-			      : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \
-			      : "0" (1), "m" (futex)			      \
+			      : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
+			      : "0" (0), "1" (1), "m" (futex)		      \
 			      : "memory"); })
 
 
@@ -133,7 +135,7 @@ extern int __lll_mutex_unlock_wait (int *__futex)
    always wakeup waiters.  */
 #define lll_mutex_cond_lock(futex) \
   (void) ({ int ignore1, ignore2;					      \
-	    __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"		      \
+	    __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
 			      "testl %0, %0\n\t"			      \
 			      "jne _L_mutex_cond_lock_%=\n\t"		      \
 			      ".subsection 1\n\t"			      \
@@ -145,29 +147,29 @@ extern int __lll_mutex_unlock_wait (int *__futex)
 			      ".size _L_mutex_cond_lock_%=,.-_L_mutex_cond_lock_%=\n"   \
 			      ".previous\n"				      \
 			      "1:"					      \
-			      : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \
-			      : "0" (2), "m" (futex)			      \
+			      : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
+			      : "0" (0), "1" (2), "m" (futex)		      \
 			      : "memory"); })
 
 
 #define lll_mutex_timedlock(futex, timeout) \
   ({ int result, ignore1, ignore2;					      \
-     __asm __volatile (LOCK_INSTR "xaddl %0, %3\n\t"			      \
+     __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
 		       "testl %0, %0\n\t"				      \
 		       "jne _L_mutex_timedlock_%=\n\t"			      \
 		       ".subsection 1\n\t"				      \
 		       ".type _L_mutex_timedlock_%=,@function\n"	      \
 		       "_L_mutex_timedlock_%=:\n\t"			      \
 		       "leal %3, %%ecx\n\t"				      \
-		       "movl %6, %%edx\n\t"				      \
+		       "movl %7, %%edx\n\t"				      \
 		       "call __lll_mutex_timedlock_wait\n\t"		      \
 		       "jmp 1f\n\t"					      \
 		       ".size _L_mutex_timedlock_%=,.-_L_mutex_timedlock_%=\n"\
 		       ".previous\n"					      \
 		       "1:"						      \
-		       : "=a" (result), "=&c" (ignore1), "=&d" (ignore2),     \
+		       : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
 			 "=m" (futex)					      \
-		       : "0" (1), "m" (futex), "m" (timeout)		      \
+		       : "0" (0), "1" (1), "m" (futex), "m" (timeout)	      \
 		       : "memory");					      \
      result; })
 
@@ -201,8 +203,8 @@ extern int __lll_mutex_unlock_wait (int *__futex)
 typedef int lll_lock_t;
 
 /* Initializers for lock.  */
-#define LLL_LOCK_INITIALIZER		(1)
-#define LLL_LOCK_INITIALIZER_LOCKED	(0)
+#define LLL_LOCK_INITIALIZER		(0)
+#define LLL_LOCK_INITIALIZER_LOCKED	(1)
 
 
 extern int __lll_lock_wait (int val, int *__futex)
@@ -213,55 +215,15 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
 
 
 /* The states of a lock are:
-    1  -  untaken
-    0  -  taken by one user
-   <0  -  taken by more users */
+    0  -  untaken
+    1  -  taken by one user
+    2  -  taken by more users */
 
 
 #if defined NOT_IN_libc || defined UP
-# define lll_trylock(futex) \
-  ({ unsigned char ret;							      \
-     __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1; setne %0"		      \
-		       : "=a" (ret), "=m" (futex)			      \
-		       : "r" (0), "m" (futex), "0" (1)			      \
-		       : "memory");					      \
-     ret; })
-
-
-# define lll_lock(futex) \
-  (void) ({ int ignore1, ignore2;					      \
-	    __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"		      \
-			      "jne _L_lock_%=\n\t"			      \
-			      ".subsection 1\n\t"			      \
-			      ".type _L_lock_%=,@function\n"		      \
-			      "_L_lock_%=:\n\t"				      \
-			      "leal %2, %%ecx\n\t"			      \
-			      "call __lll_lock_wait\n\t"		      \
-			      "jmp 1f\n\t"				      \
-			      ".size _L_lock_%=,.-_L_lock_%=\n"		      \
-			      ".previous\n"				      \
-			      "1:"					      \
-			      : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \
-			      : "0" (-1), "m" (futex)			      \
-			      : "memory"); })
-
-
-# define lll_unlock(futex) \
-  (void) ({ int ignore;							      \
-            __asm __volatile (LOCK_INSTR "addl $1,%0\n\t"		      \
-			      "jng _L_unlock_%=\n\t"			      \
-			      ".subsection 1\n\t"			      \
-			      ".type _L_unlock_%=,@function\n"		      \
-			      "_L_unlock_%=:\n\t"			      \
-			      "leal %0, %%eax\n\t"			      \
-			      "call __lll_unlock_wake\n\t"		      \
-			      "jmp 1f\n\t"				      \
-			      ".size _L_unlock_%=,.-_L_unlock_%=\n"	      \
-			      ".previous\n"				      \
-			      "1:"					      \
-			      : "=m" (futex), "=&a" (ignore)		      \
-			      : "m" (futex)				      \
-			      : "memory"); })
+# define lll_trylock(futex) lll_mutex_trylock (futex)
+# define lll_lock(futex) lll_mutex_lock (futex)
+# define lll_unlock(futex) lll_mutex_unlock (futex)
 #else
 /* Special versions of the macros for use in libc itself.  They avoid
    the lock prefix when the thread library is not used.
@@ -276,7 +238,8 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
 		       "lock\n"						      \
 		       "0:\tcmpxchgl %2, %1; setne %0"			      \
 		       : "=a" (ret), "=m" (futex)			      \
-		       : "r" (0), "m" (futex), "0" (1),			      \
+		       : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
+			 "0" (LLL_MUTEX_LOCK_INITIALIZER),		      \
 		         "i" (offsetof (tcbhead_t, multiple_threads))	      \
 		       : "memory");					      \
      ret; })
@@ -284,22 +247,23 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
 
 # define lll_lock(futex) \
   (void) ({ int ignore1, ignore2;					      \
-	    __asm __volatile ("cmpl $0, %%gs:%P5\n\t"			      \
+	    __asm __volatile ("cmpl $0, %%gs:%P6\n\t"			      \
 			      "je,pt 0f\n\t"				      \
 			      "lock\n"					      \
-			      "0:\txaddl %0, %2\n\t"			      \
-			      "jne _L_lock_%=\n\t"			      \
+			      "0:\tcmpxchgl %1, %2\n\t"			      \
+			      "testl %0, %0\n\t"			      \
+			      "jne _L_mutex_lock_%=\n\t"		      \
 			      ".subsection 1\n\t"			      \
-			      ".type _L_lock_%=,@function\n"		      \
-			      "_L_lock_%=:\n\t"				      \
+			      ".type _L_mutex_lock_%=,@function\n"	      \
+			      "_L_mutex_lock_%=:\n\t"			      \
 			      "leal %2, %%ecx\n\t"			      \
-			      "call __lll_lock_wait\n\t"		      \
-			      "jmp 2f\n\t"				      \
-			      ".size _L_lock_%=,.-_L_lock_%=\n"	      \
+			      "call __lll_mutex_lock_wait\n\t"		      \
+			      "jmp 1f\n\t"				      \
+			      ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n"   \
 			      ".previous\n"				      \
-			      "2:"					      \
-			      : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \
-			      : "0" (-1), "m" (futex),			      \
+			      "1:"					      \
+			      : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
+			      : "0" (0), "1" (1), "m" (futex),		      \
 		                "i" (offsetof (tcbhead_t, multiple_threads))  \
 			      : "memory"); })
 
@@ -309,17 +273,17 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
             __asm __volatile ("cmpl $0, %%gs:%P3\n\t"			      \
 			      "je,pt 0f\n\t"				      \
 			      "lock\n"					      \
-			      "0:\taddl $1,%0\n\t"			      \
-			      "jng _L_unlock_%=\n\t"			      \
+			      "0:\tsubl $1,%0\n\t"		      \
+			      "jne _L_mutex_unlock_%=\n\t"		      \
 			      ".subsection 1\n\t"			      \
-			      ".type _L_unlock_%=,@function\n"		      \
-			      "_L_unlock_%=:\n\t"			      \
+			      ".type _L_mutex_unlock_%=,@function\n"	      \
+			      "_L_mutex_unlock_%=:\n\t"			      \
 			      "leal %0, %%eax\n\t"			      \
-			      "call __lll_unlock_wake\n\t"		      \
-			      "jmp 2f\n\t"				      \
-			      ".size _L_unlock_%=,.-_L_unlock_%=\n"	      \
+			      "call __lll_mutex_unlock_wake\n\t"	      \
+			      "jmp 1f\n\t"				      \
+			      ".size _L_mutex_unlock_%=,.-_L_mutex_unlock_%=\n" \
 			      ".previous\n"				      \
-			      "2:"					      \
+			      "1:"					      \
 			      : "=m" (futex), "=&a" (ignore)		      \
 			      : "m" (futex),				      \
 				"i" (offsetof (tcbhead_t, multiple_threads))  \
@@ -328,7 +292,7 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
 
 
 #define lll_islocked(futex) \
-  (futex != 0)
+  (futex != LLL_LOCK_INITIALIZER)
 
 
 /* The kernel notifies a process with uses CLONE_CLEARTID via futex