* pthreadP.h (PTHREAD_ROBUST_MUTEX_PSHARED): Define.

	* pthread_mutex_lock.c: Use it instead of PTHREAD_MUTEX_PSHARED when
	dealing with robust mutexes.
	* pthread_mutex_timedlock.c: Likewise.
	* pthread_mutex_trylock.c: Likewise.
	* pthread_mutex_unlock.c: Likewise.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.

2007-08-06  Jakub Jelinek  <jakub@redhat.com>

	* pthreadP.h (PTHREAD_MUTEX_PSHARED_BIT): Define.
	(PTHREAD_MUTEX_TYPE): Mask __kind with 127.
	(PTHREAD_MUTEX_PSHARED): Define.
	* pthread_mutex_init.c (__pthread_mutex_init): Set
	PTHREAD_MUTEX_PSHARED_BIT for pshared or robust
	mutexes.
	* pthread_mutex_lock.c (LLL_MUTEX_LOCK): Take mutex as argument
	instead of its __data.__lock field, pass PTHREAD_MUTEX_PSHARED
	as second argument to lll_lock.
	(LLL_MUTEX_TRYLOCK): Take mutex as argument
	instead of its __data.__lock field.
	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
	to lll_robust_lock.
	(__pthread_mutex_lock): Update LLL_MUTEX_LOCK, LLL_MUTEX_TRYLOCK,
	LLL_ROBUST_MUTEX_LOCK users, use PTHREAD_MUTEX_TYPE (mutex)
	instead of mutex->__data.__kind directly, pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock and lll_futex_wait.
	* pthread_mutex_trylock.c (__pthread_mutex_trylock): Use
	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
	directly, pass PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock.
	(pthread_mutex_timedlock): Pass PTHREAD_MUTEX_PSHARED (mutex)
	to lll_timedlock, lll_robust_timedlock, lll_unlock and
	lll_futex_timed_wait.  Use PTHREAD_MUTEX_TYPE (mutex) instead
	of mutex->__data.__kind directly.
	* pthread_mutex_timedlock.c (pthread_mutex_timedlock): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_timedlock,
	lll_robust_timedlock, lll_unlock and lll_futex_timed_wait.  Use
	PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind directly.
	* pthread_mutex_unlock.c (__pthread_mutex_unlock_usercnt): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_unlock, lll_robust_unlock
	and lll_futex_wake.
	* pthread_mutex_setprioceiling.c (pthread_mutex_setprioceiling): Pass
	PTHREAD_MUTEX_PSHARED (mutex) to lll_futex_wait and lll_futex_wake.
	Use PTHREAD_MUTEX_TYPE (mutex) instead of mutex->__data.__kind
	directly.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c (LLL_MUTEX_LOCK):
	Take mutex as argument instead of its __data.__lock field, pass
	PTHREAD_MUTEX_PSHARED as second argument to lll_cond_lock.
	(LLL_MUTEX_TRYLOCK): Take mutex as argument instead of its
	__data.__lock field.
	(LLL_ROBUST_MUTEX_LOCK): Take mutex as argument instead of its
	__data.__lock field, pass PTHREAD_MUTEX_PSHARED as second argument
	to lll_robust_cond_lock.
	* pthread_cond_broadcast.c (__pthread_cond_broadcast): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_requeue and
	lll_futex_wake.  Don't use lll_futex_requeue if dependent mutex
	has PTHREAD_MUTEX_PSHARED_BIT bit set in its __data.__kind.
	* pthread_cond_destroy.c (__pthread_cond_destroy): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_wake and
	lll_futex_wait.
	* pthread_cond_signal.c (__pthread_cond_signal): Add pshared
	variable, pass it to lll_lock, lll_unlock, lll_futex_wake_unlock and
	lll_futex_wake.
	* pthread_cond_timedwait.c (__pthread_cond_wait): Add
	pshared variable, pass it to lll_lock, lll_unlock,
	lll_futex_timedwait and lll_futex_wake.
	* pthread_cond_wait.c (__condvar_cleanup, __pthread_cond_wait): Add
	pshared variable, pass it to lll_lock, lll_unlock, lll_futex_wait
	and lll_futex_wake.
	* sysdeps/unix/sysv/linux/alpha/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Add private argument, use __lll_private_flag
	macro.
	* sysdeps/unix/sysv/linux/ia64/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (lll_futex_requeue):
	Likewise.
	* sysdeps/unix/sysv/linux/sparc/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (lll_futex_requeue):
	Likewise.
	* sysdeps/unix/sysv/linux/s390/lowlevellock.h (lll_futex_requeue,
	lll_futex_wake_unlock): Likewise.
	(lll_futex_wake): Fix a typo.
	* sysdeps/unix/sysv/linux/pthread-pi-defines.sym (PS_BIT): Add.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
	(__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or
	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
	Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
	(__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or
	FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
	(__pthread_cond_timedwait): Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:
	(__condvar_cleanup, __pthread_cond_wait): Likewise.

1 files changed, 68 insertions, 16 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 34ef2c7b77..db2683b87b 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -49,8 +49,10 @@ __condvar_cleanup:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -81,9 +83,19 @@ __condvar_cleanup:
 	jne	4f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 	movl	$1, %r12d
@@ -98,16 +110,28 @@ __condvar_cleanup:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 	/* Wake up all waiters to make sure no signal gets lost.  */
 2:	testq	%r12, %r12
 	jnz	5f
 	addq	$cond_futex, %rdi
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -216,12 +240,20 @@ __pthread_cond_wait:
 	xorq	%r10, %r10
 	movq	%r12, %rdx
 	addq	$cond_futex-cond_lock, %rdi
-	movl	$SYS_futex, %eax
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+	cmpq	$-1, dep_mutex-cond_futex(%rdi)
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT, %eax
+	movl	$(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
 #else
-	movl	$FUTEX_WAIT, %esi
+	movl	$FUTEX_WAIT, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+# if FUTEX_WAIT != 0
+	orl	$FUTEX_WAIT, %esi
+# endif
 #endif
+	movl	$SYS_futex, %eax
 	syscall
 
 	movl	(%rsp), %edi
@@ -267,9 +299,19 @@ __pthread_cond_wait:
 	jne	17f
 
 	addq	$cond_nwaiters, %rdi
-	movl	$SYS_futex, %eax
-	movl	$FUTEX_WAKE, %esi
+	cmpq	$-1, dep_mutex-cond_nwaiters(%rdi)
 	movl	$1, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE, %eax
+	movl	$(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
+	cmove	%eax, %esi
+#else
+	movl	$0, %eax
+	movl	%fs:PRIVATE_FUTEX, %esi
+	cmove	%eax, %esi
+	orl	$FUTEX_WAKE, %esi
+#endif
+	movl	$SYS_futex, %eax
 	syscall
 	subq	$cond_nwaiters, %rdi
 
@@ -302,8 +344,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 	jmp	2b
 
@@ -312,8 +356,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	4b
 
@@ -322,8 +368,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_lock_wait
 #if cond_lock != 0
 	subq	$cond_lock, %rdi
@@ -335,8 +383,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 	jmp	11b
 
@@ -354,8 +404,10 @@ __pthread_cond_wait:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
-	/* XYZ */
+	cmpq	$-1, dep_mutex-cond_lock(%rdi)
+	movl	$LLL_PRIVATE, %eax
 	movl	$LLL_SHARED, %esi
+	cmovne	%eax, %esi
 	callq	__lll_unlock_wake
 
 13:	movq	%r10, %rax