Extend x86-64 sem_timedwait to use futex syscall with absolute timeout.

2 files changed, 120 insertions, 46 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 867493ced5..98d3a4de68 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,5 +1,9 @@
 2009-07-17  Ulrich Drepper  <drepper@redhat.com>
 
+	* sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S (sem_timedwait):
+	If possible use FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME to directly
+	use absolute timeout.
+
 	* sysdeps/unix/sysv/linux/x86_64/sem_wait.S (sem_wait): Optimize
 	handling of uncontested semaphore.
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
index c96b37375d..95762834d3 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
@@ -18,6 +18,7 @@
    02111-1307 USA.  */
 
 #include <sysdep.h>
+#include <kernel-features.h>
 #include <lowlevellock.h>
 #include <shlib-compat.h>
 #include <pthread-errnos.h>
@@ -73,8 +74,13 @@ sem_timedwait:
 	pushq	%r14
 	cfi_adjust_cfa_offset(8)
 	cfi_rel_offset(%r14, 0)
-	subq	$24, %rsp
-	cfi_adjust_cfa_offset(24)
+#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
+# define STACKFRAME 8
+#else
+# define STACKFRAME 24
+#endif
+	subq	$STACKFRAME, %rsp
+	cfi_adjust_cfa_offset(STACKFRAME)
 
 	movq	%rdi, %r12
 	movq	%rsi, %r13
@@ -87,67 +93,50 @@ sem_timedwait:
 	LOCK
 	addq	$1, NWAITERS(%r12)
 
-7:	xorl	%esi, %esi
-	movq	%rsp, %rdi
-	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
-	callq	*%rax
-
-	/* Compute relative timeout.  */
-	movq	8(%rsp), %rax
-	movl	$1000, %edi
-	mul	%rdi		/* Milli seconds to nano seconds.  */
-	movq	(%r13), %rdi
-	movq	8(%r13), %rsi
-	subq	(%rsp), %rdi
-	subq	%rax, %rsi
-	jns	5f
-	addq	$1000000000, %rsi
-	decq	%rdi
-5:	testq	%rdi, %rdi
-	movl	$ETIMEDOUT, %r14d
-	js	6f		/* Time is already up.  */
-
-	movq	%rdi, (%rsp)	/* Store relative timeout.  */
-	movq	%rsi, 8(%rsp)
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+#  ifdef PIC
+	cmpl	$0, __have_futex_clock_realtime(%rip)
+#  else
+	cmpl	$0, __have_futex_clock_realtime
+#  endif
+	je	.Lreltmo
+#endif
 
 .LcleanupSTART:
-	call	__pthread_enable_asynccancel
-	movl	%eax, 16(%rsp)
+13:	call	__pthread_enable_asynccancel
+	movl	%eax, (%rsp)
 
-	movq	%rsp, %r10
+	movq	%r13, %r10
 #if VALUE == 0
 	movq	%r12, %rdi
 #else
 	leaq	VALUE(%r12), %rdi
 #endif
-#if FUTEX_WAIT == 0
-	movl	PRIVATE(%rdi), %esi
-#else
-	movl	$FUTEX_WAIT, %esi
+	movl	$0xffffffff, %r9d
+	movl	$FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi
 	orl	PRIVATE(%rdi), %esi
-#endif
 	movl	$SYS_futex, %eax
 	xorl	%edx, %edx
 	syscall
 	movq	%rax, %r14
 
-	movl	16(%rsp), %edi
+	movl	(%rsp), %edi
 	call	__pthread_disable_asynccancel
 .LcleanupEND:
 
 	testq	%r14, %r14
-	je	9f
+	je	11f
 	cmpq	$-EWOULDBLOCK, %r14
 	jne	3f
 
-9:
+11:
 #if VALUE == 0
 	movl	(%r12), %eax
 #else
 	movl	VALUE(%r12), %eax
 #endif
-8:	testl	%eax, %eax
-	je	7b
+14:	testl	%eax, %eax
+	je	13b
 
 	leaq	-1(%rax), %rcx
 	LOCK
@@ -156,15 +145,15 @@ sem_timedwait:
 #else
 	cmpxchgl %ecx, VALUE(%r12)
 #endif
-	jne	8b
+	jne	14b
 
-	xorl	%eax, %eax
+10:	xorl	%eax, %eax
 
-10:	LOCK
+15:	LOCK
 	subq	$1, NWAITERS(%r12)
 
-	addq	$24, %rsp
-	cfi_adjust_cfa_offset(-24)
+	addq	$STACKFRAME, %rsp
+	cfi_adjust_cfa_offset(-STACKFRAME)
 	popq	%r14
 	cfi_adjust_cfa_offset(-8)
 	cfi_restore(%r14)
@@ -176,10 +165,10 @@ sem_timedwait:
 	cfi_restore(%r12)
 	retq
 
-	cfi_adjust_cfa_offset(3 * 8 + 24)
-	cfi_rel_offset(%r12, 24 + 2 * 8)
-	cfi_rel_offset(%r13, 24 + 1 * 8)
-	cfi_rel_offset(%r14, 24)
+	cfi_adjust_cfa_offset(STACKFRAME + 3 * 8)
+	cfi_rel_offset(%r12, STACKFRAME + 2 * 8)
+	cfi_rel_offset(%r13, STACKFRAME + 1 * 8)
+	cfi_rel_offset(%r14, STACKFRAME)
 3:	negq	%r14
 6:
 #if USE___THREAD
@@ -191,7 +180,82 @@ sem_timedwait:
 #endif
 
 	orl	$-1, %eax
+	jmp	15b
+
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+.Lreltmo:
+7:	xorl	%esi, %esi
+	movq	%rsp, %rdi
+	movq	$VSYSCALL_ADDR_vgettimeofday, %rax
+	callq	*%rax
+
+	/* Compute relative timeout.  */
+	movq	8(%rsp), %rax
+	movl	$1000, %edi
+	mul	%rdi		/* Milli seconds to nano seconds.  */
+	movq	(%r13), %rdi
+	movq	8(%r13), %rsi
+	subq	(%rsp), %rdi
+	subq	%rax, %rsi
+	jns	5f
+	addq	$1000000000, %rsi
+	decq	%rdi
+5:	testq	%rdi, %rdi
+	movl	$ETIMEDOUT, %r14d
+	js	6b		/* Time is already up.  */
+
+	movq	%rdi, (%rsp)	/* Store relative timeout.  */
+	movq	%rsi, 8(%rsp)
+
+.LcleanupSTART2:
+	call	__pthread_enable_asynccancel
+	movl	%eax, 16(%rsp)
+
+	movq	%rsp, %r10
+# if VALUE == 0
+	movq	%r12, %rdi
+# else
+	leaq	VALUE(%r12), %rdi
+# endif
+# if FUTEX_WAIT == 0
+	movl	PRIVATE(%rdi), %esi
+# else
+	movl	$FUTEX_WAIT, %esi
+	orl	PRIVATE(%rdi), %esi
+# endif
+	movl	$SYS_futex, %eax
+	xorl	%edx, %edx
+	syscall
+	movq	%rax, %r14
+
+	movl	16(%rsp), %edi
+	call	__pthread_disable_asynccancel
+.LcleanupEND2:
+
+	testq	%r14, %r14
+	je	9f
+	cmpq	$-EWOULDBLOCK, %r14
+	jne	3b
+
+9:
+# if VALUE == 0
+	movl	(%r12), %eax
+# else
+	movl	VALUE(%r12), %eax
+# endif
+8:	testl	%eax, %eax
+	je	7b
+
+	leaq	-1(%rax), %rcx
+	LOCK
+# if VALUE == 0
+	cmpxchgl %ecx, (%r12)
+# else
+	cmpxchgl %ecx, VALUE(%r12)
+# endif
+	jne	8b
 	jmp	10b
+#endif
 	.size	sem_timedwait,.-sem_timedwait
 
 
@@ -219,6 +283,12 @@ sem_timedwait_cleanup:
 	.uleb128 .LcleanupEND-.LcleanupSTART
 	.uleb128 sem_timedwait_cleanup-.LSTARTCODE
 	.uleb128  0
+#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
+	.uleb128 .LcleanupSTART2-.LSTARTCODE
+	.uleb128 .LcleanupEND2-.LcleanupSTART2
+	.uleb128 sem_timedwait_cleanup-.LSTARTCODE
+	.uleb128  0
+#endif
 	.uleb128 .LcallUR-.LSTARTCODE
 	.uleb128 .LENDCODE-.LcallUR
 	.uleb128 0