diff options
Diffstat (limited to 'nptl')
| -rw-r--r-- | nptl/ChangeLog | 4 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S | 162 |
2 files changed, 120 insertions, 46 deletions
diff --git a/nptl/ChangeLog b/nptl/ChangeLog index 867493ced5..98d3a4de68 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -1,5 +1,9 @@ 2009-07-17 Ulrich Drepper <drepper@redhat.com> + * sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S (sem_timedwait): + If possible use FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME to directly + use absolute timeout. + * sysdeps/unix/sysv/linux/x86_64/sem_wait.S (sem_wait): Optimize handling of uncontested semaphore. diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S index c96b37375d..95762834d3 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S @@ -18,6 +18,7 @@ 02111-1307 USA. */ #include <sysdep.h> +#include <kernel-features.h> #include <lowlevellock.h> #include <shlib-compat.h> #include <pthread-errnos.h> @@ -73,8 +74,13 @@ sem_timedwait: pushq %r14 cfi_adjust_cfa_offset(8) cfi_rel_offset(%r14, 0) - subq $24, %rsp - cfi_adjust_cfa_offset(24) +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define STACKFRAME 8 +#else +# define STACKFRAME 24 +#endif + subq $STACKFRAME, %rsp + cfi_adjust_cfa_offset(STACKFRAME) movq %rdi, %r12 movq %rsi, %r13 @@ -87,67 +93,50 @@ sem_timedwait: LOCK addq $1, NWAITERS(%r12) -7: xorl %esi, %esi - movq %rsp, %rdi - movq $VSYSCALL_ADDR_vgettimeofday, %rax - callq *%rax - - /* Compute relative timeout. */ - movq 8(%rsp), %rax - movl $1000, %edi - mul %rdi /* Milli seconds to nano seconds. */ - movq (%r13), %rdi - movq 8(%r13), %rsi - subq (%rsp), %rdi - subq %rax, %rsi - jns 5f - addq $1000000000, %rsi - decq %rdi -5: testq %rdi, %rdi - movl $ETIMEDOUT, %r14d - js 6f /* Time is already up. */ - - movq %rdi, (%rsp) /* Store relative timeout. */ - movq %rsi, 8(%rsp) +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif .LcleanupSTART: - call __pthread_enable_asynccancel - movl %eax, 16(%rsp) +13: call __pthread_enable_asynccancel + movl %eax, (%rsp) - movq %rsp, %r10 + movq %r13, %r10 #if VALUE == 0 movq %r12, %rdi #else leaq VALUE(%r12), %rdi #endif -#if FUTEX_WAIT == 0 - movl PRIVATE(%rdi), %esi -#else - movl $FUTEX_WAIT, %esi + movl $0xffffffff, %r9d + movl $FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi orl PRIVATE(%rdi), %esi -#endif movl $SYS_futex, %eax xorl %edx, %edx syscall movq %rax, %r14 - movl 16(%rsp), %edi + movl (%rsp), %edi call __pthread_disable_asynccancel .LcleanupEND: testq %r14, %r14 - je 9f + je 11f cmpq $-EWOULDBLOCK, %r14 jne 3f -9: +11: #if VALUE == 0 movl (%r12), %eax #else movl VALUE(%r12), %eax #endif -8: testl %eax, %eax - je 7b +14: testl %eax, %eax + je 13b leaq -1(%rax), %rcx LOCK @@ -156,15 +145,15 @@ sem_timedwait: #else cmpxchgl %ecx, VALUE(%r12) #endif - jne 8b + jne 14b - xorl %eax, %eax +10: xorl %eax, %eax -10: LOCK +15: LOCK subq $1, NWAITERS(%r12) - addq $24, %rsp - cfi_adjust_cfa_offset(-24) + addq $STACKFRAME, %rsp + cfi_adjust_cfa_offset(-STACKFRAME) popq %r14 cfi_adjust_cfa_offset(-8) cfi_restore(%r14) @@ -176,10 +165,10 @@ sem_timedwait: cfi_restore(%r12) retq - cfi_adjust_cfa_offset(3 * 8 + 24) - cfi_rel_offset(%r12, 24 + 2 * 8) - cfi_rel_offset(%r13, 24 + 1 * 8) - cfi_rel_offset(%r14, 24) + cfi_adjust_cfa_offset(STACKFRAME + 3 * 8) + cfi_rel_offset(%r12, STACKFRAME + 2 * 8) + cfi_rel_offset(%r13, STACKFRAME + 1 * 8) + cfi_rel_offset(%r14, STACKFRAME) 3: negq %r14 6: #if USE___THREAD @@ -191,7 +180,82 @@ sem_timedwait: #endif orl $-1, %eax + jmp 15b + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +.Lreltmo: +7: xorl %esi, %esi + movq %rsp, %rdi + movq $VSYSCALL_ADDR_vgettimeofday, %rax + callq *%rax + + /* Compute relative timeout. */ + movq 8(%rsp), %rax + movl $1000, %edi + mul %rdi /* Milli seconds to nano seconds. */ + movq (%r13), %rdi + movq 8(%r13), %rsi + subq (%rsp), %rdi + subq %rax, %rsi + jns 5f + addq $1000000000, %rsi + decq %rdi +5: testq %rdi, %rdi + movl $ETIMEDOUT, %r14d + js 6b /* Time is already up. */ + + movq %rdi, (%rsp) /* Store relative timeout. */ + movq %rsi, 8(%rsp) + +.LcleanupSTART2: + call __pthread_enable_asynccancel + movl %eax, 16(%rsp) + + movq %rsp, %r10 +# if VALUE == 0 + movq %r12, %rdi +# else + leaq VALUE(%r12), %rdi +# endif +# if FUTEX_WAIT == 0 + movl PRIVATE(%rdi), %esi +# else + movl $FUTEX_WAIT, %esi + orl PRIVATE(%rdi), %esi +# endif + movl $SYS_futex, %eax + xorl %edx, %edx + syscall + movq %rax, %r14 + + movl 16(%rsp), %edi + call __pthread_disable_asynccancel +.LcleanupEND2: + + testq %r14, %r14 + je 9f + cmpq $-EWOULDBLOCK, %r14 + jne 3b + +9: +# if VALUE == 0 + movl (%r12), %eax +# else + movl VALUE(%r12), %eax +# endif +8: testl %eax, %eax + je 7b + + leaq -1(%rax), %rcx + LOCK +# if VALUE == 0 + cmpxchgl %ecx, (%r12) +# else + cmpxchgl %ecx, VALUE(%r12) +# endif + jne 8b jmp 10b +#endif .size sem_timedwait,.-sem_timedwait @@ -219,6 +283,12 @@ sem_timedwait_cleanup: .uleb128 .LcleanupEND-.LcleanupSTART .uleb128 sem_timedwait_cleanup-.LSTARTCODE .uleb128 0 +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .uleb128 .LcleanupSTART2-.LSTARTCODE + .uleb128 .LcleanupEND2-.LcleanupSTART2 + .uleb128 sem_timedwait_cleanup-.LSTARTCODE + .uleb128 0 +#endif .uleb128 .LcallUR-.LSTARTCODE .uleb128 .LENDCODE-.LcallUR .uleb128 0 |