diff options
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S')
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S | 161 |
1 files changed, 121 insertions, 40 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S index 502f1d442f..7065cfac32 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S @@ -19,33 +19,46 @@ #include <sysdep.h> #include <pthread-errnos.h> +#include <kernel-features.h> +#include <lowlevellock.h> .text -#ifndef LOCK -# ifdef UP -# define LOCK +#ifdef __ASSUME_PRIVATE_FUTEX +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg +# define LOAD_PRIVATE_FUTEX_WAKE(reg) \ + movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg +# define LOAD_FUTEX_WAIT(reg) \ + xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg +# define LOAD_FUTEX_WAKE(reg) \ + xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg +#else +# if FUTEX_WAIT == 0 +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl %fs:PRIVATE_FUTEX, reg # else -# define LOCK lock +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT, reg # endif -#endif - -#define SYS_futex 202 -#ifndef FUTEX_WAIT -# define FUTEX_WAIT 0 -# define FUTEX_WAKE 1 -#endif - -#ifndef LOAD_FUTEX_WAIT +# define LOAD_PRIVATE_FUTEX_WAKE(reg) \ + movl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAKE, reg # if FUTEX_WAIT == 0 # define LOAD_FUTEX_WAIT(reg) \ - xorl reg, reg + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg # else # define LOAD_FUTEX_WAIT(reg) \ - movl $FUTEX_WAIT, reg + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT, reg # endif # define LOAD_FUTEX_WAKE(reg) \ - movl $FUTEX_WAKE, reg + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAKE, reg #endif @@ -53,11 +66,11 @@ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 - .globl __lll_mutex_lock_wait - .type __lll_mutex_lock_wait,@function - .hidden __lll_mutex_lock_wait + .globl __lll_lock_wait_private + .type __lll_lock_wait_private,@function + .hidden __lll_lock_wait_private .align 16 -__lll_mutex_lock_wait: +__lll_lock_wait_private: cfi_startproc pushq %r10 cfi_adjust_cfa_offset(8) @@ -67,7 +80,7 @@ __lll_mutex_lock_wait: cfi_offset(%rdx, -24) xorq %r10, %r10 /* No timeout. */ movl $2, %edx - LOAD_FUTEX_WAIT (%esi) + LOAD_PRIVATE_FUTEX_WAIT (%esi) cmpl %edx, %eax /* NB: %edx == 2 */ jne 2f @@ -89,15 +102,52 @@ __lll_mutex_lock_wait: cfi_restore(%r10) retq cfi_endproc - .size __lll_mutex_lock_wait,.-__lll_mutex_lock_wait - + .size __lll_lock_wait_private,.-__lll_lock_wait_private #ifdef NOT_IN_libc - .globl __lll_mutex_timedlock_wait - .type __lll_mutex_timedlock_wait,@function - .hidden __lll_mutex_timedlock_wait + .globl __lll_lock_wait + .type __lll_lock_wait,@function + .hidden __lll_lock_wait .align 16 -__lll_mutex_timedlock_wait: +__lll_lock_wait: + cfi_startproc + pushq %r10 + cfi_adjust_cfa_offset(8) + pushq %rdx + cfi_adjust_cfa_offset(8) + cfi_offset(%r10, -16) + cfi_offset(%rdx, -24) + xorq %r10, %r10 /* No timeout. */ + movl $2, %edx + LOAD_FUTEX_WAIT (%esi) + + cmpl %edx, %eax /* NB: %edx == 2 */ + jne 2f + +1: movl $SYS_futex, %eax + syscall + +2: movl %edx, %eax + xchgl %eax, (%rdi) /* NB: lock is implied */ + + testl %eax, %eax + jnz 1b + + popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) + popq %r10 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r10) + retq + cfi_endproc + .size __lll_lock_wait,.-__lll_lock_wait + + .globl __lll_timedlock_wait + .type __lll_timedlock_wait,@function + .hidden __lll_timedlock_wait + .align 16 +__lll_timedlock_wait: cfi_startproc /* Check for a valid timeout value. */ cmpq $1000000000, 8(%rdx) @@ -118,10 +168,12 @@ __lll_mutex_timedlock_wait: cfi_offset(%r12, -32) cfi_offset(%r13, -40) cfi_offset(%r14, -48) + pushq %rsi + cfi_adjust_cfa_offset(8) /* Stack frame for the timespec and timeval structs. */ - subq $16, %rsp - cfi_adjust_cfa_offset(16) + subq $24, %rsp + cfi_adjust_cfa_offset(24) movq %rdi, %r12 movq %rdx, %r13 @@ -162,6 +214,7 @@ __lll_mutex_timedlock_wait: je 8f movq %rsp, %r10 + movl 24(%rsp), %esi LOAD_FUTEX_WAIT (%esi) movq %r12, %rdi movl $SYS_futex, %eax @@ -174,8 +227,8 @@ __lll_mutex_timedlock_wait: cmpxchgl %edx, (%r12) jnz 7f -6: addq $16, %rsp - cfi_adjust_cfa_offset(-16) +6: addq $32, %rsp + cfi_adjust_cfa_offset(-32) popq %r14 cfi_adjust_cfa_offset(-8) cfi_restore(%r14) @@ -196,7 +249,7 @@ __lll_mutex_timedlock_wait: 3: movl $EINVAL, %eax retq - cfi_adjust_cfa_offset(56) + cfi_adjust_cfa_offset(72) cfi_offset(%r8, -16) cfi_offset(%r9, -24) cfi_offset(%r12, -32) @@ -216,15 +269,15 @@ __lll_mutex_timedlock_wait: 5: movl $ETIMEDOUT, %eax jmp 6b cfi_endproc - .size __lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait + .size __lll_timedlock_wait,.-__lll_timedlock_wait #endif - .globl __lll_mutex_unlock_wake - .type __lll_mutex_unlock_wake,@function - .hidden __lll_mutex_unlock_wake + .globl __lll_unlock_wake_private + .type __lll_unlock_wake_private,@function + .hidden __lll_unlock_wake_private .align 16 -__lll_mutex_unlock_wake: +__lll_unlock_wake_private: cfi_startproc pushq %rsi cfi_adjust_cfa_offset(8) @@ -234,7 +287,7 @@ __lll_mutex_unlock_wake: cfi_offset(%rdx, -24) movl $0, (%rdi) - LOAD_FUTEX_WAKE (%esi) + LOAD_PRIVATE_FUTEX_WAKE (%esi) movl $1, %edx /* Wake one thread. */ movl $SYS_futex, %eax syscall @@ -247,10 +300,38 @@ __lll_mutex_unlock_wake: cfi_restore(%rsi) retq cfi_endproc - .size __lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake - + .size __lll_unlock_wake_private,.-__lll_unlock_wake_private #ifdef NOT_IN_libc + .globl __lll_unlock_wake + .type __lll_unlock_wake,@function + .hidden __lll_unlock_wake + .align 16 +__lll_unlock_wake: + cfi_startproc + pushq %rsi + cfi_adjust_cfa_offset(8) + pushq %rdx + cfi_adjust_cfa_offset(8) + cfi_offset(%rsi, -16) + cfi_offset(%rdx, -24) + + movl $0, (%rdi) + LOAD_FUTEX_WAKE (%esi) + movl $1, %edx /* Wake one thread. */ + movl $SYS_futex, %eax + syscall + + popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) + popq %rsi + cfi_adjust_cfa_offset(-8) + cfi_restore(%rsi) + retq + cfi_endproc + .size __lll_unlock_wake,.-__lll_unlock_wake + .globl __lll_timedwait_tid .type __lll_timedwait_tid,@function .hidden __lll_timedwait_tid |