diff options
Diffstat (limited to 'nptl')
27 files changed, 292 insertions, 433 deletions
diff --git a/nptl/ChangeLog b/nptl/ChangeLog index 0562ae947a..8bb48ddbc8 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -3,21 +3,40 @@ * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Completely revamp the locking macros. No distinction between normal and mutex locking anymore. + * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise. * sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Rewrite mutex locking. Merge bits from lowlevelmutex.S we still need. + * sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise. * sysdeps/unix/sysv/linux/i386/i486/lowlevelmutex.S: Removed. + * sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S: Removed. * Makefile (routines): Remove libc-lowlevelmutex. (libpthread-rountines): Remove lowlevelmutex. - * pthread_barrier_wait.S: Adjust for new mutex implementation. - * pthread_cond_broadcast.S: Likewise. - * pthread_cond_timedwait.S: Likewise. - * pthread_cond_wait.S: Likewise. - * pthread_rwlock_rdlock.S: Likewise. - * pthread_rwlock_timedrdlock.S: Likewise. - * pthread_rwlock_timedwrlock.S: Likewise. - * pthread_rwlock_unlock.S: Likewise. - * pthread_rwlock_wrlock.S: Likewise. - * pthread_cond_signal.S: Likewise. Don't use requeue. + * sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S: Adjust + for new mutex implementation. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: + Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: + Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S: Likewise + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S: + Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S: + Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise. + Don't use requeue. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise. 2003-09-20 Ulrich Drepper <drepper@redhat.com> diff --git a/nptl/Makefile b/nptl/Makefile index ba568dffba..d592137caf 100644 --- a/nptl/Makefile +++ b/nptl/Makefile @@ -29,8 +29,7 @@ extra-libs := libpthread extra-libs-others := $(extra-libs) install-lib-ldscripts := libpthread.so -routines = alloca_cutoff forward libc-lowlevellock libc-lowlevelmutex \ - libc-cancellation +routines = alloca_cutoff forward libc-lowlevellock libc-cancellation shared-only-routines = forward libpthread-routines = init events version \ @@ -102,7 +101,7 @@ libpthread-routines = init events version \ cleanup_defer_compat unwind \ pt-longjmp \ cancellation \ - lowlevellock lowlevelmutex \ + lowlevellock \ pt-vfork \ ptw-write ptw-read ptw-close ptw-fcntl ptw-accept \ ptw-connect ptw-recv ptw-recvfrom ptw-recvmsg ptw-send \ diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S index 6e4b077295..c73970d45f 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S @@ -65,8 +65,7 @@ __lll_mutex_lock_wait: 2: LOCK cmpxchgl %edx, (%ebx) - testl %eax, %eax - jne,pn 1b + jnz,pn 1b popl %edx popl %ebx @@ -144,8 +143,7 @@ __lll_mutex_timedlock_wait: LOCK cmpxchgl %edx, (%ebx) - testl %eax, %eax - jne 7f + jnz 7f 6: addl $8, %esp popl %ebp diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S index 08c6e915b9..d4b61a10c1 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S @@ -46,8 +46,7 @@ pthread_barrier_wait: xorl %eax, %eax LOCK cmpxchgl %edx, MUTEX(%ebx) - testl %eax, %eax - jne 1f + jnz 1f /* One less waiter. If this was the last one needed wake everybody. */ diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S index 1049e0f4dd..cdcf6d2914 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S @@ -59,8 +59,7 @@ __pthread_cond_broadcast: #else cmpxchgl %edx, cond_lock(%ebx) #endif - testl %eax, %eax - jne 1f + jnz 1f 2: addl $wakeup_seq, %ebx movl total_seq+4-wakeup_seq(%ebx), %eax diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index d86ee054a9..3a24adf911 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -58,8 +58,7 @@ __pthread_cond_signal: #else cmpxchgl %edx, cond_lock(%edi) #endif - testl %eax, %eax - jne 1f + jnz 1f 2: leal wakeup_seq(%edi), %ebx movl total_seq+4(%edi), %eax diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S index 5799d8f862..ca5c1423c0 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S @@ -64,8 +64,7 @@ __pthread_cond_timedwait: #else cmpxchgl %edx, cond_lock(%ebx) #endif - testl %eax, %eax - jne 1f + jnz 1f /* Store the reference to the mutex. If there is already a different value in there this is a bad user bug. */ @@ -179,8 +178,7 @@ __pthread_cond_timedwait: #else cmpxchgl %edx, cond_lock(%ebx) #endif - testl %eax, %eax - jne 5f + jnz 5f 6: movl woken_seq(%ebx), %eax movl woken_seq+4(%ebx), %ecx @@ -363,8 +361,7 @@ __condvar_tw_cleanup: #else cmpxchgl %edx, cond_lock(%ebx) #endif - testl %eax, %eax - je 1f + jz 1f #if cond_lock == 0 movl %ebx, %ecx diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index db0428f527..ff3a1ccef3 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -61,8 +61,7 @@ __pthread_cond_wait: #else cmpxchgl %edx, cond_lock(%ebx) #endif - testl %eax, %eax - jne 1f + jnz 1f /* Store the reference to the mutex. If there is already a different value in there this is a bad user bug. */ @@ -123,8 +122,7 @@ __pthread_cond_wait: #else cmpxchgl %edx, cond_lock(%ebx) #endif - testl %eax, %eax - jne 5f + jnz 5f 6: movl woken_seq(%ebx), %eax movl woken_seq+4(%ebx), %ecx @@ -256,8 +254,7 @@ __condvar_w_cleanup: #else cmpxchgl %edx, cond_lock(%ebx) #endif - testl %eax, %eax - je 1f + jz 1f #if cond_lock == 0 movl %ebx, %ecx diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S index 1c149433f0..db0639d21c 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S @@ -54,8 +54,7 @@ __pthread_rwlock_rdlock: #else cmpxchgl %edx, MUTEX(%ebx) #endif - testl %eax, %eax - jne 1f + jnz 1f 2: movl WRITER(%ebx), %eax testl %eax, %eax @@ -94,8 +93,7 @@ __pthread_rwlock_rdlock: #else cmpxchgl %edx, MUTEX(%ebx) #endif - testl %eax, %eax - jne 12f + jnz 12f 13: subl $1, READERS_QUEUED(%ebx) jmp 2b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S index 409c046b7b..eb5665b432 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S @@ -58,8 +58,7 @@ pthread_rwlock_timedrdlock: #else cmpxchgl %edx, MUTEX(%ebp) #endif - testl %eax, %eax - jne 1f + jnz 1f 2: movl WRITER(%ebp), %eax testl %eax, %eax @@ -127,8 +126,7 @@ pthread_rwlock_timedrdlock: #else cmpxchgl %edx, MUTEX(%ebp) #endif - testl %eax, %eax - jne 12f + jnz 12f 13: subl $1, READERS_QUEUED(%ebp) cmpl $-ETIMEDOUT, %ecx diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S index 00d9906e59..d9db77ba05 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S @@ -58,8 +58,7 @@ pthread_rwlock_timedwrlock: #else cmpxchgl %edx, MUTEX(%ebp) #endif - testl %eax, %eax - jne 1f + jnz 1f 2: movl WRITER(%ebp), %eax testl %eax, %eax @@ -125,8 +124,7 @@ pthread_rwlock_timedwrlock: #else cmpxchgl %edx, MUTEX(%ebp) #endif - testl %eax, %eax - jne 12f + jnz 12f 13: subl $1, WRITERS_QUEUED(%ebp) cmpl $-ETIMEDOUT, %ecx diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S index 839a5eb336..64aac3255a 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S @@ -52,8 +52,7 @@ __pthread_rwlock_unlock: #else cmpxchgl %edx, MUTEX(%edi) #endif - testl %eax, %eax - jne 1f + jnz 1f 2: cmpl $0, WRITER(%edi) jne 5f diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S index 89ad6a3774..ea9cc170db 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S @@ -54,8 +54,7 @@ __pthread_rwlock_wrlock: #else cmpxchgl %edx, MUTEX(%ebx) #endif - testl %eax, %eax - jne 1f + jnz 1f 2: movl WRITER(%ebx), %eax testl %eax, %eax @@ -92,8 +91,7 @@ __pthread_rwlock_wrlock: #else cmpxchgl %edx, MUTEX(%ebx) #endif - testl %eax, %eax - jne 12f + jnz 12f 13: subl $1, WRITERS_QUEUED(%ebx) jmp 2b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index ba32591623..be00e89638 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -115,8 +115,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) #define lll_mutex_lock(futex) \ (void) ({ int ignore1, ignore2; \ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ - "testl %0, %0\n\t" \ - "jne _L_mutex_lock_%=\n\t" \ + "jnz _L_mutex_lock_%=\n\t" \ ".subsection 1\n\t" \ ".type _L_mutex_lock_%=,@function\n" \ "_L_mutex_lock_%=:\n\t" \ @@ -136,8 +135,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) #define lll_mutex_cond_lock(futex) \ (void) ({ int ignore1, ignore2; \ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ - "testl %0, %0\n\t" \ - "jne _L_mutex_cond_lock_%=\n\t" \ + "jnz _L_mutex_cond_lock_%=\n\t" \ ".subsection 1\n\t" \ ".type _L_mutex_cond_lock_%=,@function\n" \ "_L_mutex_cond_lock_%=:\n\t" \ @@ -155,8 +153,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2; \ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ - "testl %0, %0\n\t" \ - "jne _L_mutex_timedlock_%=\n\t" \ + "jnz _L_mutex_timedlock_%=\n\t" \ ".subsection 1\n\t" \ ".type _L_mutex_timedlock_%=,@function\n" \ "_L_mutex_timedlock_%=:\n\t" \ @@ -251,8 +248,7 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden; "je,pt 0f\n\t" \ "lock\n" \ "0:\tcmpxchgl %1, %2\n\t" \ - "testl %0, %0\n\t" \ - "jne _L_mutex_lock_%=\n\t" \ + "jnz _L_mutex_lock_%=\n\t" \ ".subsection 1\n\t" \ ".type _L_mutex_lock_%=,@function\n" \ "_L_mutex_lock_%=:\n\t" \ diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S index aff449b13a..a56c714aeb 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S @@ -38,34 +38,129 @@ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 - /* Modified: %rax, %rsi. */ - .globl __lll_lock_wait - .type __lll_lock_wait,@function - .hidden __lll_lock_wait + .globl __lll_mutex_lock_wait + .type __lll_mutex_lock_wait,@function + .hidden __lll_mutex_lock_wait .align 16 -__lll_lock_wait: +__lll_mutex_lock_wait: pushq %r10 pushq %rdx xorq %r10, %r10 /* No timeout. */ + movl $2, %edx + movq %r10, %rsi /* movq $FUTEX_WAIT, %rsi */ 1: - leaq -1(%rsi), %rdx /* account for the preceeded xadd. */ - movq %r10, %rsi /* movq $FUTEX_WAIT, %rsi */ + movl $1, %eax + LOCK + cmpxchgl %edx, (%rdi) + + testl %eax, %eax + je 2f + movq $SYS_futex, %rax syscall - orl $-1, %esi /* Load -1. */ - LOCK - xaddl %esi, (%rdi) - jne 1b + xorl %eax, %eax +2: LOCK + cmpxchgl %edx, (%rdi) - movl $-1, (%rdi) + jnz 1b popq %rdx popq %r10 retq - .size __lll_lock_wait,.-__lll_lock_wait + .size __lll_mutex_lock_wait,.-__lll_mutex_lock_wait + + +#ifdef NOT_IN_libc + .globl __lll_mutex_timedlock_wait + .type __lll_mutex_timedlock_wait,@function + .hidden __lll_mutex_timedlock_wait + .align 16 +__lll_mutex_timedlock_wait: + /* Check for a valid timeout value. */ + cmpq $1000000000, 8(%rdx) + jae 3f + + pushq %r12 + pushq %r13 + pushq %r14 + + /* Stack frame for the timespec and timeval structs. */ + subq $16, %rsp + + movq %rdi, %r12 + movq %rdx, %r13 + +1: + /* Get current time. */ + movq %rsp, %rdi + xorq %rsi, %rsi + movq $VSYSCALL_ADDR_vgettimeofday, %rax + /* This is a regular function call, all calleer-save registers + might be clobbered. */ + callq *%rax + + /* Compute relative timeout. */ + movq 8(%rsp), %rax + movq $1000, %rdi + mul %rdi /* Milli seconds to nano seconds. */ + movq (%r13), %rdi + movq 8(%r13), %rsi + subq (%rsp), %rdi + subq %rax, %rsi + jns 4f + addq $1000000000, %rsi + decq %rdi +4: testq %rdi, %rdi + js 5f /* Time is already up. */ + + /* Futex call. */ + movq %rdi, (%rsp) /* Store relative timeout. */ + movq %rsi, 8(%rsp) + + movl $1, %eax + movl $2, %edx + LOCK + cmpxchgl %edx, (%r12) + + testl %eax, %eax + je 8f + + movq %rsp, %r10 + xorq %rsi, %rsi /* movq $FUTEX_WAIT, %rsi */ + movq %r12, %rdi + movq $SYS_futex, %rax + syscall + movq %rax, %rcx + + movl $1, %eax + LOCK + cmpxchgl %edx, (%rdi) + jnz 7f + + movl $2, (%rdi) + xorl %eax, %eax + +8: addq $16, %rsp + popq %r14 + popq %r13 + popq %r12 + retq + + /* Check whether the time expired. */ +7: cmpq $-ETIMEDOUT, %rcx + je 5f + jmp 1b + +3: movl $EINVAL, %eax + retq + +5: movl $ETIMEDOUT, %eax + jmp 8b + .size __lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait +#endif #ifdef NOT_IN_libc @@ -88,24 +183,24 @@ lll_unlock_wake_cb: #endif - .globl __lll_unlock_wake - .type __lll_unlock_wake,@function - .hidden __lll_unlock_wake + .globl __lll_mutex_unlock_wake + .type __lll_mutex_unlock_wake,@function + .hidden __lll_mutex_unlock_wake .align 16 -__lll_unlock_wake: +__lll_mutex_unlock_wake: pushq %rsi pushq %rdx -1: movq $FUTEX_WAKE, %rsi + movl $0, (%rdi) + movq $FUTEX_WAKE, %rsi movl $1, %edx /* Wake one thread. */ movq $SYS_futex, %rax - movl %edx, (%rdi) /* Stores '$1'. */ syscall popq %rdx popq %rsi retq - .size __lll_unlock_wake,.-__lll_unlock_wake + .size __lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake #ifdef NOT_IN_libc diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index 1f1e7dd1a9..35d132113d 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -38,7 +38,8 @@ /* Initializer for compatibility lock. */ -#define LLL_MUTEX_LOCK_INITIALIZER (0) +#define LLL_MUTEX_LOCK_INITIALIZER (0) +#define LLL_MUTEX_LOCK_INITIALIZER_LOCKED (1) #define lll_futex_wait(futex, val) \ @@ -80,16 +81,16 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; ({ unsigned char ret; \ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1; setne %0" \ : "=a" (ret), "=m" (futex) \ - : "r" (1), "1" (futex), "0" (0) \ + : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\ + "0" (LLL_MUTEX_LOCK_INITIALIZER) \ : "memory"); \ ret; }) #define lll_mutex_lock(futex) \ - (void) ({ int ignore1, ignore2; \ - __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \ - "testl %0, %0\n\t" \ - "jne 1f\n\t" \ + (void) ({ int ignore1, ignore2, ignore3; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ + "jnz 1f\n\t" \ ".subsection 1\n" \ "1:\tleaq %2, %%rdi\n\t" \ "subq $128, %%rsp\n\t" \ @@ -98,16 +99,16 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; "jmp 2f\n\t" \ ".previous\n" \ "2:" \ - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \ - : "0" (1), "2" (futex) \ - : "ax", "cx", "r11", "cc", "memory"); }) + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\ + "=a" (ignore3) \ + : "0" (1), "m" (futex), "3" (0) \ + : "cx", "r11", "cc", "memory"); }) #define lll_mutex_cond_lock(futex) \ - (void) ({ int ignore1, ignore2; \ - __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \ - "testl %0, %0\n\t" \ - "jne 1f\n\t" \ + (void) ({ int ignore1, ignore2, ignore3; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ + "jnz 1f\n\t" \ ".subsection 1\n" \ "1:\tleaq %2, %%rdi\n\t" \ "subq $128, %%rsp\n\t" \ @@ -116,28 +117,28 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; "jmp 2f\n\t" \ ".previous\n" \ "2:" \ - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \ - : "0" (2), "2" (futex) \ - : "ax", "cx", "r11", "cc", "memory"); }) + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\ + "=a" (ignore3) \ + : "0" (2), "m" (futex), "3" (0) \ + : "cx", "r11", "cc", "memory"); }) #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2, ignore3; \ - __asm __volatile (LOCK_INSTR "xaddl %0, %4\n\t" \ - "testl %0, %0\n\t" \ - "jne 1f\n\t" \ + __asm __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t" \ + "jnz 1f\n\t" \ ".subsection 1\n" \ "1:\tleaq %4, %%rdi\n\t" \ - "movq %7, %%rdx\n\t" \ + "movq %8, %%rdx\n\t" \ "subq $128, %%rsp\n\t" \ "callq __lll_mutex_timedlock_wait\n\t" \ "addq $128, %%rsp\n\t" \ "jmp 2f\n\t" \ ".previous\n" \ "2:" \ - : "=a" (result), "=&D" (ignore1), "=&S" (ignore2), \ + : "=a" (result), "=&D" (ignore1), "=S" (ignore2), \ "=&d" (ignore3), "=m" (futex) \ - : "0" (1), "4" (futex), "m" (timeout) \ + : "0" (0), "2" (1), "m" (futex), "m" (timeout) \ : "memory", "cx", "cc", "r10", "r11"); \ result; }) @@ -155,12 +156,12 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; ".previous\n" \ "2:" \ : "=m" (futex), "=&D" (ignore) \ - : "0" (futex) \ + : "m" (futex) \ : "ax", "cx", "r11", "cc", "memory"); }) #define lll_mutex_islocked(futex) \ - (futex != 0) + (futex != LLL_MUTEX_LOCK_INITIALIZER) /* We have a separate internal lock implementation which is not tied @@ -170,63 +171,23 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; typedef int lll_lock_t; /* Initializers for lock. */ -#define LLL_LOCK_INITIALIZER (1) -#define LLL_LOCK_INITIALIZER_LOCKED (0) +#define LLL_LOCK_INITIALIZER (0) +#define LLL_LOCK_INITIALIZER_LOCKED (1) -extern int __lll_lock_wait (int *__futex, int val) attribute_hidden; -extern int __lll_unlock_wake (int *__futex) attribute_hidden; extern int lll_unlock_wake_cb (int *__futex) attribute_hidden; /* The states of a lock are: - 1 - untaken - 0 - taken by one user - <0 - taken by more users */ + 0 - untaken + 1 - taken by one user + 2 - taken by more users */ #if defined NOT_IN_libc || defined UP -# define lll_trylock(futex) \ - ({ unsigned char ret; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1; setne %0" \ - : "=a" (ret), "=m" (futex) \ - : "r" (0), "1" (futex), "0" (1) \ - : "memory"); \ - ret; }) - - -# define lll_lock(futex) \ - (void) ({ int ignore1, ignore2; \ - __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \ - "jne 1f\n\t" \ - ".subsection 1\n" \ - "1:\tleaq %2, %%rdi\n\t" \ - "subq $128, %%rsp\n\t" \ - "callq __lll_lock_wait\n\t" \ - "addq $128, %%rsp\n\t" \ - "jmp 2f\n\t" \ - ".previous\n" \ - "2:" \ - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \ - : "0" (-1), "2" (futex) \ - : "ax", "cx", "r11", "cc", "memory"); }) - - -# define lll_unlock(futex) \ - (void) ({ int ignore; \ - __asm __volatile (LOCK_INSTR "incl %0\n\t" \ - "jng 1f\n\t" \ - ".subsection 1\n" \ - "1:\tleaq %0, %%rdi\n\t" \ - "subq $128, %%rsp\n\t" \ - "callq __lll_unlock_wake\n\t" \ - "addq $128, %%rsp\n\t" \ - "jmp 2f\n\t" \ - ".previous\n" \ - "2:" \ - : "=m" (futex), "=&D" (ignore) \ - : "0" (futex) \ - : "ax", "cx", "r11", "cc", "memory"); }) +# define lll_trylock(futex) lll_mutex_trylock (futex) +# define lll_lock(futex) lll_mutex_lock (futex) +# define lll_unlock(futex) lll_mutex_unlock (futex) #else /* Special versions of the macros for use in libc itself. They avoid the lock prefix when the thread library is not used. @@ -240,29 +201,31 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden; "lock\n" \ "0:\tcmpxchgl %2, %1; setne %0" \ : "=a" (ret), "=m" (futex) \ - : "r" (0), "1" (futex), "0" (1) \ + : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\ + "0" (LLL_MUTEX_LOCK_INITIALIZER) \ : "memory"); \ ret; }) # define lll_lock(futex) \ - (void) ({ int ignore1, ignore2; \ + (void) ({ int ignore1, ignore2, ignore3; \ __asm __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ "je 0f\n\t" \ "lock\n" \ - "0:\txaddl %0, %2\n\t" \ - "jne 1f\n\t" \ + "0:\tcmpxchgl %0, %2\n\t" \ + "jnz 1f\n\t" \ ".subsection 1\n" \ "1:\tleaq %2, %%rdi\n\t" \ "subq $128, %%rsp\n\t" \ - "callq __lll_lock_wait\n\t" \ + "callq __lll_mutex_lock_wait\n\t" \ "addq $128, %%rsp\n\t" \ "jmp 2f\n\t" \ ".previous\n" \ "2:" \ - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \ - : "0" (-1), "2" (futex) \ - : "ax", "cx", "r11", "cc", "memory"); }) + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\ + "=a" (ignore3) \ + : "0" (1), "m" (futex), "3" (0) \ + : "cx", "r11", "cc", "memory"); }) # define lll_unlock(futex) \ @@ -270,12 +233,12 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden; __asm __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ "je 0f\n\t" \ "lock\n" \ - "0:\tincl %0\n\t" \ - "jng 1f\n\t" \ + "0:\tdecl %0\n\t" \ + "jne 1f\n\t" \ ".subsection 1\n" \ "1:\tleaq %0, %%rdi\n\t" \ "subq $128, %%rsp\n\t" \ - "callq __lll_unlock_wake\n\t" \ + "callq __lll_mutex_unlock_wake\n\t" \ "addq $128, %%rsp\n\t" \ "jmp 2f\n\t" \ ".previous\n" \ @@ -287,7 +250,7 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden; #define lll_islocked(futex) \ - (futex != 0) + (futex != LLL_MUTEX_LOCK_INITIALIZER) /* The kernel notifies a process with uses CLONE_CLEARTID via futex diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S deleted file mode 100644 index 2a2f847684..0000000000 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S +++ /dev/null @@ -1,175 +0,0 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#include <sysdep.h> -#include <pthread-errnos.h> - - .text - -#ifndef LOCK -# ifdef UP -# define LOCK -# else -# define LOCK lock -# endif -#endif - -#define SYS_futex 202 -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - -/* For the calculation see asm/vsyscall.h. */ -#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 - - - .globl __lll_mutex_lock_wait - .type __lll_mutex_lock_wait,@function - .hidden __lll_mutex_lock_wait - .align 16 -__lll_mutex_lock_wait: - pushq %r10 - pushq %rdx - - /* In the loop we are going to add 2 instead of 1 which is what - the caller did. Account for that. */ - decq %rsi - - xorq %r10, %r10 /* No timeout. */ - -1: - leaq 2(%rsi), %rdx /* account for the preceeded xadd. */ - movq %r10, %rsi /* movq $FUTEX_WAIT, %rsi */ - movq $SYS_futex, %rax - syscall - - movl $2, %esi - LOCK - xaddl %esi, (%rdi) - testl %esi, %esi - jne 1b - - popq %rdx - popq %r10 - retq - .size __lll_mutex_lock_wait,.-__lll_mutex_lock_wait - - -#ifdef NOT_IN_libc - .globl __lll_mutex_timedlock_wait - .type __lll_mutex_timedlock_wait,@function - .hidden __lll_mutex_timedlock_wait - .align 16 -__lll_mutex_timedlock_wait: - /* Check for a valid timeout value. */ - cmpq $1000000000, 8(%rdx) - jae 3f - - pushq %r12 - pushq %r13 - pushq %r14 - - /* Stack frame for the timespec and timeval structs. */ - subq $16, %rsp - - movq %rdi, %r12 - movq %rdx, %r13 - -1: leaq 1(%rax), %r14 - - /* Get current time. */ - movq %rsp, %rdi - xorq %rsi, %rsi - movq $VSYSCALL_ADDR_vgettimeofday, %rax - /* This is a regular function call, all calleer-save registers - might be clobbered. */ - callq *%rax - - /* Compute relative timeout. */ - movq 8(%rsp), %rax - movq $1000, %rdi - mul %rdi /* Milli seconds to nano seconds. */ - movq (%r13), %rdi - movq 8(%r13), %rsi - subq (%rsp), %rdi - subq %rax, %rsi - jns 4f - addq $1000000000, %rsi - decq %rdi -4: testq %rdi, %rdi - js 5f /* Time is already up. */ - - /* Futex call. */ - movq %rdi, (%rsp) /* Store relative timeout. */ - movq %rsi, 8(%rsp) - - movl %r14d, %edx - movq %rsp, %r10 - xorq %rsi, %rsi /* movq $FUTEX_WAIT, %rsi */ - movq %r12, %rdi - movq $SYS_futex, %rax - syscall - movq %rax, %rcx - - movl $1, %eax - LOCK - xaddl %eax, (%rdi) - testl %eax, %eax - jne 7f - - movl $2, (%rdi) - xorl %eax, %eax - -8: addq $16, %rsp - popq %r14 - popq %r13 - popq %r12 - retq - - /* Check whether the time expired. */ -7: cmpq $-ETIMEDOUT, %rcx - je 5f - jmp 1b - -3: movl $EINVAL, %eax - retq - -5: movl $ETIMEDOUT, %eax - jmp 8b - .size __lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait -#endif - - - .globl __lll_mutex_unlock_wake - .type __lll_mutex_unlock_wake,@function - .hidden __lll_mutex_unlock_wake - .align 16 -__lll_mutex_unlock_wake: - pushq %rsi - pushq %rdx - - movl $0, (%rdi) - movq $FUTEX_WAKE, %rsi - movl $1, %edx /* Wake one thread. */ - movq $SYS_futex, %rax - syscall - - popq %rdx - popq %rsi - retq - .size __lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S index fdd2958130..672f2cac0e 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S @@ -38,10 +38,11 @@ .align 16 pthread_barrier_wait: /* Get the mutex. */ - orl $-1, %esi + xorl %eax, %eax + movl $1, %esi LOCK - xaddl %esi, MUTEX(%rdi) - jne 1f + cmpxchgl %esi, MUTEX(%rdi) + jnz 1f /* One less waiter. If this was the last one needed wake everybody. */ @@ -57,8 +58,8 @@ pthread_barrier_wait: /* Release the mutex. */ LOCK - addl $1, MUTEX(%rdi) - jng 6f + decl MUTEX(%rdi) + jne 6f /* Wait for the remaining threads. The call will return immediately if the CURR_EVENT memory has meanwhile been changed. */ @@ -102,25 +103,25 @@ pthread_barrier_wait: waking the waiting threads since otherwise a new thread might arrive and gets waken up, too. */ LOCK - addl $1, MUTEX(%rdi) - jng 4f + decl MUTEX(%rdi) + jne 4f 5: orl $-1, %eax /* == PTHREAD_BARRIER_SERIAL_THREAD */ retq 1: addq $MUTEX, %rdi - callq __lll_lock_wait + callq __lll_mutex_lock_wait subq $MUTEX, %rdi jmp 2b 4: addq $MUTEX, %rdi - callq __lll_unlock_wake + callq __lll_mutex_unlock_wake subq $MUTEX, %rdi jmp 5b 6: addq $MUTEX, %rdi - callq __lll_unlock_wake + callq __lll_mutex_unlock_wake subq $MUTEX, %rdi jmp 7b .size pthread_barrier_wait,.-pthread_barrier_wait diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index d9185925a4..e6b2bbf19c 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -46,14 +46,14 @@ __pthread_cond_broadcast: /* Get internal lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, cond_lock(%rdi) + cmpxchgl %esi, cond_lock(%rdi) #endif - testl %esi, %esi - jne 1f + jnz 1f 2: addq $wakeup_seq, %rdi movq total_seq-wakeup_seq(%rdi), %rcx diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index d62310148d..92eadfc433 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -47,14 +47,14 @@ __pthread_cond_signal: /* Get internal lock. */ movq %rdi, %r8 movl $1, %esi + xorl %eax, %eax LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, cond_lock(%rdi) + cmpxchgl %esi, cond_lock(%rdi) #endif - testl %esi, %esi - jne 1f + jnz 1f 2: addq $wakeup_seq, %rdi movq total_seq(%r8), %rcx @@ -65,21 +65,11 @@ __pthread_cond_signal: addq $1, (%rdi) /* Wake up one thread. */ - movq $FUTEX_REQUEUE, %rsi + movq $FUTEX_WAKE, %rsi movq $SYS_futex, %rax - xorq %rdx, %rdx - movq $1, %r10 + movq $1, %rdx syscall -#ifndef __ASSUME_FUTEX_REQUEUE - cmpq $-EINVAL, %rax - je 7f -#endif - - /* If we moved a thread we in any case have to make the syscall. */ - testq %rax, %rax - jne 5f - /* Unlock. */ 4: LOCK #if cond_lock == 0 @@ -112,15 +102,6 @@ __pthread_cond_signal: #endif callq __lll_mutex_unlock_wake jmp 6b - -#ifndef __ASSUME_FUTEX_REQUEUE -7: /* The futex requeue functionality is not available. */ - movq $1, %rdx - movq $FUTEX_WAKE, %rsi - movq $SYS_futex, %rax - syscall - jmp 4b -#endif .size __pthread_cond_signal, .-__pthread_cond_signal versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index b2b4449554..c0c8f55cfb 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -80,14 +80,14 @@ __pthread_cond_timedwait: /* Get internal lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, cond_lock(%rdi) + cmpxchgl %esi, cond_lock(%rdi) #endif - testl %esi, %esi - jne 1f + jnz 1f /* Unlock the mutex. */ 2: movq 16(%rsp), %rdi @@ -188,13 +188,13 @@ __pthread_cond_timedwait: /* Lock. */ movq 8(%rsp), %rdi movl $1, %esi + xorl %eax, %eax LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, cond_lock(%rdi) + cmpxchgl %esi, cond_lock(%rdi) #endif - testl %esi, %esi jne 5f 6: movq woken_seq(%rdi), %rax diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 3942faf1ad..d8681ebe6b 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -44,14 +44,14 @@ __condvar_cleanup: movq %rdi, %r8 movq 8(%rdi), %rdi movl $1, %esi + xorl %eax, %eax LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, cond_lock(%rdi) + cmpxchgl %esi, cond_lock(%rdi) #endif - testl %esi, %esi - je 1f + jz 1f #if cond_lock != 0 addq $cond_lock, %rdi @@ -125,13 +125,13 @@ __pthread_cond_wait: /* Get internal lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, cond_lock(%rdi) + cmpxchgl %esi, cond_lock(%rdi) #endif - testl %esi, %esi jne 1f /* Unlock the mutex. */ @@ -186,14 +186,14 @@ __pthread_cond_wait: /* Lock. */ movq 8(%rsp), %rdi movl $1, %esi + xorl %eax, %eax LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, cond_lock(%rdi) + cmpxchgl %esi, cond_lock(%rdi) #endif - testl %esi, %esi - jne 5f + jnz 5f 6: movq woken_seq(%rdi), %rax diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S index 0fd093e7dd..43c8cae34c 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S @@ -43,14 +43,14 @@ __pthread_rwlock_rdlock: /* Get the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, MUTEX(%rdi) + cmpxchgl %esi, MUTEX(%rdi) #endif - testl %esi, %esi - jne 1f + jnz 1f 2: movl WRITER(%rdi), %eax testl %eax, %eax @@ -82,14 +82,14 @@ __pthread_rwlock_rdlock: /* Reget the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, MUTEX(%rdi) + cmpxchgl %esi, MUTEX(%rdi) #endif - testl %esi, %esi - jne 12f + jnz 12f 13: decl READERS_QUEUED(%rdi) jmp 2b diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S index 582e030abc..924e20868f 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S @@ -53,14 +53,14 @@ pthread_rwlock_timedrdlock: /* Get the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, MUTEX(%rdi) + cmpxchgl %esi, MUTEX(%rdi) #endif - testl %esi, %esi - jne 1f + jnz 1f 2: movl WRITER(%r12), %eax testl %eax, %eax @@ -123,14 +123,14 @@ pthread_rwlock_timedrdlock: /* Reget the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%r12) + cmpxchgl %esi, (%r12) #else - xaddl %esi, MUTEX(%r12) + cmpxchgl %esi, MUTEX(%r12) #endif - testl %esi, %esi - jne 12f + jnz 12f 13: decl READERS_QUEUED(%r12) cmpq $-ETIMEDOUT, %rdx diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S index e7acab538a..ccaef47070 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S @@ -52,14 +52,14 @@ pthread_rwlock_timedwrlock: /* Get the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, MUTEX(%rdi) + cmpxchgl %esi, MUTEX(%rdi) #endif - testl %esi, %esi - jne 1f + jnz 1f 2: movl WRITER(%r12), %eax testl %eax, %eax @@ -119,14 +119,14 @@ pthread_rwlock_timedwrlock: /* Reget the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%r12) + cmpxchgl %esi, (%r12) #else - xaddl %esi, MUTEX(%r12) + cmpxchgl %esi, MUTEX(%r12) #endif - testl %esi, %esi - jne 12f + jnz 12f 13: decl WRITERS_QUEUED(%r12) cmpq $-ETIMEDOUT, %rdx diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S index a3d2351b68..89652cc5d7 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S @@ -40,14 +40,14 @@ __pthread_rwlock_unlock: /* Get the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, MUTEX(%rdi) + cmpxchgl %esi, MUTEX(%rdi) #endif - testl %esi, %esi - jne 1f + jnz 1f 2: cmpq $0, WRITER(%rdi) jne 5f diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S index f866d2e271..1fcb07eaef 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S @@ -43,14 +43,14 @@ __pthread_rwlock_wrlock: /* Get the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, MUTEX(%rdi) + cmpxchgl %esi, MUTEX(%rdi) #endif - testl %esi, %esi - jne 1f + jnz 1f 2: movl WRITER(%rdi), %eax testl %eax, %eax @@ -80,14 +80,14 @@ __pthread_rwlock_wrlock: /* Reget the lock. */ movl $1, %esi + xorl %eax, %eax LOCK #if MUTEX == 0 - xaddl %esi, (%rdi) + cmpxchgl %esi, (%rdi) #else - xaddl %esi, MUTEX(%rdi) + cmpxchgl %esi, MUTEX(%rdi) #endif - testl %esi, %esi - jne 12f + jnz 12f 13: decl WRITERS_QUEUED(%rdi) jmp 2b |