From 69431c9a21f7393f34330a27df1630520930789e Mon Sep 17 00:00:00 2001 From: Ulrich Drepper Date: Mon, 26 May 2003 02:47:39 +0000 Subject: Update. 2003-05-25 Ulrich Drepper * sysdeps/unix/sysv/linux/kernel-features.h: Define __ASSUME_FUTEX_REQUEUE for >= 2.5.70. * math/test-fenv.c (feexcp_nomask_test): Fix comment. --- nptl/sysdeps/unix/sysv/linux/Makefile | 2 +- .../unix/sysv/linux/i386/bits/pthreadtypes.h | 1 + .../sysv/linux/i386/i486/pthread_cond_broadcast.S | 39 +++++++++++++-- .../sysv/linux/i386/i486/pthread_cond_signal.S | 55 +++++++++++++++++----- .../sysv/linux/i386/i486/pthread_cond_timedwait.S | 13 +++-- .../unix/sysv/linux/i386/i486/pthread_cond_wait.S | 21 +++++---- nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h | 18 +++++++ nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h | 46 ++++++++++++++++-- nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym | 1 + .../unix/sysv/linux/pthread_mutex_cond_lock.c | 6 +++ .../unix/sysv/linux/x86_64/bits/pthreadtypes.h | 1 + nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h | 19 ++++++++ .../sysv/linux/x86_64/pthread_cond_broadcast.S | 27 ++++++++++- .../unix/sysv/linux/x86_64/pthread_cond_signal.S | 41 +++++++++++++--- .../sysv/linux/x86_64/pthread_cond_timedwait.S | 10 ++-- .../unix/sysv/linux/x86_64/pthread_cond_wait.S | 16 +++---- 16 files changed, 261 insertions(+), 55 deletions(-) create mode 100644 nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c (limited to 'nptl/sysdeps/unix/sysv') diff --git a/nptl/sysdeps/unix/sysv/linux/Makefile b/nptl/sysdeps/unix/sysv/linux/Makefile index 5701887c06..510232c9c6 100644 --- a/nptl/sysdeps/unix/sysv/linux/Makefile +++ b/nptl/sysdeps/unix/sysv/linux/Makefile @@ -21,7 +21,7 @@ ifeq ($(subdir),nptl) sysdep_routines += register-atfork unregister-atfork libc_pthread_init \ libc_multiple_threads -libpthread-sysdep_routines += pt-fork +libpthread-sysdep_routines += pt-fork pthread_mutex_cond_lock gen-as-const-headers += lowlevelcond.sym lowlevelrwlock.sym lowlevelbarrier.sym endif diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h index 0834894c25..4b0f11adc5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h @@ -78,6 +78,7 @@ typedef union unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; + void *__mutex; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; long long int __align; diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S index 0fa402a95e..06821ad376 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S @@ -20,6 +20,7 @@ #include #include #include +#include #ifdef UP # define LOCK @@ -30,6 +31,9 @@ #define SYS_futex 240 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 + +#define EINVAL 22 .text @@ -41,8 +45,10 @@ __pthread_cond_broadcast: pushl %ebx + pushl %esi + pushl %edi - movl 8(%esp), %ebx + movl 16(%esp), %ebx /* Get internal lock. */ movl $1, %eax @@ -69,18 +75,34 @@ __pthread_cond_broadcast: 3: movl %ecx, (%ebx) movl %eax, 4(%ebx) + /* Get the address of the mutex used. */ + movl dep_mutex-wakeup_seq(%ebx), %edi + /* Unlock. */ LOCK subl $1, cond_lock-wakeup_seq(%ebx) jne 7f /* Wake up all threads. */ -8: movl $FUTEX_WAKE, %ecx +8: movl $FUTEX_REQUEUE, %ecx movl $SYS_futex, %eax - movl $0x7fffffff, %edx + movl $0x7fffffff, %esi + movl $1, %edx + /* Get the address of the futex involved. */ +# if MUTEX_FUTEX != 0 + addl $MUTEX_FUTEX, %edi +# endif ENTER_KERNEL +#ifndef __ASSUME_FUTEX_REQUEUE + cmpl $-EINVAL, %eax + je 9f +10: +#endif + xorl %eax, %eax + popl %edi + popl %esi popl %ebx ret @@ -91,6 +113,8 @@ __pthread_cond_broadcast: jne 5f 6: xorl %eax, %eax + popl %edi + popl %esi popl %ebx ret @@ -113,6 +137,15 @@ __pthread_cond_broadcast: 7: leal cond_lock-wakeup_seq(%ebx), %eax call __lll_mutex_unlock_wake jmp 8b + +#ifndef __ASSUME_FUTEX_REQUEUE +9: /* The futex requeue functionality is not available. */ + movl $0x7fffffff, %edx + movl $FUTEX_WAKE, %ecx + movl $SYS_futex, %eax + ENTER_KERNEL + jmp 10b +#endif .size __pthread_cond_broadcast, .-__pthread_cond_broadcast versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index 5465d7b2a8..ed25c554d2 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -20,6 +20,7 @@ #include #include #include +#include #ifdef UP # define LOCK @@ -30,6 +31,9 @@ #define SYS_futex 240 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 + +#define EINVAL 22 .text @@ -41,23 +45,25 @@ __pthread_cond_signal: pushl %ebx + pushl %esi + pushl %edi - movl 8(%esp), %ebx + movl 16(%esp), %edi /* Get internal lock. */ movl $1, %eax LOCK #if cond_lock == 0 - xaddl %eax, (%ebx) + xaddl %eax, (%edi) #else - xaddl %eax, cond_lock(%ebx) + xaddl %eax, cond_lock(%edi) #endif testl %eax, %eax jne 1f -2: addl $wakeup_seq, %ebx - movl total_seq+4-wakeup_seq(%ebx), %eax - movl total_seq-wakeup_seq(%ebx), %ecx +2: leal wakeup_seq(%edi), %ebx + movl total_seq+4(%edi), %eax + movl total_seq(%edi), %ecx cmpl 4(%ebx), %eax ja 3f jb 4f @@ -68,18 +74,30 @@ __pthread_cond_signal: 3: addl $1, (%ebx) adcl $0, 4(%ebx) - /* Wake up one thread. */ - movl $FUTEX_WAKE, %ecx + /* Wake up one thread by moving it to the internal lock futex. */ + movl $FUTEX_REQUEUE, %ecx movl $SYS_futex, %eax - movl %ecx, %edx /* movl $1, %edx */ + xorl %edx, %edx + movl $1, %esi ENTER_KERNEL +#ifndef __ASSUME_FUTEX_REQUEUE + cmpl $-EINVAL, %eax + je 7f +#endif + + /* If we moved a thread we in any case have to make the syscall. */ + testl %eax, %eax + jne 5f + /* Unlock. */ 4: LOCK - subl $1, cond_lock-wakeup_seq(%ebx) + subl $1, (%edi) jne 5f 6: xorl %eax, %eax + popl %edi + popl %esi popl %ebx ret @@ -93,11 +111,24 @@ __pthread_cond_signal: call __lll_mutex_lock_wait jmp 2b - /* Unlock in loop requires waekup. */ + /* Unlock in loop requires wakeup. */ 5: - leal cond_lock-wakeup_seq(%ebx), %eax +#if cond_lock == 0 + movl %edi, %eax +#else + leal cond_lock(%edi), %eax +#endif call __lll_mutex_unlock_wake jmp 6b + +#ifndef __ASSUME_FUTEX_REQUEUE +7: /* The futex requeue functionality is not available. */ + movl $1, %edx + movl $FUTEX_WAKE, %ecx + movl $SYS_futex, %eax + ENTER_KERNEL + jmp 4b +#endif .size __pthread_cond_signal, .-__pthread_cond_signal versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S index d9bffe5139..5eec268b29 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S @@ -66,8 +66,13 @@ __pthread_cond_timedwait: testl %eax, %eax jne 1f + /* Store the reference to the mutex. If there is already a + different value in there this is a bad user bug. */ +2: movl 24(%esp), %eax + movl %eax, dep_mutex(%ebx) + /* Unlock the mutex. */ -2: pushl 24(%esp) + pushl %eax .Lpush4: call __pthread_mutex_unlock_internal @@ -113,8 +118,8 @@ __pthread_cond_timedwait: #endif jne 3f -4: leal 8(%esp), %eax - call __pthread_enable_asynccancel_2 +4: call __pthread_enable_asynccancel + movl %eax, 8(%esp) /* Get the current time. */ movl %ebx, %edx @@ -230,7 +235,7 @@ __pthread_cond_timedwait: movl %edx, %gs:CLEANUP /* Trick ahead: (%esp) contains the address of the mutex. */ - call __pthread_mutex_lock_internal + call __pthread_mutex_cond_lock addl $44, %esp .Laddl: diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index 1ac6c1a7d6..61d3d8d225 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -83,15 +83,11 @@ __condvar_cleanup: movl $0x7fffffff, %edx ENTER_KERNEL - /* Lock the mutex unless asynchronous cancellation is in effect. */ - testl $2, 8(%esi) - jne 3f - pushl (%esi) - call __pthread_mutex_lock_internal + call __pthread_mutex_cond_lock popl %eax -3: popl %esi + popl %esi popl %ebx ret .size __condvar_cleanup, .-__condvar_cleanup @@ -125,8 +121,13 @@ __pthread_cond_wait: testl %eax, %eax jne 1f + /* Store the reference to the mutex. If there is already a + different value in there this is a bad user bug. */ +2: movl 20(%esp), %eax + movl %eax, dep_mutex(%ebx) + /* Unlock the mutex. */ -2: pushl 20(%esp) + pushl %eax .Lpush4: call __pthread_mutex_unlock_internal @@ -171,8 +172,8 @@ __pthread_cond_wait: #endif jne 3f -4: leal 8(%esp), %eax - call __pthread_enable_asynccancel_2 +4: call __pthread_enable_asynccancel + movl %eax, 8(%esp) movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */ movl %edi, %edx @@ -229,7 +230,7 @@ __pthread_cond_wait: movl %edx, %gs:CLEANUP /* Trick ahead: (%esp) contains the address of the mutex. */ - call __pthread_mutex_lock_internal + call __pthread_mutex_cond_lock addl $36, %esp .Laddl: diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 5fd50b97db..8923afbfed 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -126,6 +126,24 @@ extern int __lll_mutex_unlock_wait (int *__futex) : "memory"); }) +/* Special version of lll_mutex_lock which causes the unlock function to + always wakeup waiters. */ +#define lll_mutex_cond_lock(futex) \ + (void) ({ int ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \ + "testl %0, %0\n\t" \ + "jne 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleal %2, %%ecx\n\t" \ + "call __lll_mutex_lock_wait\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \ + : "0" (2), "2" (futex) \ + : "memory"); }) + + #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2; \ __asm __volatile (LOCK_INSTR "xaddl %0, %3\n\t" \ diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h index 92c0b5c524..b1941e07c6 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h @@ -29,6 +29,7 @@ #define SYS_futex 1230 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -64,7 +65,7 @@ "=r" (__o0), "=r" (__o1), "=r" (__o2), "=r" (__o3) \ : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \ "5" (__o2), "6" (__o3) \ - : lll_futex_clobbers); \ + : "out4", lll_futex_clobbers); \ __r10 == -1 ? -__r8 : __r8; \ }) @@ -83,10 +84,33 @@ "=r" (__o0), "=r" (__o1), "=r" (__o2) \ : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \ "5" (__o2) \ - : "out3", lll_futex_clobbers); \ + : "out3", "out4", lll_futex_clobbers); \ __r10 == -1 ? -__r8 : __r8; \ }) + +#define lll_futex_requeue(futex, nr_wake, nr_move, mutex) \ + ({ \ + register long int __o0 asm ("out0") = (long int) (futex); \ + register long int __o1 asm ("out1") = FUTEX_REQUEUE; \ + register long int __o2 asm ("out2") = (long int) (nr_wake); \ + register long int __o3 asm ("out3") = (long int) (nr_move); \ + register long int __o4 asm ("out4") = (long int) (mutex); \ + register long int __r8 asm ("r8"); \ + register long int __r10 asm ("r10"); \ + register long int __r15 asm ("r15") = SYS_futex; \ + \ + __asm __volatile ("break %7;;" \ + : "=r" (__r8), "=r" (__r10), "=r" (__r15), \ + "=r" (__o0), "=r" (__o1), "=r" (__o2), "r" (__o3), \ + "=r" (__o4) \ + : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \ + "5" (__o2), "6" (__o3), "7" (__o4) \ + : lll_futex_clobbers); \ + __r8; \ + }) + + static inline int __attribute__ ((always_inline)) __lll_mutex_trylock (int *futex) @@ -111,6 +135,18 @@ __lll_mutex_lock (int *futex) #define lll_mutex_lock(futex) __lll_mutex_lock (&(futex)) +static inline void +__attribute__ ((always_inline)) +__lll_mutex_cond_lock (int *futex) +{ + int val = atomic_exchange_and_add (futex, 2); + + if (__builtin_expect (val != 0, 0)) + __lll_lock_wait (futex, val); +} +#define lll_mutex_cond_lock(futex) __lll_mutex_cond_lock (&(futex)) + + extern int __lll_timedlock_wait (int *futex, int val, const struct timespec *) attribute_hidden; @@ -140,7 +176,11 @@ __lll_mutex_unlock (int *futex) if (__builtin_expect (val > 1, 0)) lll_futex_wake (futex, 1); } -#define lll_mutex_unlock(futex) __lll_mutex_unlock(&(futex)) +#define lll_mutex_unlock(futex) \ + __lll_mutex_unlock(&(futex)) + +#define lll_mutex_unlock_force(futex) \ + lll_futex_wake (&(futex), 1) #define lll_mutex_islocked(futex) \ (futex != 0) diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym index 5eb535e157..1463e0810e 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym @@ -8,3 +8,4 @@ cond_clock offsetof (pthread_cond_t, __data.__clock) total_seq offsetof (pthread_cond_t, __data.__total_seq) wakeup_seq offsetof (pthread_cond_t, __data.__wakeup_seq) woken_seq offsetof (pthread_cond_t, __data.__woken_seq) +dep_mutex offsetof (pthread_cond_t, __data.__mutex) diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c new file mode 100644 index 0000000000..893a5e932d --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c @@ -0,0 +1,6 @@ +#include + +#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock(mutex) +#define __pthread_mutex_lock __pthread_mutex_cond_lock + +#include diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h index 47d856cae4..7c12db6fc6 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h @@ -78,6 +78,7 @@ typedef union unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; + void *__mutex; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; long int __align; diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index 8e1742b671..136dc574e0 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -102,6 +102,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; : "0" (1), "2" (futex) \ : "cx", "r11", "cc", "memory"); }) + +#define lll_mutex_cond_lock(futex) \ + (void) ({ int ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \ + "testl %0, %0\n\t" \ + "jne 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %2, %%rdi\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_mutex_lock_wait\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \ + : "0" (2), "2" (futex) \ + : "cx", "r11", "cc", "memory"); }) + + #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2, ignore3; \ __asm __volatile (LOCK_INSTR "xaddl %0, %4\n\t" \ diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index cc12f5407c..66edb9ac85 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -20,6 +20,7 @@ #include #include #include +#include #ifdef UP # define LOCK @@ -30,6 +31,9 @@ #define SYS_futex 202 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 + +#define EINVAL 22 .text @@ -60,17 +64,27 @@ __pthread_cond_broadcast: woken up. */ movq %rcx, (%rdi) + /* Get the address of the mutex used. */ + movq dep_mutex-wakeup_seq(%rdi), %r8 + /* Unlock. */ LOCK decl cond_lock-wakeup_seq(%rdi) jne 7f /* Wake up all threads. */ -8: movq $FUTEX_WAKE, %rsi +8: movq $FUTEX_REQUEUE, %rsi movq $SYS_futex, %rax - movl $0x7fffffff, %edx + movl $1, %edx + movq $0x7fffffff, %r10 syscall +#ifndef __ASSUME_FUTEX_REQUEUE + cmpq $-EINVAL, %eax + je 9f +10: +#endif + xorl %eax, %eax retq @@ -104,6 +118,15 @@ __pthread_cond_broadcast: callq __lll_mutex_unlock_wake subq $cond_lock-wakeup_seq, %rdi jmp 8b + +#ifndef __ASSUME_FUTEX_REQUEUE +9: /* The futex requeue functionality is not available. */ + movq $0x7fffffff, %rdx + movq $FUTEX_WAKE, %rsi + movq $SYS_futex, %rax + syscall + jmp 10b +#endif .size __pthread_cond_broadcast, .-__pthread_cond_broadcast versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index 11635baa22..709fcf4ca8 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -20,6 +20,7 @@ #include #include #include +#include #ifdef UP # define LOCK @@ -30,6 +31,9 @@ #define SYS_futex 202 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 + +#define EINVAL 22 .text @@ -44,15 +48,15 @@ __pthread_cond_signal: movl $1, %esi LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + xaddl %esi, (%r8) #else - xaddl %esi, cond_lock(%rdi) + xaddl %esi, cond_lock(%r8) #endif testl %esi, %esi jne 1f -2: addq $wakeup_seq, %rdi - movq total_seq-wakeup_seq(%rdi), %rcx +2: leaq wakeup_seq(%r8), %rdi + movq total_seq(%r8), %rcx cmpq (%rdi), %rcx jbe 4f @@ -62,12 +66,22 @@ __pthread_cond_signal: /* Wake up one thread. */ movq $FUTEX_WAKE, %rsi movq $SYS_futex, %rax - movq %rsi, %rdx /* movl $1, %edx */ + xorq %rdx, %rdx + movq $1, %r10 syscall +#ifndef __ASSUME_FUTEX_REQUEUE + cmpq $-EINVAL, %rax + je 7f +#endif + + /* If we moved a thread we in any case have to make the syscall. */ + testq %rax, %rax + jne 5f + /* Unlock. */ 4: LOCK - decl cond_lock-wakeup_seq(%rdi) + decl cond_lock(%r8) jne 5f 6: xorl %eax, %eax @@ -85,9 +99,22 @@ __pthread_cond_signal: jmp 2b /* Unlock in loop requires waekup. */ -5: addq $cond_lock-wakeup_seq, %rdi +5: +#if cond_lock != 0 + addq $cond_lock-wakeup_seq, %rdi +#else + movq %r8, %rdi +#endif callq __lll_mutex_unlock_wake jmp 6b + +#ifndef __ASSUME_FUTEX_REQUEUE +7: /* The futex requeue functionality is not available. */ + movq $1, %rdx + movq $FUTEX_WAKE, %esi + movq $SYS_futex, %rax + syscall + jmp 4b .size __pthread_cond_signal, .-__pthread_cond_signal versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 151395048a..c8fd4ea129 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -88,7 +88,9 @@ __pthread_cond_timedwait: jne 1f /* Unlock the mutex. */ -2: movq 16(%rsp), %rdi +2: movq %rdi, %rax + movq 16(%rsp), %rdi + movq %rdi, dep_mutex(%rax) callq __pthread_mutex_unlock_internal testl %eax, %eax @@ -121,8 +123,8 @@ __pthread_cond_timedwait: #endif jne 3f -4: movq %rsp, %rdi - callq __pthread_enable_asynccancel_2 +4: callq __pthread_enable_asynccancel + movq %rax, (%rsp) /* Get the current time. */ #ifdef __NR_clock_gettime @@ -227,7 +229,7 @@ __pthread_cond_timedwait: movq %rdx, %fs:CLEANUP movq 16(%rsp), %rdi - callq __pthread_mutex_lock_internal + callq __pthread_mutex_cond_lock testq %rax, %rax cmoveq %r14, %rax diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 6cad2812f0..5189972fc7 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -84,12 +84,8 @@ __condvar_cleanup: movq $SYS_futex, %rax syscall - /* Lock the mutex unless asynchronous cancellation is in effect. */ - testq $2, (%r8) - jne 3f - movq 16(%r8), %rdi - callq __pthread_mutex_lock_internal + callq __pthread_mutex_cond_lock 3: retq .size __condvar_cleanup, .-__condvar_cleanup @@ -137,7 +133,9 @@ __pthread_cond_wait: jne 1f /* Unlock the mutex. */ -2: movq 16(%rsp), %rdi +2: movq %rdi, %rax + movq 16(%rsp), %rdi + movq %rdi, dep_mutex(%rax) callq __pthread_mutex_unlock_internal testl %eax, %eax @@ -170,8 +168,8 @@ __pthread_cond_wait: #endif jne 3f -4: movq %rsp, %rdi - callq __pthread_enable_asynccancel_2 +4: callq __pthread_enable_asynccancel + movq %rax, (%rsp) movq 8(%rsp), %rdi xorq %r10, %r10 @@ -221,7 +219,7 @@ __pthread_cond_wait: movq %rdx, %fs:CLEANUP movq 16(%rsp), %rdi - callq __pthread_mutex_lock_internal + callq __pthread_mutex_cond_lock 14: addq $64, %rsp .Laddq: -- cgit 1.4.1