From 75fcceded2cfc65e4879521fff4db6a620a96363 Mon Sep 17 00:00:00 2001 From: Ulrich Drepper Date: Thu, 3 Jun 2004 16:04:11 +0000 Subject: Update. 2004-06-03 Ulrich Drepper * sysdeps/i386/i486/bits/atomic.h: Optimize a bit. --- .../unix/sysv/linux/alpha/bits/pthreadtypes.h | 3 +- nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h | 12 +++-- .../unix/sysv/linux/i386/bits/pthreadtypes.h | 3 +- .../sysv/linux/i386/i486/pthread_cond_broadcast.S | 53 +++++++++++++--------- .../sysv/linux/i386/i486/pthread_cond_signal.S | 11 +++-- .../unix/sysv/linux/i386/i486/pthread_cond_wait.S | 29 +++++++----- .../unix/sysv/linux/ia64/bits/pthreadtypes.h | 3 +- nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h | 11 +++-- nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym | 1 + .../unix/sysv/linux/powerpc/bits/pthreadtypes.h | 3 +- .../sysdeps/unix/sysv/linux/powerpc/lowlevellock.h | 12 +++-- .../unix/sysv/linux/s390/bits/pthreadtypes.h | 3 +- nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h | 24 ++++++---- nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h | 3 ++ .../unix/sysv/linux/sparc/bits/pthreadtypes.h | 3 +- nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h | 12 +++-- .../unix/sysv/linux/x86_64/bits/pthreadtypes.h | 3 +- .../sysv/linux/x86_64/pthread_cond_broadcast.S | 38 +++++++++------- .../unix/sysv/linux/x86_64/pthread_cond_signal.S | 13 ++---- .../sysv/linux/x86_64/pthread_cond_timedwait.S | 16 ++++--- .../unix/sysv/linux/x86_64/pthread_cond_wait.S | 26 ++++++----- 21 files changed, 167 insertions(+), 115 deletions(-) (limited to 'nptl/sysdeps/unix/sysv/linux') diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h index 1bb2968120..62c853cee9 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h @@ -76,11 +76,12 @@ typedef union struct { int __lock; - int __clock; + unsigned int __futex; unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; + int __clock; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h index a7079a8490..fd4a7ca4bb 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h @@ -30,6 +30,7 @@ #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -61,14 +62,15 @@ INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret : __ret; \ }) -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex) \ +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ - __ret = INTERNAL_SYSCALL (futex, __err, 5, \ - (futexp), FUTEX_REQUEUE, (nr_wake), (nr_move), \ - (mutex)); \ - INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret : __ret; \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, \ + (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ + (nr_move), (mutex), (val)); \ + INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h index 9da84c66d6..8d7858a071 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h @@ -76,11 +76,12 @@ typedef union struct { int __lock; - int __clock; + unsigned int __futex; unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; + int __clock; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S index 456f3dbfb1..5471c1c927 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S @@ -32,6 +32,7 @@ #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 #define EINVAL 22 @@ -47,8 +48,9 @@ __pthread_cond_broadcast: pushl %ebx pushl %esi pushl %edi + pushl %ebp - movl 16(%esp), %ebx + movl 20(%esp), %ebx /* Get internal lock. */ movl $1, %edx @@ -61,29 +63,31 @@ __pthread_cond_broadcast: #endif jnz 1f -2: addl $wakeup_seq, %ebx - movl total_seq+4-wakeup_seq(%ebx), %eax - movl total_seq-wakeup_seq(%ebx), %ecx - cmpl 4(%ebx), %eax +2: addl $cond_futex, %ebx + movl total_seq+4-cond_futex(%ebx), %eax + movl total_seq-cond_futex(%ebx), %ebp + cmpl wakeup_seq+4-cond_futex(%ebx), %eax ja 3f jb 4f - cmpl (%ebx), %ecx + cmpl wakeup_seq-cond_futex(%ebx), %ebp jna 4f /* Cause all currently waiting threads to recognize they are woken up. */ -3: movl %ecx, (%ebx) - movl %eax, 4(%ebx) - movl %ecx, woken_seq-wakeup_seq(%ebx) - movl %eax, woken_seq-wakeup_seq+4(%ebx) - addl $1, broadcast_seq-wakeup_seq(%ebx) +3: movl %ebp, wakeup_seq-cond_futex(%ebx) + movl %eax, wakeup_seq-cond_futex+4(%ebx) + movl %ebp, woken_seq-cond_futex(%ebx) + movl %eax, woken_seq-cond_futex+4(%ebx) + addl %ebp, %ebp + addl $1, broadcast_seq-cond_futex(%ebx) + movl %ebp, (%ebx) /* Get the address of the mutex used. */ - movl dep_mutex-wakeup_seq(%ebx), %edi + movl dep_mutex-cond_futex(%ebx), %edi /* Unlock. */ LOCK - subl $1, cond_lock-wakeup_seq(%ebx) + subl $1, cond_lock-cond_futex(%ebx) jne 7f /* Don't use requeue for pshared condvars. */ @@ -91,7 +95,7 @@ __pthread_cond_broadcast: je 9f /* Wake up all threads. */ - movl $FUTEX_REQUEUE, %ecx + movl $FUTEX_CMP_REQUEUE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %esi movl $1, %edx @@ -99,14 +103,18 @@ __pthread_cond_broadcast: # if MUTEX_FUTEX != 0 addl $MUTEX_FUTEX, %edi # endif - ENTER_KERNEL +/* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for sysenter. + ENTER_KERNEL */ + int $0x80 -#ifndef __ASSUME_FUTEX_REQUEUE - cmpl $-EINVAL, %eax - je 9f -#endif + /* For any kind of error, which mainly is EAGAIN, we try again + with WAKE. The general test also covers running on old + kernels. */ + cmpl $0xfffff001, %eax + jae 9f 10: xorl %eax, %eax + popl %ebp popl %edi popl %esi popl %ebx @@ -115,10 +123,11 @@ __pthread_cond_broadcast: .align 16 /* Unlock. */ 4: LOCK - subl $1, cond_lock-wakeup_seq(%ebx) + subl $1, cond_lock-cond_futex(%ebx) jne 5f 6: xorl %eax, %eax + popl %ebp popl %edi popl %esi popl %ebx @@ -135,12 +144,12 @@ __pthread_cond_broadcast: jmp 2b /* Unlock in loop requires waekup. */ -5: leal cond_lock-wakeup_seq(%ebx), %eax +5: leal cond_lock-cond_futex(%ebx), %eax call __lll_mutex_unlock_wake jmp 6b /* Unlock in loop requires waekup. */ -7: leal cond_lock-wakeup_seq(%ebx), %eax +7: leal cond_lock-cond_futex(%ebx), %eax call __lll_mutex_unlock_wake jmp 8b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index 4722b4c0e0..3c5a1db59c 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -60,22 +60,23 @@ __pthread_cond_signal: #endif jnz 1f -2: leal wakeup_seq(%edi), %ebx +2: leal cond_futex(%edi), %ebx movl total_seq+4(%edi), %eax movl total_seq(%edi), %ecx - cmpl 4(%ebx), %eax + cmpl wakeup_seq+4(%edi), %eax #if cond_lock != 0 /* Must use leal to preserve the flags. */ leal cond_lock(%edi), %edi #endif ja 3f jb 4f - cmpl (%ebx), %ecx + cmpl wakeup_seq-cond_futex(%ebx), %ecx jbe 4f /* Bump the wakeup number. */ -3: addl $1, (%ebx) - adcl $0, 4(%ebx) +3: addl $1, wakeup_seq-cond_futex(%ebx) + adcl $0, wakeup_seq-cond_futex+4(%ebx) + addl $1, (%ebx) /* Wake up one thread. */ movl $FUTEX_WAKE, %ecx diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index 74e3172ab0..3fe7f8c17a 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -79,6 +79,7 @@ __pthread_cond_wait: addl $1, total_seq(%ebx) adcl $0, total_seq+4(%ebx) + addl $1, cond_futex(%ebx) #define FRAME_SIZE 16 subl $FRAME_SIZE, %esp @@ -92,8 +93,10 @@ __pthread_cond_wait: movl %edx, 8(%esp) movl %eax, 12(%esp) +8: movl cond_futex(%ebx), %edi + /* Unlock. */ -8: LOCK + LOCK #if cond_lock == 0 subl $1, (%ebx) #else @@ -107,12 +110,12 @@ __pthread_cond_wait: movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */ movl %edi, %edx - addl $wakeup_seq, %ebx -.Ladd_wakeup: + addl $cond_futex, %ebx +.Ladd_cond_futex: movl $SYS_futex, %eax ENTER_KERNEL - subl $wakeup_seq, %ebx -.Lsub_wakeup: + subl $cond_futex, %ebx +.Lsub_cond_futex: movl (%esp), %eax call __pthread_disable_asynccancel @@ -246,7 +249,7 @@ versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, .type __condvar_w_cleanup2, @function __condvar_w_cleanup2: - subl $wakeup_seq, %ebx + subl $cond_futex, %ebx .size __condvar_w_cleanup2, .-__condvar_w_cleanup2 .LSbl4: .type __condvar_w_cleanup, @function @@ -278,6 +281,8 @@ __condvar_w_cleanup: addl $1, wakeup_seq(%ebx) adcl $0, wakeup_seq+4(%ebx) + addl $1, cond_futex(%ebx) + addl $1, woken_seq(%ebx) adcl $0, woken_seq+4(%ebx) @@ -297,7 +302,7 @@ __condvar_w_cleanup: call __lll_mutex_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ -2: addl $wakeup_seq, %ebx +2: addl $cond_futex, %ebx movl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx @@ -323,15 +328,15 @@ __condvar_w_cleanup: .uleb128 .Lcstend-.Lcstbegin .Lcstbegin: .long .LcleanupSTART-.LSTARTCODE - .long .Ladd_wakeup-.LcleanupSTART + .long .Ladd_cond_futex-.LcleanupSTART .long __condvar_w_cleanup-.LSTARTCODE .uleb128 0 - .long .Ladd_wakeup-.LSTARTCODE - .long .Lsub_wakeup-.Ladd_wakeup + .long .Ladd_cond_futex-.LSTARTCODE + .long .Lsub_cond_futex-.Ladd_cond_futex .long __condvar_w_cleanup2-.LSTARTCODE .uleb128 0 - .long .Lsub_wakeup-.LSTARTCODE - .long .LcleanupEND-.Lsub_wakeup + .long .Lsub_cond_futex-.LSTARTCODE + .long .LcleanupEND-.Lsub_cond_futex .long __condvar_w_cleanup-.LSTARTCODE .uleb128 0 .long .LcallUR-.LSTARTCODE diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h index bdf3ee741b..958af47162 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h @@ -76,11 +76,12 @@ typedef union struct { int __lock; - int __clock; + unsigned int __futex; unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; + int __clock; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h index 6ed21fdc14..e13358ffef 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h @@ -30,6 +30,7 @@ #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 /* Delay in spinlock loop. */ #define BUSY_WAIT_NOP asm ("hint @pause") @@ -52,11 +53,13 @@ _r10 == -1 ? -_retval : _retval; \ }) -#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex) \ +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \ ({ \ - DO_INLINE_SYSCALL(futex, 5, (long) (ftx), FUTEX_REQUEUE, (int) (nr_wake), \ - (int) (nr_move), (long) (mutex)); \ - _r10 == -1 ? -_retval : _retval; \ + DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_CMP_REQUEUE, \ + (int) (nr_wake), (int) (nr_move), (long) (mutex), \ + (int) val); \ + _r10 == -1; \ }) diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym index d0559da86f..2e193e6d2e 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym @@ -4,6 +4,7 @@ -- cond_lock offsetof (pthread_cond_t, __data.__lock) +cond_futex offsetof (pthread_cond_t, __data.__futex) cond_clock offsetof (pthread_cond_t, __data.__clock) total_seq offsetof (pthread_cond_t, __data.__total_seq) wakeup_seq offsetof (pthread_cond_t, __data.__wakeup_seq) diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h index 794d888a13..5a000d30af 100644 --- a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h @@ -96,11 +96,12 @@ typedef union struct { int __lock; - int __clock; + unsigned int __futex; unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; + int __clock; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h index 8e3f46d534..ead89ba931 100644 --- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h @@ -30,6 +30,7 @@ #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -64,15 +65,16 @@ INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret; \ }) -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex) \ +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ \ - __ret = INTERNAL_SYSCALL (futex, __err, 5, \ - (futexp), FUTEX_REQUEUE, (nr_wake), (nr_move), \ - (mutex)); \ - INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret; \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, \ + (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ + (nr_move), (mutex), (val)); \ + INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) #ifdef UP diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h index 02b061b05c..142286451c 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h @@ -95,11 +95,12 @@ typedef union struct { int __lock; - int __clock; + unsigned int __futex; unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; + int __clock; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h index 8b460eb98f..60679cca86 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h @@ -29,6 +29,7 @@ #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -82,22 +83,29 @@ }) -#define lll_futex_requeue(futex, nr_wake, nr_move, mutex) \ +/* Returns non-zero if error happened, zero if success. */ +#if 0 +/* FIXME: s390 only supports up to 5 argument syscalls. Once FUTEX_CMP_REQUEUE + kernel interface for s390 is finalized, adjust this. */ +#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val) \ ({ \ - register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ - register unsigned long int __r3 asm ("3") = FUTEX_REQUEUE; \ - register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \ - register unsigned long int __r5 asm ("5") = (long int) (nr_move); \ - register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex); \ - register unsigned long __result asm ("2"); \ + register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \ + register unsigned long int __r3 asm ("3") = FUTEX_CMP_REQUEUE; \ + register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \ + register unsigned long int __r5 asm ("5") = (long int) (nr_move); \ + register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex); \ + register unsigned long __result asm ("2"); \ \ __asm __volatile ("svc %b1" \ : "=d" (__result) \ : "i" (SYS_futex), "0" (__r2), "d" (__r3), \ "d" (__r4), "d" (__r5), "d" (__r6) \ : "cc", "memory" ); \ - __result; \ + __result > -4096UL; \ }) +#else +#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val) 1 +#endif #define lll_compare_and_swap(futex, oldval, newval, operation) \ diff --git a/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h index 00ff1b20fb..d9376d45a0 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h @@ -89,6 +89,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden; __asm __volatile ("\ .align 2\n\ mova 1f,r0\n\ + nop\n\ mov r15,r1\n\ mov #-8,r15\n\ 0: mov.l @%2,%0\n\ @@ -108,6 +109,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden; __asm __volatile ("\ .align 2\n\ mova 1f,r0\n\ + nop\n\ mov r15,r1\n\ mov #-8,r15\n\ 0: mov.l @%2,%0\n\ @@ -125,6 +127,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden; __asm __volatile ("\ .align 2\n\ mova 1f,r0\n\ + nop\n\ mov r15,r1\n\ mov #-8,r15\n\ 0: mov.l @%2,%0\n\ diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h index b0c1372bba..e86dd000a1 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h @@ -96,11 +96,12 @@ typedef union struct { int __lock; - int __clock; + unsigned int __futex; unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; + int __clock; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h index 05bc5a2974..4626aec524 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h @@ -29,6 +29,7 @@ #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -63,15 +64,16 @@ __ret; \ }) -#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex) \ +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \ ({ \ INTERNAL_SYSCALL_DECL (__err); \ long int __ret; \ \ - __ret = INTERNAL_SYSCALL (futex, __err, 5, \ - (futexp), FUTEX_REQUEUE, (nr_wake), (nr_move), \ - (mutex)); \ - __ret; \ + __ret = INTERNAL_SYSCALL (futex, __err, 6, \ + (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \ + (nr_move), (mutex), (val)); \ + INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ }) #ifdef __sparc32_atomic_do_lock diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h index e29c77529e..661e476aa7 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h @@ -76,11 +76,12 @@ typedef union struct { int __lock; - int __clock; + unsigned int __futex; unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; + int __clock; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index 6d29d761fa..e8d7bd9bb6 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -32,6 +32,7 @@ #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 #define EINVAL 22 @@ -55,39 +56,42 @@ __pthread_cond_broadcast: #endif jnz 1f -2: addq $wakeup_seq, %rdi - movq total_seq-wakeup_seq(%rdi), %rcx - cmpq (%rdi), %rcx +2: addq $cond_futex, %rdi + movq total_seq-cond_futex(%rdi), %r9 + cmpq wakeup_seq-cond_futex(%rdi), %r9 jna 4f /* Cause all currently waiting threads to recognize they are woken up. */ - movq %rcx, (%rdi) - movq %rcx, woken_seq-wakeup_seq(%rdi) - incl broadcast_seq-wakeup_seq(%rdi) + movq %r9, wakeup_seq-cond_futex(%rdi) + movq %r9, woken_seq-cond_futex(%rdi) + addq %r9, %r9 + movl %r9d, (%rdi) + incl broadcast_seq-cond_futex(%rdi) /* Get the address of the mutex used. */ - movq dep_mutex-wakeup_seq(%rdi), %r8 + movq dep_mutex-cond_futex(%rdi), %r8 /* Unlock. */ LOCK - decl cond_lock-wakeup_seq(%rdi) + decl cond_lock-cond_futex(%rdi) jne 7f 8: cmpq $-1, %r8 je 9f /* Wake up all threads. */ - movq $FUTEX_REQUEUE, %rsi + movq $FUTEX_CMP_REQUEUE, %rsi movq $SYS_futex, %rax movl $1, %edx movq $0x7fffffff, %r10 syscall -#ifndef __ASSUME_FUTEX_REQUEUE - cmpq $-EINVAL, %rax - je 9f -#endif + /* For any kind of error, which mainly is EAGAIN, we try again + with WAKE. The general test also covers running on old + kernels. */ + cmpq $-4095, %rax + jae 9f 10: xorl %eax, %eax retq @@ -95,7 +99,7 @@ __pthread_cond_broadcast: .align 16 /* Unlock. */ 4: LOCK - decl cond_lock-wakeup_seq(%rdi) + decl cond_lock-cond_futex(%rdi) jne 5f 6: xorl %eax, %eax @@ -113,14 +117,14 @@ __pthread_cond_broadcast: jmp 2b /* Unlock in loop requires wakeup. */ -5: addq $cond_lock-wakeup_seq, %rdi +5: addq $cond_lock-cond_futex, %rdi callq __lll_mutex_unlock_wake jmp 6b /* Unlock in loop requires wakeup. */ -7: addq $cond_lock-wakeup_seq, %rdi +7: addq $cond_lock-cond_futex, %rdi callq __lll_mutex_unlock_wake - subq $cond_lock-wakeup_seq, %rdi + subq $cond_lock-cond_futex, %rdi jmp 8b 9: /* The futex requeue functionality is not available. */ diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index 92eadfc433..62bb74cc1a 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -56,13 +56,14 @@ __pthread_cond_signal: #endif jnz 1f -2: addq $wakeup_seq, %rdi +2: addq $cond_futex, %rdi movq total_seq(%r8), %rcx - cmpq (%rdi), %rcx + cmpq wakeup_seq(%r8), %rcx jbe 4f /* Bump the wakeup number. */ - addq $1, (%rdi) + addq $1, wakeup_seq(%r8) + addl $1, (%rdi) /* Wake up one thread. */ movq $FUTEX_WAKE, %rsi @@ -95,11 +96,7 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: -#if cond_lock != 0 - addq $cond_lock-wakeup_seq, %rdi -#else movq %r8, %rdi -#endif callq __lll_mutex_unlock_wake jmp 6b .size __pthread_cond_signal, .-__pthread_cond_signal diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 80cbf7e430..e75f05e07f 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -105,6 +105,7 @@ __pthread_cond_timedwait: movq 8(%rsp), %rdi incq total_seq(%rdi) + incl cond_futex(%rdi) /* Install cancellation handler. */ #ifdef PIC @@ -118,9 +119,9 @@ __pthread_cond_timedwait: /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi - movq wakeup_seq(%rdi), %r12 + movq wakeup_seq(%rdi), %r9 movl broadcast_seq(%rdi), %edx - movq %r12, 40(%rsp) + movq %r9, 40(%rsp) movl %edx, 4(%rsp) /* Get the current time. */ @@ -172,6 +173,8 @@ __pthread_cond_timedwait: 21: movq %rcx, 24(%rsp) movq %rdx, 32(%rsp) + movl cond_futex(%rdi), %r12d + /* Unlock. */ LOCK #if cond_lock == 0 @@ -187,7 +190,7 @@ __pthread_cond_timedwait: leaq 24(%rsp), %r10 xorq %rsi, %rsi /* movq $FUTEX_WAIT, %rsi */ movq %r12, %rdx - addq $wakeup_seq-cond_lock, %rdi + addq $cond_futex, %rdi movq $SYS_futex, %rax syscall movq %rax, %r14 @@ -211,21 +214,22 @@ __pthread_cond_timedwait: movq woken_seq(%rdi), %rax - movq wakeup_seq(%rdi), %r12 + movq wakeup_seq(%rdi), %r9 cmpl 4(%rsp), %edx jne 23f - cmpq 40(%rsp), %r12 + cmpq 40(%rsp), %r9 jbe 15f - cmpq %rax, %r12 + cmpq %rax, %r9 ja 9f 15: cmpq $-ETIMEDOUT, %r14 jne 8b 13: incq wakeup_seq(%rdi) + incl cond_futex(%rdi) movq $ETIMEDOUT, %r14 jmp 14f diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 065eb11813..9e7da301d3 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -69,6 +69,8 @@ __condvar_cleanup: incq woken_seq(%rdi) + incl cond_futex(%rdi) + 3: LOCK #if cond_lock == 0 decl (%rdi) @@ -82,7 +84,7 @@ __condvar_cleanup: callq __lll_mutex_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ -2: addq $wakeup_seq, %rdi +2: addq $cond_futex, %rdi movq $FUTEX_WAKE, %rsi movl $0x7fffffff, %edx movq $SYS_futex, %rax @@ -154,6 +156,7 @@ __pthread_cond_wait: movq 8(%rsp), %rdi incq total_seq(%rdi) + incl cond_futex(%rdi) /* Install cancellation handler. */ #ifdef PIC @@ -167,13 +170,14 @@ __pthread_cond_wait: /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi - movq wakeup_seq(%rdi), %r12 + movq wakeup_seq(%rdi), %r9 movl broadcast_seq(%rdi), %edx - movq %r12, 24(%rsp) + movq %r9, 24(%rsp) movl %edx, 4(%rsp) /* Unlock. */ -8: LOCK +8: movl cond_futex(%rdi), %r12d + LOCK #if cond_lock == 0 decl (%rdi) #else @@ -187,7 +191,7 @@ __pthread_cond_wait: movq 8(%rsp), %rdi xorq %r10, %r10 movq %r12, %rdx - addq $wakeup_seq-cond_lock, %rdi + addq $cond_futex-cond_lock, %rdi movq $SYS_futex, %rax movq %r10, %rsi /* movq $FUTEX_WAIT, %rsi */ syscall @@ -211,15 +215,15 @@ __pthread_cond_wait: movq woken_seq(%rdi), %rax - movq wakeup_seq(%rdi), %r12 + movq wakeup_seq(%rdi), %r9 cmpl 4(%rsp), %edx jne 16f - cmpq 24(%rsp), %r12 + cmpq 24(%rsp), %r9 jbe 8b - cmpq %rax, %r12 + cmpq %rax, %r9 jna 8b incq woken_seq(%rdi) @@ -359,8 +363,8 @@ versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, .byte 0x40+.Lsubq-.Lpush_r12 # DW_CFA_advance_loc+N .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 2 # DW_CFA_advance_loc1 - .byte .Laddq-.Lsubq + .byte 3 # DW_CFA_advance_loc2 + .2byte .Laddq-.Lsubq .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 .byte 0x40+.Lpop_r12-.Laddq # DW_CFA_advance_loc+N -- cgit 1.4.1