diff options
27 files changed, 377 insertions, 69 deletions
diff --git a/ChangeLog b/ChangeLog index 0dbe72a6be..d8376fbffc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2003-05-25 Ulrich Drepper <drepper@redhat.com> + + * sysdeps/unix/sysv/linux/kernel-features.h: Define + __ASSUME_FUTEX_REQUEUE for >= 2.5.70. + 2003-05-22 Andreas Jaeger <aj@suse.de> * sysdeps/unix/sysv/linux/x86_64/gettimeofday.S: Add CFI @@ -12,7 +17,7 @@ 2003-05-21 H.J. Lu <hongjiu.lu@intel.com> - * math/test-fenv.c (feexcp_nomask_test): Fix comment + * math/test-fenv.c (feexcp_nomask_test): Fix comment. (feexcp_mask_test): Likewise. 2003-05-21 Ulrich Drepper <drepper@redhat.com> diff --git a/nptl/ChangeLog b/nptl/ChangeLog index 0f4d0855a2..7a0af3bfde 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -1,3 +1,52 @@ +2003-05-25 Ulrich Drepper <drepper@redhat.com> + + * sysdeps/pthread/pthread_cond_broadcast.c: Try using FUTEX_REQUEUE + instead of FUTEX_WAIT. + * sysdeps/pthread/pthread_cond_signal.c: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise. + * sysdeps/pthread/pthread_cond_timedwait.c: Remember mutex which was + used in condvar structure. Call __pthread_mutex_cond_lock instead + of __pthread_mutex_lock_internal. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise. + * sysdeps/pthread/pthread_cond_wait.c: Likewise. + (__condvar_cleanup): Always call __pthread_mutex_cond_lock. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise. + * sysdeps/unix/sysv/linux/Makefile (libpthread-sysdep_routines): + Add pthread_mutex_cond_lock. + * sysdeps/unix/sysv/linux/lowlevelcond.sym: Add dep_mutex. + * sysdeps/unix/sysv/linux/pthread_cond_mutex_lock.c: New file. + * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Define + lll_mutex_cond_lock. + * sysdeps/unix/sysv/linux/ia64/lowlevellock.h: Likewise. + * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise. + * sysdeps/unix/sysv/linux/i386/bits/pthread_types.h (pthread_cond_t): + Add __mutex field. + * sysdeps/unix/sysv/linux/ia64/bits/pthread_types.h: Likewise. + * sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h: Likewise. + + * sysdeps/i386/tcb-offsets.sym: Define MUTEX_FUTEX. + * sysdeps/x86_64/tcb-offsets.sym: Likewise. + + * pthreadP.h: Declare __pthread_mutex_cond_lock. + * pthread_mutex_lock.c: Define LLL_MUTEX_LOCK if not already defined. + Use it instead of lll_mutex_lock. If __pthread_mutex_lock is a + macro don't define aliases. + + * cancellation.c: Remove __pthread_enable_asynccancel_2. + * pthreadP.h: Remove declaration of __pthread_enable_asynccancel_2. + * sysdeps/pthread/pthread_cond_timedwait.c: Use + __pthread_enable_asynccancel instead of __pthread_enable_asynccancel_2. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise. + * sysdeps/pthread/pthread_cond_wait.c: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise. + 2003-05-17 Ulrich Drepper <drepper@redhat.com> * sem_open.c: Fix one endless loop. Implement correct semantics diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h index 71f1a7fede..fe982af87b 100644 --- a/nptl/pthreadP.h +++ b/nptl/pthreadP.h @@ -241,6 +241,8 @@ extern int __pthread_mutex_destroy_internal (pthread_mutex_t *__mutex); extern int __pthread_mutex_trylock (pthread_mutex_t *_mutex); extern int __pthread_mutex_lock (pthread_mutex_t *__mutex); extern int __pthread_mutex_lock_internal (pthread_mutex_t *__mutex); +extern int __pthread_mutex_cond_lock (pthread_mutex_t *__mutex) + attribute_hidden; extern int __pthread_mutex_unlock (pthread_mutex_t *__mutex); extern int __pthread_mutex_unlock_internal (pthread_mutex_t *__mutex); extern int __pthread_mutexattr_init (pthread_mutexattr_t *attr); @@ -320,8 +322,6 @@ extern int __pthread_kill (pthread_t threadid, int signo); extern void __pthread_exit (void *value); extern int __pthread_setcanceltype (int type, int *oldtype); extern int __pthread_enable_asynccancel (void) attribute_hidden; -extern void __pthread_enable_asynccancel_2 (int *oldvalp) - internal_function attribute_hidden; extern void __pthread_disable_asynccancel (int oldtype) internal_function attribute_hidden; diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c index f70445acfc..4fdb13718c 100644 --- a/nptl/pthread_mutex_lock.c +++ b/nptl/pthread_mutex_lock.c @@ -22,6 +22,11 @@ #include <lowlevellock.h> +#ifndef LLL_MUTEX_LOCK +# define LLL_MUTEX_LOCK(mutex) lll_mutex_lock (mutex) +#endif + + int __pthread_mutex_lock (mutex) pthread_mutex_t *mutex; @@ -45,7 +50,7 @@ __pthread_mutex_lock (mutex) else { /* We have to get the mutex. */ - lll_mutex_lock (mutex->__data.__lock); + LLL_MUTEX_LOCK (mutex->__data.__lock); /* Record the ownership. */ mutex->__data.__owner = id; @@ -66,7 +71,7 @@ __pthread_mutex_lock (mutex) case PTHREAD_MUTEX_TIMED_NP: case PTHREAD_MUTEX_ADAPTIVE_NP: /* Normal mutex. */ - lll_mutex_lock (mutex->__data.__lock); + LLL_MUTEX_LOCK (mutex->__data.__lock); /* Record the ownership. */ mutex->__data.__owner = id; break; @@ -74,5 +79,7 @@ __pthread_mutex_lock (mutex) return 0; } +#ifndef __pthread_mutex_lock strong_alias (__pthread_mutex_lock, pthread_mutex_lock) strong_alias (__pthread_mutex_lock, __pthread_mutex_lock_internal) +#endif diff --git a/nptl/sysdeps/i386/tcb-offsets.sym b/nptl/sysdeps/i386/tcb-offsets.sym index 6a5c16c0ee..562ac70d4b 100644 --- a/nptl/sysdeps/i386/tcb-offsets.sym +++ b/nptl/sysdeps/i386/tcb-offsets.sym @@ -6,3 +6,4 @@ MULTIPLE_THREADS_OFFSET offsetof (tcbhead_t, multiple_threads) SYSINFO_OFFSET offsetof (tcbhead_t, sysinfo) CLEANUP offsetof (struct pthread, cleanup) CLEANUP_PREV offsetof (struct _pthread_cleanup_buffer, __prev) +MUTEX_FUTEX offsetof (pthread_mutex_t, __data.__lock) diff --git a/nptl/sysdeps/pthread/pthread_cond_broadcast.c b/nptl/sysdeps/pthread/pthread_cond_broadcast.c index 1076fe3c50..f34f58cc99 100644 --- a/nptl/sysdeps/pthread/pthread_cond_broadcast.c +++ b/nptl/sysdeps/pthread/pthread_cond_broadcast.c @@ -25,6 +25,7 @@ #include <pthreadP.h> #include <shlib-compat.h> +#include <kernel-features.h> int @@ -54,7 +55,16 @@ __pthread_cond_broadcast (cond) #endif /* Wake everybody. */ - lll_futex_wake (futex, INT_MAX); + pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; + if (__builtin_expect (lll_futex_requeue (futex, 1, MAX_INT, + &mut->__data.__lock) == -EINVAL, + 0)) + { + /* The requeue functionality is not available. */ +#ifndef __ASSUME_FUTEX_REQUEUE + lll_futex_wake (futex, MAX_INT); +#endif + } /* That's all. */ return 0; diff --git a/nptl/sysdeps/pthread/pthread_cond_signal.c b/nptl/sysdeps/pthread/pthread_cond_signal.c index 1a035fe05f..b9d8af07b2 100644 --- a/nptl/sysdeps/pthread/pthread_cond_signal.c +++ b/nptl/sysdeps/pthread/pthread_cond_signal.c @@ -25,13 +25,15 @@ #include <pthreadP.h> #include <shlib-compat.h> +#include <kernel-features.h> + int __pthread_cond_signal (cond) pthread_cond_t *cond; { /* Make sure we are alone. */ - lll_mutex_lock(cond->__data.__lock); + lll_mutex_lock (cond->__data.__lock); /* Are there any waiters to be woken? */ if (cond->__data.__total_seq > cond->__data.__wakeup_seq) @@ -50,7 +52,22 @@ __pthread_cond_signal (cond) #endif /* Wake one. */ - lll_futex_wake (futex, 1); + int r = lll_futex_requeue (futex, 0, 1, &cond->__data.__lock); + if (__builtin_expect (r == -EINVAL, 0)) + { + /* The requeue functionality is not available. */ +#ifndef __ASSUME_FUTEX_REQUEUE + lll_futex_wake (futex, 1); +#endif + } + else if (r != 0) + { + /* We always have to make the syscall if requeue actually + moved a thread. */ + lll_mutex_unlock_force (cond->__data.__lock); + + return 0; + } } /* We are done. */ diff --git a/nptl/sysdeps/pthread/pthread_cond_timedwait.c b/nptl/sysdeps/pthread/pthread_cond_timedwait.c index 23cf0acb99..4dd6f2e02a 100644 --- a/nptl/sysdeps/pthread/pthread_cond_timedwait.c +++ b/nptl/sysdeps/pthread/pthread_cond_timedwait.c @@ -66,6 +66,10 @@ __pthread_cond_timedwait (cond, mutex, abstime) /* We have one new user of the condvar. */ ++cond->__data.__total_seq; + /* Remember the mutex we are using here. If there is already a + different address store this is a bad user bug. */ + cond->__data.__mutex = mutex; + /* Prepare structure passed to cancellation handler. */ cbuffer.cond = cond; cbuffer.mutex = mutex; @@ -145,7 +149,7 @@ __pthread_cond_timedwait (cond, mutex, abstime) lll_mutex_unlock (cond->__data.__lock); /* Enable asynchronous cancellation. Required by the standard. */ - __pthread_enable_asynccancel_2 (&cbuffer.oldtype); + cbuffer.oldtype = __pthread_enable_asynccancel (); /* Wait until woken by signal or broadcast. Note that we truncate the 'val' value to 32 bits. */ @@ -184,7 +188,7 @@ __pthread_cond_timedwait (cond, mutex, abstime) __pthread_cleanup_pop (&buffer, 0); /* Get the mutex before returning. */ - err = __pthread_mutex_lock_internal (mutex); + err = __pthread_mutex_cond_lock (mutex); return err ?: result; } diff --git a/nptl/sysdeps/pthread/pthread_cond_wait.c b/nptl/sysdeps/pthread/pthread_cond_wait.c index 708566be03..da94cc2d6b 100644 --- a/nptl/sysdeps/pthread/pthread_cond_wait.c +++ b/nptl/sysdeps/pthread/pthread_cond_wait.c @@ -65,8 +65,7 @@ __condvar_cleanup (void *arg) /* Get the mutex before returning unless asynchronous cancellation is in effect. */ - if (!(cbuffer->oldtype & CANCELTYPE_BITMASK)) - __pthread_mutex_lock_internal (cbuffer->mutex); + __pthread_mutex_cond_lock (cbuffer->mutex); } @@ -93,6 +92,10 @@ __pthread_cond_wait (cond, mutex) /* We have one new user of the condvar. */ ++cond->__data.__total_seq; + /* Remember the mutex we are using here. If there is already a + different address store this is a bad user bug. */ + cond->__data.__mutex = mutex; + /* Prepare structure passed to cancellation handler. */ cbuffer.cond = cond; cbuffer.mutex = mutex; @@ -123,7 +126,7 @@ __pthread_cond_wait (cond, mutex) lll_mutex_unlock (cond->__data.__lock); /* Enable asynchronous cancellation. Required by the standard. */ - __pthread_enable_asynccancel_2 (&cbuffer.oldtype); + cbuffer.oldtype = __pthread_enable_asynccancel (); /* Wait until woken by signal or broadcast. Note that we truncate the 'val' value to 32 bits. */ @@ -150,7 +153,7 @@ __pthread_cond_wait (cond, mutex) __pthread_cleanup_pop (&buffer, 0); /* Get the mutex before returning. */ - return __pthread_mutex_lock_internal (mutex); + return __pthread_mutex_cond_lock (mutex); } versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, diff --git a/nptl/sysdeps/unix/sysv/linux/Makefile b/nptl/sysdeps/unix/sysv/linux/Makefile index 5701887c06..510232c9c6 100644 --- a/nptl/sysdeps/unix/sysv/linux/Makefile +++ b/nptl/sysdeps/unix/sysv/linux/Makefile @@ -21,7 +21,7 @@ ifeq ($(subdir),nptl) sysdep_routines += register-atfork unregister-atfork libc_pthread_init \ libc_multiple_threads -libpthread-sysdep_routines += pt-fork +libpthread-sysdep_routines += pt-fork pthread_mutex_cond_lock gen-as-const-headers += lowlevelcond.sym lowlevelrwlock.sym lowlevelbarrier.sym endif diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h index 0834894c25..4b0f11adc5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h @@ -78,6 +78,7 @@ typedef union unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; + void *__mutex; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; long long int __align; diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S index 0fa402a95e..06821ad376 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S @@ -20,6 +20,7 @@ #include <sysdep.h> #include <shlib-compat.h> #include <lowlevelcond.h> +#include <kernel-features.h> #ifdef UP # define LOCK @@ -30,6 +31,9 @@ #define SYS_futex 240 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 + +#define EINVAL 22 .text @@ -41,8 +45,10 @@ __pthread_cond_broadcast: pushl %ebx + pushl %esi + pushl %edi - movl 8(%esp), %ebx + movl 16(%esp), %ebx /* Get internal lock. */ movl $1, %eax @@ -69,18 +75,34 @@ __pthread_cond_broadcast: 3: movl %ecx, (%ebx) movl %eax, 4(%ebx) + /* Get the address of the mutex used. */ + movl dep_mutex-wakeup_seq(%ebx), %edi + /* Unlock. */ LOCK subl $1, cond_lock-wakeup_seq(%ebx) jne 7f /* Wake up all threads. */ -8: movl $FUTEX_WAKE, %ecx +8: movl $FUTEX_REQUEUE, %ecx movl $SYS_futex, %eax - movl $0x7fffffff, %edx + movl $0x7fffffff, %esi + movl $1, %edx + /* Get the address of the futex involved. */ +# if MUTEX_FUTEX != 0 + addl $MUTEX_FUTEX, %edi +# endif ENTER_KERNEL +#ifndef __ASSUME_FUTEX_REQUEUE + cmpl $-EINVAL, %eax + je 9f +10: +#endif + xorl %eax, %eax + popl %edi + popl %esi popl %ebx ret @@ -91,6 +113,8 @@ __pthread_cond_broadcast: jne 5f 6: xorl %eax, %eax + popl %edi + popl %esi popl %ebx ret @@ -113,6 +137,15 @@ __pthread_cond_broadcast: 7: leal cond_lock-wakeup_seq(%ebx), %eax call __lll_mutex_unlock_wake jmp 8b + +#ifndef __ASSUME_FUTEX_REQUEUE +9: /* The futex requeue functionality is not available. */ + movl $0x7fffffff, %edx + movl $FUTEX_WAKE, %ecx + movl $SYS_futex, %eax + ENTER_KERNEL + jmp 10b +#endif .size __pthread_cond_broadcast, .-__pthread_cond_broadcast versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index 5465d7b2a8..ed25c554d2 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -20,6 +20,7 @@ #include <sysdep.h> #include <shlib-compat.h> #include <lowlevelcond.h> +#include <kernel-features.h> #ifdef UP # define LOCK @@ -30,6 +31,9 @@ #define SYS_futex 240 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 + +#define EINVAL 22 .text @@ -41,23 +45,25 @@ __pthread_cond_signal: pushl %ebx + pushl %esi + pushl %edi - movl 8(%esp), %ebx + movl 16(%esp), %edi /* Get internal lock. */ movl $1, %eax LOCK #if cond_lock == 0 - xaddl %eax, (%ebx) + xaddl %eax, (%edi) #else - xaddl %eax, cond_lock(%ebx) + xaddl %eax, cond_lock(%edi) #endif testl %eax, %eax jne 1f -2: addl $wakeup_seq, %ebx - movl total_seq+4-wakeup_seq(%ebx), %eax - movl total_seq-wakeup_seq(%ebx), %ecx +2: leal wakeup_seq(%edi), %ebx + movl total_seq+4(%edi), %eax + movl total_seq(%edi), %ecx cmpl 4(%ebx), %eax ja 3f jb 4f @@ -68,18 +74,30 @@ __pthread_cond_signal: 3: addl $1, (%ebx) adcl $0, 4(%ebx) - /* Wake up one thread. */ - movl $FUTEX_WAKE, %ecx + /* Wake up one thread by moving it to the internal lock futex. */ + movl $FUTEX_REQUEUE, %ecx movl $SYS_futex, %eax - movl %ecx, %edx /* movl $1, %edx */ + xorl %edx, %edx + movl $1, %esi ENTER_KERNEL +#ifndef __ASSUME_FUTEX_REQUEUE + cmpl $-EINVAL, %eax + je 7f +#endif + + /* If we moved a thread we in any case have to make the syscall. */ + testl %eax, %eax + jne 5f + /* Unlock. */ 4: LOCK - subl $1, cond_lock-wakeup_seq(%ebx) + subl $1, (%edi) jne 5f 6: xorl %eax, %eax + popl %edi + popl %esi popl %ebx ret @@ -93,11 +111,24 @@ __pthread_cond_signal: call __lll_mutex_lock_wait jmp 2b - /* Unlock in loop requires waekup. */ + /* Unlock in loop requires wakeup. */ 5: - leal cond_lock-wakeup_seq(%ebx), %eax +#if cond_lock == 0 + movl %edi, %eax +#else + leal cond_lock(%edi), %eax +#endif call __lll_mutex_unlock_wake jmp 6b + +#ifndef __ASSUME_FUTEX_REQUEUE +7: /* The futex requeue functionality is not available. */ + movl $1, %edx + movl $FUTEX_WAKE, %ecx + movl $SYS_futex, %eax + ENTER_KERNEL + jmp 4b +#endif .size __pthread_cond_signal, .-__pthread_cond_signal versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S index d9bffe5139..5eec268b29 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S @@ -66,8 +66,13 @@ __pthread_cond_timedwait: testl %eax, %eax jne 1f + /* Store the reference to the mutex. If there is already a + different value in there this is a bad user bug. */ +2: movl 24(%esp), %eax + movl %eax, dep_mutex(%ebx) + /* Unlock the mutex. */ -2: pushl 24(%esp) + pushl %eax .Lpush4: call __pthread_mutex_unlock_internal @@ -113,8 +118,8 @@ __pthread_cond_timedwait: #endif jne 3f -4: leal 8(%esp), %eax - call __pthread_enable_asynccancel_2 +4: call __pthread_enable_asynccancel + movl %eax, 8(%esp) /* Get the current time. */ movl %ebx, %edx @@ -230,7 +235,7 @@ __pthread_cond_timedwait: movl %edx, %gs:CLEANUP /* Trick ahead: (%esp) contains the address of the mutex. */ - call __pthread_mutex_lock_internal + call __pthread_mutex_cond_lock addl $44, %esp .Laddl: diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index 1ac6c1a7d6..61d3d8d225 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -83,15 +83,11 @@ __condvar_cleanup: movl $0x7fffffff, %edx ENTER_KERNEL - /* Lock the mutex unless asynchronous cancellation is in effect. */ - testl $2, 8(%esi) - jne 3f - pushl (%esi) - call __pthread_mutex_lock_internal + call __pthread_mutex_cond_lock popl %eax -3: popl %esi + popl %esi popl %ebx ret .size __condvar_cleanup, .-__condvar_cleanup @@ -125,8 +121,13 @@ __pthread_cond_wait: testl %eax, %eax jne 1f + /* Store the reference to the mutex. If there is already a + different value in there this is a bad user bug. */ +2: movl 20(%esp), %eax + movl %eax, dep_mutex(%ebx) + /* Unlock the mutex. */ -2: pushl 20(%esp) + pushl %eax .Lpush4: call __pthread_mutex_unlock_internal @@ -171,8 +172,8 @@ __pthread_cond_wait: #endif jne 3f -4: leal 8(%esp), %eax - call __pthread_enable_asynccancel_2 +4: call __pthread_enable_asynccancel + movl %eax, 8(%esp) movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */ movl %edi, %edx @@ -229,7 +230,7 @@ __pthread_cond_wait: movl %edx, %gs:CLEANUP /* Trick ahead: (%esp) contains the address of the mutex. */ - call __pthread_mutex_lock_internal + call __pthread_mutex_cond_lock addl $36, %esp .Laddl: diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 5fd50b97db..8923afbfed 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -126,6 +126,24 @@ extern int __lll_mutex_unlock_wait (int *__futex) : "memory"); }) +/* Special version of lll_mutex_lock which causes the unlock function to + always wakeup waiters. */ +#define lll_mutex_cond_lock(futex) \ + (void) ({ int ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \ + "testl %0, %0\n\t" \ + "jne 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleal %2, %%ecx\n\t" \ + "call __lll_mutex_lock_wait\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=a" (ignore1), "=&c" (ignore2), "=m" (futex) \ + : "0" (2), "2" (futex) \ + : "memory"); }) + + #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2; \ __asm __volatile (LOCK_INSTR "xaddl %0, %3\n\t" \ diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h index 92c0b5c524..b1941e07c6 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h @@ -29,6 +29,7 @@ #define SYS_futex 1230 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) @@ -64,7 +65,7 @@ "=r" (__o0), "=r" (__o1), "=r" (__o2), "=r" (__o3) \ : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \ "5" (__o2), "6" (__o3) \ - : lll_futex_clobbers); \ + : "out4", lll_futex_clobbers); \ __r10 == -1 ? -__r8 : __r8; \ }) @@ -83,10 +84,33 @@ "=r" (__o0), "=r" (__o1), "=r" (__o2) \ : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \ "5" (__o2) \ - : "out3", lll_futex_clobbers); \ + : "out3", "out4", lll_futex_clobbers); \ __r10 == -1 ? -__r8 : __r8; \ }) + +#define lll_futex_requeue(futex, nr_wake, nr_move, mutex) \ + ({ \ + register long int __o0 asm ("out0") = (long int) (futex); \ + register long int __o1 asm ("out1") = FUTEX_REQUEUE; \ + register long int __o2 asm ("out2") = (long int) (nr_wake); \ + register long int __o3 asm ("out3") = (long int) (nr_move); \ + register long int __o4 asm ("out4") = (long int) (mutex); \ + register long int __r8 asm ("r8"); \ + register long int __r10 asm ("r10"); \ + register long int __r15 asm ("r15") = SYS_futex; \ + \ + __asm __volatile ("break %7;;" \ + : "=r" (__r8), "=r" (__r10), "=r" (__r15), \ + "=r" (__o0), "=r" (__o1), "=r" (__o2), "r" (__o3), \ + "=r" (__o4) \ + : "i" (0x100000), "2" (__r15), "3" (__o0), "4" (__o1), \ + "5" (__o2), "6" (__o3), "7" (__o4) \ + : lll_futex_clobbers); \ + __r8; \ + }) + + static inline int __attribute__ ((always_inline)) __lll_mutex_trylock (int *futex) @@ -111,6 +135,18 @@ __lll_mutex_lock (int *futex) #define lll_mutex_lock(futex) __lll_mutex_lock (&(futex)) +static inline void +__attribute__ ((always_inline)) +__lll_mutex_cond_lock (int *futex) +{ + int val = atomic_exchange_and_add (futex, 2); + + if (__builtin_expect (val != 0, 0)) + __lll_lock_wait (futex, val); +} +#define lll_mutex_cond_lock(futex) __lll_mutex_cond_lock (&(futex)) + + extern int __lll_timedlock_wait (int *futex, int val, const struct timespec *) attribute_hidden; @@ -140,7 +176,11 @@ __lll_mutex_unlock (int *futex) if (__builtin_expect (val > 1, 0)) lll_futex_wake (futex, 1); } -#define lll_mutex_unlock(futex) __lll_mutex_unlock(&(futex)) +#define lll_mutex_unlock(futex) \ + __lll_mutex_unlock(&(futex)) + +#define lll_mutex_unlock_force(futex) \ + lll_futex_wake (&(futex), 1) #define lll_mutex_islocked(futex) \ (futex != 0) diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym index 5eb535e157..1463e0810e 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym @@ -8,3 +8,4 @@ cond_clock offsetof (pthread_cond_t, __data.__clock) total_seq offsetof (pthread_cond_t, __data.__total_seq) wakeup_seq offsetof (pthread_cond_t, __data.__wakeup_seq) woken_seq offsetof (pthread_cond_t, __data.__woken_seq) +dep_mutex offsetof (pthread_cond_t, __data.__mutex) diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c new file mode 100644 index 0000000000..893a5e932d --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c @@ -0,0 +1,6 @@ +#include <pthreadP.h> + +#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock(mutex) +#define __pthread_mutex_lock __pthread_mutex_cond_lock + +#include <nptl/pthread_mutex_lock.c> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h index 47d856cae4..7c12db6fc6 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h @@ -78,6 +78,7 @@ typedef union unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; + void *__mutex; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; long int __align; diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index 8e1742b671..136dc574e0 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -102,6 +102,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; : "0" (1), "2" (futex) \ : "cx", "r11", "cc", "memory"); }) + +#define lll_mutex_cond_lock(futex) \ + (void) ({ int ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t" \ + "testl %0, %0\n\t" \ + "jne 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %2, %%rdi\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_mutex_lock_wait\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \ + : "0" (2), "2" (futex) \ + : "cx", "r11", "cc", "memory"); }) + + #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2, ignore3; \ __asm __volatile (LOCK_INSTR "xaddl %0, %4\n\t" \ diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index cc12f5407c..66edb9ac85 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -20,6 +20,7 @@ #include <sysdep.h> #include <shlib-compat.h> #include <lowlevelcond.h> +#include <kernel-features.h> #ifdef UP # define LOCK @@ -30,6 +31,9 @@ #define SYS_futex 202 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 + +#define EINVAL 22 .text @@ -60,17 +64,27 @@ __pthread_cond_broadcast: woken up. */ movq %rcx, (%rdi) + /* Get the address of the mutex used. */ + movq dep_mutex-wakeup_seq(%rdi), %r8 + /* Unlock. */ LOCK decl cond_lock-wakeup_seq(%rdi) jne 7f /* Wake up all threads. */ -8: movq $FUTEX_WAKE, %rsi +8: movq $FUTEX_REQUEUE, %rsi movq $SYS_futex, %rax - movl $0x7fffffff, %edx + movl $1, %edx + movq $0x7fffffff, %r10 syscall +#ifndef __ASSUME_FUTEX_REQUEUE + cmpq $-EINVAL, %eax + je 9f +10: +#endif + xorl %eax, %eax retq @@ -104,6 +118,15 @@ __pthread_cond_broadcast: callq __lll_mutex_unlock_wake subq $cond_lock-wakeup_seq, %rdi jmp 8b + +#ifndef __ASSUME_FUTEX_REQUEUE +9: /* The futex requeue functionality is not available. */ + movq $0x7fffffff, %rdx + movq $FUTEX_WAKE, %rsi + movq $SYS_futex, %rax + syscall + jmp 10b +#endif .size __pthread_cond_broadcast, .-__pthread_cond_broadcast versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index 11635baa22..709fcf4ca8 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -20,6 +20,7 @@ #include <sysdep.h> #include <shlib-compat.h> #include <lowlevelcond.h> +#include <kernel-features.h> #ifdef UP # define LOCK @@ -30,6 +31,9 @@ #define SYS_futex 202 #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_REQUEUE 3 + +#define EINVAL 22 .text @@ -44,15 +48,15 @@ __pthread_cond_signal: movl $1, %esi LOCK #if cond_lock == 0 - xaddl %esi, (%rdi) + xaddl %esi, (%r8) #else - xaddl %esi, cond_lock(%rdi) + xaddl %esi, cond_lock(%r8) #endif testl %esi, %esi jne 1f -2: addq $wakeup_seq, %rdi - movq total_seq-wakeup_seq(%rdi), %rcx +2: leaq wakeup_seq(%r8), %rdi + movq total_seq(%r8), %rcx cmpq (%rdi), %rcx jbe 4f @@ -62,12 +66,22 @@ __pthread_cond_signal: /* Wake up one thread. */ movq $FUTEX_WAKE, %rsi movq $SYS_futex, %rax - movq %rsi, %rdx /* movl $1, %edx */ + xorq %rdx, %rdx + movq $1, %r10 syscall +#ifndef __ASSUME_FUTEX_REQUEUE + cmpq $-EINVAL, %rax + je 7f +#endif + + /* If we moved a thread we in any case have to make the syscall. */ + testq %rax, %rax + jne 5f + /* Unlock. */ 4: LOCK - decl cond_lock-wakeup_seq(%rdi) + decl cond_lock(%r8) jne 5f 6: xorl %eax, %eax @@ -85,9 +99,22 @@ __pthread_cond_signal: jmp 2b /* Unlock in loop requires waekup. */ -5: addq $cond_lock-wakeup_seq, %rdi +5: +#if cond_lock != 0 + addq $cond_lock-wakeup_seq, %rdi +#else + movq %r8, %rdi +#endif callq __lll_mutex_unlock_wake jmp 6b + +#ifndef __ASSUME_FUTEX_REQUEUE +7: /* The futex requeue functionality is not available. */ + movq $1, %rdx + movq $FUTEX_WAKE, %esi + movq $SYS_futex, %rax + syscall + jmp 4b .size __pthread_cond_signal, .-__pthread_cond_signal versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal, GLIBC_2_3_2) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 151395048a..c8fd4ea129 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -88,7 +88,9 @@ __pthread_cond_timedwait: jne 1f /* Unlock the mutex. */ -2: movq 16(%rsp), %rdi +2: movq %rdi, %rax + movq 16(%rsp), %rdi + movq %rdi, dep_mutex(%rax) callq __pthread_mutex_unlock_internal testl %eax, %eax @@ -121,8 +123,8 @@ __pthread_cond_timedwait: #endif jne 3f -4: movq %rsp, %rdi - callq __pthread_enable_asynccancel_2 +4: callq __pthread_enable_asynccancel + movq %rax, (%rsp) /* Get the current time. */ #ifdef __NR_clock_gettime @@ -227,7 +229,7 @@ __pthread_cond_timedwait: movq %rdx, %fs:CLEANUP movq 16(%rsp), %rdi - callq __pthread_mutex_lock_internal + callq __pthread_mutex_cond_lock testq %rax, %rax cmoveq %r14, %rax diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 6cad2812f0..5189972fc7 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -84,12 +84,8 @@ __condvar_cleanup: movq $SYS_futex, %rax syscall - /* Lock the mutex unless asynchronous cancellation is in effect. */ - testq $2, (%r8) - jne 3f - movq 16(%r8), %rdi - callq __pthread_mutex_lock_internal + callq __pthread_mutex_cond_lock 3: retq .size __condvar_cleanup, .-__condvar_cleanup @@ -137,7 +133,9 @@ __pthread_cond_wait: jne 1f /* Unlock the mutex. */ -2: movq 16(%rsp), %rdi +2: movq %rdi, %rax + movq 16(%rsp), %rdi + movq %rdi, dep_mutex(%rax) callq __pthread_mutex_unlock_internal testl %eax, %eax @@ -170,8 +168,8 @@ __pthread_cond_wait: #endif jne 3f -4: movq %rsp, %rdi - callq __pthread_enable_asynccancel_2 +4: callq __pthread_enable_asynccancel + movq %rax, (%rsp) movq 8(%rsp), %rdi xorq %r10, %r10 @@ -221,7 +219,7 @@ __pthread_cond_wait: movq %rdx, %fs:CLEANUP movq 16(%rsp), %rdi - callq __pthread_mutex_lock_internal + callq __pthread_mutex_cond_lock 14: addq $64, %rsp .Laddq: diff --git a/nptl/sysdeps/x86_64/tcb-offsets.sym b/nptl/sysdeps/x86_64/tcb-offsets.sym index 046ad5bee8..e2abc02647 100644 --- a/nptl/sysdeps/x86_64/tcb-offsets.sym +++ b/nptl/sysdeps/x86_64/tcb-offsets.sym @@ -4,3 +4,4 @@ SELF offsetof (tcbhead_t, self) CLEANUP offsetof (struct pthread, cleanup) CLEANUP_PREV offsetof (struct _pthread_cleanup_buffer, __prev) +MUTEX_FUTEX offsetof (pthread_mutex_t, __data.__lock) diff --git a/sysdeps/unix/sysv/linux/kernel-features.h b/sysdeps/unix/sysv/linux/kernel-features.h index 56b68ffb5a..a95559de94 100644 --- a/sysdeps/unix/sysv/linux/kernel-features.h +++ b/sysdeps/unix/sysv/linux/kernel-features.h @@ -294,3 +294,8 @@ #if __LINUX_KERNEL_VERSION >= 132405 && defined __i386__ # define __ASSUME_VSYSCALL 1 #endif + +/* The requeue futex functionality was introduced in 2.5.70. */ +#if __LINUX_KERNEL_VERSION >= 132422 +# define __ASSUME_FUTEX_REQUEUE 1 +#endif |