diff options
author | Jakub Jelinek <jakub@redhat.com> | 2006-02-17 16:15:56 +0000 |
---|---|---|
committer | Jakub Jelinek <jakub@redhat.com> | 2006-02-17 16:15:56 +0000 |
commit | 9c09cd93597a91d6b0b59c8813be97d530e7551c (patch) | |
tree | cd7ea8e6dc157e92ffd5f20dfa997e8be5b7da8b /nptl | |
parent | 7d8db4cd587349f485fb85f4bea9e3011a3e55d7 (diff) | |
download | glibc-9c09cd93597a91d6b0b59c8813be97d530e7551c.tar.gz glibc-9c09cd93597a91d6b0b59c8813be97d530e7551c.tar.xz glibc-9c09cd93597a91d6b0b59c8813be97d530e7551c.zip |
Updated to fedora-glibc-20060217T1609
Diffstat (limited to 'nptl')
22 files changed, 1276 insertions, 187 deletions
diff --git a/nptl/ChangeLog b/nptl/ChangeLog index 73bcfaa84f..454295fb11 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -1,5 +1,20 @@ +2006-02-17 Kaz Kojima <kkojima@rr.iij4u.or.jp> + + * sysdeps/unix/sysv/linux/sh/lowlevellock.h: Add lll_robust_mutex_* + definitions. + * sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S: New file. + +2006-02-17 Ulrich Drepper <drepper@redhat.com> + + * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h + (lll_robust_mutex_unlock): Avoid unnecessary wakeups. + * sysdeps/unix/sysv/linux/i386/lowlevellock.h + (lll_robust_mutex_unlock): Likewise. + 2006-02-13 Jakub Jelinek <jakub@redhat.com> + * descr.h [!__PTHREAD_MUTEX_HAVE_PREV] (DEQUEUE_MUTEX): + Set robust_list.__next rather than robust_list. * sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (__pthread_list_t): New typedef. (pthread_mutex_t): Replace __next and __prev fields with __list. @@ -19,6 +34,33 @@ (__pthread_slist_t): New typedef. (pthread_mutex_t): Replace __next field with __list. +2006-02-15 Ulrich Drepper <drepper@redhat.com> + + * pthreadp.h: Define PTHREAD_MUTEX_INCONSISTENT instead of + PTHREAD_MUTEX_OWNERDEAD. + (PTHREAD_MUTEX_ROBUST_PRIVATE_NP): Define as 16, not 256. + Define FUTEX_WAITERS, FUTEX_OWNER_DIED, FUTEX_TID_MASK. + * Makefile (libpthread-routines): Add lowlevelrobustlock. + * pthread_create.c (start_thread): Very much simplify robust_list loop. + * pthread_mutex_consistent.c: Inconsistent mutex have __owner now set + to PTHREAD_MUTEX_INCONSISTENT. + * pthread_mutex_destroy.c: Allow destroying of inconsistent mutexes. + * pthread_mutex_lock.c: Reimplement robust mutex handling. + * pthread_mutex_trylock.c: Likewise. + * pthread_mutex_timedlock.c: Likewise. + * pthread_mutex_unlock.c: Likewise. + * sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise. + * sysdeps/unix/sysv/linux/Makefile (gen-as-const-headers): Add + lowlevelrobustlock.sym. + * sysdeps/unix/sysv/linux/lowlevelrobustlock.sym: New file. + * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Add lll_robust_mutex_* + definitions. + * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S: New file. + * sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S: New file. + * sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S: New file. + * sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S: New file. + 2006-02-12 Ulrich Drepper <drepper@redhat.com> * allocatestack.c (allocate_stack): Initialize robust_list. diff --git a/nptl/Makefile b/nptl/Makefile index 2b245dfb31..60203cd114 100644 --- a/nptl/Makefile +++ b/nptl/Makefile @@ -100,7 +100,7 @@ libpthread-routines = init vars events version \ cleanup_defer_compat unwind \ pt-longjmp pt-cleanup\ cancellation \ - lowlevellock \ + lowlevellock lowlevelrobustlock \ pt-vfork \ ptw-write ptw-read ptw-close ptw-fcntl ptw-accept \ ptw-connect ptw-recv ptw-recvfrom ptw-recvmsg ptw-send \ diff --git a/nptl/descr.h b/nptl/descr.h index 6984138537..d5491c1355 100644 --- a/nptl/descr.h +++ b/nptl/descr.h @@ -166,7 +166,7 @@ struct pthread do { \ __pthread_slist_t *runp = THREAD_GETMEM (THREAD_SELF, robust_list.__next);\ if (runp == &mutex->__data.__list) \ - THREAD_SETMEM (THREAD_SELF, robust_list, runp->__next); \ + THREAD_SETMEM (THREAD_SELF, robust_list.__next, runp->__next); \ else \ { \ while (runp->__next != &mutex->__data.__list) \ diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h index 61b7176159..77d8f5ad24 100644 --- a/nptl/pthreadP.h +++ b/nptl/pthreadP.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -52,7 +52,7 @@ /* Magic cookie representing robust mutex with dead owner. */ -#define PTHREAD_MUTEX_OWNERDEAD INT_MAX +#define PTHREAD_MUTEX_INCONSISTENT INT_MAX /* Magic cookie representing not recoverable robust mutex. */ #define PTHREAD_MUTEX_NOTRECOVERABLE (INT_MAX - 1) @@ -60,7 +60,7 @@ /* Internal mutex type value. */ enum { - PTHREAD_MUTEX_ROBUST_PRIVATE_NP = 256, + PTHREAD_MUTEX_ROBUST_PRIVATE_NP = 16, PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP = PTHREAD_MUTEX_ROBUST_PRIVATE_NP | PTHREAD_MUTEX_RECURSIVE_NP, PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP @@ -77,6 +77,12 @@ enum (PTHREAD_MUTEXATTR_FLAG_ROBUST | PTHREAD_MUTEXATTR_FLAG_PSHARED) +/* Bits used in robust mutex implementation. */ +#define FUTEX_WAITERS 0x80000000 +#define FUTEX_OWNER_DIED 0x40000000 +#define FUTEX_TID_MASK 0x1fffffff + + /* Internal variables. */ diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c index b1253b2243..f3d90ecebf 100644 --- a/nptl/pthread_create.c +++ b/nptl/pthread_create.c @@ -324,17 +324,12 @@ start_thread (void *arg) ((char *) robust - offsetof (struct __pthread_mutex_s, __list)); robust = robust->__next; - assert (lll_mutex_islocked (this->__lock)); - this->__count = 0; - --this->__nusers; - assert (this->__owner != PTHREAD_MUTEX_NOTRECOVERABLE); - this->__owner = PTHREAD_MUTEX_OWNERDEAD; this->__list.__next = NULL; #ifdef __PTHREAD_MUTEX_HAVE_PREV this->__list.__prev = NULL; #endif - lll_mutex_unlock (this->__lock); + lll_robust_mutex_dead (this->__lock); } while (robust != &pd->robust_list); diff --git a/nptl/pthread_mutex_consistent.c b/nptl/pthread_mutex_consistent.c index 2edfe8a5a6..0cfe972da0 100644 --- a/nptl/pthread_mutex_consistent.c +++ b/nptl/pthread_mutex_consistent.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2006 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2005. @@ -27,10 +27,10 @@ pthread_mutex_consistent_np (mutex) { /* Test whether this is a robust mutex with a dead owner. */ if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_PRIVATE_NP) == 0 - || mutex->__data.__owner != -THREAD_GETMEM (THREAD_SELF, tid)) + || mutex->__data.__owner != PTHREAD_MUTEX_INCONSISTENT) return EINVAL; - mutex->__data.__owner = -mutex->__data.__owner; + mutex->__data.__owner = THREAD_GETMEM (THREAD_SELF, tid); return 0; } diff --git a/nptl/pthread_mutex_destroy.c b/nptl/pthread_mutex_destroy.c index 2bf76a9da3..19a647a846 100644 --- a/nptl/pthread_mutex_destroy.c +++ b/nptl/pthread_mutex_destroy.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2006 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -26,9 +26,17 @@ __pthread_mutex_destroy (mutex) pthread_mutex_t *mutex; { if (mutex->__data.__nusers != 0) - return EBUSY; + { + if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_PRIVATE_NP) != 0 + && (mutex->__data.__lock & FUTEX_OWNER_DIED) != 0 + && mutex->__data.__nusers == 1) + goto dead_robust_mutex; + + return EBUSY; + } /* Set to an invalid value. */ + dead_robust_mutex: mutex->__data.__kind = -1; return 0; diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c index 420711a4d4..dd22567c71 100644 --- a/nptl/pthread_mutex_lock.c +++ b/nptl/pthread_mutex_lock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -27,6 +27,7 @@ #ifndef LLL_MUTEX_LOCK # define LLL_MUTEX_LOCK(mutex) lll_mutex_lock (mutex) # define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_trylock (mutex) +# define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_mutex_lock (mutex, id) #endif @@ -36,6 +37,7 @@ __pthread_mutex_lock (mutex) { assert (sizeof (mutex->__size) >= sizeof (mutex->__data)); + int oldval; pid_t id = THREAD_GETMEM (THREAD_SELF, tid); int retval = 0; @@ -107,60 +109,83 @@ __pthread_mutex_lock (mutex) break; case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP: - /* Check whether we already hold the mutex. */ - if (abs (mutex->__data.__owner) == id) + case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP: + case PTHREAD_MUTEX_ROBUST_PRIVATE_NP: + case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP: + oldval = mutex->__data.__lock; + do { - /* Just bump the counter. */ - if (__builtin_expect (mutex->__data.__count + 1 == 0, 0)) - /* Overflow of the counter. */ - return EAGAIN; - - ++mutex->__data.__count; - - return 0; - } - - /* We have to get the mutex. */ - LLL_MUTEX_LOCK (mutex->__data.__lock); + if ((oldval & FUTEX_OWNER_DIED) != 0) + { + /* The previous owner died. Try locking the mutex. */ + int newval; + while ((newval + = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, + id, oldval)) + != oldval) + { + if ((newval & FUTEX_OWNER_DIED) == 0) + goto normal; + oldval = newval; + } - mutex->__data.__count = 1; + /* We got the mutex. */ + mutex->__data.__count = 1; + /* But it is inconsistent unless marked otherwise. */ + mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; + + ENQUEUE_MUTEX (mutex); + + /* Note that we deliberately exit here. If we fall + through to the end of the function __nusers would be + incremented which is not correct because the old + owner has to be discounted. If we are not supposed + to increment __nusers we actually have to decrement + it here. */ +#ifdef NO_INCR + --mutex->__data.__nusers; +#endif - goto robust; + return EOWNERDEAD; + } - case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP: - /* Check whether we already hold the mutex. */ - if (__builtin_expect (abs (mutex->__data.__owner) == id, 0)) - return EDEADLK; + normal: + /* Check whether we already hold the mutex. */ + if (__builtin_expect ((mutex->__data.__lock & FUTEX_TID_MASK) + == id, 0)) + { + if (mutex->__data.__kind + == PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP) + return EDEADLK; - /* FALLTHROUGH */ + if (mutex->__data.__kind + == PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP) + { + /* Just bump the counter. */ + if (__builtin_expect (mutex->__data.__count + 1 == 0, 0)) + /* Overflow of the counter. */ + return EAGAIN; - case PTHREAD_MUTEX_ROBUST_PRIVATE_NP: - case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP: - LLL_MUTEX_LOCK (mutex->__data.__lock); + ++mutex->__data.__count; - robust: - if (__builtin_expect (mutex->__data.__owner - == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) - { - /* This mutex is now not recoverable. */ - mutex->__data.__count = 0; - lll_mutex_unlock (mutex->__data.__lock); - return ENOTRECOVERABLE; - } + return 0; + } + } - /* This mutex is either healthy or we can try to recover it. */ - assert (mutex->__data.__owner == 0 - || mutex->__data.__owner == PTHREAD_MUTEX_OWNERDEAD); + oldval = LLL_ROBUST_MUTEX_LOCK (mutex->__data.__lock, id); - if (__builtin_expect (mutex->__data.__owner - == PTHREAD_MUTEX_OWNERDEAD, 0)) - { - retval = EOWNERDEAD; - /* We signal ownership of a not yet recovered robust mutex - by storing the negative thread ID. */ - id = -id; + if (__builtin_expect (mutex->__data.__owner + == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) + { + /* This mutex is now not recoverable. */ + mutex->__data.__count = 0; + lll_mutex_unlock (mutex->__data.__lock); + return ENOTRECOVERABLE; + } } + while ((oldval & FUTEX_OWNER_DIED) != 0); + mutex->__data.__count = 1; ENQUEUE_MUTEX (mutex); break; diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c index bc4ead765d..b69caedc7c 100644 --- a/nptl/pthread_mutex_timedlock.c +++ b/nptl/pthread_mutex_timedlock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -28,6 +28,7 @@ pthread_mutex_timedlock (mutex, abstime) pthread_mutex_t *mutex; const struct timespec *abstime; { + int oldval; pid_t id = THREAD_GETMEM (THREAD_SELF, tid); int result = 0; @@ -103,67 +104,83 @@ pthread_mutex_timedlock (mutex, abstime) break; case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP: - /* Check whether we already hold the mutex. */ - if (abs (mutex->__data.__owner) == id) + case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP: + case PTHREAD_MUTEX_ROBUST_PRIVATE_NP: + case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP: + oldval = mutex->__data.__lock; + do { - /* Just bump the counter. */ - if (__builtin_expect (mutex->__data.__count + 1 == 0, 0)) - /* Overflow of the counter. */ - return EAGAIN; - - ++mutex->__data.__count; + if ((oldval & FUTEX_OWNER_DIED) != 0) + { + /* The previous owner died. Try locking the mutex. */ + int newval; + while ((newval + = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, + id, oldval)) + != oldval) + { + if ((newval & FUTEX_OWNER_DIED) == 0) + goto normal; + oldval = newval; + } - goto out; - } + /* We got the mutex. */ + mutex->__data.__count = 1; + /* But it is inconsistent unless marked otherwise. */ + mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; - /* We have to get the mutex. */ - result = lll_mutex_timedlock (mutex->__data.__lock, abstime); + ENQUEUE_MUTEX (mutex); - if (result != 0) - goto out; + /* Note that we deliberately exist here. If we fall + through to the end of the function __nusers would be + incremented which is not correct because the old + owner has to be discounted. */ + return EOWNERDEAD; + } - /* Only locked once so far. */ - mutex->__data.__count = 1; - goto robust; + normal: + /* Check whether we already hold the mutex. */ + if (__builtin_expect ((mutex->__data.__lock & FUTEX_TID_MASK) + == id, 0)) + { + if (mutex->__data.__kind + == PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP) + return EDEADLK; - case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP: - /* Check whether we already hold the mutex. */ - if (__builtin_expect (abs (mutex->__data.__owner) == id, 0)) - return EDEADLK; + if (mutex->__data.__kind + == PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP) + { + /* Just bump the counter. */ + if (__builtin_expect (mutex->__data.__count + 1 == 0, 0)) + /* Overflow of the counter. */ + return EAGAIN; - /* FALLTHROUGH */ + ++mutex->__data.__count; - case PTHREAD_MUTEX_ROBUST_PRIVATE_NP: - case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP: - result = lll_mutex_timedlock (mutex->__data.__lock, abstime); + return 0; + } + } - if (result != 0) - goto out; + result = lll_robust_mutex_timedlock (mutex->__data.__lock, abstime, + id); - robust: - if (__builtin_expect (mutex->__data.__owner - == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) - { - /* This mutex is now not recoverable. */ - mutex->__data.__count = 0; - lll_mutex_unlock (mutex->__data.__lock); - return ENOTRECOVERABLE; - } + if (__builtin_expect (mutex->__data.__owner + == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) + { + /* This mutex is now not recoverable. */ + mutex->__data.__count = 0; + lll_mutex_unlock (mutex->__data.__lock); + return ENOTRECOVERABLE; + } - /* This mutex is either healthy or we can try to recover it. */ - assert (mutex->__data.__owner == 0 - || mutex->__data.__owner == PTHREAD_MUTEX_OWNERDEAD); + if (result == ETIMEDOUT || result == EINVAL) + goto out; - if (__builtin_expect (mutex->__data.__owner - == PTHREAD_MUTEX_OWNERDEAD, 0)) - { - result = EOWNERDEAD; - /* We signal ownership of a not yet recovered robust mutex - by storing the negative thread ID. */ - mutex->__data.__owner = -id; - ++mutex->__data.__nusers; + oldval = result; } + while ((oldval & FUTEX_OWNER_DIED) != 0); + mutex->__data.__count = 1; ENQUEUE_MUTEX (mutex); break; diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c index ae73ecc39b..5a13ea6925 100644 --- a/nptl/pthread_mutex_trylock.c +++ b/nptl/pthread_mutex_trylock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2006 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -28,6 +28,7 @@ int __pthread_mutex_trylock (mutex) pthread_mutex_t *mutex; { + int oldval; pid_t id = THREAD_GETMEM (THREAD_SELF, tid); switch (__builtin_expect (mutex->__data.__kind, PTHREAD_MUTEX_TIMED_NP)) @@ -77,73 +78,88 @@ __pthread_mutex_trylock (mutex) case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP: - /* Check whether we already hold the mutex. */ - if (abs (mutex->__data.__owner) == id) - { - /* Just bump the counter. */ - if (__builtin_expect (mutex->__data.__count + 1 == 0, 0)) - /* Overflow of the counter. */ - return EAGAIN; - - ++mutex->__data.__count; - - return 0; - } - - /* We have to get the mutex. */ - if (lll_mutex_trylock (mutex->__data.__lock) == 0) - { - mutex->__data.__count = 1; - - goto robust; - } - - break; - case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP: - /* Check whether we already hold the mutex. */ - if (__builtin_expect (abs (mutex->__data.__owner) == id, 0)) - return EDEADLK; - - /* FALLTHROUGH */ - case PTHREAD_MUTEX_ROBUST_PRIVATE_NP: case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP: - if (lll_mutex_trylock (mutex->__data.__lock) != 0) - break; - - robust: - if (__builtin_expect (mutex->__data.__owner - == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) - { - /* This mutex is now not recoverable. */ - mutex->__data.__count = 0; - lll_mutex_unlock (mutex->__data.__lock); - return ENOTRECOVERABLE; - } - - /* This mutex is either healthy or we can try to recover it. */ - assert (mutex->__data.__owner == 0 - || mutex->__data.__owner == PTHREAD_MUTEX_OWNERDEAD); - - /* Record the ownership. */ - int retval = 0; - if (__builtin_expect (mutex->__data.__owner - == PTHREAD_MUTEX_OWNERDEAD, 0)) + oldval = mutex->__data.__lock; + do { - retval = EOWNERDEAD; - /* We signal ownership of a not yet recovered robust - mutex by storing the negative thread ID. */ - id = -id; + if ((oldval & FUTEX_OWNER_DIED) != 0) + { + /* The previous owner died. Try locking the mutex. */ + int newval; + while ((newval + = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, + id, oldval)) + != oldval) + { + if ((newval & FUTEX_OWNER_DIED) == 0) + goto normal; + oldval = newval; + } + + /* We got the mutex. */ + mutex->__data.__count = 1; + /* But it is inconsistent unless marked otherwise. */ + mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; + + ENQUEUE_MUTEX (mutex); + + /* Note that we deliberately exist here. If we fall + through to the end of the function __nusers would be + incremented which is not correct because the old + owner has to be discounted. */ + return EOWNERDEAD; + } + + normal: + /* Check whether we already hold the mutex. */ + if (__builtin_expect ((mutex->__data.__lock & FUTEX_TID_MASK) + == id, 0)) + { + if (mutex->__data.__kind + == PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP) + return EDEADLK; + + if (mutex->__data.__kind + == PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP) + { + /* Just bump the counter. */ + if (__builtin_expect (mutex->__data.__count + 1 == 0, 0)) + /* Overflow of the counter. */ + return EAGAIN; + + ++mutex->__data.__count; + + return 0; + } + } + + oldval = lll_robust_mutex_trylock (mutex->__data.__lock, id); + if (oldval != 0 && (oldval & FUTEX_OWNER_DIED) == 0) + return EBUSY; + + robust: + if (__builtin_expect (mutex->__data.__owner + == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) + { + /* This mutex is now not recoverable. */ + mutex->__data.__count = 0; + if (oldval == id) + lll_mutex_unlock (mutex->__data.__lock); + return ENOTRECOVERABLE; + } } + while ((oldval & FUTEX_OWNER_DIED) != 0); ENQUEUE_MUTEX (mutex); mutex->__data.__owner = id; ++mutex->__data.__nusers; + mutex->__data.__count = 1; + + return 0; - return retval -; default: /* Correct code cannot set any other type. */ return EINVAL; diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c index 4d87381065..d41eefe34c 100644 --- a/nptl/pthread_mutex_unlock.c +++ b/nptl/pthread_mutex_unlock.c @@ -41,23 +41,32 @@ __pthread_mutex_unlock_usercnt (mutex, decr) if (--mutex->__data.__count != 0) /* We still hold the mutex. */ return 0; - break; + goto normal; case PTHREAD_MUTEX_ERRORCHECK_NP: /* Error checking mutex. */ if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid) || ! lll_mutex_islocked (mutex->__data.__lock)) return EPERM; - break; + /* FALLTHROUGH */ case PTHREAD_MUTEX_TIMED_NP: case PTHREAD_MUTEX_ADAPTIVE_NP: - /* Normal mutex. Nothing special to do. */ + /* Always reset the owner field. */ + normal: + mutex->__data.__owner = 0; + if (decr) + /* One less user. */ + --mutex->__data.__nusers; + + /* Unlock. */ + lll_mutex_unlock (mutex->__data.__lock); break; case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP: /* Recursive mutex. */ - if (mutex->__data.__owner == -THREAD_GETMEM (THREAD_SELF, tid)) + if ((mutex->__data.__lock & FUTEX_TID_MASK) + == THREAD_GETMEM (THREAD_SELF, tid)) { if (--mutex->__data.__count != 0) /* We still hold the mutex. */ @@ -78,7 +87,8 @@ __pthread_mutex_unlock_usercnt (mutex, decr) case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP: case PTHREAD_MUTEX_ROBUST_PRIVATE_NP: case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP: - if (abs (mutex->__data.__owner) != THREAD_GETMEM (THREAD_SELF, tid) + if ((mutex->__data.__lock & FUTEX_TID_MASK) + != THREAD_GETMEM (THREAD_SELF, tid) || ! lll_mutex_islocked (mutex->__data.__lock)) return EPERM; @@ -86,15 +96,21 @@ __pthread_mutex_unlock_usercnt (mutex, decr) making the state consistent, mark the mutex as unrecoverable and make all waiters. */ if (__builtin_expect (mutex->__data.__owner - == -THREAD_GETMEM (THREAD_SELF, tid) - || (mutex->__data.__owner - == PTHREAD_MUTEX_NOTRECOVERABLE), 0)) + == PTHREAD_MUTEX_INCONSISTENT, 0)) notrecoverable: newowner = PTHREAD_MUTEX_NOTRECOVERABLE; robust: /* Remove mutex from the list. */ DEQUEUE_MUTEX (mutex); + + mutex->__data.__owner = newowner; + if (decr) + /* One less user. */ + --mutex->__data.__nusers; + + /* Unlock. */ + lll_robust_mutex_unlock (mutex->__data.__lock); break; default: @@ -102,15 +118,6 @@ __pthread_mutex_unlock_usercnt (mutex, decr) return EINVAL; } - /* Always reset the owner field. */ - mutex->__data.__owner = newowner; - if (decr) - /* One less user. */ - --mutex->__data.__nusers; - - /* Unlock. */ - lll_mutex_unlock (mutex->__data.__lock); - return 0; } diff --git a/nptl/sysdeps/unix/sysv/linux/Makefile b/nptl/sysdeps/unix/sysv/linux/Makefile index 14f513c127..88dce1a886 100644 --- a/nptl/sysdeps/unix/sysv/linux/Makefile +++ b/nptl/sysdeps/unix/sysv/linux/Makefile @@ -1,4 +1,4 @@ -# Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +# Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. # This file is part of the GNU C Library. # Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -24,7 +24,8 @@ sysdep_routines += register-atfork unregister-atfork libc_pthread_init \ libpthread-sysdep_routines += pt-fork pthread_mutex_cond_lock gen-as-const-headers += lowlevelcond.sym lowlevelrwlock.sym \ - lowlevelbarrier.sym unwindbuf.sym + lowlevelbarrier.sym unwindbuf.sym \ + lowlevelrobustlock.sym endif ifeq ($(subdir),posix) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S new file mode 100644 index 0000000000..1c516c7424 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S @@ -0,0 +1,186 @@ +/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include <sysdep.h> +#include <pthread-errnos.h> +#include <lowlevelrobustlock.h> + + .text + +#ifndef LOCK +# ifdef UP +# define LOCK +# else +# define LOCK lock +# endif +#endif + +#define SYS_gettimeofday __NR_gettimeofday +#define SYS_futex 240 +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_WAITERS 0x80000000 +#define FUTEX_OWNER_DIED 0x40000000 + + + .globl __lll_robust_mutex_lock_wait + .type __lll_robust_mutex_lock_wait,@function + .hidden __lll_robust_mutex_lock_wait + .align 16 +__lll_robust_mutex_lock_wait: + pushl %edx + pushl %ebx + pushl %esi + + movl %ecx, %ebx + xorl %esi, %esi /* No timeout. */ + xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ + +4: movl %eax, %edx + orl $FUTEX_WAITERS, %edx + + testl $FUTEX_OWNER_DIED, %eax + jnz 3f + + cmpl %edx, %eax /* NB: %edx == 2 */ + je 1f + + LOCK + cmpxchgl %edx, (%ebx) + jnz 2f + +1: movl $SYS_futex, %eax + ENTER_KERNEL + + movl (%ebx), %eax + +2: test %eax, %eax + jne 4b + + movl %gs:TID, %edx + LOCK + cmpxchgl %edx, (%ebx) + jnz 4b + /* NB: %eax == 0 */ + +3: popl %esi + popl %ebx + popl %edx + ret + .size __lll_robust_mutex_lock_wait,.-__lll_robust_mutex_lock_wait + + + .globl __lll_robust_mutex_timedlock_wait + .type __lll_robust_mutex_timedlock_wait,@function + .hidden __lll_robust_mutex_timedlock_wait + .align 16 +__lll_robust_mutex_timedlock_wait: + /* Check for a valid timeout value. */ + cmpl $1000000000, 4(%edx) + jae 3f + + pushl %edi + pushl %esi + pushl %ebx + pushl %ebp + + /* Stack frame for the timespec and timeval structs. */ + subl $12, %esp + + movl %ecx, %ebp + movl %edx, %edi + +1: movl %eax, 8(%esp) + + /* Get current time. */ + movl %esp, %ebx + xorl %ecx, %ecx + movl $SYS_gettimeofday, %eax + ENTER_KERNEL + + /* Compute relative timeout. */ + movl 4(%esp), %eax + movl $1000, %edx + mul %edx /* Milli seconds to nano seconds. */ + movl (%edi), %ecx + movl 4(%edi), %edx + subl (%esp), %ecx + subl %eax, %edx + jns 4f + addl $1000000000, %edx + subl $1, %ecx +4: testl %ecx, %ecx + js 8f /* Time is already up. */ + + /* Store relative timeout. */ + movl %ecx, (%esp) + movl %edx, 4(%esp) + + movl %ebp, %ebx + + movl 8(%esp), %edx + movl %edx, %eax + orl $FUTEX_WAITERS, %edx + + testl $FUTEX_OWNER_DIED, %eax + jnz 6f + + cmpl %eax, %edx + je 2f + + LOCK + cmpxchgl %edx, (%ebx) + movl $0, %ecx /* Must use mov to avoid changing cc. */ + jnz 5f + +2: + /* Futex call. */ + movl %esp, %esi + xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ + movl $SYS_futex, %eax + ENTER_KERNEL + movl %eax, %ecx + + movl (%ebx), %eax + +5: testl %eax, %eax + jne 7f + + movl %gs:TID, %edx + LOCK + cmpxchgl %edx, (%ebx) + jnz 7f + +6: addl $12, %esp + popl %ebp + popl %ebx + popl %esi + popl %edi + ret + + /* Check whether the time expired. */ +7: cmpl $-ETIMEDOUT, %ecx + jne 1b + +8: movl $ETIMEDOUT, %eax + jmp 6b + +3: movl $EINVAL, %eax + ret + .size __lll_robust_mutex_timedlock_wait,.-__lll_robust_mutex_timedlock_wait diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S new file mode 100644 index 0000000000..f768e16a7d --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S @@ -0,0 +1,20 @@ +/* Copyright (C) 2002, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include "../i486/lowlevelrobustlock.S" diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S new file mode 100644 index 0000000000..f768e16a7d --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S @@ -0,0 +1,20 @@ +/* Copyright (C) 2002, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include "../i486/lowlevelrobustlock.S" diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index b233d9747f..afdba009a0 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -139,6 +139,16 @@ extern int __lll_mutex_unlock_wake (int *__futex) ret; }) +#define lll_robust_mutex_trylock(futex, id) \ + ({ int ret; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ + : "=a" (ret), "=m" (futex) \ + : "r" (id), "m" (futex), \ + "0" (LLL_MUTEX_LOCK_INITIALIZER) \ + : "memory"); \ + ret; }) + + #define lll_mutex_cond_trylock(futex) \ ({ int ret; \ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ @@ -167,6 +177,25 @@ extern int __lll_mutex_unlock_wake (int *__futex) : "memory"); }) +#define lll_robust_mutex_lock(futex, id) \ + ({ int result, ignore; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ + "jnz _L_mutex_lock_%=\n\t" \ + ".subsection 1\n\t" \ + ".type _L_mutex_lock_%=,@function\n" \ + "_L_mutex_lock_%=:\n\t" \ + "leal %2, %%ecx\n\t" \ + "call __lll_robust_mutex_lock_wait\n\t" \ + "jmp 1f\n\t" \ + ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n" \ + ".previous\n" \ + "1:" \ + : "=a" (result), "=c" (ignore), "=m" (futex) \ + : "0" (0), "1" (id), "m" (futex) \ + : "memory"); \ + result; }) + + /* Special version of lll_mutex_lock which causes the unlock function to always wakeup waiters. */ #define lll_mutex_cond_lock(futex) \ @@ -187,6 +216,25 @@ extern int __lll_mutex_unlock_wake (int *__futex) : "memory"); }) +#define lll_robust_mutex_cond_lock(futex, id) \ + ({ int result, ignore; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ + "jnz _L_mutex_cond_lock_%=\n\t" \ + ".subsection 1\n\t" \ + ".type _L_mutex_cond_lock_%=,@function\n" \ + "_L_mutex_cond_lock_%=:\n\t" \ + "leal %2, %%ecx\n\t" \ + "call __lll_robust_mutex_lock_wait\n\t" \ + "jmp 1f\n\t" \ + ".size _L_mutex_cond_lock_%=,.-_L_mutex_cond_lock_%=\n"\ + ".previous\n" \ + "1:" \ + : "=a" (result), "=c" (ignore), "=m" (futex) \ + : "0" (0), "1" (id | FUTEX_WAITERS), "m" (futex) \ + : "memory"); \ + result; }) + + #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2; \ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ @@ -208,9 +256,30 @@ extern int __lll_mutex_unlock_wake (int *__futex) result; }) +#define lll_robust_mutex_timedlock(futex, timeout, id) \ + ({ int result, ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ + "jnz _L_mutex_timedlock_%=\n\t" \ + ".subsection 1\n\t" \ + ".type _L_mutex_timedlock_%=,@function\n" \ + "_L_mutex_timedlock_%=:\n\t" \ + "leal %3, %%ecx\n\t" \ + "movl %7, %%edx\n\t" \ + "call __lll_robust_mutex_timedlock_wait\n\t" \ + "jmp 1f\n\t" \ + ".size _L_mutex_timedlock_%=,.-_L_mutex_timedlock_%=\n"\ + ".previous\n" \ + "1:" \ + : "=a" (result), "=c" (ignore1), "=&d" (ignore2), \ + "=m" (futex) \ + : "0" (0), "1" (id), "m" (futex), "m" (timeout) \ + : "memory"); \ + result; }) + + #define lll_mutex_unlock(futex) \ (void) ({ int ignore; \ - __asm __volatile (LOCK_INSTR "subl $1,%0\n\t" \ + __asm __volatile (LOCK_INSTR "subl $1, %0\n\t" \ "jne _L_mutex_unlock_%=\n\t" \ ".subsection 1\n\t" \ ".type _L_mutex_unlock_%=,@function\n" \ @@ -226,6 +295,53 @@ extern int __lll_mutex_unlock_wake (int *__futex) : "memory"); }) +#define lll_robust_mutex_unlock(futex) \ + (void) ({ int ignore; \ + __asm __volatile (LOCK_INSTR "andl %2, %0\n\t" \ + "jne _L_mutex_unlock_%=\n\t" \ + ".subsection 1\n\t" \ + ".type _L_mutex_unlock_%=,@function\n" \ + "_L_mutex_unlock_%=:\n\t" \ + "leal %0, %%eax\n\t" \ + "call __lll_mutex_unlock_wake\n\t" \ + "jmp 1f\n\t" \ + ".size _L_mutex_unlock_%=,.-_L_mutex_unlock_%=\n" \ + ".previous\n" \ + "1:" \ + : "=m" (futex), "=&a" (ignore) \ + : "i" (FUTEX_WAITERS), "m" (futex) \ + : "memory"); }) + + +#define lll_robust_mutex_dead(futex) \ + (void) ({ int __ignore; \ + register int _nr asm ("edx") = 1; \ + __asm __volatile (LOCK_INSTR "orl %5, (%2)\n\t" \ + LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__ignore) \ + : "0" (SYS_futex), LLL_EBX_REG (&(futex)), \ + "c" (FUTEX_WAKE), "d" (_nr), \ + "i" (FUTEX_OWNER_DIED), \ + "i" (offsetof (tcbhead_t, sysinfo))); }) + + +#define lll_futex_wake(futex, nr) \ + do { \ + int __ignore; \ + register __typeof (nr) _nr asm ("edx") = (nr); \ + __asm __volatile (LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__ignore) \ + : "0" (SYS_futex), LLL_EBX_REG (futex), \ + "c" (FUTEX_WAKE), "d" (_nr), \ + "i" (0) /* phony, to align next arg's number */, \ + "i" (offsetof (tcbhead_t, sysinfo))); \ + } while (0) + + #define lll_mutex_islocked(futex) \ (futex != 0) diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym new file mode 100644 index 0000000000..2f1e9da52b --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym @@ -0,0 +1,6 @@ +#include <stddef.h> +#include <pthreadP.h> + +-- + +TID offsetof (struct pthread, tid) diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c index 990db87416..a97351f880 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c +++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c @@ -1,7 +1,8 @@ #include <pthreadP.h> -#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock(mutex) -#define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_cond_trylock(mutex) +#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock (mutex) +#define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_cond_trylock (mutex) +#define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_mutex_cond_lock (mutex, id) #define __pthread_mutex_lock __pthread_mutex_cond_lock #define NO_INCR diff --git a/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h index e3e3777f44..92f93cd5f5 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h @@ -62,6 +62,28 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden; : "r0", "r1", "r2", "t", "memory"); \ __result; }) +#define lll_robust_mutex_trylock(futex, id) \ + ({ unsigned char __result; \ + __asm __volatile ("\ + .align 2\n\ + mova 1f,r0\n\ + nop\n\ + mov r15,r1\n\ + mov #-8,r15\n\ + 0: mov.l @%1,r2\n\ + cmp/eq r2,%3\n\ + bf 1f\n\ + mov.l %2,@%1\n\ + 1: mov r1,r15\n\ + mov #-1,%0\n\ + negc %0,%0"\ + : "=r" (__result) \ + : "r" (&(futex)), \ + "r" (id), \ + "r" (LLL_MUTEX_LOCK_INITIALIZER) \ + : "r0", "r1", "r2", "t", "memory"); \ + __result; }) + #define lll_mutex_cond_trylock(futex) \ ({ unsigned char __result; \ __asm __volatile ("\ @@ -102,6 +124,25 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden; if (__result) \ __lll_mutex_lock_wait (__result, __futex); }) +#define lll_robust_mutex_lock(futex, id) \ + ({ int __result, val, *__futex = &(futex); \ + __asm __volatile ("\ + .align 2\n\ + mova 1f,r0\n\ + nop\n\ + mov r15,r1\n\ + mov #-8,r15\n\ + 0: mov.l @%2,%0\n\ + tst %0,%0\n\ + bf 1f\n\ + mov.l %1,@%2\n\ + 1: mov r1,r15"\ + : "=&r" (__result) : "r" (id), "r" (__futex) \ + : "r0", "r1", "t", "memory"); \ + if (__result) \ + __result = __lll_robust_mutex_lock_wait (__result, __futex); \ + __result; }) + /* Special version of lll_mutex_lock which causes the unlock function to always wakeup waiters. */ #define lll_mutex_cond_lock(futex) \ @@ -122,6 +163,25 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden; if (__result) \ __lll_mutex_lock_wait (__result, __futex); }) +#define lll_robust_mutex_cond_lock(futex, id) \ + ({ int __result, val, *__futex = &(futex); \ + __asm __volatile ("\ + .align 2\n\ + mova 1f,r0\n\ + nop\n\ + mov r15,r1\n\ + mov #-8,r15\n\ + 0: mov.l @%2,%0\n\ + tst %0,%0\n\ + bf 1f\n\ + mov.l %1,@%2\n\ + 1: mov r1,r15"\ + : "=&r" (__result) : "r" (id | FUTEX_WAITERS), "r" (__futex) \ + : "r0", "r1", "t", "memory"); \ + if (__result) \ + __result = __lll_robust_mutex_lock_wait (__result, __futex); \ + __result; }) + #define lll_mutex_timedlock(futex, timeout) \ ({ int __result, val, *__futex = &(futex); \ __asm __volatile ("\ @@ -141,6 +201,26 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden; __result = __lll_mutex_timedlock_wait (__result, __futex, timeout); \ __result; }) +#define lll_robust_mutex_timedlock(futex, timeout, id) \ + ({ int __result, val, *__futex = &(futex); \ + __asm __volatile ("\ + .align 2\n\ + mova 1f,r0\n\ + nop\n\ + mov r15,r1\n\ + mov #-8,r15\n\ + 0: mov.l @%2,%0\n\ + tst %0,%0\n\ + bf 1f\n\ + mov.l %1,@%2\n\ + 1: mov r1,r15"\ + : "=&r" (__result) : "r" (id), "r" (__futex) \ + : "r0", "r1", "t", "memory"); \ + if (__result) \ + __result = __lll_robust_mutex_timedlock_wait (__result, __futex, \ + timeout); \ + __result; }) + #define lll_mutex_unlock(futex) \ (void) ({ int __result, *__futex = &(futex); \ __asm __volatile ("\ @@ -157,6 +237,37 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden; if (__result) \ __lll_mutex_unlock_wake (__futex); }) +#define lll_robust_mutex_unlock(futex) \ + (void) ({ int __result, *__futex = &(futex); \ + __asm __volatile ("\ + .align 2\n\ + mova 1f,r0\n\ + mov r15,r1\n\ + mov #-6,r15\n\ + 0: mov.l @%1,%0\n\ + and %2,%0\n\ + mov.l %0,@%1\n\ + 1: mov r1,r15"\ + : "=&r" (__result) : "r" (__futex), "r" (FUTEX_TID_MASK) \ + : "r0", "r1", "memory"); \ + if (__result) \ + __lll_mutex_unlock_wake (__futex); }) + +#define lll_robust_mutex_dead(futex) \ + (void) ({ int __ignore, *__futex = &(futex); \ + __asm __volatile ("\ + .align 2\n\ + mova 1f,r0\n\ + mov r15,r1\n\ + mov #-6,r15\n\ + 0: mov.l @%1,%0\n\ + or %2,%0\n\ + mov.l %0,@%1\n\ + 1: mov r1,r15"\ + : "=&r" (__ignore) : "r" (__futex), "r" (FUTEX_OWNER_DIED) \ + : "r0", "r1", "memory"); \ + lll_futex_wake (__futex, 1); }) + #define lll_mutex_islocked(futex) \ (futex != 0) diff --git a/nptl/sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S new file mode 100644 index 0000000000..c57d3cff18 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/sh/lowlevelrobustlock.S @@ -0,0 +1,224 @@ +/* Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include <sysdep.h> +#include <pthread-errnos.h> +#include <lowlevelrobustlock.h> +#include "lowlevel-atomic.h" + + .text + +#define SYS_gettimeofday __NR_gettimeofday +#define SYS_futex 240 +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_WAITERS 0x80000000 +#define FUTEX_OWNER_DIED 0x40000000 + + + .globl __lll_robust_mutex_lock_wait + .type __lll_robust_mutex_lock_wait,@function + .hidden __lll_robust_mutex_lock_wait + .align 5 + cfi_startproc +__lll_robust_mutex_lock_wait: + mov.l r8, @-r15 + cfi_adjust_cfa_offset(4) + cfi_rel_offset (r8, 0) + mov r5, r8 + mov #0, r7 /* No timeout. */ + mov #FUTEX_WAIT, r5 + +4: + mov r4, r6 + mov.l .L_FUTEX_WAITERS, r0 + or r0, r6 + shlr r0 /* r0 = FUTEX_OWNER_DIED */ + tst r0, r4 + bf/s 3f + cmp/eq r4, r6 + bt 1f + + CMPXCHG (r4, @r8, r6, r2) + bf 2f + +1: + mov r8, r4 + mov #SYS_futex, r3 + extu.b r3, r3 + trapa #0x14 + SYSCALL_INST_PAD + + mov.l @r8, r2 + +2: + tst r2, r2 + bf/s 4b + mov r2, r4 + + stc gbr, r1 + mov.w .Ltidoff, r2 + add r2, r1 + mov.l @r1, r6 + mov #0, r3 + CMPXCHG (r3, @r8, r6, r4) + bf 4b + mov #0, r4 + +3: + mov.l @r15+, r8 + ret + mov r4, r0 + cfi_endproc + .align 2 +.L_FUTEX_WAITERS: + .long FUTEX_WAITERS +.Ltidoff: + .word TID - TLS_PRE_TCB_SIZE + .size __lll_robust_mutex_lock_wait,.-__lll_robust_mutex_lock_wait + + + .globl __lll_robust_mutex_timedlock_wait + .type __lll_robust_mutex_timedlock_wait,@function + .hidden __lll_robust_mutex_timedlock_wait + .align 5 + cfi_startproc +__lll_robust_mutex_timedlock_wait: + /* Check for a valid timeout value. */ + mov.l @(4,r6), r1 + mov.l .L1g, r0 + cmp/hs r0, r1 + bt 3f + + mov.l r10, @-r15 + cfi_adjust_cfa_offset(4) + cfi_rel_offset (r10, 0) + mov.l r9, @-r15 + cfi_adjust_cfa_offset(4) + cfi_rel_offset (r9, 0) + mov.l r8, @-r15 + cfi_adjust_cfa_offset(4) + cfi_rel_offset (r8, 0) + mov r4, r10 + mov r6, r9 + mov r5, r8 + + /* Stack frame for the timespec and timeval structs. */ + add #-8, r15 + cfi_adjust_cfa_offset(8) + +1: + /* Get current time. */ + mov r15, r4 + mov #0, r5 + mov #SYS_gettimeofday, r3 + trapa #0x12 + SYSCALL_INST_PAD + + /* Compute relative timeout. */ + mov.l @(4,r15), r0 + mov.w .L1k, r1 + dmulu.l r0, r1 /* Micro seconds to nano seconds. */ + mov.l @r9, r2 + mov.l @(4,r9), r3 + mov.l @r15, r0 + sts macl, r1 + sub r0, r2 + clrt + subc r1, r3 + bf 4f + mov.l .L1g, r1 + add r1, r3 + add #-1, r2 +4: + cmp/pz r2 + bf 8f /* Time is already up. */ + + mov.l r2, @r15 /* Store relative timeout. */ + mov.l r3, @(4,r15) + + mov r10, r6 + mov.l .L_FUTEX_WAITERS2, r0 + or r0, r6 + shlr r0 /* r0 = FUTEX_OWNER_DIED */ + tst r0, r4 + bf/s 6f + cmp/eq r4, r6 + bt 2f + + CMPXCHG (r4, @r8, r6, r2) + bf/s 5f + mov #0, r5 + +2: + mov r8, r4 + mov #FUTEX_WAIT, r5 + mov r10, r6 + mov r15, r7 + mov #SYS_futex, r3 + extu.b r3, r3 + trapa #0x14 + SYSCALL_INST_PAD + mov r0, r5 + + mov.l @r8, r2 + +5: + tst r2, r2 + bf/s 7f + mov r2, r10 + + stc gbr, r1 + mov.w .Ltidoff2, r2 + add r2, r1 + mov.l @r1, r4 + mov #0, r3 + CMPXCHG (r3, @r8, r4, r10) + bf 7f + mov #0, r0 + +6: + add #8, r15 + mov.l @r15+, r8 + mov.l @r15+, r9 + rts + mov.l @r15+, r10 + +7: + /* Check whether the time expired. */ + mov #-ETIMEDOUT, r1 + cmp/eq r5, r1 + bf 1b + +8: + bra 6b + mov #ETIMEDOUT, r0 +3: + rts + mov #EINVAL, r0 + cfi_endproc + .align 2 +.L_FUTEX_WAITERS2: + .long FUTEX_WAITERS +.L1g: + .long 1000000000 +.Ltidoff2: + .word TID - TLS_PRE_TCB_SIZE +.L1k: + .word 1000 + .size __lll_robust_mutex_timedlock_wait,.-__lll_robust_mutex_timedlock_wait diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index ebcfe6eef1..95e6923d4d 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -111,6 +111,16 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; ret; }) +#define lll_robust_mutex_trylock(futex, id) \ + ({ int ret; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ + : "=a" (ret), "=m" (futex) \ + : "r" (id), "m" (futex), \ + "0" (LLL_MUTEX_LOCK_INITIALIZER) \ + : "memory"); \ + ret; }) + + #define lll_mutex_cond_trylock(futex) \ ({ int ret; \ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ @@ -139,6 +149,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; : "cx", "r11", "cc", "memory"); }) +#define lll_robust_mutex_lock(futex, id) \ + ({ int result, ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %2, %%rdi\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_robust_mutex_lock_wait\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ + "=a" (result) \ + : "0" (id), "m" (futex), "3" (0) \ + : "cx", "r11", "cc", "memory"); \ + result; }) + + #define lll_mutex_cond_lock(futex) \ (void) ({ int ignore1, ignore2, ignore3; \ __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ @@ -157,6 +186,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; : "cx", "r11", "cc", "memory"); }) +#define lll_robust_mutex_cond_lock(futex, id) \ + ({ int result, ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %2, %%rdi\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_robust_mutex_lock_wait\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ + "=a" (result) \ + : "0" (id | FUTEX_WAITERS), "m" (futex), "3" (0) \ + : "cx", "r11", "cc", "memory"); \ + result; }) + + #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2, ignore3; \ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t" \ @@ -177,6 +225,26 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; result; }) +#define lll_robust_mutex_timedlock(futex, timeout, id) \ + ({ int result, ignore1, ignore2, ignore3; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %4, %%rdi\n\t" \ + "movq %8, %%rdx\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_robust_mutex_timedlock_wait\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=a" (result), "=&D" (ignore1), "=S" (ignore2), \ + "=&d" (ignore3), "=m" (futex) \ + : "0" (0), "2" (id), "m" (futex), "m" (timeout) \ + : "memory", "cx", "cc", "r10", "r11"); \ + result; }) + + #define lll_mutex_unlock(futex) \ (void) ({ int ignore; \ __asm __volatile (LOCK_INSTR "decl %0\n\t" \ @@ -194,6 +262,34 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; : "ax", "cx", "r11", "cc", "memory"); }) +#define lll_robust_mutex_unlock(futex) \ + (void) ({ int ignore; \ + __asm __volatile (LOCK_INSTR "andl %2, %0\n\t" \ + "jne 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %0, %%rdi\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_mutex_unlock_wake\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=m" (futex), "=&D" (ignore) \ + : "i" (FUTEX_WAITERS), "m" (futex) \ + : "ax", "cx", "r11", "cc", "memory"); }) + + +#define lll_robust_mutex_dead(futex) \ + (void) ({ int ignore; \ + __asm __volatile (LOCK_INSTR "orl %3, (%2)\n\t" \ + "syscall" \ + : "=m" (futex), "=a" (ignore) \ + : "D" (&(futex)), "i" (FUTEX_OWNER_DIED), \ + "S" (FUTEX_WAKE), "1" (__NR_futex), \ + "d" (1) \ + : "cx", "r11", "cc", "memory"); }) + + #define lll_mutex_islocked(futex) \ (futex != LLL_MUTEX_LOCK_INITIALIZER) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S new file mode 100644 index 0000000000..7bb9191691 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S @@ -0,0 +1,192 @@ +/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include <sysdep.h> +#include <pthread-errnos.h> +#include <lowlevelrobustlock.h> + + .text + +#ifndef LOCK +# ifdef UP +# define LOCK +# else +# define LOCK lock +# endif +#endif + +#define SYS_futex 202 +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_WAITERS 0x80000000 +#define FUTEX_OWNER_DIED 0x40000000 + +/* For the calculation see asm/vsyscall.h. */ +#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 + + + .globl __lll_robust_mutex_lock_wait + .type __lll_robust_mutex_lock_wait,@function + .hidden __lll_robust_mutex_lock_wait + .align 16 +__lll_robust_mutex_lock_wait: + pushq %r10 + pushq %rdx + + xorq %r10, %r10 /* No timeout. */ +#if FUTEX_WAIT == 0 + xorl %esi, %esi +#else + movl $FUTEX_WAIT, %esi +#endif + +4: movl %eax, %edx + orl $FUTEX_WAITERS, %edx + + testl $FUTEX_OWNER_DIED, %eax + jnz 3f + + cmpl %edx, %eax + je 1f + + LOCK + cmpxchgl %edx, (%rdi) + jnz 2f + +1: movl $SYS_futex, %eax + syscall + + movl (%rdi), %eax + +2: testl %eax, %eax + jne 4b + + movl %fs:TID, %edx + LOCK + cmpxchgl %edx, (%rdi) + jnz 4b + /* NB: %rax == 0 */ + +3: popq %rdx + popq %r10 + retq + .size __lll_robust_mutex_lock_wait,.-__lll_robust_mutex_lock_wait + + + .globl __lll_robust_mutex_timedlock_wait + .type __lll_robust_mutex_timedlock_wait,@function + .hidden __lll_robust_mutex_timedlock_wait + .align 16 +__lll_robust_mutex_timedlock_wait: + /* Check for a valid timeout value. */ + cmpq $1000000000, 8(%rdx) + jae 3f + + pushq %r8 + pushq %r9 + pushq %r12 + pushq %r13 + + /* Stack frame for the timespec and timeval structs. */ + subq $24, %rsp + + movq %rdi, %r12 + movq %rdx, %r13 + +1: movq %rax, 16(%rsp) + + /* Get current time. */ + movq %rsp, %rdi + xorl %esi, %esi + movq $VSYSCALL_ADDR_vgettimeofday, %rax + /* This is a regular function call, all caller-save registers + might be clobbered. */ + callq *%rax + + /* Compute relative timeout. */ + movq 8(%rsp), %rax + movl $1000, %edi + mul %rdi /* Milli seconds to nano seconds. */ + movq (%r13), %rdi + movq 8(%r13), %rsi + subq (%rsp), %rdi + subq %rax, %rsi + jns 4f + addq $1000000000, %rsi + decq %rdi +4: testq %rdi, %rdi + js 8f /* Time is already up. */ + + /* Futex call. */ + movq %rdi, (%rsp) /* Store relative timeout. */ + movq %rsi, 8(%rsp) + + movq 16(%rsp), %rdx + movl %edx, %eax + orl $FUTEX_WAITERS, %edx + + testl $FUTEX_OWNER_DIED, %eax + jnz 6f + + cmpl %eax, %edx + je 2f + + LOCK + cmpxchgl %edx, (%r12) + movq $0, %rcx /* Must use mov to avoid changing cc. */ + jnz 5f + +2: movq %rsp, %r10 +#if FUTEX_WAIT == 0 + xorl %esi, %esi +#else + movl $FUTEX_WAIT, %esi +#endif + movq %r12, %rdi + movl $SYS_futex, %eax + syscall + movq %rax, %rcx + + movl (%r12), %eax + +5: testl %eax, %eax + jne 7f + + movl %fs:TID, %edx + LOCK + cmpxchgl %edx, (%r12) + jnz 7f + +6: addq $24, %rsp + popq %r13 + popq %r12 + popq %r9 + popq %r8 + retq + + /* Check whether the time expired. */ +7: cmpq $-ETIMEDOUT, %rcx + jne 1b + +8: movl $ETIMEDOUT, %eax + jmp 6b + +3: movl $EINVAL, %eax + retq + .size __lll_robust_mutex_timedlock_wait,.-__lll_robust_mutex_timedlock_wait |