diff options
Diffstat (limited to 'nptl')
| -rw-r--r-- | nptl/allocatestack.c | 16 | ||||
| -rw-r--r-- | nptl/cancellation.c | 4 | ||||
| -rw-r--r-- | nptl/nptl-init.c | 5 | ||||
| -rw-r--r-- | nptl/pthread_barrier_init.c | 14 | ||||
| -rw-r--r-- | nptl/pthread_barrier_wait.c | 16 | ||||
| -rw-r--r-- | nptl/pthread_barrierattr_setpshared.c | 7 | ||||
| -rw-r--r-- | nptl/pthread_condattr_setpshared.c | 7 | ||||
| -rw-r--r-- | nptl/pthread_create.c | 14 | ||||
| -rw-r--r-- | nptl/pthread_mutexattr_setpshared.c | 7 | ||||
| -rw-r--r-- | nptl/pthread_once.c | 12 | ||||
| -rw-r--r-- | nptl/pthread_rwlock_init.c | 7 | ||||
| -rw-r--r-- | nptl/pthread_rwlock_rdlock.c | 18 | ||||
| -rw-r--r-- | nptl/pthread_rwlock_timedrdlock.c | 57 | ||||
| -rw-r--r-- | nptl/pthread_rwlock_timedwrlock.c | 56 | ||||
| -rw-r--r-- | nptl/pthread_rwlock_tryrdlock.c | 6 | ||||
| -rw-r--r-- | nptl/pthread_rwlock_unlock.c | 11 | ||||
| -rw-r--r-- | nptl/pthread_rwlock_wrlock.c | 11 | ||||
| -rw-r--r-- | nptl/pthread_rwlockattr_setpshared.c | 7 | ||||
| -rw-r--r-- | nptl/sem_init.c | 27 | ||||
| -rw-r--r-- | nptl/sem_open.c | 11 | ||||
| -rw-r--r-- | nptl/sem_post.c | 28 | ||||
| -rw-r--r-- | nptl/sem_wait.c | 1 | ||||
| -rw-r--r-- | nptl/sem_waitcommon.c | 115 | ||||
| -rw-r--r-- | nptl/unregister-atfork.c | 3 |
24 files changed, 142 insertions, 318 deletions
diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c index 8e620c46e9..c56a4df12c 100644 --- a/nptl/allocatestack.c +++ b/nptl/allocatestack.c @@ -29,6 +29,7 @@ #include <tls.h> #include <list.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <kernel-features.h> #include <stack-aliasing.h> @@ -987,7 +988,7 @@ setxid_mark_thread (struct xid_command *cmdp, struct pthread *t) if (t->setxid_futex == -1 && ! atomic_compare_and_exchange_bool_acq (&t->setxid_futex, -2, -1)) do - lll_futex_wait (&t->setxid_futex, -2, LLL_PRIVATE); + futex_wait_simple (&t->setxid_futex, -2, FUTEX_PRIVATE); while (t->setxid_futex == -2); /* Don't let the thread exit before the setxid handler runs. */ @@ -1005,7 +1006,7 @@ setxid_mark_thread (struct xid_command *cmdp, struct pthread *t) if ((ch & SETXID_BITMASK) == 0) { t->setxid_futex = 1; - lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE); } return; } @@ -1032,7 +1033,7 @@ setxid_unmark_thread (struct xid_command *cmdp, struct pthread *t) /* Release the futex just in case. */ t->setxid_futex = 1; - lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE); } @@ -1141,7 +1142,8 @@ __nptl_setxid (struct xid_command *cmdp) int cur = cmdp->cntr; while (cur != 0) { - lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE); + futex_wait_simple ((unsigned int *) &cmdp->cntr, cur, + FUTEX_PRIVATE); cur = cmdp->cntr; } } @@ -1251,7 +1253,8 @@ __wait_lookup_done (void) continue; do - lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE); + futex_wait_simple ((unsigned int *) gscope_flagp, + THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE); while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); } @@ -1273,7 +1276,8 @@ __wait_lookup_done (void) continue; do - lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE); + futex_wait_simple ((unsigned int *) gscope_flagp, + THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE); while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); } diff --git a/nptl/cancellation.c b/nptl/cancellation.c index deac1ebb3e..2bd31686fd 100644 --- a/nptl/cancellation.c +++ b/nptl/cancellation.c @@ -19,6 +19,7 @@ #include <setjmp.h> #include <stdlib.h> #include "pthreadP.h" +#include <futex-internal.h> /* The next two functions are similar to pthread_setcanceltype() but @@ -93,7 +94,8 @@ __pthread_disable_asynccancel (int oldtype) while (__builtin_expect ((newval & (CANCELING_BITMASK | CANCELED_BITMASK)) == CANCELING_BITMASK, 0)) { - lll_futex_wait (&self->cancelhandling, newval, LLL_PRIVATE); + futex_wait_simple ((unsigned int *) &self->cancelhandling, newval, + FUTEX_PRIVATE); newval = THREAD_GETMEM (self, cancelhandling); } } diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c index 8a511610cd..c043fb50ab 100644 --- a/nptl/nptl-init.c +++ b/nptl/nptl-init.c @@ -34,6 +34,7 @@ #include <shlib-compat.h> #include <smp.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <kernel-features.h> #include <libc-internal.h> #include <pthread-pids.h> @@ -279,10 +280,10 @@ sighandler_setxid (int sig, siginfo_t *si, void *ctx) /* And release the futex. */ self->setxid_futex = 1; - lll_futex_wake (&self->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&self->setxid_futex, 1, FUTEX_PRIVATE); if (atomic_decrement_val (&__xidcmd->cntr) == 0) - lll_futex_wake (&__xidcmd->cntr, 1, LLL_PRIVATE); + futex_wake ((unsigned int *) &__xidcmd->cntr, 1, FUTEX_PRIVATE); } #endif diff --git a/nptl/pthread_barrier_init.c b/nptl/pthread_barrier_init.c index 5ea9fad291..8fe15ecba5 100644 --- a/nptl/pthread_barrier_init.c +++ b/nptl/pthread_barrier_init.c @@ -36,6 +36,7 @@ __pthread_barrier_init (barrier, attr, count) { struct pthread_barrier *ibarrier; + /* XXX EINVAL is not specified by POSIX as a possible error code. */ if (__glibc_unlikely (count == 0)) return EINVAL; @@ -44,11 +45,6 @@ __pthread_barrier_init (barrier, attr, count) ? iattr = (struct pthread_barrierattr *) attr : &default_barrierattr); - if (iattr->pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (iattr->pshared != PTHREAD_PROCESS_SHARED, 0)) - /* Invalid attribute. */ - return EINVAL; - ibarrier = (struct pthread_barrier *) barrier; /* Initialize the individual fields. */ @@ -57,14 +53,10 @@ __pthread_barrier_init (barrier, attr, count) ibarrier->init_count = count; ibarrier->curr_event = 0; -#ifdef __ASSUME_PRIVATE_FUTEX + /* XXX Don't use FUTEX_SHARED or FUTEX_PRIVATE as long as there are still + assembly implementations that expect the value determined below. */ ibarrier->private = (iattr->pshared != PTHREAD_PROCESS_PRIVATE ? 0 : FUTEX_PRIVATE_FLAG); -#else - ibarrier->private = (iattr->pshared != PTHREAD_PROCESS_PRIVATE - ? 0 : THREAD_GETMEM (THREAD_SELF, - header.private_futex)); -#endif return 0; } diff --git a/nptl/pthread_barrier_wait.c b/nptl/pthread_barrier_wait.c index d69a929ef6..2b34e3097b 100644 --- a/nptl/pthread_barrier_wait.c +++ b/nptl/pthread_barrier_wait.c @@ -19,6 +19,7 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthreadP.h> @@ -29,9 +30,12 @@ __pthread_barrier_wait (barrier) { struct pthread_barrier *ibarrier = (struct pthread_barrier *) barrier; int result = 0; + int lll_private = ibarrier->private ^ FUTEX_PRIVATE_FLAG; + int futex_private = (lll_private == LLL_PRIVATE + ? FUTEX_PRIVATE : FUTEX_SHARED); /* Make sure we are alone. */ - lll_lock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG); + lll_lock (ibarrier->lock, lll_private); /* One more arrival. */ --ibarrier->left; @@ -44,8 +48,7 @@ __pthread_barrier_wait (barrier) ++ibarrier->curr_event; /* Wake up everybody. */ - lll_futex_wake (&ibarrier->curr_event, INT_MAX, - ibarrier->private ^ FUTEX_PRIVATE_FLAG); + futex_wake (&ibarrier->curr_event, INT_MAX, futex_private); /* This is the thread which finished the serialization. */ result = PTHREAD_BARRIER_SERIAL_THREAD; @@ -57,12 +60,11 @@ __pthread_barrier_wait (barrier) unsigned int event = ibarrier->curr_event; /* Before suspending, make the barrier available to others. */ - lll_unlock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG); + lll_unlock (ibarrier->lock, lll_private); /* Wait for the event counter of the barrier to change. */ do - lll_futex_wait (&ibarrier->curr_event, event, - ibarrier->private ^ FUTEX_PRIVATE_FLAG); + futex_wait_simple (&ibarrier->curr_event, event, futex_private); while (event == ibarrier->curr_event); } @@ -72,7 +74,7 @@ __pthread_barrier_wait (barrier) /* If this was the last woken thread, unlock. */ if (atomic_increment_val (&ibarrier->left) == init_count) /* We are done. */ - lll_unlock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG); + lll_unlock (ibarrier->lock, lll_private); return result; } diff --git a/nptl/pthread_barrierattr_setpshared.c b/nptl/pthread_barrierattr_setpshared.c index 86d72c5d87..eeaee5d430 100644 --- a/nptl/pthread_barrierattr_setpshared.c +++ b/nptl/pthread_barrierattr_setpshared.c @@ -18,6 +18,7 @@ #include <errno.h> #include "pthreadP.h" +#include <futex-internal.h> int @@ -27,9 +28,9 @@ pthread_barrierattr_setpshared (attr, pshared) { struct pthread_barrierattr *iattr; - if (pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0)) - return EINVAL; + int err = futex_supports_pshared (pshared); + if (err != 0) + return err; iattr = (struct pthread_barrierattr *) attr; diff --git a/nptl/pthread_condattr_setpshared.c b/nptl/pthread_condattr_setpshared.c index 3a760cfdf1..a015403cf5 100644 --- a/nptl/pthread_condattr_setpshared.c +++ b/nptl/pthread_condattr_setpshared.c @@ -18,15 +18,16 @@ #include <errno.h> #include <pthreadP.h> +#include <futex-internal.h> int pthread_condattr_setpshared (attr, pshared) pthread_condattr_t *attr; int pshared; { - if (pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0)) - return EINVAL; + int err = futex_supports_pshared (pshared); + if (err != 0) + return err; int *valuep = &((struct pthread_condattr *) attr)->value; diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c index 71a56193e6..d10f4ea800 100644 --- a/nptl/pthread_create.c +++ b/nptl/pthread_create.c @@ -31,6 +31,7 @@ #include <kernel-features.h> #include <exit-thread.h> #include <default-sched.h> +#include <futex-internal.h> #include <shlib-compat.h> @@ -269,7 +270,7 @@ START_THREAD_DEFN /* Allow setxid from now onwards. */ if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2)) - lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE); #ifdef __NR_set_robust_list # ifndef __ASSUME_SET_ROBUST_LIST @@ -414,7 +415,8 @@ START_THREAD_DEFN this->__list.__next = NULL; atomic_or (&this->__lock, FUTEX_OWNER_DIED); - lll_futex_wake (&this->__lock, 1, /* XYZ */ LLL_SHARED); + futex_wake ((unsigned int *) &this->__lock, 1, + /* XYZ */ FUTEX_SHARED); } while (robust != (void *) &pd->robust_head); } @@ -442,7 +444,11 @@ START_THREAD_DEFN /* Some other thread might call any of the setXid functions and expect us to reply. In this case wait until we did that. */ do - lll_futex_wait (&pd->setxid_futex, 0, LLL_PRIVATE); + /* XXX This differs from the typical futex_wait_simple pattern in that + the futex_wait condition (setxid_futex) is different from the + condition used in the surrounding loop (cancelhandling). We need + to check and document why this is correct. */ + futex_wait_simple (&pd->setxid_futex, 0, FUTEX_PRIVATE); while (pd->cancelhandling & SETXID_BITMASK); /* Reset the value so that the stack can be reused. */ @@ -683,7 +689,7 @@ __pthread_create_2_1 (newthread, attr, start_routine, arg) stillborn thread. */ if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2)) - lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE); /* Free the resources. */ __deallocate_stack (pd); diff --git a/nptl/pthread_mutexattr_setpshared.c b/nptl/pthread_mutexattr_setpshared.c index 77ee9f794f..62fd168265 100644 --- a/nptl/pthread_mutexattr_setpshared.c +++ b/nptl/pthread_mutexattr_setpshared.c @@ -18,6 +18,7 @@ #include <errno.h> #include <pthreadP.h> +#include <futex-internal.h> int @@ -27,9 +28,9 @@ pthread_mutexattr_setpshared (attr, pshared) { struct pthread_mutexattr *iattr; - if (pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0)) - return EINVAL; + int err = futex_supports_pshared (pshared); + if (err != 0) + return err; iattr = (struct pthread_mutexattr *) attr; diff --git a/nptl/pthread_once.c b/nptl/pthread_once.c index fe6d923794..3c5bc33622 100644 --- a/nptl/pthread_once.c +++ b/nptl/pthread_once.c @@ -17,7 +17,7 @@ <http://www.gnu.org/licenses/>. */ #include "pthreadP.h" -#include <lowlevellock.h> +#include <futex-internal.h> #include <atomic.h> @@ -35,7 +35,7 @@ clear_once_control (void *arg) get interrupted (see __pthread_once), so all we need to relay to other threads is the state being reset again. */ atomic_store_relaxed (once_control, 0); - lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE); + futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE); } @@ -100,8 +100,10 @@ __pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void)) is set and __PTHREAD_ONCE_DONE is not. */ if (val == newval) { - /* Same generation, some other thread was faster. Wait. */ - lll_futex_wait (once_control, newval, LLL_PRIVATE); + /* Same generation, some other thread was faster. Wait and + retry. */ + futex_wait_simple ((unsigned int *) once_control, + (unsigned int) newval, FUTEX_PRIVATE); continue; } } @@ -122,7 +124,7 @@ __pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void)) atomic_store_release (once_control, __PTHREAD_ONCE_DONE); /* Wake up all other threads. */ - lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE); + futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE); break; } diff --git a/nptl/pthread_rwlock_init.c b/nptl/pthread_rwlock_init.c index 23ee6c659d..99ab5e35cb 100644 --- a/nptl/pthread_rwlock_init.c +++ b/nptl/pthread_rwlock_init.c @@ -58,15 +58,8 @@ __pthread_rwlock_init (rwlock, attr) If the pshared value is in locking functions XORed with avail we get the expected result. */ -#ifdef __ASSUME_PRIVATE_FUTEX rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE ? 0 : FUTEX_PRIVATE_FLAG); -#else - rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE - ? 0 - : THREAD_GETMEM (THREAD_SELF, - header.private_futex)); -#endif return 0; } diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c index 004a386fd5..eb7ac8d226 100644 --- a/nptl/pthread_rwlock_rdlock.c +++ b/nptl/pthread_rwlock_rdlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> @@ -32,6 +33,8 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock) { int result = 0; bool wake = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; /* Lock is taken in caller. */ @@ -60,9 +63,10 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock) /* Free the lock. */ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - /* Wait for the writer to finish. */ - lll_futex_wait (&rwlock->__data.__readers_wakeup, waitval, - rwlock->__data.__shared); + /* Wait for the writer to finish. We do not check the return value + because we decide how to continue based on the state of the rwlock. */ + futex_wait_simple (&rwlock->__data.__readers_wakeup, waitval, + futex_shared); /* Get the lock. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); @@ -103,8 +107,7 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock) lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); if (wake) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } @@ -117,6 +120,8 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock) { int result = 0; bool wake = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; LIBC_PROBE (rdlock_entry, 1, rwlock); @@ -164,8 +169,7 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock) lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); if (wake) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } diff --git a/nptl/pthread_rwlock_timedrdlock.c b/nptl/pthread_rwlock_timedrdlock.c index 63fb313762..93d235e9e1 100644 --- a/nptl/pthread_rwlock_timedrdlock.c +++ b/nptl/pthread_rwlock_timedrdlock.c @@ -19,10 +19,10 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <sys/time.h> -#include <kernel-features.h> #include <stdbool.h> @@ -34,6 +34,8 @@ pthread_rwlock_timedrdlock (rwlock, abstime) { int result = 0; bool wake = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; /* Make sure we are alone. */ lll_lock(rwlock->__data.__lock, rwlock->__data.__shared); @@ -91,38 +93,6 @@ pthread_rwlock_timedrdlock (rwlock, abstime) break; } - /* Work around the fact that the kernel rejects negative timeout values - despite them being valid. */ - if (__glibc_unlikely (abstime->tv_sec < 0)) - { - result = ETIMEDOUT; - break; - } - -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ - || !defined lll_futex_timed_wait_bitset) - /* Get the current time. So far we support only one clock. */ - struct timeval tv; - (void) __gettimeofday (&tv, NULL); - - /* Convert the absolute timeout value to a relative timeout. */ - struct timespec rt; - rt.tv_sec = abstime->tv_sec - tv.tv_sec; - rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000; - if (rt.tv_nsec < 0) - { - rt.tv_nsec += 1000000000; - --rt.tv_sec; - } - /* Did we already time out? */ - if (rt.tv_sec < 0) - { - /* Yep, return with an appropriate error. */ - result = ETIMEDOUT; - break; - } -#endif - /* Remember that we are a reader. */ if (++rwlock->__data.__nr_readers_queued == 0) { @@ -137,17 +107,11 @@ pthread_rwlock_timedrdlock (rwlock, abstime) /* Free the lock. */ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - /* Wait for the writer to finish. */ -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ - || !defined lll_futex_timed_wait_bitset) - err = lll_futex_timed_wait (&rwlock->__data.__readers_wakeup, - waitval, &rt, rwlock->__data.__shared); -#else - err = lll_futex_timed_wait_bitset (&rwlock->__data.__readers_wakeup, - waitval, abstime, - FUTEX_CLOCK_REALTIME, - rwlock->__data.__shared); -#endif + /* Wait for the writer to finish. We handle ETIMEDOUT below; on other + return values, we decide how to continue based on the state of the + rwlock. */ + err = futex_abstimed_wait (&rwlock->__data.__readers_wakeup, waitval, + abstime, futex_shared); /* Get the lock. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); @@ -155,7 +119,7 @@ pthread_rwlock_timedrdlock (rwlock, abstime) --rwlock->__data.__nr_readers_queued; /* Did the futex call time out? */ - if (err == -ETIMEDOUT) + if (err == ETIMEDOUT) { /* Yep, report it. */ result = ETIMEDOUT; @@ -167,8 +131,7 @@ pthread_rwlock_timedrdlock (rwlock, abstime) lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); if (wake) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } diff --git a/nptl/pthread_rwlock_timedwrlock.c b/nptl/pthread_rwlock_timedwrlock.c index c54253450b..615623a75d 100644 --- a/nptl/pthread_rwlock_timedwrlock.c +++ b/nptl/pthread_rwlock_timedwrlock.c @@ -19,10 +19,10 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <sys/time.h> -#include <kernel-features.h> #include <stdbool.h> @@ -34,6 +34,8 @@ pthread_rwlock_timedwrlock (rwlock, abstime) { int result = 0; bool wake_readers = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; /* Make sure we are alone. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); @@ -71,37 +73,6 @@ pthread_rwlock_timedwrlock (rwlock, abstime) break; } - /* Work around the fact that the kernel rejects negative timeout values - despite them being valid. */ - if (__glibc_unlikely (abstime->tv_sec < 0)) - { - result = ETIMEDOUT; - break; - } - -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ - || !defined lll_futex_timed_wait_bitset) - /* Get the current time. So far we support only one clock. */ - struct timeval tv; - (void) __gettimeofday (&tv, NULL); - - /* Convert the absolute timeout value to a relative timeout. */ - struct timespec rt; - rt.tv_sec = abstime->tv_sec - tv.tv_sec; - rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000; - if (rt.tv_nsec < 0) - { - rt.tv_nsec += 1000000000; - --rt.tv_sec; - } - /* Did we already time out? */ - if (rt.tv_sec < 0) - { - result = ETIMEDOUT; - break; - } -#endif - /* Remember that we are a writer. */ if (++rwlock->__data.__nr_writers_queued == 0) { @@ -116,17 +87,11 @@ pthread_rwlock_timedwrlock (rwlock, abstime) /* Free the lock. */ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - /* Wait for the writer or reader(s) to finish. */ -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ - || !defined lll_futex_timed_wait_bitset) - err = lll_futex_timed_wait (&rwlock->__data.__writer_wakeup, - waitval, &rt, rwlock->__data.__shared); -#else - err = lll_futex_timed_wait_bitset (&rwlock->__data.__writer_wakeup, - waitval, abstime, - FUTEX_CLOCK_REALTIME, - rwlock->__data.__shared); -#endif + /* Wait for the writer or reader(s) to finish. We handle ETIMEDOUT + below; on other return values, we decide how to continue based on + the state of the rwlock. */ + err = futex_abstimed_wait (&rwlock->__data.__writer_wakeup, waitval, + abstime, futex_shared); /* Get the lock. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); @@ -135,7 +100,7 @@ pthread_rwlock_timedwrlock (rwlock, abstime) --rwlock->__data.__nr_writers_queued; /* Did the futex call time out? */ - if (err == -ETIMEDOUT) + if (err == ETIMEDOUT) { result = ETIMEDOUT; /* If we prefer writers, it can have happened that readers blocked @@ -166,8 +131,7 @@ pthread_rwlock_timedwrlock (rwlock, abstime) /* Might be required after timeouts. */ if (wake_readers) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c index cde123fd5d..256188a14a 100644 --- a/nptl/pthread_rwlock_tryrdlock.c +++ b/nptl/pthread_rwlock_tryrdlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include "pthreadP.h" #include <lowlevellock.h> +#include <futex-internal.h> #include <elide.h> #include <stdbool.h> @@ -28,6 +29,8 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock) { int result = EBUSY; bool wake = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; if (ELIDE_TRYLOCK (rwlock->__data.__rwelision, rwlock->__data.__lock == 0 @@ -63,8 +66,7 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock) lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); if (wake) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c index d2ad4b0375..bdd115d6bd 100644 --- a/nptl/pthread_rwlock_unlock.c +++ b/nptl/pthread_rwlock_unlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> @@ -29,6 +30,9 @@ int __pthread_rwlock_unlock (pthread_rwlock_t *rwlock) { + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; + LIBC_PROBE (rwlock_unlock, 1, rwlock); if (ELIDE_UNLOCK (rwlock->__data.__writer == 0 @@ -51,16 +55,15 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock) { ++rwlock->__data.__writer_wakeup; lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - lll_futex_wake (&rwlock->__data.__writer_wakeup, 1, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__writer_wakeup, 1, futex_shared); return 0; } else if (rwlock->__data.__nr_readers_queued) { ++rwlock->__data.__readers_wakeup; lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, + futex_shared); return 0; } } diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c index 835a62f0eb..60fa909340 100644 --- a/nptl/pthread_rwlock_wrlock.c +++ b/nptl/pthread_rwlock_wrlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> @@ -30,6 +31,8 @@ static int __attribute__((noinline)) __pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock) { int result = 0; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; /* Caller has taken the lock. */ @@ -58,9 +61,11 @@ __pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock) /* Free the lock. */ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - /* Wait for the writer or reader(s) to finish. */ - lll_futex_wait (&rwlock->__data.__writer_wakeup, waitval, - rwlock->__data.__shared); + /* Wait for the writer or reader(s) to finish. We do not check the + return value because we decide how to continue based on the state of + the rwlock. */ + futex_wait_simple (&rwlock->__data.__writer_wakeup, waitval, + futex_shared); /* Get the lock. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); diff --git a/nptl/pthread_rwlockattr_setpshared.c b/nptl/pthread_rwlockattr_setpshared.c index 4b4b31093b..0369209a94 100644 --- a/nptl/pthread_rwlockattr_setpshared.c +++ b/nptl/pthread_rwlockattr_setpshared.c @@ -18,6 +18,7 @@ #include <errno.h> #include "pthreadP.h" +#include <futex-internal.h> int @@ -27,9 +28,9 @@ pthread_rwlockattr_setpshared (attr, pshared) { struct pthread_rwlockattr *iattr; - if (pshared != PTHREAD_PROCESS_SHARED - && __builtin_expect (pshared != PTHREAD_PROCESS_PRIVATE, 0)) - return EINVAL; + int err = futex_supports_pshared (pshared); + if (err != 0) + return err; iattr = (struct pthread_rwlockattr *) attr; diff --git a/nptl/sem_init.c b/nptl/sem_init.c index 575b661f62..bd1b592420 100644 --- a/nptl/sem_init.c +++ b/nptl/sem_init.c @@ -21,22 +21,7 @@ #include <shlib-compat.h> #include "semaphoreP.h" #include <kernel-features.h> - -/* Returns FUTEX_PRIVATE if pshared is zero and private futexes are supported; - returns FUTEX_SHARED otherwise. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline int -futex_private_if_supported (int pshared) -{ - if (pshared != 0) - return LLL_SHARED; -#ifdef __ASSUME_PRIVATE_FUTEX - return LLL_PRIVATE; -#else - return THREAD_GETMEM (THREAD_SELF, header.private_futex) - ^ FUTEX_PRIVATE_FLAG; -#endif -} +#include <futex-internal.h> int @@ -48,6 +33,13 @@ __new_sem_init (sem_t *sem, int pshared, unsigned int value) __set_errno (EINVAL); return -1; } + pshared = pshared != 0 ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE; + int err = futex_supports_pshared (pshared); + if (err != 0) + { + __set_errno (err); + return -1; + } /* Map to the internal type. */ struct new_sem *isem = (struct new_sem *) sem; @@ -60,7 +52,8 @@ __new_sem_init (sem_t *sem, int pshared, unsigned int value) isem->nwaiters = 0; #endif - isem->private = futex_private_if_supported (pshared); + isem->private = (pshared == PTHREAD_PROCESS_PRIVATE + ? FUTEX_PRIVATE : FUTEX_SHARED); return 0; } diff --git a/nptl/sem_open.c b/nptl/sem_open.c index ecd051a547..5ee8b71aa5 100644 --- a/nptl/sem_open.c +++ b/nptl/sem_open.c @@ -30,6 +30,7 @@ #include <sys/stat.h> #include "semaphoreP.h" #include <shm-directory.h> +#include <futex-internal.h> /* Comparison function for search of existing mapping. */ @@ -141,6 +142,14 @@ sem_open (const char *name, int oflag, ...) int fd; sem_t *result; + /* Check that shared futexes are supported. */ + int err = futex_supports_pshared (PTHREAD_PROCESS_SHARED); + if (err != 0) + { + __set_errno (err); + return SEM_FAILED; + } + /* Create the name of the final file in local variable SHM_NAME. */ SHM_GET_NAME (EINVAL, SEM_FAILED, SEM_SHM_PREFIX); @@ -201,7 +210,7 @@ sem_open (const char *name, int oflag, ...) sem.newsem.nwaiters = 0; #endif /* This always is a shared semaphore. */ - sem.newsem.private = LLL_SHARED; + sem.newsem.private = FUTEX_SHARED; /* Initialize the remaining bytes as well. */ memset ((char *) &sem.initsem + sizeof (struct new_sem), '\0', diff --git a/nptl/sem_post.c b/nptl/sem_post.c index b6d30b514f..06d835907e 100644 --- a/nptl/sem_post.c +++ b/nptl/sem_post.c @@ -20,37 +20,13 @@ #include <atomic.h> #include <errno.h> #include <sysdep.h> -#include <lowlevellock.h> +#include <lowlevellock.h> /* lll_futex* used by the old code. */ +#include <futex-internal.h> #include <internaltypes.h> #include <semaphore.h> #include <shlib-compat.h> -/* Wrapper for lll_futex_wake, with error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline void -futex_wake (unsigned int* futex, int processes_to_wake, int private) -{ - int res = lll_futex_wake (futex, processes_to_wake, private); - /* No error. Ignore the number of woken processes. */ - if (res >= 0) - return; - switch (res) - { - case -EFAULT: /* Could have happened due to memory reuse. */ - case -EINVAL: /* Could be either due to incorrect alignment (a bug in - glibc or in the application) or due to memory being - reused for a PI futex. We cannot distinguish between the - two causes, and one of them is correct use, so we do not - act in this case. */ - return; - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - /* See sem_wait for an explanation of the algorithm. */ int diff --git a/nptl/sem_wait.c b/nptl/sem_wait.c index c1fd10c9d0..fce7ed43ee 100644 --- a/nptl/sem_wait.c +++ b/nptl/sem_wait.c @@ -17,6 +17,7 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ +#include <lowlevellock.h> /* lll_futex* used by the old code. */ #include "sem_waitcommon.c" int diff --git a/nptl/sem_waitcommon.c b/nptl/sem_waitcommon.c index 772425d33e..d3702c7b4f 100644 --- a/nptl/sem_waitcommon.c +++ b/nptl/sem_waitcommon.c @@ -20,7 +20,7 @@ #include <kernel-features.h> #include <errno.h> #include <sysdep.h> -#include <lowlevellock.h> +#include <futex-internal.h> #include <internaltypes.h> #include <semaphore.h> #include <sys/time.h> @@ -29,110 +29,6 @@ #include <shlib-compat.h> #include <atomic.h> -/* Wrapper for lll_futex_wait with absolute timeout and error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline int -futex_abstimed_wait (unsigned int* futex, unsigned int expected, - const struct timespec* abstime, int private, bool cancel) -{ - int err, oldtype; - if (abstime == NULL) - { - if (cancel) - oldtype = __pthread_enable_asynccancel (); - err = lll_futex_wait (futex, expected, private); - if (cancel) - __pthread_disable_asynccancel (oldtype); - } - else - { -#if (defined __ASSUME_FUTEX_CLOCK_REALTIME \ - && defined lll_futex_timed_wait_bitset) - /* The Linux kernel returns EINVAL for this, but in userspace - such a value is valid. */ - if (abstime->tv_sec < 0) - return ETIMEDOUT; -#else - struct timeval tv; - struct timespec rt; - int sec, nsec; - - /* Get the current time. */ - __gettimeofday (&tv, NULL); - - /* Compute relative timeout. */ - sec = abstime->tv_sec - tv.tv_sec; - nsec = abstime->tv_nsec - tv.tv_usec * 1000; - if (nsec < 0) - { - nsec += 1000000000; - --sec; - } - - /* Already timed out? */ - if (sec < 0) - return ETIMEDOUT; - - /* Do wait. */ - rt.tv_sec = sec; - rt.tv_nsec = nsec; -#endif - if (cancel) - oldtype = __pthread_enable_asynccancel (); -#if (defined __ASSUME_FUTEX_CLOCK_REALTIME \ - && defined lll_futex_timed_wait_bitset) - err = lll_futex_timed_wait_bitset (futex, expected, abstime, - FUTEX_CLOCK_REALTIME, private); -#else - err = lll_futex_timed_wait (futex, expected, &rt, private); -#endif - if (cancel) - __pthread_disable_asynccancel (oldtype); - } - switch (err) - { - case 0: - case -EAGAIN: - case -EINTR: - case -ETIMEDOUT: - return -err; - - case -EFAULT: /* Must have been caused by a glibc or application bug. */ - case -EINVAL: /* Either due to wrong alignment or due to the timeout not - being normalized. Must have been caused by a glibc or - application bug. */ - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - -/* Wrapper for lll_futex_wake, with error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline void -futex_wake (unsigned int* futex, int processes_to_wake, int private) -{ - int res = lll_futex_wake (futex, processes_to_wake, private); - /* No error. Ignore the number of woken processes. */ - if (res >= 0) - return; - switch (res) - { - case -EFAULT: /* Could have happened due to memory reuse. */ - case -EINVAL: /* Could be either due to incorrect alignment (a bug in - glibc or in the application) or due to memory being - reused for a PI futex. We cannot distinguish between the - two causes, and one of them is correct use, so we do not - act in this case. */ - return; - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - /* The semaphore provides two main operations: sem_post adds a token to the semaphore; sem_wait grabs a token from the semaphore, potentially waiting @@ -220,11 +116,12 @@ do_futex_wait (struct new_sem *sem, const struct timespec *abstime) int err; #if __HAVE_64B_ATOMICS - err = futex_abstimed_wait ((unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0, - abstime, sem->private, true); + err = futex_abstimed_wait_cancelable ( + (unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0, abstime, + sem->private); #else - err = futex_abstimed_wait (&sem->value, SEM_NWAITERS_MASK, abstime, - sem->private, true); + err = futex_abstimed_wait_cancelable (&sem->value, SEM_NWAITERS_MASK, + abstime, sem->private); #endif return err; diff --git a/nptl/unregister-atfork.c b/nptl/unregister-atfork.c index 3838cb7dee..6d08ed737e 100644 --- a/nptl/unregister-atfork.c +++ b/nptl/unregister-atfork.c @@ -20,6 +20,7 @@ #include <stdlib.h> #include <fork.h> #include <atomic.h> +#include <futex-internal.h> void @@ -114,7 +115,7 @@ __unregister_atfork (dso_handle) atomic_decrement (&deleted->handler->refcntr); unsigned int val; while ((val = deleted->handler->refcntr) != 0) - lll_futex_wait (&deleted->handler->refcntr, val, LLL_PRIVATE); + futex_wait_simple (&deleted->handler->refcntr, val, FUTEX_PRIVATE); deleted = deleted->next; } |