diff options
40 files changed, 963 insertions, 492 deletions
diff --git a/ChangeLog b/ChangeLog index 91c963200a..a8bab48b86 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,73 @@ +2015-07-10 Torvald Riegel <triegel@redhat.com> + + * sysdeps/nptl/futex-internal.h: New file. + * sysdeps/nacl/futex-internal.h: New file. + * sysdeps/unix/sysv/linux/futex-internal.h: New file. + * nptl/allocatestack.c (setxid_mark_thread): Use futex wrappers with + error checking. + (setxid_unmark_thread): Likewise. + (__nptl_setxid): Likewise. + (__wait_lookup_done): Likewise. + * nptl/cancellation.c (__pthread_disable_asynccancel): Likewise. + * nptl/nptl-init.c (sighandler_setxid): Likewise. + * nptl/pthread_create.c (START_THREAD_DEFN): Likewise. + * nptl/pthread_once.c (clear_once_control): Likewise. + (__pthread_once_slow): Likewise. + * nptl/unregister-atfork.c (__unregister_atfork): Likewise. + * sysdeps/nacl/exit-thread.h (__exit_thread): Likewise. + * sysdeps/nptl/aio_misc.h (AIO_MISC_NOTIFY, AIO_MISC_WAIT): Likewise. + * sysdeps/nptl/fork.c (__libc_fork): Likewise. + * sysdeps/nptl/gai_misc.h (GAI_MISC_NOTIFY, GAI_MISC_WAIT): Likewise. + * nptl/pthread_rwlock_rdlock.c (__pthread_rwlock_rdlock_slow): + Likewise. + (__pthread_rwlock_rdlock): Likewise. + * nptl/pthread_rwlock_timedrdlock.c (pthread_rwlock_timedrdlock): + Likewise. + * nptl/pthread_rwlock_timedwrlock.c (pthread_rwlock_timedwrlock): + Likewise. + * nptl/pthread_rwlock_tryrdlock.c (__pthread_rwlock_tryrdlock): + Likewise. + * nptl/pthread_rwlock_unlock.c (__pthread_rwlock_unlock): Likewise. + * nptl/pthread_rwlock_wrlock.c (__pthread_rwlock_wrlock_slow: + Likewise. + * nptl/pthread_rwlock_init (__pthread_rwlock_init): Remove + __ASSUME_PRIVATE_FUTEX check. + * nptl/pthread_rwlockattr_setpshared (pthread_rwlockattr_setpshared): + Check that shared futexes are supported. + * nptl/pthread_barrier_wait.c (pthread_barrier_wait): Use futex + wrappers with error checking. + * nptl/pthread_barrier_init.c (pthread_barrier_init): Add comments, + remove attribute sanity check and __ASSUME_PRIVATE_FUTEX check. + * nptl/pthread_barrierattr_setpshared.c + (pthread_barrierattr_setpshared): Check that shared futexes are + supported. + * nptl/pthread_condattr_setpshared.c (pthread_condattr_setpshared): + Likewise. + * nptl/pthread_mutexattr_setpshared.c (pthread_mutexattr_setpshared): + Likewise. + * nptl/sem_init.c (futex_private_if_supported): Remove. + (__new_sem_init): Adapt and check that shared futexes are supported. + * nptl/sem_open.c (sem_open): Likewise. + * nptl/sem_post.c (futex_wake): Remove. + * nptl/sem_waitcommon.c (futex_abstimed_wait, futex_wake): Remove. + (do_futex_wait): Use futex wrappers with error checking. + * nptl/sem_wait.c: Include lowlevellock.h. + * sysdeps/sparc/nptl/pthread_barrier_init.c (__pthread_barrier_init): + Use futex_supports_pshared. + * sysdeps/sparc/nptl/pthread_barrier_wait.c (pthread_barrier_wait): + Use futex wrappers with error checking. + * sysdeps/sparc/sparc32/pthread_barrier_wait.c (pthread_barrier_wait): + Likewise. + * sysdeps/sparc/sparc32/sem_init.c (futex_private_if_supported): Remove. + * sysdeps/sparc/sparc32/sem_post.c (futex_wake): Likewise. + * sysdeps/sparc/sparc32/sem_open.c (sem_open): Use FUTEX_SHARED. + * sysdeps/sparc/sparc32/sem_waitcommon.c (futex_abstimed_wait): Remove. + (futex_wake): Likewise. + (sem_assume_only_signals_cause_futex_EINTR): Likewise. + (do_futex_wait): Use futex wrappers with error checking. + (__new_sem_wait_slow): Update EINTR handling. + * sysdeps/sparc/sparc32/sem_wait.c: Include lowlevellock.h. + 2015-07-09 Martin Sebor <msebor@redhat.com> [BZ #18435] diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c index 8e620c46e9..c56a4df12c 100644 --- a/nptl/allocatestack.c +++ b/nptl/allocatestack.c @@ -29,6 +29,7 @@ #include <tls.h> #include <list.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <kernel-features.h> #include <stack-aliasing.h> @@ -987,7 +988,7 @@ setxid_mark_thread (struct xid_command *cmdp, struct pthread *t) if (t->setxid_futex == -1 && ! atomic_compare_and_exchange_bool_acq (&t->setxid_futex, -2, -1)) do - lll_futex_wait (&t->setxid_futex, -2, LLL_PRIVATE); + futex_wait_simple (&t->setxid_futex, -2, FUTEX_PRIVATE); while (t->setxid_futex == -2); /* Don't let the thread exit before the setxid handler runs. */ @@ -1005,7 +1006,7 @@ setxid_mark_thread (struct xid_command *cmdp, struct pthread *t) if ((ch & SETXID_BITMASK) == 0) { t->setxid_futex = 1; - lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE); } return; } @@ -1032,7 +1033,7 @@ setxid_unmark_thread (struct xid_command *cmdp, struct pthread *t) /* Release the futex just in case. */ t->setxid_futex = 1; - lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE); } @@ -1141,7 +1142,8 @@ __nptl_setxid (struct xid_command *cmdp) int cur = cmdp->cntr; while (cur != 0) { - lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE); + futex_wait_simple ((unsigned int *) &cmdp->cntr, cur, + FUTEX_PRIVATE); cur = cmdp->cntr; } } @@ -1251,7 +1253,8 @@ __wait_lookup_done (void) continue; do - lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE); + futex_wait_simple ((unsigned int *) gscope_flagp, + THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE); while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); } @@ -1273,7 +1276,8 @@ __wait_lookup_done (void) continue; do - lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE); + futex_wait_simple ((unsigned int *) gscope_flagp, + THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE); while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); } diff --git a/nptl/cancellation.c b/nptl/cancellation.c index deac1ebb3e..2bd31686fd 100644 --- a/nptl/cancellation.c +++ b/nptl/cancellation.c @@ -19,6 +19,7 @@ #include <setjmp.h> #include <stdlib.h> #include "pthreadP.h" +#include <futex-internal.h> /* The next two functions are similar to pthread_setcanceltype() but @@ -93,7 +94,8 @@ __pthread_disable_asynccancel (int oldtype) while (__builtin_expect ((newval & (CANCELING_BITMASK | CANCELED_BITMASK)) == CANCELING_BITMASK, 0)) { - lll_futex_wait (&self->cancelhandling, newval, LLL_PRIVATE); + futex_wait_simple ((unsigned int *) &self->cancelhandling, newval, + FUTEX_PRIVATE); newval = THREAD_GETMEM (self, cancelhandling); } } diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c index 8a511610cd..c043fb50ab 100644 --- a/nptl/nptl-init.c +++ b/nptl/nptl-init.c @@ -34,6 +34,7 @@ #include <shlib-compat.h> #include <smp.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <kernel-features.h> #include <libc-internal.h> #include <pthread-pids.h> @@ -279,10 +280,10 @@ sighandler_setxid (int sig, siginfo_t *si, void *ctx) /* And release the futex. */ self->setxid_futex = 1; - lll_futex_wake (&self->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&self->setxid_futex, 1, FUTEX_PRIVATE); if (atomic_decrement_val (&__xidcmd->cntr) == 0) - lll_futex_wake (&__xidcmd->cntr, 1, LLL_PRIVATE); + futex_wake ((unsigned int *) &__xidcmd->cntr, 1, FUTEX_PRIVATE); } #endif diff --git a/nptl/pthread_barrier_init.c b/nptl/pthread_barrier_init.c index 5ea9fad291..8fe15ecba5 100644 --- a/nptl/pthread_barrier_init.c +++ b/nptl/pthread_barrier_init.c @@ -36,6 +36,7 @@ __pthread_barrier_init (barrier, attr, count) { struct pthread_barrier *ibarrier; + /* XXX EINVAL is not specified by POSIX as a possible error code. */ if (__glibc_unlikely (count == 0)) return EINVAL; @@ -44,11 +45,6 @@ __pthread_barrier_init (barrier, attr, count) ? iattr = (struct pthread_barrierattr *) attr : &default_barrierattr); - if (iattr->pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (iattr->pshared != PTHREAD_PROCESS_SHARED, 0)) - /* Invalid attribute. */ - return EINVAL; - ibarrier = (struct pthread_barrier *) barrier; /* Initialize the individual fields. */ @@ -57,14 +53,10 @@ __pthread_barrier_init (barrier, attr, count) ibarrier->init_count = count; ibarrier->curr_event = 0; -#ifdef __ASSUME_PRIVATE_FUTEX + /* XXX Don't use FUTEX_SHARED or FUTEX_PRIVATE as long as there are still + assembly implementations that expect the value determined below. */ ibarrier->private = (iattr->pshared != PTHREAD_PROCESS_PRIVATE ? 0 : FUTEX_PRIVATE_FLAG); -#else - ibarrier->private = (iattr->pshared != PTHREAD_PROCESS_PRIVATE - ? 0 : THREAD_GETMEM (THREAD_SELF, - header.private_futex)); -#endif return 0; } diff --git a/nptl/pthread_barrier_wait.c b/nptl/pthread_barrier_wait.c index d69a929ef6..2b34e3097b 100644 --- a/nptl/pthread_barrier_wait.c +++ b/nptl/pthread_barrier_wait.c @@ -19,6 +19,7 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthreadP.h> @@ -29,9 +30,12 @@ __pthread_barrier_wait (barrier) { struct pthread_barrier *ibarrier = (struct pthread_barrier *) barrier; int result = 0; + int lll_private = ibarrier->private ^ FUTEX_PRIVATE_FLAG; + int futex_private = (lll_private == LLL_PRIVATE + ? FUTEX_PRIVATE : FUTEX_SHARED); /* Make sure we are alone. */ - lll_lock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG); + lll_lock (ibarrier->lock, lll_private); /* One more arrival. */ --ibarrier->left; @@ -44,8 +48,7 @@ __pthread_barrier_wait (barrier) ++ibarrier->curr_event; /* Wake up everybody. */ - lll_futex_wake (&ibarrier->curr_event, INT_MAX, - ibarrier->private ^ FUTEX_PRIVATE_FLAG); + futex_wake (&ibarrier->curr_event, INT_MAX, futex_private); /* This is the thread which finished the serialization. */ result = PTHREAD_BARRIER_SERIAL_THREAD; @@ -57,12 +60,11 @@ __pthread_barrier_wait (barrier) unsigned int event = ibarrier->curr_event; /* Before suspending, make the barrier available to others. */ - lll_unlock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG); + lll_unlock (ibarrier->lock, lll_private); /* Wait for the event counter of the barrier to change. */ do - lll_futex_wait (&ibarrier->curr_event, event, - ibarrier->private ^ FUTEX_PRIVATE_FLAG); + futex_wait_simple (&ibarrier->curr_event, event, futex_private); while (event == ibarrier->curr_event); } @@ -72,7 +74,7 @@ __pthread_barrier_wait (barrier) /* If this was the last woken thread, unlock. */ if (atomic_increment_val (&ibarrier->left) == init_count) /* We are done. */ - lll_unlock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG); + lll_unlock (ibarrier->lock, lll_private); return result; } diff --git a/nptl/pthread_barrierattr_setpshared.c b/nptl/pthread_barrierattr_setpshared.c index 86d72c5d87..eeaee5d430 100644 --- a/nptl/pthread_barrierattr_setpshared.c +++ b/nptl/pthread_barrierattr_setpshared.c @@ -18,6 +18,7 @@ #include <errno.h> #include "pthreadP.h" +#include <futex-internal.h> int @@ -27,9 +28,9 @@ pthread_barrierattr_setpshared (attr, pshared) { struct pthread_barrierattr *iattr; - if (pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0)) - return EINVAL; + int err = futex_supports_pshared (pshared); + if (err != 0) + return err; iattr = (struct pthread_barrierattr *) attr; diff --git a/nptl/pthread_condattr_setpshared.c b/nptl/pthread_condattr_setpshared.c index 3a760cfdf1..a015403cf5 100644 --- a/nptl/pthread_condattr_setpshared.c +++ b/nptl/pthread_condattr_setpshared.c @@ -18,15 +18,16 @@ #include <errno.h> #include <pthreadP.h> +#include <futex-internal.h> int pthread_condattr_setpshared (attr, pshared) pthread_condattr_t *attr; int pshared; { - if (pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0)) - return EINVAL; + int err = futex_supports_pshared (pshared); + if (err != 0) + return err; int *valuep = &((struct pthread_condattr *) attr)->value; diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c index 71a56193e6..d10f4ea800 100644 --- a/nptl/pthread_create.c +++ b/nptl/pthread_create.c @@ -31,6 +31,7 @@ #include <kernel-features.h> #include <exit-thread.h> #include <default-sched.h> +#include <futex-internal.h> #include <shlib-compat.h> @@ -269,7 +270,7 @@ START_THREAD_DEFN /* Allow setxid from now onwards. */ if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2)) - lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE); #ifdef __NR_set_robust_list # ifndef __ASSUME_SET_ROBUST_LIST @@ -414,7 +415,8 @@ START_THREAD_DEFN this->__list.__next = NULL; atomic_or (&this->__lock, FUTEX_OWNER_DIED); - lll_futex_wake (&this->__lock, 1, /* XYZ */ LLL_SHARED); + futex_wake ((unsigned int *) &this->__lock, 1, + /* XYZ */ FUTEX_SHARED); } while (robust != (void *) &pd->robust_head); } @@ -442,7 +444,11 @@ START_THREAD_DEFN /* Some other thread might call any of the setXid functions and expect us to reply. In this case wait until we did that. */ do - lll_futex_wait (&pd->setxid_futex, 0, LLL_PRIVATE); + /* XXX This differs from the typical futex_wait_simple pattern in that + the futex_wait condition (setxid_futex) is different from the + condition used in the surrounding loop (cancelhandling). We need + to check and document why this is correct. */ + futex_wait_simple (&pd->setxid_futex, 0, FUTEX_PRIVATE); while (pd->cancelhandling & SETXID_BITMASK); /* Reset the value so that the stack can be reused. */ @@ -683,7 +689,7 @@ __pthread_create_2_1 (newthread, attr, start_routine, arg) stillborn thread. */ if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2)) - lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE); + futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE); /* Free the resources. */ __deallocate_stack (pd); diff --git a/nptl/pthread_mutexattr_setpshared.c b/nptl/pthread_mutexattr_setpshared.c index 77ee9f794f..62fd168265 100644 --- a/nptl/pthread_mutexattr_setpshared.c +++ b/nptl/pthread_mutexattr_setpshared.c @@ -18,6 +18,7 @@ #include <errno.h> #include <pthreadP.h> +#include <futex-internal.h> int @@ -27,9 +28,9 @@ pthread_mutexattr_setpshared (attr, pshared) { struct pthread_mutexattr *iattr; - if (pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0)) - return EINVAL; + int err = futex_supports_pshared (pshared); + if (err != 0) + return err; iattr = (struct pthread_mutexattr *) attr; diff --git a/nptl/pthread_once.c b/nptl/pthread_once.c index fe6d923794..3c5bc33622 100644 --- a/nptl/pthread_once.c +++ b/nptl/pthread_once.c @@ -17,7 +17,7 @@ <http://www.gnu.org/licenses/>. */ #include "pthreadP.h" -#include <lowlevellock.h> +#include <futex-internal.h> #include <atomic.h> @@ -35,7 +35,7 @@ clear_once_control (void *arg) get interrupted (see __pthread_once), so all we need to relay to other threads is the state being reset again. */ atomic_store_relaxed (once_control, 0); - lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE); + futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE); } @@ -100,8 +100,10 @@ __pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void)) is set and __PTHREAD_ONCE_DONE is not. */ if (val == newval) { - /* Same generation, some other thread was faster. Wait. */ - lll_futex_wait (once_control, newval, LLL_PRIVATE); + /* Same generation, some other thread was faster. Wait and + retry. */ + futex_wait_simple ((unsigned int *) once_control, + (unsigned int) newval, FUTEX_PRIVATE); continue; } } @@ -122,7 +124,7 @@ __pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void)) atomic_store_release (once_control, __PTHREAD_ONCE_DONE); /* Wake up all other threads. */ - lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE); + futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE); break; } diff --git a/nptl/pthread_rwlock_init.c b/nptl/pthread_rwlock_init.c index 23ee6c659d..99ab5e35cb 100644 --- a/nptl/pthread_rwlock_init.c +++ b/nptl/pthread_rwlock_init.c @@ -58,15 +58,8 @@ __pthread_rwlock_init (rwlock, attr) If the pshared value is in locking functions XORed with avail we get the expected result. */ -#ifdef __ASSUME_PRIVATE_FUTEX rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE ? 0 : FUTEX_PRIVATE_FLAG); -#else - rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE - ? 0 - : THREAD_GETMEM (THREAD_SELF, - header.private_futex)); -#endif return 0; } diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c index 004a386fd5..eb7ac8d226 100644 --- a/nptl/pthread_rwlock_rdlock.c +++ b/nptl/pthread_rwlock_rdlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> @@ -32,6 +33,8 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock) { int result = 0; bool wake = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; /* Lock is taken in caller. */ @@ -60,9 +63,10 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock) /* Free the lock. */ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - /* Wait for the writer to finish. */ - lll_futex_wait (&rwlock->__data.__readers_wakeup, waitval, - rwlock->__data.__shared); + /* Wait for the writer to finish. We do not check the return value + because we decide how to continue based on the state of the rwlock. */ + futex_wait_simple (&rwlock->__data.__readers_wakeup, waitval, + futex_shared); /* Get the lock. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); @@ -103,8 +107,7 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock) lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); if (wake) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } @@ -117,6 +120,8 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock) { int result = 0; bool wake = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; LIBC_PROBE (rdlock_entry, 1, rwlock); @@ -164,8 +169,7 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock) lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); if (wake) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } diff --git a/nptl/pthread_rwlock_timedrdlock.c b/nptl/pthread_rwlock_timedrdlock.c index 63fb313762..93d235e9e1 100644 --- a/nptl/pthread_rwlock_timedrdlock.c +++ b/nptl/pthread_rwlock_timedrdlock.c @@ -19,10 +19,10 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <sys/time.h> -#include <kernel-features.h> #include <stdbool.h> @@ -34,6 +34,8 @@ pthread_rwlock_timedrdlock (rwlock, abstime) { int result = 0; bool wake = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; /* Make sure we are alone. */ lll_lock(rwlock->__data.__lock, rwlock->__data.__shared); @@ -91,38 +93,6 @@ pthread_rwlock_timedrdlock (rwlock, abstime) break; } - /* Work around the fact that the kernel rejects negative timeout values - despite them being valid. */ - if (__glibc_unlikely (abstime->tv_sec < 0)) - { - result = ETIMEDOUT; - break; - } - -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ - || !defined lll_futex_timed_wait_bitset) - /* Get the current time. So far we support only one clock. */ - struct timeval tv; - (void) __gettimeofday (&tv, NULL); - - /* Convert the absolute timeout value to a relative timeout. */ - struct timespec rt; - rt.tv_sec = abstime->tv_sec - tv.tv_sec; - rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000; - if (rt.tv_nsec < 0) - { - rt.tv_nsec += 1000000000; - --rt.tv_sec; - } - /* Did we already time out? */ - if (rt.tv_sec < 0) - { - /* Yep, return with an appropriate error. */ - result = ETIMEDOUT; - break; - } -#endif - /* Remember that we are a reader. */ if (++rwlock->__data.__nr_readers_queued == 0) { @@ -137,17 +107,11 @@ pthread_rwlock_timedrdlock (rwlock, abstime) /* Free the lock. */ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - /* Wait for the writer to finish. */ -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ - || !defined lll_futex_timed_wait_bitset) - err = lll_futex_timed_wait (&rwlock->__data.__readers_wakeup, - waitval, &rt, rwlock->__data.__shared); -#else - err = lll_futex_timed_wait_bitset (&rwlock->__data.__readers_wakeup, - waitval, abstime, - FUTEX_CLOCK_REALTIME, - rwlock->__data.__shared); -#endif + /* Wait for the writer to finish. We handle ETIMEDOUT below; on other + return values, we decide how to continue based on the state of the + rwlock. */ + err = futex_abstimed_wait (&rwlock->__data.__readers_wakeup, waitval, + abstime, futex_shared); /* Get the lock. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); @@ -155,7 +119,7 @@ pthread_rwlock_timedrdlock (rwlock, abstime) --rwlock->__data.__nr_readers_queued; /* Did the futex call time out? */ - if (err == -ETIMEDOUT) + if (err == ETIMEDOUT) { /* Yep, report it. */ result = ETIMEDOUT; @@ -167,8 +131,7 @@ pthread_rwlock_timedrdlock (rwlock, abstime) lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); if (wake) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } diff --git a/nptl/pthread_rwlock_timedwrlock.c b/nptl/pthread_rwlock_timedwrlock.c index c54253450b..615623a75d 100644 --- a/nptl/pthread_rwlock_timedwrlock.c +++ b/nptl/pthread_rwlock_timedwrlock.c @@ -19,10 +19,10 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <sys/time.h> -#include <kernel-features.h> #include <stdbool.h> @@ -34,6 +34,8 @@ pthread_rwlock_timedwrlock (rwlock, abstime) { int result = 0; bool wake_readers = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; /* Make sure we are alone. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); @@ -71,37 +73,6 @@ pthread_rwlock_timedwrlock (rwlock, abstime) break; } - /* Work around the fact that the kernel rejects negative timeout values - despite them being valid. */ - if (__glibc_unlikely (abstime->tv_sec < 0)) - { - result = ETIMEDOUT; - break; - } - -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ - || !defined lll_futex_timed_wait_bitset) - /* Get the current time. So far we support only one clock. */ - struct timeval tv; - (void) __gettimeofday (&tv, NULL); - - /* Convert the absolute timeout value to a relative timeout. */ - struct timespec rt; - rt.tv_sec = abstime->tv_sec - tv.tv_sec; - rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000; - if (rt.tv_nsec < 0) - { - rt.tv_nsec += 1000000000; - --rt.tv_sec; - } - /* Did we already time out? */ - if (rt.tv_sec < 0) - { - result = ETIMEDOUT; - break; - } -#endif - /* Remember that we are a writer. */ if (++rwlock->__data.__nr_writers_queued == 0) { @@ -116,17 +87,11 @@ pthread_rwlock_timedwrlock (rwlock, abstime) /* Free the lock. */ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - /* Wait for the writer or reader(s) to finish. */ -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ - || !defined lll_futex_timed_wait_bitset) - err = lll_futex_timed_wait (&rwlock->__data.__writer_wakeup, - waitval, &rt, rwlock->__data.__shared); -#else - err = lll_futex_timed_wait_bitset (&rwlock->__data.__writer_wakeup, - waitval, abstime, - FUTEX_CLOCK_REALTIME, - rwlock->__data.__shared); -#endif + /* Wait for the writer or reader(s) to finish. We handle ETIMEDOUT + below; on other return values, we decide how to continue based on + the state of the rwlock. */ + err = futex_abstimed_wait (&rwlock->__data.__writer_wakeup, waitval, + abstime, futex_shared); /* Get the lock. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); @@ -135,7 +100,7 @@ pthread_rwlock_timedwrlock (rwlock, abstime) --rwlock->__data.__nr_writers_queued; /* Did the futex call time out? */ - if (err == -ETIMEDOUT) + if (err == ETIMEDOUT) { result = ETIMEDOUT; /* If we prefer writers, it can have happened that readers blocked @@ -166,8 +131,7 @@ pthread_rwlock_timedwrlock (rwlock, abstime) /* Might be required after timeouts. */ if (wake_readers) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c index cde123fd5d..256188a14a 100644 --- a/nptl/pthread_rwlock_tryrdlock.c +++ b/nptl/pthread_rwlock_tryrdlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include "pthreadP.h" #include <lowlevellock.h> +#include <futex-internal.h> #include <elide.h> #include <stdbool.h> @@ -28,6 +29,8 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock) { int result = EBUSY; bool wake = false; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; if (ELIDE_TRYLOCK (rwlock->__data.__rwelision, rwlock->__data.__lock == 0 @@ -63,8 +66,7 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock) lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); if (wake) - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); return result; } diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c index d2ad4b0375..bdd115d6bd 100644 --- a/nptl/pthread_rwlock_unlock.c +++ b/nptl/pthread_rwlock_unlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> @@ -29,6 +30,9 @@ int __pthread_rwlock_unlock (pthread_rwlock_t *rwlock) { + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; + LIBC_PROBE (rwlock_unlock, 1, rwlock); if (ELIDE_UNLOCK (rwlock->__data.__writer == 0 @@ -51,16 +55,15 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock) { ++rwlock->__data.__writer_wakeup; lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - lll_futex_wake (&rwlock->__data.__writer_wakeup, 1, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__writer_wakeup, 1, futex_shared); return 0; } else if (rwlock->__data.__nr_readers_queued) { ++rwlock->__data.__readers_wakeup; lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, - rwlock->__data.__shared); + futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, + futex_shared); return 0; } } diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c index 835a62f0eb..60fa909340 100644 --- a/nptl/pthread_rwlock_wrlock.c +++ b/nptl/pthread_rwlock_wrlock.c @@ -19,6 +19,7 @@ #include <errno.h> #include <sysdep.h> #include <lowlevellock.h> +#include <futex-internal.h> #include <pthread.h> #include <pthreadP.h> #include <stap-probe.h> @@ -30,6 +31,8 @@ static int __attribute__((noinline)) __pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock) { int result = 0; + int futex_shared = + rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; /* Caller has taken the lock. */ @@ -58,9 +61,11 @@ __pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock) /* Free the lock. */ lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - /* Wait for the writer or reader(s) to finish. */ - lll_futex_wait (&rwlock->__data.__writer_wakeup, waitval, - rwlock->__data.__shared); + /* Wait for the writer or reader(s) to finish. We do not check the + return value because we decide how to continue based on the state of + the rwlock. */ + futex_wait_simple (&rwlock->__data.__writer_wakeup, waitval, + futex_shared); /* Get the lock. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); diff --git a/nptl/pthread_rwlockattr_setpshared.c b/nptl/pthread_rwlockattr_setpshared.c index 4b4b31093b..0369209a94 100644 --- a/nptl/pthread_rwlockattr_setpshared.c +++ b/nptl/pthread_rwlockattr_setpshared.c @@ -18,6 +18,7 @@ #include <errno.h> #include "pthreadP.h" +#include <futex-internal.h> int @@ -27,9 +28,9 @@ pthread_rwlockattr_setpshared (attr, pshared) { struct pthread_rwlockattr *iattr; - if (pshared != PTHREAD_PROCESS_SHARED - && __builtin_expect (pshared != PTHREAD_PROCESS_PRIVATE, 0)) - return EINVAL; + int err = futex_supports_pshared (pshared); + if (err != 0) + return err; iattr = (struct pthread_rwlockattr *) attr; diff --git a/nptl/sem_init.c b/nptl/sem_init.c index 575b661f62..bd1b592420 100644 --- a/nptl/sem_init.c +++ b/nptl/sem_init.c @@ -21,22 +21,7 @@ #include <shlib-compat.h> #include "semaphoreP.h" #include <kernel-features.h> - -/* Returns FUTEX_PRIVATE if pshared is zero and private futexes are supported; - returns FUTEX_SHARED otherwise. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline int -futex_private_if_supported (int pshared) -{ - if (pshared != 0) - return LLL_SHARED; -#ifdef __ASSUME_PRIVATE_FUTEX - return LLL_PRIVATE; -#else - return THREAD_GETMEM (THREAD_SELF, header.private_futex) - ^ FUTEX_PRIVATE_FLAG; -#endif -} +#include <futex-internal.h> int @@ -48,6 +33,13 @@ __new_sem_init (sem_t *sem, int pshared, unsigned int value) __set_errno (EINVAL); return -1; } + pshared = pshared != 0 ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE; + int err = futex_supports_pshared (pshared); + if (err != 0) + { + __set_errno (err); + return -1; + } /* Map to the internal type. */ struct new_sem *isem = (struct new_sem *) sem; @@ -60,7 +52,8 @@ __new_sem_init (sem_t *sem, int pshared, unsigned int value) isem->nwaiters = 0; #endif - isem->private = futex_private_if_supported (pshared); + isem->private = (pshared == PTHREAD_PROCESS_PRIVATE + ? FUTEX_PRIVATE : FUTEX_SHARED); return 0; } diff --git a/nptl/sem_open.c b/nptl/sem_open.c index ecd051a547..5ee8b71aa5 100644 --- a/nptl/sem_open.c +++ b/nptl/sem_open.c @@ -30,6 +30,7 @@ #include <sys/stat.h> #include "semaphoreP.h" #include <shm-directory.h> +#include <futex-internal.h> /* Comparison function for search of existing mapping. */ @@ -141,6 +142,14 @@ sem_open (const char *name, int oflag, ...) int fd; sem_t *result; + /* Check that shared futexes are supported. */ + int err = futex_supports_pshared (PTHREAD_PROCESS_SHARED); + if (err != 0) + { + __set_errno (err); + return SEM_FAILED; + } + /* Create the name of the final file in local variable SHM_NAME. */ SHM_GET_NAME (EINVAL, SEM_FAILED, SEM_SHM_PREFIX); @@ -201,7 +210,7 @@ sem_open (const char *name, int oflag, ...) sem.newsem.nwaiters = 0; #endif /* This always is a shared semaphore. */ - sem.newsem.private = LLL_SHARED; + sem.newsem.private = FUTEX_SHARED; /* Initialize the remaining bytes as well. */ memset ((char *) &sem.initsem + sizeof (struct new_sem), '\0', diff --git a/nptl/sem_post.c b/nptl/sem_post.c index b6d30b514f..06d835907e 100644 --- a/nptl/sem_post.c +++ b/nptl/sem_post.c @@ -20,37 +20,13 @@ #include <atomic.h> #include <errno.h> #include <sysdep.h> -#include <lowlevellock.h> +#include <lowlevellock.h> /* lll_futex* used by the old code. */ +#include <futex-internal.h> #include <internaltypes.h> #include <semaphore.h> #include <shlib-compat.h> -/* Wrapper for lll_futex_wake, with error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline void -futex_wake (unsigned int* futex, int processes_to_wake, int private) -{ - int res = lll_futex_wake (futex, processes_to_wake, private); - /* No error. Ignore the number of woken processes. */ - if (res >= 0) - return; - switch (res) - { - case -EFAULT: /* Could have happened due to memory reuse. */ - case -EINVAL: /* Could be either due to incorrect alignment (a bug in - glibc or in the application) or due to memory being - reused for a PI futex. We cannot distinguish between the - two causes, and one of them is correct use, so we do not - act in this case. */ - return; - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - /* See sem_wait for an explanation of the algorithm. */ int diff --git a/nptl/sem_wait.c b/nptl/sem_wait.c index c1fd10c9d0..fce7ed43ee 100644 --- a/nptl/sem_wait.c +++ b/nptl/sem_wait.c @@ -17,6 +17,7 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ +#include <lowlevellock.h> /* lll_futex* used by the old code. */ #include "sem_waitcommon.c" int diff --git a/nptl/sem_waitcommon.c b/nptl/sem_waitcommon.c index 772425d33e..d3702c7b4f 100644 --- a/nptl/sem_waitcommon.c +++ b/nptl/sem_waitcommon.c @@ -20,7 +20,7 @@ #include <kernel-features.h> #include <errno.h> #include <sysdep.h> -#include <lowlevellock.h> +#include <futex-internal.h> #include <internaltypes.h> #include <semaphore.h> #include <sys/time.h> @@ -29,110 +29,6 @@ #include <shlib-compat.h> #include <atomic.h> -/* Wrapper for lll_futex_wait with absolute timeout and error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline int -futex_abstimed_wait (unsigned int* futex, unsigned int expected, - const struct timespec* abstime, int private, bool cancel) -{ - int err, oldtype; - if (abstime == NULL) - { - if (cancel) - oldtype = __pthread_enable_asynccancel (); - err = lll_futex_wait (futex, expected, private); - if (cancel) - __pthread_disable_asynccancel (oldtype); - } - else - { -#if (defined __ASSUME_FUTEX_CLOCK_REALTIME \ - && defined lll_futex_timed_wait_bitset) - /* The Linux kernel returns EINVAL for this, but in userspace - such a value is valid. */ - if (abstime->tv_sec < 0) - return ETIMEDOUT; -#else - struct timeval tv; - struct timespec rt; - int sec, nsec; - - /* Get the current time. */ - __gettimeofday (&tv, NULL); - - /* Compute relative timeout. */ - sec = abstime->tv_sec - tv.tv_sec; - nsec = abstime->tv_nsec - tv.tv_usec * 1000; - if (nsec < 0) - { - nsec += 1000000000; - --sec; - } - - /* Already timed out? */ - if (sec < 0) - return ETIMEDOUT; - - /* Do wait. */ - rt.tv_sec = sec; - rt.tv_nsec = nsec; -#endif - if (cancel) - oldtype = __pthread_enable_asynccancel (); -#if (defined __ASSUME_FUTEX_CLOCK_REALTIME \ - && defined lll_futex_timed_wait_bitset) - err = lll_futex_timed_wait_bitset (futex, expected, abstime, - FUTEX_CLOCK_REALTIME, private); -#else - err = lll_futex_timed_wait (futex, expected, &rt, private); -#endif - if (cancel) - __pthread_disable_asynccancel (oldtype); - } - switch (err) - { - case 0: - case -EAGAIN: - case -EINTR: - case -ETIMEDOUT: - return -err; - - case -EFAULT: /* Must have been caused by a glibc or application bug. */ - case -EINVAL: /* Either due to wrong alignment or due to the timeout not - being normalized. Must have been caused by a glibc or - application bug. */ - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - -/* Wrapper for lll_futex_wake, with error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline void -futex_wake (unsigned int* futex, int processes_to_wake, int private) -{ - int res = lll_futex_wake (futex, processes_to_wake, private); - /* No error. Ignore the number of woken processes. */ - if (res >= 0) - return; - switch (res) - { - case -EFAULT: /* Could have happened due to memory reuse. */ - case -EINVAL: /* Could be either due to incorrect alignment (a bug in - glibc or in the application) or due to memory being - reused for a PI futex. We cannot distinguish between the - two causes, and one of them is correct use, so we do not - act in this case. */ - return; - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - /* The semaphore provides two main operations: sem_post adds a token to the semaphore; sem_wait grabs a token from the semaphore, potentially waiting @@ -220,11 +116,12 @@ do_futex_wait (struct new_sem *sem, const struct timespec *abstime) int err; #if __HAVE_64B_ATOMICS - err = futex_abstimed_wait ((unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0, - abstime, sem->private, true); + err = futex_abstimed_wait_cancelable ( + (unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0, abstime, + sem->private); #else - err = futex_abstimed_wait (&sem->value, SEM_NWAITERS_MASK, abstime, - sem->private, true); + err = futex_abstimed_wait_cancelable (&sem->value, SEM_NWAITERS_MASK, + abstime, sem->private); #endif return err; diff --git a/nptl/unregister-atfork.c b/nptl/unregister-atfork.c index 3838cb7dee..6d08ed737e 100644 --- a/nptl/unregister-atfork.c +++ b/nptl/unregister-atfork.c @@ -20,6 +20,7 @@ #include <stdlib.h> #include <fork.h> #include <atomic.h> +#include <futex-internal.h> void @@ -114,7 +115,7 @@ __unregister_atfork (dso_handle) atomic_decrement (&deleted->handler->refcntr); unsigned int val; while ((val = deleted->handler->refcntr) != 0) - lll_futex_wait (&deleted->handler->refcntr, val, LLL_PRIVATE); + futex_wait_simple (&deleted->handler->refcntr, val, FUTEX_PRIVATE); deleted = deleted->next; } diff --git a/sysdeps/nacl/exit-thread.h b/sysdeps/nacl/exit-thread.h index c809405cf1..915f93dd0d 100644 --- a/sysdeps/nacl/exit-thread.h +++ b/sysdeps/nacl/exit-thread.h @@ -18,7 +18,7 @@ #include <assert.h> #include <atomic.h> -#include <lowlevellock.h> +#include <futex-internal.h> #include <nacl-interfaces.h> #include <nptl/pthreadP.h> @@ -64,7 +64,7 @@ __exit_thread (void) assert (NACL_EXITING_TID > 0); atomic_store_relaxed (&pd->tid, NACL_EXITING_TID); - lll_futex_wake (&pd->tid, 1, LLL_PRIVATE); + futex_wake ((unsigned int *) &pd->tid, 1, FUTEX_PRIVATE); } /* This clears PD->tid some time after the thread stack can never diff --git a/sysdeps/nacl/futex-internal.h b/sysdeps/nacl/futex-internal.h new file mode 100644 index 0000000000..593bb9d9bf --- /dev/null +++ b/sysdeps/nacl/futex-internal.h @@ -0,0 +1,248 @@ +/* futex operations for glibc-internal use. NaCl version. + Copyright (C) 2014-2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef FUTEX_INTERNAL_H +#define FUTEX_INTERNAL_H + +#include <sysdeps/nptl/futex-internal.h> +#include <errno.h> +#include <lowlevellock-futex.h> +#include <nacl-interfaces.h> +#include <nptl/pthreadP.h> + +/* See sysdeps/nptl/futex-internal.h for documentation; this file only + contains NaCl-specific comments. + + There is no support yet for shared futexes nor for exact relative + timeouts. */ + +/* See sysdeps/nptl/futex-internal.h for constraints on the value of the + FUTEX_PRIVATE and FUTEX_SHARED constants. + Shared futexes are not yet supported, and we never allow clients to + actually request shared futexes. Therefore, we do not need a different + value. */ +#undef FUTEX_SHARED +#define FUTEX_SHARED FUTEX_PRIVATE + +/* FUTEX_SHARED is not yet supported. */ +static __always_inline int +futex_supports_pshared (int pshared) +{ + if (__glibc_likely (pshared == PTHREAD_PROCESS_PRIVATE)) + return 0; + else if (pshared == PTHREAD_PROCESS_SHARED) + return ENOTSUP; + else + return EINVAL; +} + +/* Relative timeouts are only emulated via absolute timeouts using the + system clock. */ +static __always_inline bool +futex_supports_exact_relative_timeouts (void) +{ + return false; +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_wait (unsigned int *futex_word, unsigned int expected, int private) +{ + int err = lll_futex_timed_wait (futex_word, expected, NULL, private); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + return -err; + + case -ETIMEDOUT: /* Cannot have happened as we provided no timeout. */ + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_wait_cancelable (unsigned int *futex_word, unsigned int expected, + int private) +{ + int oldtype; + oldtype = __pthread_enable_asynccancel (); + int err = lll_futex_timed_wait (futex_word, expected, NULL, private); + __pthread_disable_asynccancel (oldtype); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + return -err; + + case -ETIMEDOUT: /* Cannot have happened as we provided no timeout. */ + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_reltimed_wait (unsigned int *futex_word, unsigned int expected, + const struct timespec *reltime, int private) +{ + int err = lll_futex_timed_wait (futex_word, expected, reltime, private); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + case -ETIMEDOUT: + return -err; + + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_reltimed_wait_cancelable (unsigned int *futex_word, + unsigned int expected, + const struct timespec *reltime, int private) +{ + int oldtype; + oldtype = __pthread_enable_asynccancel (); + int err = lll_futex_timed_wait (futex_word, expected, reltime, private); + __pthread_disable_asynccancel (oldtype); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + case -ETIMEDOUT: + return -err; + + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_abstimed_wait (unsigned int *futex_word, unsigned int expected, + const struct timespec *abstime, int private) +{ + int err = __nacl_irt_futex.futex_wait_abs ((volatile int *) futex_word, + expected, abstime); + switch (err) + { + case 0: + case EAGAIN: + case EINTR: + case ETIMEDOUT: + return err; + + case EFAULT: /* Must have been caused by a glibc or application bug. */ + case EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_abstimed_wait_cancelable (unsigned int *futex_word, + unsigned int expected, + const struct timespec *abstime, int private) +{ + int oldtype; + oldtype = __pthread_enable_asynccancel (); + int err = __nacl_irt_futex.futex_wait_abs ((volatile int *) futex_word, + expected, abstime); + __pthread_disable_asynccancel (oldtype); + switch (err) + { + case 0: + case EAGAIN: + case EINTR: + case ETIMEDOUT: + return err; + + case EFAULT: /* Must have been caused by a glibc or application bug. */ + case EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline void +futex_wake (unsigned int *futex_word, int processes_to_wake, int private) +{ + int res = lll_futex_wake (futex_word, processes_to_wake, private); + /* No error. Ignore the number of woken processes. */ + if (res >= 0) + return; + switch (res) + { + case -EFAULT: /* Could have happened due to memory reuse. */ + case -EINVAL: /* Could be either due to incorrect alignment (a bug in + glibc or in the application) or due to memory being + reused for a PI futex. We cannot distinguish between the + two causes, and one of them is correct use, so we do not + act in this case. */ + return; + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +#endif /* futex-internal.h */ diff --git a/sysdeps/nptl/aio_misc.h b/sysdeps/nptl/aio_misc.h index d5d1c08d0f..4a6ebfc9b3 100644 --- a/sysdeps/nptl/aio_misc.h +++ b/sysdeps/nptl/aio_misc.h @@ -22,14 +22,14 @@ #include <assert.h> #include <nptl/pthreadP.h> -#include <lowlevellock.h> +#include <futex-internal.h> #define DONT_NEED_AIO_MISC_COND 1 #define AIO_MISC_NOTIFY(waitlist) \ do { \ if (*waitlist->counterp > 0 && --*waitlist->counterp == 0) \ - lll_futex_wake (waitlist->counterp, 1, LLL_PRIVATE); \ + futex_wake ((unsigned int *) waitlist->counterp, 1, FUTEX_PRIVATE); \ } while (0) #define AIO_MISC_WAIT(result, futex, timeout, cancel) \ @@ -48,9 +48,9 @@ int status; \ do \ { \ - status = lll_futex_timed_wait (futexaddr, oldval, timeout, \ - LLL_PRIVATE); \ - if (status != -EWOULDBLOCK) \ + status = futex_reltimed_wait ((unsigned int *) futexaddr, oldval, \ + timeout, FUTEX_PRIVATE); \ + if (status != EAGAIN) \ break; \ \ oldval = *futexaddr; \ @@ -60,12 +60,12 @@ if (cancel) \ LIBC_CANCEL_RESET (oldtype); \ \ - if (status == -EINTR) \ + if (status == EINTR) \ result = EINTR; \ - else if (status == -ETIMEDOUT) \ + else if (status == ETIMEDOUT) \ result = EAGAIN; \ else \ - assert (status == 0 || status == -EWOULDBLOCK); \ + assert (status == 0 || status == EAGAIN); \ \ pthread_mutex_lock (&__aio_requests_mutex); \ } \ diff --git a/sysdeps/nptl/fork.c b/sysdeps/nptl/fork.c index 74482b7a38..2b9ae4b571 100644 --- a/sysdeps/nptl/fork.c +++ b/sysdeps/nptl/fork.c @@ -30,6 +30,7 @@ #include <nptl/pthreadP.h> #include <fork.h> #include <arch-fork.h> +#include <futex-internal.h> static void @@ -219,7 +220,7 @@ __libc_fork (void) if (atomic_decrement_and_test (&allp->handler->refcntr) && allp->handler->need_signal) - lll_futex_wake (&allp->handler->refcntr, 1, LLL_PRIVATE); + futex_wake (&allp->handler->refcntr, 1, FUTEX_PRIVATE); allp = allp->next; } diff --git a/sysdeps/nptl/futex-internal.h b/sysdeps/nptl/futex-internal.h new file mode 100644 index 0000000000..4f8c8fe72d --- /dev/null +++ b/sysdeps/nptl/futex-internal.h @@ -0,0 +1,203 @@ +/* futex operations for glibc-internal use. Stub version; do not include + this file directly. + Copyright (C) 2014-2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef STUB_FUTEX_INTERNAL_H +#define STUB_FUTEX_INTERNAL_H + +#include <sys/time.h> +#include <stdio.h> +#include <stdbool.h> +#include <libc-internal.h> + +/* This file defines futex operations used internally in glibc. A futex + consists of the so-called futex word in userspace, which is of type + unsigned int and represents an application-specific condition, and kernel + state associated with this particular futex word (e.g., wait queues). The + futex operations we provide are wrappers for the futex syscalls and add + glibc-specific error checking of the syscall return value. We abort on + error codes that are caused by bugs in glibc or in the calling application, + or when an error code is not known. We return error codes that can arise + in correct executions to the caller. Each operation calls out exactly the + return values that callers need to handle. + + The private flag must be either FUTEX_PRIVATE or FUTEX_SHARED. + FUTEX_PRIVATE is always supported, and the implementation can internally + use FUTEX_SHARED when FUTEX_PRIVATE is requested. FUTEX_SHARED is not + necessarily supported (use futex_supports_pshared to detect this). + + We expect callers to only use these operations if futexes and the + specific futex operations being used are supported (e.g., FUTEX_SHARED). + + Given that waking other threads waiting on a futex involves concurrent + accesses to the futex word, you must use atomic operations to access the + futex word. + + Both absolute and relative timeouts can be used. An absolute timeout + expires when the given specific point in time on the CLOCK_REALTIME clock + passes, or when it already has passed. A relative timeout expires when + the given duration of time on the CLOCK_MONOTONIC clock passes. Relative + timeouts may be imprecise (see futex_supports_exact_relative_timeouts). + + Due to POSIX requirements on when synchronization data structures such + as mutexes or semaphores can be destroyed and due to the futex design + having separate fast/slow paths for wake-ups, we need to consider that + futex_wake calls might effectively target a data structure that has been + destroyed and reused for another object, or unmapped; thus, some + errors or spurious wake-ups can happen in correct executions that would + not be possible in a program using just a single futex whose lifetime + does not end before the program terminates. For background, see: + https://sourceware.org/ml/libc-alpha/2014-04/msg00075.html + https://lkml.org/lkml/2014/11/27/472 */ + +/* Defined this way for interoperability with lowlevellock. + FUTEX_PRIVATE must be zero because the initializers for pthread_mutex_t, + pthread_rwlock_t, and pthread_cond_t initialize the respective field of + those structures to zero, and we want FUTEX_PRIVATE to be the default. */ +#define FUTEX_PRIVATE LLL_PRIVATE +#define FUTEX_SHARED LLL_SHARED +#if FUTEX_PRIVATE != 0 +# error FUTEX_PRIVATE must be equal to 0 +#endif + +/* Returns EINVAL if PSHARED is neither PTHREAD_PROCESS_PRIVATE nor + PTHREAD_PROCESS_SHARED; otherwise, returns 0 if PSHARED is supported, and + ENOTSUP if not. */ +static __always_inline int +futex_supports_pshared (int pshared); + +/* Returns true if relative timeouts are robust to concurrent changes to the + system clock. If this returns false, relative timeouts can still be used + but might be effectively longer or shorter than requested. */ +static __always_inline bool +futex_supports_exact_relative_timeouts (void); + +/* Atomically wrt other futex operations on the same futex, this blocks iff + the value *FUTEX_WORD matches the expected value. This is + semantically equivalent to: + l = <get lock associated with futex> (FUTEX_WORD); + wait_flag = <get wait_flag associated with futex> (FUTEX_WORD); + lock (l); + val = atomic_load_relaxed (FUTEX_WORD); + if (val != expected) { unlock (l); return EAGAIN; } + atomic_store_relaxed (wait_flag, true); + unlock (l); + // Now block; can time out in futex_time_wait (see below) + while (atomic_load_relaxed(wait_flag) && !<spurious wake-up>); + + Note that no guarantee of a happens-before relation between a woken + futex_wait and a futex_wake is documented; however, this does not matter + in practice because we have to consider spurious wake-ups (see below), + and thus would not be able to reliably reason about which futex_wake woke + us. + + Returns 0 if woken by a futex operation or spuriously. (Note that due to + the POSIX requirements mentioned above, we need to conservatively assume + that unrelated futex_wake operations could wake this futex; it is easiest + to just be prepared for spurious wake-ups.) + Returns EAGAIN if the futex word did not match the expected value. + Returns EINTR if waiting was interrupted by a signal. + + Note that some previous code in glibc assumed the underlying futex + operation (e.g., syscall) to start with or include the equivalent of a + seq_cst fence; this allows one to avoid an explicit seq_cst fence before + a futex_wait call when synchronizing similar to Dekker synchronization. + However, we make no such guarantee here. */ +static __always_inline int +futex_wait (unsigned int *futex_word, unsigned int expected, int private); + +/* Like futex_wait but does not provide any indication why we stopped waiting. + Thus, when this function returns, you have to always check FUTEX_WORD to + determine whether you need to continue waiting, and you cannot detect + whether the waiting was interrupted by a signal. Example use: + while (atomic_load_relaxed (&futex_word) == 23) + futex_wait_simple (&futex_word, 23, FUTEX_PRIVATE); + This is common enough to make providing this wrapper worthwhile. */ +static __always_inline void +futex_wait_simple (unsigned int *futex_word, unsigned int expected, + int private) +{ + ignore_value (futex_wait (futex_word, expected, private)); +} + + +/* Like futex_wait but is a POSIX cancellation point. */ +static __always_inline int +futex_wait_cancelable (unsigned int *futex_word, unsigned int expected, + int private); + +/* Like futex_wait, but will eventually time out (i.e., stop being + blocked) after the duration of time provided (i.e., RELTIME) has + passed. The caller must provide a normalized RELTIME. RELTIME can also + equal NULL, in which case this function behaves equivalent to futex_wait. + + Returns the same values as futex_wait under those same conditions; + additionally, returns ETIMEDOUT if the timeout expired. + */ +static __always_inline int +futex_reltimed_wait (unsigned int* futex_word, unsigned int expected, + const struct timespec* reltime, int private); + +/* Like futex_reltimed_wait but is a POSIX cancellation point. */ +static __always_inline int +futex_reltimed_wait_cancelable (unsigned int* futex_word, + unsigned int expected, + const struct timespec* reltime, int private); + +/* Like futex_reltimed_wait, but the provided timeout (ABSTIME) is an + absolute point in time; a call will time out after this point in time. */ +static __always_inline int +futex_abstimed_wait (unsigned int* futex_word, unsigned int expected, + const struct timespec* abstime, int private); + +/* Like futex_reltimed_wait but is a POSIX cancellation point. */ +static __always_inline int +futex_abstimed_wait_cancelable (unsigned int* futex_word, + unsigned int expected, + const struct timespec* abstime, int private); + +/* Atomically wrt other futex operations on the same futex, this unblocks the + specified number of processes, or all processes blocked on this futex if + there are fewer than the specified number. Semantically, this is + equivalent to: + l = <get lock associated with futex> (FUTEX_WORD); + lock (l); + for (res = 0; PROCESSES_TO_WAKE > 0; PROCESSES_TO_WAKE--, res++) { + if (<no process blocked on futex>) break; + wf = <get wait_flag of a process blocked on futex> (FUTEX_WORD); + // No happens-before guarantee with woken futex_wait (see above) + atomic_store_relaxed (wf, 0); + } + return res; + + Note that we need to support futex_wake calls to past futexes whose memory + has potentially been reused due to POSIX' requirements on synchronization + object destruction (see above); therefore, we must not report or abort + on most errors. */ +static __always_inline void +futex_wake (unsigned int* futex_word, int processes_to_wake, int private); + +/* Calls __libc_fatal with an error message. Convenience function for + concrete implementations of the futex interface. */ +static __always_inline __attribute__ ((__noreturn__)) void +futex_fatal_error (void) +{ + __libc_fatal ("The futex facility returned an unexpected error code."); +} + +#endif /* futex-internal.h */ diff --git a/sysdeps/nptl/gai_misc.h b/sysdeps/nptl/gai_misc.h index a34dbc0dd9..96c8fa0f91 100644 --- a/sysdeps/nptl/gai_misc.h +++ b/sysdeps/nptl/gai_misc.h @@ -23,14 +23,14 @@ #include <assert.h> #include <signal.h> #include <nptl/pthreadP.h> -#include <lowlevellock.h> +#include <futex-internal.h> #define DONT_NEED_GAI_MISC_COND 1 #define GAI_MISC_NOTIFY(waitlist) \ do { \ if (*waitlist->counterp > 0 && --*waitlist->counterp == 0) \ - lll_futex_wake (waitlist->counterp, 1, LLL_PRIVATE); \ + futex_wake ((unsigned int *) waitlist->counterp, 1, FUTEX_PRIVATE); \ } while (0) #define GAI_MISC_WAIT(result, futex, timeout, cancel) \ @@ -49,9 +49,9 @@ int status; \ do \ { \ - status = lll_futex_timed_wait (futexaddr, oldval, timeout, \ - LLL_PRIVATE); \ - if (status != -EWOULDBLOCK) \ + status = futex_reltimed_wait ((unsigned int *) futexaddr, oldval, \ + timeout, FUTEX_PRIVATE); \ + if (status != EAGAIN) \ break; \ \ oldval = *futexaddr; \ @@ -61,12 +61,12 @@ if (cancel) \ LIBC_CANCEL_RESET (oldtype); \ \ - if (status == -EINTR) \ + if (status == EINTR) \ result = EINTR; \ - else if (status == -ETIMEDOUT) \ + else if (status == ETIMEDOUT) \ result = EAGAIN; \ else \ - assert (status == 0 || status == -EWOULDBLOCK); \ + assert (status == 0 || status == EAGAIN); \ \ pthread_mutex_lock (&__gai_requests_mutex); \ } \ diff --git a/sysdeps/sparc/nptl/pthread_barrier_init.c b/sysdeps/sparc/nptl/pthread_barrier_init.c index aa21a63f2d..86ec7d0348 100644 --- a/sysdeps/sparc/nptl/pthread_barrier_init.c +++ b/sysdeps/sparc/nptl/pthread_barrier_init.c @@ -35,10 +35,9 @@ __pthread_barrier_init (barrier, attr, count) struct pthread_barrierattr *iattr = (struct pthread_barrierattr *) attr; if (iattr != NULL) { - if (iattr->pshared != PTHREAD_PROCESS_PRIVATE - && __builtin_expect (iattr->pshared != PTHREAD_PROCESS_SHARED, 0)) - /* Invalid attribute. */ - return EINVAL; + int err = futex_supports_pshared (iattr->pshared); + if (err != 0) + return err; } ibarrier = (union sparc_pthread_barrier *) barrier; diff --git a/sysdeps/sparc/nptl/pthread_barrier_wait.c b/sysdeps/sparc/nptl/pthread_barrier_wait.c index dd4c336dc2..9e9806a31a 100644 --- a/sysdeps/sparc/nptl/pthread_barrier_wait.c +++ b/sysdeps/sparc/nptl/pthread_barrier_wait.c @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <pthreadP.h> #include <sparc-nptl.h> +#include <futex-internal.h> /* Wait on barrier. */ int @@ -31,6 +32,7 @@ __pthread_barrier_wait (barrier) = (union sparc_pthread_barrier *) barrier; int result = 0; int private = ibarrier->s.pshared ? LLL_SHARED : LLL_PRIVATE; + int futex_private = ibarrier->s.pshared ? FUTEX_SHARED : FUTEX_PRIVATE; /* Make sure we are alone. */ lll_lock (ibarrier->b.lock, private); @@ -46,7 +48,7 @@ __pthread_barrier_wait (barrier) ++ibarrier->b.curr_event; /* Wake up everybody. */ - lll_futex_wake (&ibarrier->b.curr_event, INT_MAX, private); + futex_wake (&ibarrier->b.curr_event, INT_MAX, futex_private); /* This is the thread which finished the serialization. */ result = PTHREAD_BARRIER_SERIAL_THREAD; @@ -62,7 +64,7 @@ __pthread_barrier_wait (barrier) /* Wait for the event counter of the barrier to change. */ do - lll_futex_wait (&ibarrier->b.curr_event, event, private); + futex_wait_simple (&ibarrier->b.curr_event, event, futex_private); while (event == ibarrier->b.curr_event); } diff --git a/sysdeps/sparc/sparc32/pthread_barrier_wait.c b/sysdeps/sparc/sparc32/pthread_barrier_wait.c index 81d22b0653..5e1aa1159c 100644 --- a/sysdeps/sparc/sparc32/pthread_barrier_wait.c +++ b/sysdeps/sparc/sparc32/pthread_barrier_wait.c @@ -21,6 +21,7 @@ #include <lowlevellock.h> #include <pthreadP.h> #include <sparc-nptl.h> +#include <futex-internal.h> /* Wait on barrier. */ int @@ -31,6 +32,7 @@ __pthread_barrier_wait (barrier) = (union sparc_pthread_barrier *) barrier; int result = 0; int private = ibarrier->s.pshared ? LLL_SHARED : LLL_PRIVATE; + int futex_private = ibarrier->s.pshared ? FUTEX_SHARED : FUTEX_PRIVATE; /* Make sure we are alone. */ lll_lock (ibarrier->b.lock, private); @@ -46,7 +48,7 @@ __pthread_barrier_wait (barrier) ++ibarrier->b.curr_event; /* Wake up everybody. */ - lll_futex_wake (&ibarrier->b.curr_event, INT_MAX, private); + futex_wake (&ibarrier->b.curr_event, INT_MAX, futex_private); /* This is the thread which finished the serialization. */ result = PTHREAD_BARRIER_SERIAL_THREAD; @@ -62,7 +64,7 @@ __pthread_barrier_wait (barrier) /* Wait for the event counter of the barrier to change. */ do - lll_futex_wait (&ibarrier->b.curr_event, event, private); + futex_wait_simple (&ibarrier->b.curr_event, event, futex_private); while (event == ibarrier->b.curr_event); } diff --git a/sysdeps/sparc/sparc32/sem_init.c b/sysdeps/sparc/sparc32/sem_init.c index 7c46cee900..1c7c455617 100644 --- a/sysdeps/sparc/sparc32/sem_init.c +++ b/sysdeps/sparc/sparc32/sem_init.c @@ -20,23 +20,7 @@ #include <semaphore.h> #include <shlib-compat.h> #include "semaphoreP.h" -#include <kernel-features.h> - -/* Returns FUTEX_PRIVATE if pshared is zero and private futexes are supported; - returns FUTEX_SHARED otherwise. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline int -futex_private_if_supported (int pshared) -{ - if (pshared != 0) - return LLL_SHARED; -#ifdef __ASSUME_PRIVATE_FUTEX - return LLL_PRIVATE; -#else - return THREAD_GETMEM (THREAD_SELF, header.private_futex) - ^ FUTEX_PRIVATE_FLAG; -#endif -} +#include <futex-internal.h> int @@ -49,6 +33,14 @@ __new_sem_init (sem_t *sem, int pshared, unsigned int value) return -1; } + pshared = pshared != 0 ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE; + int err = futex_supports_pshared (pshared); + if (err != 0) + { + __set_errno (err); + return -1; + } + /* Map to the internal type. */ struct new_sem *isem = (struct new_sem *) sem; @@ -57,7 +49,8 @@ __new_sem_init (sem_t *sem, int pshared, unsigned int value) isem->pad = 0; isem->nwaiters = 0; - isem->private = futex_private_if_supported (pshared); + isem->private = (pshared == PTHREAD_PROCESS_PRIVATE + ? FUTEX_PRIVATE : FUTEX_SHARED); return 0; } diff --git a/sysdeps/sparc/sparc32/sem_open.c b/sysdeps/sparc/sparc32/sem_open.c index 2698d195ba..16cb9ad591 100644 --- a/sysdeps/sparc/sparc32/sem_open.c +++ b/sysdeps/sparc/sparc32/sem_open.c @@ -199,7 +199,7 @@ sem_open (const char *name, int oflag, ...) sem.newsem.nwaiters = 0; /* This always is a shared semaphore. */ - sem.newsem.private = LLL_SHARED; + sem.newsem.private = FUTEX_SHARED; /* Initialize the remaining bytes as well. */ memset ((char *) &sem.initsem + sizeof (struct new_sem), '\0', diff --git a/sysdeps/sparc/sparc32/sem_post.c b/sysdeps/sparc/sparc32/sem_post.c index c9f85a06f5..fd1a2fe910 100644 --- a/sysdeps/sparc/sparc32/sem_post.c +++ b/sysdeps/sparc/sparc32/sem_post.c @@ -23,34 +23,10 @@ #include <lowlevellock.h> #include <internaltypes.h> #include <semaphore.h> +#include <futex-internal.h> #include <shlib-compat.h> -/* Wrapper for lll_futex_wake, with error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline void -futex_wake (unsigned int* futex, int processes_to_wake, int private) -{ - int res = lll_futex_wake (futex, processes_to_wake, private); - /* No error. Ignore the number of woken processes. */ - if (res >= 0) - return; - switch (res) - { - case -EFAULT: /* Could have happened due to memory reuse. */ - case -EINVAL: /* Could be either due to incorrect alignment (a bug in - glibc or in the application) or due to memory being - reused for a PI futex. We cannot distinguish between the - two causes, and one of them is correct use, so we do not - act in this case. */ - return; - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - /* See sem_wait for an explanation of the algorithm. */ int diff --git a/sysdeps/sparc/sparc32/sem_wait.c b/sysdeps/sparc/sparc32/sem_wait.c index c1fd10c9d0..fce7ed43ee 100644 --- a/sysdeps/sparc/sparc32/sem_wait.c +++ b/sysdeps/sparc/sparc32/sem_wait.c @@ -17,6 +17,7 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ +#include <lowlevellock.h> /* lll_futex* used by the old code. */ #include "sem_waitcommon.c" int diff --git a/sysdeps/sparc/sparc32/sem_waitcommon.c b/sysdeps/sparc/sparc32/sem_waitcommon.c index 9c1c6a5af7..9e43d45431 100644 --- a/sysdeps/sparc/sparc32/sem_waitcommon.c +++ b/sysdeps/sparc/sparc32/sem_waitcommon.c @@ -19,7 +19,7 @@ #include <errno.h> #include <sysdep.h> -#include <lowlevellock.h> +#include <futex-internal.h> #include <internaltypes.h> #include <semaphore.h> #include <sys/time.h> @@ -28,104 +28,6 @@ #include <shlib-compat.h> #include <atomic.h> -/* Wrapper for lll_futex_wait with absolute timeout and error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline int -futex_abstimed_wait (unsigned int* futex, unsigned int expected, - const struct timespec* abstime, int private, bool cancel) -{ - int err, oldtype; - if (abstime == NULL) - { - if (cancel) - oldtype = __pthread_enable_asynccancel (); - err = lll_futex_wait (futex, expected, private); - if (cancel) - __pthread_disable_asynccancel (oldtype); - } - else - { - struct timeval tv; - struct timespec rt; - int sec, nsec; - - /* Get the current time. */ - __gettimeofday (&tv, NULL); - - /* Compute relative timeout. */ - sec = abstime->tv_sec - tv.tv_sec; - nsec = abstime->tv_nsec - tv.tv_usec * 1000; - if (nsec < 0) - { - nsec += 1000000000; - --sec; - } - - /* Already timed out? */ - if (sec < 0) - return ETIMEDOUT; - - /* Do wait. */ - rt.tv_sec = sec; - rt.tv_nsec = nsec; - if (cancel) - oldtype = __pthread_enable_asynccancel (); - err = lll_futex_timed_wait (futex, expected, &rt, private); - if (cancel) - __pthread_disable_asynccancel (oldtype); - } - switch (err) - { - case 0: - case -EAGAIN: - case -EINTR: - case -ETIMEDOUT: - return -err; - - case -EFAULT: /* Must have been caused by a glibc or application bug. */ - case -EINVAL: /* Either due to wrong alignment or due to the timeout not - being normalized. Must have been caused by a glibc or - application bug. */ - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - -/* Wrapper for lll_futex_wake, with error checking. - TODO Remove when cleaning up the futex API throughout glibc. */ -static __always_inline void -futex_wake (unsigned int* futex, int processes_to_wake, int private) -{ - int res = lll_futex_wake (futex, processes_to_wake, private); - /* No error. Ignore the number of woken processes. */ - if (res >= 0) - return; - switch (res) - { - case -EFAULT: /* Could have happened due to memory reuse. */ - case -EINVAL: /* Could be either due to incorrect alignment (a bug in - glibc or in the application) or due to memory being - reused for a PI futex. We cannot distinguish between the - two causes, and one of them is correct use, so we do not - act in this case. */ - return; - case -ENOSYS: /* Must have been caused by a glibc bug. */ - /* No other errors are documented at this time. */ - default: - abort (); - } -} - - -/* Set this to true if you assume that, in contrast to current Linux futex - documentation, lll_futex_wake can return -EINTR only if interrupted by a - signal, not spuriously due to some other reason. - TODO Discuss EINTR conditions with the Linux kernel community. For - now, we set this to true to not change behavior of semaphores compared - to previous glibc builds. */ -static const int sem_assume_only_signals_cause_futex_EINTR = 1; static void __sem_wait_32_finish (struct new_sem *sem); @@ -149,8 +51,8 @@ do_futex_wait (struct new_sem *sem, const struct timespec *abstime) { int err; - err = futex_abstimed_wait (&sem->value, SEM_NWAITERS_MASK, abstime, - sem->private, true); + err = futex_abstimed_wait_cancelable (&sem->value, SEM_NWAITERS_MASK, + abstime, sem->private); return err; } @@ -202,8 +104,7 @@ __new_sem_wait_slow (struct new_sem *sem, const struct timespec *abstime) __sparc32_atomic_do_unlock24(&sem->pad); err = do_futex_wait(sem, abstime); - if (err == ETIMEDOUT || - (err == EINTR && sem_assume_only_signals_cause_futex_EINTR)) + if (err == ETIMEDOUT || err == EINTR) { __set_errno (err); err = -1; diff --git a/sysdeps/unix/sysv/linux/futex-internal.h b/sysdeps/unix/sysv/linux/futex-internal.h new file mode 100644 index 0000000000..aca0911252 --- /dev/null +++ b/sysdeps/unix/sysv/linux/futex-internal.h @@ -0,0 +1,251 @@ +/* futex operations for glibc-internal use. Linux version. + Copyright (C) 2014-2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef FUTEX_INTERNAL_H +#define FUTEX_INTERNAL_H + +#include <sysdeps/nptl/futex-internal.h> +#include <errno.h> +#include <lowlevellock-futex.h> +#include <nptl/pthreadP.h> + +/* See sysdeps/nptl/futex-internal.h for documentation; this file only + contains Linux-specific comments. + + The Linux kernel treats provides absolute timeouts based on the + CLOCK_REALTIME clock and relative timeouts measured against the + CLOCK_MONOTONIC clock. + + We expect a Linux kernel version of 2.6.22 or more recent (since this + version, EINTR is not returned on spurious wake-ups anymore). */ + +/* FUTEX_SHARED is always supported by the Linux kernel. */ +static __always_inline int +futex_supports_pshared (int pshared) +{ + if (__glibc_likely (pshared == PTHREAD_PROCESS_PRIVATE)) + return 0; + else if (pshared == PTHREAD_PROCESS_SHARED) + return 0; + else + return EINVAL; +} + +/* The Linux kernel supports relative timeouts measured against the + CLOCK_MONOTONIC clock. */ +static __always_inline bool +futex_supports_exact_relative_timeouts (void) +{ + return true; +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_wait (unsigned int *futex_word, unsigned int expected, int private) +{ + int err = lll_futex_timed_wait (futex_word, expected, NULL, private); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + return -err; + + case -ETIMEDOUT: /* Cannot have happened as we provided no timeout. */ + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_wait_cancelable (unsigned int *futex_word, unsigned int expected, + int private) +{ + int oldtype; + oldtype = __pthread_enable_asynccancel (); + int err = lll_futex_timed_wait (futex_word, expected, NULL, private); + __pthread_disable_asynccancel (oldtype); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + return -err; + + case -ETIMEDOUT: /* Cannot have happened as we provided no timeout. */ + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_reltimed_wait (unsigned int *futex_word, unsigned int expected, + const struct timespec *reltime, int private) +{ + int err = lll_futex_timed_wait (futex_word, expected, reltime, private); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + case -ETIMEDOUT: + return -err; + + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_reltimed_wait_cancelable (unsigned int *futex_word, + unsigned int expected, + const struct timespec *reltime, int private) +{ + int oldtype; + oldtype = __pthread_enable_asynccancel (); + int err = lll_futex_timed_wait (futex_word, expected, reltime, private); + __pthread_disable_asynccancel (oldtype); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + case -ETIMEDOUT: + return -err; + + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_abstimed_wait (unsigned int *futex_word, unsigned int expected, + const struct timespec *abstime, int private) +{ + /* Work around the fact that the kernel rejects negative timeout values + despite them being valid. */ + if (__glibc_unlikely ((abstime != NULL) && (abstime->tv_sec < 0))) + return ETIMEDOUT; + int err = lll_futex_timed_wait_bitset (futex_word, expected, abstime, + FUTEX_CLOCK_REALTIME, private); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + case -ETIMEDOUT: + return -err; + + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline int +futex_abstimed_wait_cancelable (unsigned int *futex_word, + unsigned int expected, + const struct timespec *abstime, int private) +{ + /* Work around the fact that the kernel rejects negative timeout values + despite them being valid. */ + if (__glibc_unlikely ((abstime != NULL) && (abstime->tv_sec < 0))) + return ETIMEDOUT; + int oldtype; + oldtype = __pthread_enable_asynccancel (); + int err = lll_futex_timed_wait_bitset (futex_word, expected, abstime, + FUTEX_CLOCK_REALTIME, private); + __pthread_disable_asynccancel (oldtype); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + case -ETIMEDOUT: + return -err; + + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* See sysdeps/nptl/futex-internal.h for details. */ +static __always_inline void +futex_wake (unsigned int *futex_word, int processes_to_wake, int private) +{ + int res = lll_futex_wake (futex_word, processes_to_wake, private); + /* No error. Ignore the number of woken processes. */ + if (res >= 0) + return; + switch (res) + { + case -EFAULT: /* Could have happened due to memory reuse. */ + case -EINVAL: /* Could be either due to incorrect alignment (a bug in + glibc or in the application) or due to memory being + reused for a PI futex. We cannot distinguish between the + two causes, and one of them is correct use, so we do not + act in this case. */ + return; + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +#endif /* futex-internal.h */ |