diff options
| -rw-r--r-- | nptl/pthread_mutex_init.c | 13 | ||||
| -rw-r--r-- | nptl/pthread_mutex_lock.c | 24 | ||||
| -rw-r--r-- | nptl/pthread_mutex_timedlock.c | 60 | ||||
| -rw-r--r-- | nptl/pthread_mutex_trylock.c | 8 | ||||
| -rw-r--r-- | nptl/pthread_mutex_unlock.c | 6 | ||||
| -rw-r--r-- | sysdeps/nptl/futex-internal.h | 86 | ||||
| -rw-r--r-- | sysdeps/nptl/lowlevellock-futex.h | 9 |
7 files changed, 137 insertions, 69 deletions
diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c index fe4eeee37c..20800b80f5 100644 --- a/nptl/pthread_mutex_init.c +++ b/nptl/pthread_mutex_init.c @@ -24,6 +24,7 @@ #include "pthreadP.h" #include <atomic.h> #include <pthread-offsets.h> +#include <futex-internal.h> #include <stap-probe.h> @@ -37,19 +38,13 @@ static const struct pthread_mutexattr default_mutexattr = static bool prio_inherit_missing (void) { -#ifdef __NR_futex static int tpi_supported; - if (__glibc_unlikely (tpi_supported == 0)) + if (__glibc_unlikely (atomic_load_relaxed (&tpi_supported) == 0)) { - int lock = 0; - INTERNAL_SYSCALL_DECL (err); - int ret = INTERNAL_SYSCALL (futex, err, 4, &lock, FUTEX_UNLOCK_PI, 0, 0); - assert (INTERNAL_SYSCALL_ERROR_P (ret, err)); - tpi_supported = INTERNAL_SYSCALL_ERRNO (ret, err) == ENOSYS ? -1 : 1; + int e = futex_unlock_pi (&(unsigned int){0}, 0); + atomic_store_relaxed (&tpi_supported, e == ENOSYS ? -1 : 1); } return __glibc_unlikely (tpi_supported < 0); -#endif - return true; } int diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c index ace436d5a6..05bba50666 100644 --- a/nptl/pthread_mutex_lock.c +++ b/nptl/pthread_mutex_lock.c @@ -24,7 +24,7 @@ #include <not-cancel.h> #include "pthreadP.h" #include <atomic.h> -#include <lowlevellock.h> +#include <futex-internal.h> #include <stap-probe.h> #ifndef lll_lock_elision @@ -416,21 +416,16 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex) int private = (robust ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex) : PTHREAD_MUTEX_PSHARED (mutex)); - INTERNAL_SYSCALL_DECL (__err); - int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, - __lll_private_flag (FUTEX_LOCK_PI, - private), 1, 0); - - if (INTERNAL_SYSCALL_ERROR_P (e, __err) - && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH - || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)) + int e = futex_lock_pi ((unsigned int *) &mutex->__data.__lock, + NULL, private); + if (e == ESRCH || e == EDEADLK) { - assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK + assert (e != EDEADLK || (kind != PTHREAD_MUTEX_ERRORCHECK_NP && kind != PTHREAD_MUTEX_RECURSIVE_NP)); /* ESRCH can happen only for non-robust PI mutexes where the owner of the lock died. */ - assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust); + assert (e != ESRCH || !robust); /* Delay the thread indefinitely. */ while (1) @@ -479,11 +474,8 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex) /* This mutex is now not recoverable. */ mutex->__data.__count = 0; - INTERNAL_SYSCALL_DECL (__err); - INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, - __lll_private_flag (FUTEX_UNLOCK_PI, - PTHREAD_ROBUST_MUTEX_PSHARED (mutex)), - 0, 0); + futex_unlock_pi ((unsigned int *) &mutex->__data.__lock, + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); /* To the kernel, this will be visible after the kernel has acquired the mutex in the syscall. */ diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c index 76b93bddc6..eb4baae93b 100644 --- a/nptl/pthread_mutex_timedlock.c +++ b/nptl/pthread_mutex_timedlock.c @@ -25,6 +25,7 @@ #include <atomic.h> #include <lowlevellock.h> #include <not-cancel.h> +#include <futex-internal.h> #include <stap-probe.h> @@ -377,39 +378,29 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex, int private = (robust ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex) : PTHREAD_MUTEX_PSHARED (mutex)); - INTERNAL_SYSCALL_DECL (__err); - - int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, - __lll_private_flag (FUTEX_LOCK_PI, - private), 1, - abstime); - if (INTERNAL_SYSCALL_ERROR_P (e, __err)) + int e = futex_lock_pi ((unsigned int *) &mutex->__data.__lock, + abstime, private); + if (e == ETIMEDOUT) + return ETIMEDOUT; + else if (e == ESRCH || e == EDEADLK) { - if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT) - return ETIMEDOUT; - - if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH - || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK) - { - assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK - || (kind != PTHREAD_MUTEX_ERRORCHECK_NP - && kind != PTHREAD_MUTEX_RECURSIVE_NP)); - /* ESRCH can happen only for non-robust PI mutexes where - the owner of the lock died. */ - assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH - || !robust); - - /* Delay the thread until the timeout is reached. - Then return ETIMEDOUT. */ - do - e = lll_timedwait (&(int){0}, 0, clockid, abstime, - private); - while (e != ETIMEDOUT); - return ETIMEDOUT; - } - - return INTERNAL_SYSCALL_ERRNO (e, __err); + assert (e != EDEADLK + || (kind != PTHREAD_MUTEX_ERRORCHECK_NP + && kind != PTHREAD_MUTEX_RECURSIVE_NP)); + /* ESRCH can happen only for non-robust PI mutexes where + the owner of the lock died. */ + assert (e != ESRCH || !robust); + + /* Delay the thread until the timeout is reached. Then return + ETIMEDOUT. */ + do + e = lll_timedwait (&(int){0}, 0, clockid, abstime, + private); + while (e != ETIMEDOUT); + return ETIMEDOUT; } + else if (e != 0) + return e; oldval = mutex->__data.__lock; @@ -447,11 +438,8 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex, /* This mutex is now not recoverable. */ mutex->__data.__count = 0; - INTERNAL_SYSCALL_DECL (__err); - INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, - __lll_private_flag (FUTEX_UNLOCK_PI, - PTHREAD_ROBUST_MUTEX_PSHARED (mutex)), - 0, 0); + futex_unlock_pi ((unsigned int *) &mutex->__data.__lock, + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); /* To the kernel, this will be visible after the kernel has acquired the mutex in the syscall. */ diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c index 87e87c013a..d24bb58a8b 100644 --- a/nptl/pthread_mutex_trylock.c +++ b/nptl/pthread_mutex_trylock.c @@ -21,6 +21,7 @@ #include <stdlib.h> #include "pthreadP.h" #include <lowlevellock.h> +#include <futex-internal.h> #ifndef lll_trylock_elision #define lll_trylock_elision(a,t) lll_trylock(a) @@ -346,11 +347,8 @@ __pthread_mutex_trylock (pthread_mutex_t *mutex) /* This mutex is now not recoverable. */ mutex->__data.__count = 0; - INTERNAL_SYSCALL_DECL (__err); - INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, - __lll_private_flag (FUTEX_UNLOCK_PI, - PTHREAD_ROBUST_MUTEX_PSHARED (mutex)), - 0, 0); + futex_unlock_pi ((unsigned int *) &mutex->__data.__lock, + PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); /* To the kernel, this will be visible after the kernel has acquired the mutex in the syscall. */ diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c index 71038f92e4..53f8b868e4 100644 --- a/nptl/pthread_mutex_unlock.c +++ b/nptl/pthread_mutex_unlock.c @@ -22,6 +22,7 @@ #include "pthreadP.h" #include <lowlevellock.h> #include <stap-probe.h> +#include <futex-internal.h> #ifndef lll_unlock_elision #define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; }) @@ -277,9 +278,8 @@ __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr) if (((l & FUTEX_WAITERS) != 0) || (l != THREAD_GETMEM (THREAD_SELF, tid))) { - INTERNAL_SYSCALL_DECL (__err); - INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock, - __lll_private_flag (FUTEX_UNLOCK_PI, private)); + futex_unlock_pi ((unsigned int *) &mutex->__data.__lock, + private); break; } } diff --git a/sysdeps/nptl/futex-internal.h b/sysdeps/nptl/futex-internal.h index 76921466f0..4feb772f14 100644 --- a/sysdeps/nptl/futex-internal.h +++ b/sysdeps/nptl/futex-internal.h @@ -381,4 +381,90 @@ futex_wake (unsigned int* futex_word, int processes_to_wake, int private) } } +/* The operation checks the value of the futex, if the value is 0, then + it is atomically set to the caller's thread ID. If the futex value is + nonzero, it is atomically sets the FUTEX_WAITERS bit, which signals wrt + other futex owner that it cannot unlock the futex in user space by + atomically by setting its value to 0. + + If more than one wait operations is issued, the enqueueing of the waiters + are done in descending priority order. + + The ABSTIME arguments provides an absolute timeout (measured against the + CLOCK_REALTIME clock). If TIMEOUT is NULL, the operation will block + indefinitely. + + Returns: + + - 0 if woken by a PI unlock operation or spuriously. + - EAGAIN if the futex owner thread ID is about to exit, but has not yet + handled the state cleanup. + - EDEADLK if the futex is already locked by the caller. + - ESRCH if the thread ID int he futex does not exist. + - EINVAL is the state is corrupted or if there is a waiter on the + futex. + - ETIMEDOUT if the ABSTIME expires. +*/ +static __always_inline int +futex_lock_pi (unsigned int *futex_word, const struct timespec *abstime, + int private) +{ + int err = lll_futex_timed_lock_pi (futex_word, abstime, private); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + case -ETIMEDOUT: + case -ESRCH: + case -EDEADLK: + case -EINVAL: /* This indicates either state corruption or that the kernel + found a waiter on futex address which is waiting via + FUTEX_WAIT or FUTEX_WAIT_BITSET. This is reported on + some futex_lock_pi usage (pthread_mutex_timedlock for + instance). */ + return -err; + + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + case -ENOSYS: /* Must have been caused by a glibc bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + +/* Wakes the top priority waiter that called a futex_lock_pi operation on + the futex. + + Returns the same values as futex_lock_pi under those same conditions; + additionally, returns EPERM when the caller is not allowed to attach + itself to the futex. */ +static __always_inline int +futex_unlock_pi (unsigned int *futex_word, int private) +{ + int err = lll_futex_timed_unlock_pi (futex_word, private); + switch (err) + { + case 0: + case -EAGAIN: + case -EINTR: + case -ETIMEDOUT: + case -ESRCH: + case -EDEADLK: + case -ENOSYS: + case -EPERM: /* The caller is not allowed to attach itself to the futex. + Used to check if PI futexes are supported by the + kernel. */ + return -err; + + case -EINVAL: /* Either due to wrong alignment or due to the timeout not + being normalized. Must have been caused by a glibc or + application bug. */ + case -EFAULT: /* Must have been caused by a glibc or application bug. */ + /* No other errors are documented at this time. */ + default: + futex_fatal_error (); + } +} + #endif /* futex-internal.h */ diff --git a/sysdeps/nptl/lowlevellock-futex.h b/sysdeps/nptl/lowlevellock-futex.h index 392277e8a5..ff0fd4edc4 100644 --- a/sysdeps/nptl/lowlevellock-futex.h +++ b/sysdeps/nptl/lowlevellock-futex.h @@ -140,6 +140,15 @@ /* Priority Inheritance support. */ +#define lll_futex_timed_lock_pi(futexp, abstime, private) \ + lll_futex_syscall (4, futexp, \ + __lll_private_flag (FUTEX_LOCK_PI, private), \ + 0, abstime) + +#define lll_futex_timed_unlock_pi(futexp, private) \ + lll_futex_syscall (4, futexp, \ + __lll_private_flag (FUTEX_UNLOCK_PI, private), \ + 0, 0) /* Like lll_futex_wait (FUTEXP, VAL, PRIVATE) but with the expectation that lll_futex_cmp_requeue_pi (FUTEXP, _, _, MUTEX, _, PRIVATE) will |