From a68cafa11c500d8a49a3014c43c5152859d037ae Mon Sep 17 00:00:00 2001 From: Adhemerval Zanella Date: Tue, 17 May 2016 10:16:39 -0300 Subject: Add runtime check for __ASSUME_REQUEUE_PI (BZ# 18463) This patch adds a runtime check for requeue priority futexes (FUTEX_{WAIT,CMP}_REQUEUE_PI) for configurations that do not define __ASSUME_REQUEUE_PI. It uses the same check uses for priority lock/unlock support, where issuing a futex operation with FUTEX_UNLOCK_PI returns if kernel has or not internal 'futex_cmpxchg_enabled' support (which is also used on requeue priority futexes operations). For architectures that already have __ASSUME_REQUEUE_PI the code flow does not change, 'use_requeue_pi' returns the previous chec logic. Also, if 'futex_cmpxchg_enabled' is not supported by the kernel, the previous logic will be used (by issuing a default futex operation). Tested on ARM (v3.8 kernel) and x86_64. * nptl/pthreadP.h (prio_inherit_missing): Change name to __prio_inherit_missing. (USE_REQUEUE_PI): Remove define. (use_requeue_pi): New function. * nptl/pthread_cond_broadcast.c [__ASSUME_REQUEUE_PI] (__pthread_cond_broadcast): Remove ifdef. (USE_REQUEUE_PI): Change to use_requeue_pi. * nptl/pthread_cond_signal.c [__ASSUME_REQUEUE_PI] (__pthread_cond_signal): Remove ifdef. (USE_REQUEUE_PI): Change to use_requeue_pi. * nptl/pthread_cond_timedwait.c [__ASSUME_REQUEUE_PI] (__pthread_cond_timedwait): Remove ifdef. (USE_REQUEUE_PI): Change to use_requeue_pi. ( nptl/pthread_cond_wait.c [__ASSUME_REQUEUE_PI] (__pthread_cond_wait): Remove ifdef. (USE_REQUEUE_PI): Change to use_requeue_pi. * nptl/pthread_mutex_init.c (prio_inherit_missing): Remove 'static' qualifier and change name to __prio_inherit_missing. --- ChangeLog | 19 +++++++++++++++++++ nptl/pthreadP.h | 23 ++++++++++++++--------- nptl/pthread_cond_broadcast.c | 5 ++--- nptl/pthread_cond_signal.c | 5 ++--- nptl/pthread_cond_timedwait.c | 11 ++++------- nptl/pthread_cond_wait.c | 13 +++++-------- nptl/pthread_mutex_init.c | 7 ++++--- 7 files changed, 50 insertions(+), 33 deletions(-) diff --git a/ChangeLog b/ChangeLog index aadf824afe..4f035c460d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,25 @@ 2016-06-03 Adhemerval Zanella + * nptl/pthreadP.h (prio_inherit_missing): Change name to + __prio_inherit_missing. + (USE_REQUEUE_PI): Remove define. + (use_requeue_pi): New function. + * nptl/pthread_cond_broadcast.c [__ASSUME_REQUEUE_PI] + (__pthread_cond_broadcast): Remove ifdef. + (USE_REQUEUE_PI): Change to use_requeue_pi. + * nptl/pthread_cond_signal.c [__ASSUME_REQUEUE_PI] + (__pthread_cond_signal): Remove ifdef. + (USE_REQUEUE_PI): Change to use_requeue_pi. + * nptl/pthread_cond_timedwait.c [__ASSUME_REQUEUE_PI] + (__pthread_cond_timedwait): Remove ifdef. + (USE_REQUEUE_PI): Change to use_requeue_pi. + ( nptl/pthread_cond_wait.c [__ASSUME_REQUEUE_PI] + (__pthread_cond_wait): Remove ifdef. + (USE_REQUEUE_PI): Change to use_requeue_pi. + * nptl/pthread_mutex_init.c (prio_inherit_missing): Remove 'static' + qualifier and change name to __prio_inherit_missing. + * nptl/nptl-init.c [__ASSUME_SET_ROBUST_LIST] (__set_robust_list_avail): Remove define. [__NR_set_robust_list] (__pthread_initialize_minimal_internal): diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h index a4a6ee1906..ee52b0c601 100644 --- a/nptl/pthreadP.h +++ b/nptl/pthreadP.h @@ -580,17 +580,22 @@ extern void __wait_lookup_done (void) attribute_hidden; # define PTHREAD_STATIC_FN_REQUIRE(name) __asm (".globl " #name); #endif +extern bool __prio_inherit_missing (void); + /* Test if the mutex is suitable for the FUTEX_WAIT_REQUEUE_PI operation. */ -#if (defined lll_futex_wait_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) -# define USE_REQUEUE_PI(mut) \ - ((mut) && (mut) != (void *) ~0l \ - && (((mut)->__data.__kind \ - & (PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NORMAL_NP)) \ - == PTHREAD_MUTEX_PRIO_INHERIT_NP)) -#else -# define USE_REQUEUE_PI(mut) 0 +static inline bool +use_requeue_pi (pthread_mutex_t *mut) +{ +#ifndef __ASSUME_REQUEUE_PI + if (__prio_inherit_missing ()) + return false; #endif + return (mut) && + (mut) != (void *) ~0l && + (((mut)->__data.__kind + & (PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NORMAL_NP)) == + PTHREAD_MUTEX_PRIO_INHERIT_NP); +} /* Returns 0 if POL is a valid scheduling policy. */ static inline int diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c index 881d098c05..ce75b3c7cd 100644 --- a/nptl/pthread_cond_broadcast.c +++ b/nptl/pthread_cond_broadcast.c @@ -61,9 +61,8 @@ __pthread_cond_broadcast (cond) || PTHREAD_MUTEX_PSHARED (mut) & PTHREAD_MUTEX_PSHARED_BIT) goto wake_all; -#if (defined lll_futex_cmp_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) - if (USE_REQUEUE_PI (mut)) +#ifdef lll_futex_cmp_requeue_pi + if (use_requeue_pi (mut)) { if (lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, INT_MAX, &mut->__data.__lock, futex_val, diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c index ba32f40532..0133ffe825 100644 --- a/nptl/pthread_cond_signal.c +++ b/nptl/pthread_cond_signal.c @@ -47,11 +47,10 @@ __pthread_cond_signal (cond) ++cond->__data.__wakeup_seq; ++cond->__data.__futex; -#if (defined lll_futex_cmp_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) +#ifdef lll_futex_cmp_requeue_pi pthread_mutex_t *mut = cond->__data.__mutex; - if (USE_REQUEUE_PI (mut) + if (use_requeue_pi (mut) /* This can only really fail with a ENOSYS, since nobody can modify futex while we have the cond_lock. */ && lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0, diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c index bf80467845..3553b362e6 100644 --- a/nptl/pthread_cond_timedwait.c +++ b/nptl/pthread_cond_timedwait.c @@ -63,8 +63,7 @@ __pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, int pshared = (cond->__data.__mutex == (void *) ~0l) ? LLL_SHARED : LLL_PRIVATE; -#if (defined lll_futex_timed_wait_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) +#ifdef lll_futex_timed_wait_requeue_pi int pi_flag = 0; #endif @@ -161,8 +160,7 @@ __pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, /* REQUEUE_PI was implemented after FUTEX_CLOCK_REALTIME, so it is sufficient to check just the former. */ -#if (defined lll_futex_timed_wait_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) +#ifdef lll_futex_timed_wait_requeue_pi /* If pi_flag remained 1 then it means that we had the lock and the mutex but a spurious waker raced ahead of us. Give back the mutex before going into wait again. */ @@ -171,7 +169,7 @@ __pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, __pthread_mutex_cond_lock_adjust (mutex); __pthread_mutex_unlock_usercnt (mutex, 0); } - pi_flag = USE_REQUEUE_PI (mutex); + pi_flag = use_requeue_pi (mutex); if (pi_flag) { @@ -250,8 +248,7 @@ __pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, __pthread_cleanup_pop (&buffer, 0); /* Get the mutex before returning. */ -#if (defined lll_futex_timed_wait_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) +#ifdef lll_futex_timed_wait_requeue_pi if (pi_flag) { __pthread_mutex_cond_lock_adjust (mutex); diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c index 0d6558b642..9db01d243d 100644 --- a/nptl/pthread_cond_wait.c +++ b/nptl/pthread_cond_wait.c @@ -86,7 +86,7 @@ __condvar_cleanup (void *arg) /* Get the mutex before returning unless asynchronous cancellation is in effect. We don't try to get the mutex if we already own it. */ - if (!(USE_REQUEUE_PI (cbuffer->mutex)) + if (!(use_requeue_pi (cbuffer->mutex)) || ((cbuffer->mutex->__data.__lock & FUTEX_TID_MASK) != THREAD_GETMEM (THREAD_SELF, tid))) { @@ -106,8 +106,7 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) int pshared = (cond->__data.__mutex == (void *) ~0l) ? LLL_SHARED : LLL_PRIVATE; -#if (defined lll_futex_wait_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) +#ifdef lll_futex_wait_requeue_pi int pi_flag = 0; #endif @@ -160,8 +159,7 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) /* Enable asynchronous cancellation. Required by the standard. */ cbuffer.oldtype = __pthread_enable_asynccancel (); -#if (defined lll_futex_wait_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) +#ifdef lll_futex_wait_requeue_pi /* If pi_flag remained 1 then it means that we had the lock and the mutex but a spurious waker raced ahead of us. Give back the mutex before going into wait again. */ @@ -170,7 +168,7 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) __pthread_mutex_cond_lock_adjust (mutex); __pthread_mutex_unlock_usercnt (mutex, 0); } - pi_flag = USE_REQUEUE_PI (mutex); + pi_flag = use_requeue_pi (mutex); if (pi_flag) { @@ -221,8 +219,7 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) __pthread_cleanup_pop (&buffer, 0); /* Get the mutex before returning. Not needed for PI. */ -#if (defined lll_futex_wait_requeue_pi \ - && defined __ASSUME_REQUEUE_PI) +#ifdef lll_futex_wait_requeue_pi if (pi_flag) { __pthread_mutex_cond_lock_adjust (mutex); diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c index 2bc1f881fa..3e4c06e9f9 100644 --- a/nptl/pthread_mutex_init.c +++ b/nptl/pthread_mutex_init.c @@ -33,8 +33,9 @@ static const struct pthread_mutexattr default_mutexattr = }; -static bool -prio_inherit_missing (void) +bool +attribute_hidden +__prio_inherit_missing (void) { #ifdef __NR_futex static int tpi_supported; @@ -73,7 +74,7 @@ __pthread_mutex_init (mutex, mutexattr) break; case PTHREAD_PRIO_INHERIT << PTHREAD_MUTEXATTR_PROTOCOL_SHIFT: - if (__glibc_unlikely (prio_inherit_missing ())) + if (__glibc_unlikely (__prio_inherit_missing ())) return ENOTSUP; break; -- cgit 1.4.1