Refactor PI mutexes internal definitions

This patch adds the generic futex_lock_pi and futex_unlock_pi to wrap
around the syscall machinery required to issue the syscall calls. It
simplifies a bit the futex code required to implement PI mutexes.

No function changes, checked on x86_64-linux-gnu.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>

7 files changed, 137 insertions, 69 deletions

diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c
index fe4eeee37c..20800b80f5 100644
--- a/nptl/pthread_mutex_init.c
+++ b/nptl/pthread_mutex_init.c
@@ -24,6 +24,7 @@
 #include "pthreadP.h"
 #include <atomic.h>
 #include <pthread-offsets.h>
+#include <futex-internal.h>
 
 #include <stap-probe.h>
 
@@ -37,19 +38,13 @@ static const struct pthread_mutexattr default_mutexattr =
 static bool
 prio_inherit_missing (void)
 {
-#ifdef __NR_futex
   static int tpi_supported;
-  if (__glibc_unlikely (tpi_supported == 0))
+  if (__glibc_unlikely (atomic_load_relaxed (&tpi_supported) == 0))
     {
-      int lock = 0;
-      INTERNAL_SYSCALL_DECL (err);
-      int ret = INTERNAL_SYSCALL (futex, err, 4, &lock, FUTEX_UNLOCK_PI, 0, 0);
-      assert (INTERNAL_SYSCALL_ERROR_P (ret, err));
-      tpi_supported = INTERNAL_SYSCALL_ERRNO (ret, err) == ENOSYS ? -1 : 1;
+      int e = futex_unlock_pi (&(unsigned int){0}, 0);
+      atomic_store_relaxed (&tpi_supported, e == ENOSYS ? -1 : 1);
     }
   return __glibc_unlikely (tpi_supported < 0);
-#endif
-  return true;
 }
 
 int
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index ace436d5a6..05bba50666 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -24,7 +24,7 @@
 #include <not-cancel.h>
 #include "pthreadP.h"
 #include <atomic.h>
-#include <lowlevellock.h>
+#include <futex-internal.h>
 #include <stap-probe.h>
 
 #ifndef lll_lock_elision
@@ -416,21 +416,16 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 	    int private = (robust
 			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 			   : PTHREAD_MUTEX_PSHARED (mutex));
-	    INTERNAL_SYSCALL_DECL (__err);
-	    int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-				      __lll_private_flag (FUTEX_LOCK_PI,
-							  private), 1, 0);
-
-	    if (INTERNAL_SYSCALL_ERROR_P (e, __err)
-		&& (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
-		    || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
+	    int e = futex_lock_pi ((unsigned int *) &mutex->__data.__lock,
+				   NULL, private);
+	    if (e == ESRCH || e == EDEADLK)
 	      {
-		assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
+		assert (e != EDEADLK
 			|| (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 			    && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 		/* ESRCH can happen only for non-robust PI mutexes where
 		   the owner of the lock died.  */
-		assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
+		assert (e != ESRCH || !robust);
 
 		/* Delay the thread indefinitely.  */
 		while (1)
@@ -479,11 +474,8 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 	    /* This mutex is now not recoverable.  */
 	    mutex->__data.__count = 0;
 
-	    INTERNAL_SYSCALL_DECL (__err);
-	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-			      __lll_private_flag (FUTEX_UNLOCK_PI,
-						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
-			      0, 0);
+	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
+			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 
 	    /* To the kernel, this will be visible after the kernel has
 	       acquired the mutex in the syscall.  */
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index 76b93bddc6..eb4baae93b 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -25,6 +25,7 @@
 #include <atomic.h>
 #include <lowlevellock.h>
 #include <not-cancel.h>
+#include <futex-internal.h>
 
 #include <stap-probe.h>
 
@@ -377,39 +378,29 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex,
 	    int private = (robust
 			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 			   : PTHREAD_MUTEX_PSHARED (mutex));
-	    INTERNAL_SYSCALL_DECL (__err);
-
-	    int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-				      __lll_private_flag (FUTEX_LOCK_PI,
-							  private), 1,
-				      abstime);
-	    if (INTERNAL_SYSCALL_ERROR_P (e, __err))
+	    int e = futex_lock_pi ((unsigned int *) &mutex->__data.__lock,
+				   abstime, private);
+	    if (e == ETIMEDOUT)
+	      return ETIMEDOUT;
+	    else if (e == ESRCH || e == EDEADLK)
 	      {
-		if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
-		  return ETIMEDOUT;
-
-		if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
-		    || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
-		  {
-		    assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
-			    || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
-				&& kind != PTHREAD_MUTEX_RECURSIVE_NP));
-		    /* ESRCH can happen only for non-robust PI mutexes where
-		       the owner of the lock died.  */
-		    assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
-			    || !robust);
-
-		    /* Delay the thread until the timeout is reached.
-		       Then return ETIMEDOUT.  */
-		    do
-		      e = lll_timedwait (&(int){0}, 0, clockid, abstime,
-					 private);
-		    while (e != ETIMEDOUT);
-		    return ETIMEDOUT;
-		  }
-
-		return INTERNAL_SYSCALL_ERRNO (e, __err);
+		assert (e != EDEADLK
+			|| (kind != PTHREAD_MUTEX_ERRORCHECK_NP
+			   && kind != PTHREAD_MUTEX_RECURSIVE_NP));
+		/* ESRCH can happen only for non-robust PI mutexes where
+		   the owner of the lock died.  */
+		assert (e != ESRCH || !robust);
+
+		/* Delay the thread until the timeout is reached. Then return
+		   ETIMEDOUT.  */
+		do
+		  e = lll_timedwait (&(int){0}, 0, clockid, abstime,
+				     private);
+		while (e != ETIMEDOUT);
+		return ETIMEDOUT;
 	      }
+	    else if (e != 0)
+	      return e;
 
 	    oldval = mutex->__data.__lock;
 
@@ -447,11 +438,8 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex,
 	    /* This mutex is now not recoverable.  */
 	    mutex->__data.__count = 0;
 
-	    INTERNAL_SYSCALL_DECL (__err);
-	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-			      __lll_private_flag (FUTEX_UNLOCK_PI,
-						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
-			      0, 0);
+	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
+			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 
 	    /* To the kernel, this will be visible after the kernel has
 	       acquired the mutex in the syscall.  */
diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c
index 87e87c013a..d24bb58a8b 100644
--- a/nptl/pthread_mutex_trylock.c
+++ b/nptl/pthread_mutex_trylock.c
@@ -21,6 +21,7 @@
 #include <stdlib.h>
 #include "pthreadP.h"
 #include <lowlevellock.h>
+#include <futex-internal.h>
 
 #ifndef lll_trylock_elision
 #define lll_trylock_elision(a,t) lll_trylock(a)
@@ -346,11 +347,8 @@ __pthread_mutex_trylock (pthread_mutex_t *mutex)
 	    /* This mutex is now not recoverable.  */
 	    mutex->__data.__count = 0;
 
-	    INTERNAL_SYSCALL_DECL (__err);
-	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-			      __lll_private_flag (FUTEX_UNLOCK_PI,
-						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
-			      0, 0);
+	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
+			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 
 	    /* To the kernel, this will be visible after the kernel has
 	       acquired the mutex in the syscall.  */
diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c
index 71038f92e4..53f8b868e4 100644
--- a/nptl/pthread_mutex_unlock.c
+++ b/nptl/pthread_mutex_unlock.c
@@ -22,6 +22,7 @@
 #include "pthreadP.h"
 #include <lowlevellock.h>
 #include <stap-probe.h>
+#include <futex-internal.h>
 
 #ifndef lll_unlock_elision
 #define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
@@ -277,9 +278,8 @@ __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
 	  if (((l & FUTEX_WAITERS) != 0)
 	      || (l != THREAD_GETMEM (THREAD_SELF, tid)))
 	    {
-	      INTERNAL_SYSCALL_DECL (__err);
-	      INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
-				__lll_private_flag (FUTEX_UNLOCK_PI, private));
+	      futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
+			       private);
 	      break;
 	    }
 	}
diff --git a/sysdeps/nptl/futex-internal.h b/sysdeps/nptl/futex-internal.h
index 76921466f0..4feb772f14 100644
--- a/sysdeps/nptl/futex-internal.h
+++ b/sysdeps/nptl/futex-internal.h
@@ -381,4 +381,90 @@ futex_wake (unsigned int* futex_word, int processes_to_wake, int private)
     }
 }
 
+/* The operation checks the value of the futex, if the value is 0, then
+   it is atomically set to the caller's thread ID.  If the futex value is
+   nonzero, it is atomically sets the FUTEX_WAITERS bit, which signals wrt
+   other futex owner that it cannot unlock the futex in user space by
+   atomically by setting its value to 0.
+
+   If more than one wait operations is issued, the enqueueing of the waiters
+   are done in descending priority order.
+
+   The ABSTIME arguments provides an absolute timeout (measured against the
+   CLOCK_REALTIME clock).  If TIMEOUT is NULL, the operation will block
+   indefinitely.
+
+   Returns:
+
+     - 0 if woken by a PI unlock operation or spuriously.
+     - EAGAIN if the futex owner thread ID is about to exit, but has not yet
+       handled the state cleanup.
+     - EDEADLK if the futex is already locked by the caller.
+     - ESRCH if the thread ID int he futex does not exist.
+     - EINVAL is the state is corrupted or if there is a waiter on the
+       futex.
+     - ETIMEDOUT if the ABSTIME expires.
+*/
+static __always_inline int
+futex_lock_pi (unsigned int *futex_word, const struct timespec *abstime,
+	       int private)
+{
+  int err = lll_futex_timed_lock_pi (futex_word, abstime, private);
+  switch (err)
+    {
+    case 0:
+    case -EAGAIN:
+    case -EINTR:
+    case -ETIMEDOUT:
+    case -ESRCH:
+    case -EDEADLK:
+    case -EINVAL: /* This indicates either state corruption or that the kernel
+		     found a waiter on futex address which is waiting via
+		     FUTEX_WAIT or FUTEX_WAIT_BITSET.  This is reported on
+		     some futex_lock_pi usage (pthread_mutex_timedlock for
+		     instance).  */
+      return -err;
+
+    case -EFAULT: /* Must have been caused by a glibc or application bug.  */
+    case -ENOSYS: /* Must have been caused by a glibc bug.  */
+    /* No other errors are documented at this time.  */
+    default:
+      futex_fatal_error ();
+    }
+}
+
+/* Wakes the top priority waiter that called a futex_lock_pi operation on
+   the futex.
+
+   Returns the same values as futex_lock_pi under those same conditions;
+   additionally, returns EPERM when the caller is not allowed to attach
+   itself to the futex.  */
+static __always_inline int
+futex_unlock_pi (unsigned int *futex_word, int private)
+{
+  int err = lll_futex_timed_unlock_pi (futex_word, private);
+  switch (err)
+    {
+    case 0:
+    case -EAGAIN:
+    case -EINTR:
+    case -ETIMEDOUT:
+    case -ESRCH:
+    case -EDEADLK:
+    case -ENOSYS:
+    case -EPERM:  /*  The caller is not allowed to attach itself to the futex.
+		      Used to check if PI futexes are supported by the
+		      kernel.  */
+      return -err;
+
+    case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+		     being normalized.  Must have been caused by a glibc or
+		     application bug.  */
+    case -EFAULT: /* Must have been caused by a glibc or application bug.  */
+    /* No other errors are documented at this time.  */
+    default:
+      futex_fatal_error ();
+    }
+}
+
 #endif  /* futex-internal.h */
diff --git a/sysdeps/nptl/lowlevellock-futex.h b/sysdeps/nptl/lowlevellock-futex.h
index 392277e8a5..ff0fd4edc4 100644
--- a/sysdeps/nptl/lowlevellock-futex.h
+++ b/sysdeps/nptl/lowlevellock-futex.h
@@ -140,6 +140,15 @@
 
 
 /* Priority Inheritance support.  */
+#define lll_futex_timed_lock_pi(futexp, abstime, private) 		\
+  lll_futex_syscall (4, futexp,						\
+		     __lll_private_flag (FUTEX_LOCK_PI, private),	\
+		     0, abstime)
+
+#define lll_futex_timed_unlock_pi(futexp, private) 			\
+  lll_futex_syscall (4, futexp,						\
+		     __lll_private_flag (FUTEX_UNLOCK_PI, private),	\
+		     0, 0)
 
 /* Like lll_futex_wait (FUTEXP, VAL, PRIVATE) but with the expectation
    that lll_futex_cmp_requeue_pi (FUTEXP, _, _, MUTEX, _, PRIVATE) will