nptl: Use FUTEX_LOCK_PI2 when available

This patch uses the new futex PI operation provided by Linux v5.14
when it is required.

The futex_lock_pi64() is moved to futex-internal.c (since it used on
two different places and its code size might be large depending of the
kernel configuration) and clockid is added as an argument.

Co-authored-by: Kurt Kanzenbach <kurt@linutronix.de>

5 files changed, 72 insertions, 56 deletions

diff --git a/nptl/futex-internal.c b/nptl/futex-internal.c
index e74647a9d4..58605b2fca 100644
--- a/nptl/futex-internal.c
+++ b/nptl/futex-internal.c
@@ -140,3 +140,66 @@ __futex_abstimed_wait_cancelable64 (unsigned int* futex_word,
                                        abstime, private, true);
 }
 libc_hidden_def (__futex_abstimed_wait_cancelable64)
+
+int
+__futex_lock_pi64 (int *futex_word, clockid_t clockid,
+		   const struct __timespec64 *abstime, int private)
+{
+  int err;
+
+  unsigned int clockbit = clockid == CLOCK_REALTIME
+			  ? FUTEX_CLOCK_REALTIME : 0;
+  int op_pi2 = __lll_private_flag (FUTEX_LOCK_PI2 | clockbit, private);
+#if __ASSUME_FUTEX_LOCK_PI2
+  /* Assume __ASSUME_TIME64_SYSCALLS since FUTEX_LOCK_PI2 was added later.  */
+  err = INTERNAL_SYSCALL_CALL (futex_time64, futex_word, op_pi2, 0, abstime);
+#else
+  /* FUTEX_LOCK_PI does not support clock selection, so for CLOCK_MONOTONIC
+     the only option is to use FUTEX_LOCK_PI2.  */
+  int op_pi1 = __lll_private_flag (FUTEX_LOCK_PI, private);
+  int op_pi = abstime != NULL && clockid != CLOCK_REALTIME ? op_pi2 : op_pi1;
+
+# ifdef __ASSUME_TIME64_SYSCALLS
+  err = INTERNAL_SYSCALL_CALL (futex_time64, futex_word, op_pi, 0, abstime);
+# else
+  bool need_time64 = abstime != NULL && !in_time_t_range (abstime->tv_sec);
+  if (need_time64)
+    err = INTERNAL_SYSCALL_CALL (futex_time64, futex_word, op_pi, 0, abstime);
+  else
+    {
+      struct timespec ts32, *pts32 = NULL;
+      if (abstime != NULL)
+	{
+	  ts32 = valid_timespec64_to_timespec (*abstime);
+	  pts32 = &ts32;
+	}
+      err = INTERNAL_SYSCALL_CALL (futex, futex_word, op_pi, 0, pts32);
+    }
+# endif	 /* __ASSUME_TIME64_SYSCALLS */
+   /* FUTEX_LOCK_PI2 is not available on this kernel.  */
+   if (err == -ENOSYS)
+     err = -EINVAL;
+#endif /* __ASSUME_FUTEX_LOCK_PI2  */
+
+  switch (err)
+    {
+    case 0:
+    case -EAGAIN:
+    case -EINTR:
+    case -ETIMEDOUT:
+    case -ESRCH:
+    case -EDEADLK:
+    case -EINVAL: /* This indicates either state corruption or that the kernel
+                     found a waiter on futex address which is waiting via
+                     FUTEX_WAIT or FUTEX_WAIT_BITSET.  This is reported on
+                     some futex_lock_pi usage (pthread_mutex_timedlock for
+                     instance).  */
+      return -err;
+
+    case -EFAULT: /* Must have been caused by a glibc or application bug.  */
+    case -ENOSYS: /* Must have been caused by a glibc bug.  */
+    /* No other errors are documented at this time.  */
+    default:
+      futex_fatal_error ();
+    }
+}
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index fbe8a3cd21..2bd41767e0 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -421,7 +421,8 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 	    int private = (robust
 			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 			   : PTHREAD_MUTEX_PSHARED (mutex));
-	    int e = futex_lock_pi64 (&mutex->__data.__lock, NULL, private);
+	    int e = __futex_lock_pi64 (&mutex->__data.__lock, 0 /* ununsed  */,
+				       NULL, private);
 	    if (e == ESRCH || e == EDEADLK)
 	      {
 		assert (e != EDEADLK
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index bf2af6b048..a695faeb7e 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -369,7 +369,8 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex,
 	    int private = (robust
 			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 			   : PTHREAD_MUTEX_PSHARED (mutex));
-	    int e = futex_lock_pi64 (&mutex->__data.__lock, abstime, private);
+	    int e = __futex_lock_pi64 (&mutex->__data.__lock, clockid, abstime,
+				       private);
 	    if (e == ETIMEDOUT)
 	      return ETIMEDOUT;
 	    else if (e == ESRCH || e == EDEADLK)
diff --git a/sysdeps/nptl/futex-internal.h b/sysdeps/nptl/futex-internal.h
index 79a366604d..a46730cad6 100644
--- a/sysdeps/nptl/futex-internal.h
+++ b/sysdeps/nptl/futex-internal.h
@@ -236,8 +236,8 @@ futex_wake (unsigned int* futex_word, int processes_to_wake, int private)
    are done in descending priority order.
 
    The ABSTIME arguments provides an absolute timeout (measured against the
-   CLOCK_REALTIME clock).  If TIMEOUT is NULL, the operation will block
-   indefinitely.
+   CLOCK_REALTIME or CLOCK_MONOTONIC clock).  If TIMEOUT is NULL, the operation
+   will block indefinitely.
 
    Returns:
 
@@ -250,58 +250,8 @@ futex_wake (unsigned int* futex_word, int processes_to_wake, int private)
        futex.
      - ETIMEDOUT if the ABSTIME expires.
 */
-static __always_inline int
-futex_lock_pi64 (int *futex_word, const struct __timespec64 *abstime,
-                 int private)
-{
-  int err;
-#ifdef __ASSUME_TIME64_SYSCALLS
-  err = INTERNAL_SYSCALL_CALL (futex_time64, futex_word,
-			       __lll_private_flag (FUTEX_LOCK_PI, private), 0,
-			       abstime);
-#else
-  bool need_time64 = abstime != NULL && !in_time_t_range (abstime->tv_sec);
-  if (need_time64)
-    {
-      err = INTERNAL_SYSCALL_CALL (futex_time64, futex_word,
-				   __lll_private_flag (FUTEX_LOCK_PI, private),
-				   0, abstime);
-      if (err == -ENOSYS)
-	err = -EOVERFLOW;
-    }
-  else
-    {
-      struct timespec ts32;
-      if (abstime != NULL)
-        ts32 = valid_timespec64_to_timespec (*abstime);
-
-      err = INTERNAL_SYSCALL_CALL (futex, futex_word, __lll_private_flag
-                                   (FUTEX_LOCK_PI, private), 0,
-                                   abstime != NULL ? &ts32 : NULL);
-    }
-#endif
-  switch (err)
-    {
-    case 0:
-    case -EAGAIN:
-    case -EINTR:
-    case -ETIMEDOUT:
-    case -ESRCH:
-    case -EDEADLK:
-    case -EINVAL: /* This indicates either state corruption or that the kernel
-		     found a waiter on futex address which is waiting via
-		     FUTEX_WAIT or FUTEX_WAIT_BITSET.  This is reported on
-		     some futex_lock_pi usage (pthread_mutex_timedlock for
-		     instance).  */
-      return -err;
-
-    case -EFAULT: /* Must have been caused by a glibc or application bug.  */
-    case -ENOSYS: /* Must have been caused by a glibc bug.  */
-    /* No other errors are documented at this time.  */
-    default:
-      futex_fatal_error ();
-    }
-}
+int __futex_lock_pi64 (int *futex_word, clockid_t clockid,
+		       const struct __timespec64 *abstime, int private);
 
 /* Wakes the top priority waiter that called a futex_lock_pi operation on
    the futex.
diff --git a/sysdeps/nptl/lowlevellock-futex.h b/sysdeps/nptl/lowlevellock-futex.h
index 66ebfe50f4..abda179e0d 100644
--- a/sysdeps/nptl/lowlevellock-futex.h
+++ b/sysdeps/nptl/lowlevellock-futex.h
@@ -38,6 +38,7 @@
 #define FUTEX_WAKE_BITSET	10
 #define FUTEX_WAIT_REQUEUE_PI   11
 #define FUTEX_CMP_REQUEUE_PI    12
+#define FUTEX_LOCK_PI2		13
 #define FUTEX_PRIVATE_FLAG	128
 #define FUTEX_CLOCK_REALTIME	256