Add and use new glibc-internal futex API.

This adds new functions for futex operations, starting with wait,
abstimed_wait, reltimed_wait, wake.  They add documentation and error
checking according to the current draft of the Linux kernel futex manpage.

Waiting with absolute or relative timeouts is split into separate functions.
This allows for removing a few cases of code duplication in pthreads code,
which uses absolute timeouts; also, it allows us to put platform-specific
code to go from an absolute to a relative timeout into the platform-specific
futex abstractions..

Futex operations that can be canceled are also split out into separate
functions suffixed by "_cancelable".

There are separate versions for both Linux and NaCl; while they currently
differ only slightly, my expectation is that the separate versions of
lowlevellock-futex.h will eventually be merged into futex-internal.h
when we get to move the lll_ functions over to the new futex API.

24 files changed, 142 insertions, 318 deletions

diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c
index 8e620c46e9..c56a4df12c 100644
--- a/nptl/allocatestack.c
+++ b/nptl/allocatestack.c
@@ -29,6 +29,7 @@
 #include <tls.h>
 #include <list.h>
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <kernel-features.h>
 #include <stack-aliasing.h>
 
@@ -987,7 +988,7 @@ setxid_mark_thread (struct xid_command *cmdp, struct pthread *t)
   if (t->setxid_futex == -1
       && ! atomic_compare_and_exchange_bool_acq (&t->setxid_futex, -2, -1))
     do
-      lll_futex_wait (&t->setxid_futex, -2, LLL_PRIVATE);
+      futex_wait_simple (&t->setxid_futex, -2, FUTEX_PRIVATE);
     while (t->setxid_futex == -2);
 
   /* Don't let the thread exit before the setxid handler runs.  */
@@ -1005,7 +1006,7 @@ setxid_mark_thread (struct xid_command *cmdp, struct pthread *t)
 	  if ((ch & SETXID_BITMASK) == 0)
 	    {
 	      t->setxid_futex = 1;
-	      lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE);
+	      futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE);
 	    }
 	  return;
 	}
@@ -1032,7 +1033,7 @@ setxid_unmark_thread (struct xid_command *cmdp, struct pthread *t)
 
   /* Release the futex just in case.  */
   t->setxid_futex = 1;
-  lll_futex_wake (&t->setxid_futex, 1, LLL_PRIVATE);
+  futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE);
 }
 
 
@@ -1141,7 +1142,8 @@ __nptl_setxid (struct xid_command *cmdp)
       int cur = cmdp->cntr;
       while (cur != 0)
 	{
-	  lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE);
+	  futex_wait_simple ((unsigned int *) &cmdp->cntr, cur,
+			     FUTEX_PRIVATE);
 	  cur = cmdp->cntr;
 	}
     }
@@ -1251,7 +1253,8 @@ __wait_lookup_done (void)
 	continue;
 
       do
-	lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
+	futex_wait_simple ((unsigned int *) gscope_flagp,
+			   THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
     }
 
@@ -1273,7 +1276,8 @@ __wait_lookup_done (void)
 	continue;
 
       do
-	lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
+	futex_wait_simple ((unsigned int *) gscope_flagp,
+			   THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
     }
 
diff --git a/nptl/cancellation.c b/nptl/cancellation.c
index deac1ebb3e..2bd31686fd 100644
--- a/nptl/cancellation.c
+++ b/nptl/cancellation.c
@@ -19,6 +19,7 @@
 #include <setjmp.h>
 #include <stdlib.h>
 #include "pthreadP.h"
+#include <futex-internal.h>
 
 
 /* The next two functions are similar to pthread_setcanceltype() but
@@ -93,7 +94,8 @@ __pthread_disable_asynccancel (int oldtype)
   while (__builtin_expect ((newval & (CANCELING_BITMASK | CANCELED_BITMASK))
 			   == CANCELING_BITMASK, 0))
     {
-      lll_futex_wait (&self->cancelhandling, newval, LLL_PRIVATE);
+      futex_wait_simple ((unsigned int *) &self->cancelhandling, newval,
+			 FUTEX_PRIVATE);
       newval = THREAD_GETMEM (self, cancelhandling);
     }
 }
diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c
index 8a511610cd..c043fb50ab 100644
--- a/nptl/nptl-init.c
+++ b/nptl/nptl-init.c
@@ -34,6 +34,7 @@
 #include <shlib-compat.h>
 #include <smp.h>
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <kernel-features.h>
 #include <libc-internal.h>
 #include <pthread-pids.h>
@@ -279,10 +280,10 @@ sighandler_setxid (int sig, siginfo_t *si, void *ctx)
 
   /* And release the futex.  */
   self->setxid_futex = 1;
-  lll_futex_wake (&self->setxid_futex, 1, LLL_PRIVATE);
+  futex_wake (&self->setxid_futex, 1, FUTEX_PRIVATE);
 
   if (atomic_decrement_val (&__xidcmd->cntr) == 0)
-    lll_futex_wake (&__xidcmd->cntr, 1, LLL_PRIVATE);
+    futex_wake ((unsigned int *) &__xidcmd->cntr, 1, FUTEX_PRIVATE);
 }
 #endif
 
diff --git a/nptl/pthread_barrier_init.c b/nptl/pthread_barrier_init.c
index 5ea9fad291..8fe15ecba5 100644
--- a/nptl/pthread_barrier_init.c
+++ b/nptl/pthread_barrier_init.c
@@ -36,6 +36,7 @@ __pthread_barrier_init (barrier, attr, count)
 {
   struct pthread_barrier *ibarrier;
 
+  /* XXX EINVAL is not specified by POSIX as a possible error code.  */
   if (__glibc_unlikely (count == 0))
     return EINVAL;
 
@@ -44,11 +45,6 @@ __pthread_barrier_init (barrier, attr, count)
        ? iattr = (struct pthread_barrierattr *) attr
        : &default_barrierattr);
 
-  if (iattr->pshared != PTHREAD_PROCESS_PRIVATE
-      && __builtin_expect (iattr->pshared != PTHREAD_PROCESS_SHARED, 0))
-    /* Invalid attribute.  */
-    return EINVAL;
-
   ibarrier = (struct pthread_barrier *) barrier;
 
   /* Initialize the individual fields.  */
@@ -57,14 +53,10 @@ __pthread_barrier_init (barrier, attr, count)
   ibarrier->init_count = count;
   ibarrier->curr_event = 0;
 
-#ifdef __ASSUME_PRIVATE_FUTEX
+  /* XXX Don't use FUTEX_SHARED or FUTEX_PRIVATE as long as there are still
+     assembly implementations that expect the value determined below.  */
   ibarrier->private = (iattr->pshared != PTHREAD_PROCESS_PRIVATE
 		       ? 0 : FUTEX_PRIVATE_FLAG);
-#else
-  ibarrier->private = (iattr->pshared != PTHREAD_PROCESS_PRIVATE
-		       ? 0 : THREAD_GETMEM (THREAD_SELF,
-					    header.private_futex));
-#endif
 
   return 0;
 }
diff --git a/nptl/pthread_barrier_wait.c b/nptl/pthread_barrier_wait.c
index d69a929ef6..2b34e3097b 100644
--- a/nptl/pthread_barrier_wait.c
+++ b/nptl/pthread_barrier_wait.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include <sysdep.h>
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <pthreadP.h>
 
 
@@ -29,9 +30,12 @@ __pthread_barrier_wait (barrier)
 {
   struct pthread_barrier *ibarrier = (struct pthread_barrier *) barrier;
   int result = 0;
+  int lll_private = ibarrier->private ^ FUTEX_PRIVATE_FLAG;
+  int futex_private = (lll_private == LLL_PRIVATE
+		       ? FUTEX_PRIVATE : FUTEX_SHARED);
 
   /* Make sure we are alone.  */
-  lll_lock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+  lll_lock (ibarrier->lock, lll_private);
 
   /* One more arrival.  */
   --ibarrier->left;
@@ -44,8 +48,7 @@ __pthread_barrier_wait (barrier)
       ++ibarrier->curr_event;
 
       /* Wake up everybody.  */
-      lll_futex_wake (&ibarrier->curr_event, INT_MAX,
-		      ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+      futex_wake (&ibarrier->curr_event, INT_MAX, futex_private);
 
       /* This is the thread which finished the serialization.  */
       result = PTHREAD_BARRIER_SERIAL_THREAD;
@@ -57,12 +60,11 @@ __pthread_barrier_wait (barrier)
       unsigned int event = ibarrier->curr_event;
 
       /* Before suspending, make the barrier available to others.  */
-      lll_unlock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+      lll_unlock (ibarrier->lock, lll_private);
 
       /* Wait for the event counter of the barrier to change.  */
       do
-	lll_futex_wait (&ibarrier->curr_event, event,
-			ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+	futex_wait_simple (&ibarrier->curr_event, event, futex_private);
       while (event == ibarrier->curr_event);
     }
 
@@ -72,7 +74,7 @@ __pthread_barrier_wait (barrier)
   /* If this was the last woken thread, unlock.  */
   if (atomic_increment_val (&ibarrier->left) == init_count)
     /* We are done.  */
-    lll_unlock (ibarrier->lock, ibarrier->private ^ FUTEX_PRIVATE_FLAG);
+    lll_unlock (ibarrier->lock, lll_private);
 
   return result;
 }
diff --git a/nptl/pthread_barrierattr_setpshared.c b/nptl/pthread_barrierattr_setpshared.c
index 86d72c5d87..eeaee5d430 100644
--- a/nptl/pthread_barrierattr_setpshared.c
+++ b/nptl/pthread_barrierattr_setpshared.c
@@ -18,6 +18,7 @@
 
 #include <errno.h>
 #include "pthreadP.h"
+#include <futex-internal.h>
 
 
 int
@@ -27,9 +28,9 @@ pthread_barrierattr_setpshared (attr, pshared)
 {
   struct pthread_barrierattr *iattr;
 
-  if (pshared != PTHREAD_PROCESS_PRIVATE
-      && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0))
-    return EINVAL;
+  int err = futex_supports_pshared (pshared);
+  if (err != 0)
+    return err;
 
   iattr = (struct pthread_barrierattr *) attr;
 
diff --git a/nptl/pthread_condattr_setpshared.c b/nptl/pthread_condattr_setpshared.c
index 3a760cfdf1..a015403cf5 100644
--- a/nptl/pthread_condattr_setpshared.c
+++ b/nptl/pthread_condattr_setpshared.c
@@ -18,15 +18,16 @@
 
 #include <errno.h>
 #include <pthreadP.h>
+#include <futex-internal.h>
 
 int
 pthread_condattr_setpshared (attr, pshared)
      pthread_condattr_t *attr;
      int pshared;
 {
-  if (pshared != PTHREAD_PROCESS_PRIVATE
-      && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0))
-    return EINVAL;
+  int err = futex_supports_pshared (pshared);
+  if (err != 0)
+    return err;
 
   int *valuep = &((struct pthread_condattr *) attr)->value;
 
diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c
index 71a56193e6..d10f4ea800 100644
--- a/nptl/pthread_create.c
+++ b/nptl/pthread_create.c
@@ -31,6 +31,7 @@
 #include <kernel-features.h>
 #include <exit-thread.h>
 #include <default-sched.h>
+#include <futex-internal.h>
 
 #include <shlib-compat.h>
 
@@ -269,7 +270,7 @@ START_THREAD_DEFN
 
   /* Allow setxid from now onwards.  */
   if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2))
-    lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE);
+    futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE);
 
 #ifdef __NR_set_robust_list
 # ifndef __ASSUME_SET_ROBUST_LIST
@@ -414,7 +415,8 @@ START_THREAD_DEFN
 	  this->__list.__next = NULL;
 
 	  atomic_or (&this->__lock, FUTEX_OWNER_DIED);
-	  lll_futex_wake (&this->__lock, 1, /* XYZ */ LLL_SHARED);
+	  futex_wake ((unsigned int *) &this->__lock, 1,
+		      /* XYZ */ FUTEX_SHARED);
 	}
       while (robust != (void *) &pd->robust_head);
     }
@@ -442,7 +444,11 @@ START_THREAD_DEFN
       /* Some other thread might call any of the setXid functions and expect
 	 us to reply.  In this case wait until we did that.  */
       do
-	lll_futex_wait (&pd->setxid_futex, 0, LLL_PRIVATE);
+	/* XXX This differs from the typical futex_wait_simple pattern in that
+	   the futex_wait condition (setxid_futex) is different from the
+	   condition used in the surrounding loop (cancelhandling).  We need
+	   to check and document why this is correct.  */
+	futex_wait_simple (&pd->setxid_futex, 0, FUTEX_PRIVATE);
       while (pd->cancelhandling & SETXID_BITMASK);
 
       /* Reset the value so that the stack can be reused.  */
@@ -683,7 +689,7 @@ __pthread_create_2_1 (newthread, attr, start_routine, arg)
 	     stillborn thread.  */
 	  if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0)
 				== -2))
-	    lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE);
+	    futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE);
 
 	  /* Free the resources.  */
 	  __deallocate_stack (pd);
diff --git a/nptl/pthread_mutexattr_setpshared.c b/nptl/pthread_mutexattr_setpshared.c
index 77ee9f794f..62fd168265 100644
--- a/nptl/pthread_mutexattr_setpshared.c
+++ b/nptl/pthread_mutexattr_setpshared.c
@@ -18,6 +18,7 @@
 
 #include <errno.h>
 #include <pthreadP.h>
+#include <futex-internal.h>
 
 
 int
@@ -27,9 +28,9 @@ pthread_mutexattr_setpshared (attr, pshared)
 {
   struct pthread_mutexattr *iattr;
 
-  if (pshared != PTHREAD_PROCESS_PRIVATE
-      && __builtin_expect (pshared != PTHREAD_PROCESS_SHARED, 0))
-    return EINVAL;
+  int err = futex_supports_pshared (pshared);
+  if (err != 0)
+    return err;
 
   iattr = (struct pthread_mutexattr *) attr;
 
diff --git a/nptl/pthread_once.c b/nptl/pthread_once.c
index fe6d923794..3c5bc33622 100644
--- a/nptl/pthread_once.c
+++ b/nptl/pthread_once.c
@@ -17,7 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include "pthreadP.h"
-#include <lowlevellock.h>
+#include <futex-internal.h>
 #include <atomic.h>
 
 
@@ -35,7 +35,7 @@ clear_once_control (void *arg)
      get interrupted (see __pthread_once), so all we need to relay to other
      threads is the state being reset again.  */
   atomic_store_relaxed (once_control, 0);
-  lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE);
+  futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE);
 }
 
 
@@ -100,8 +100,10 @@ __pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
 	     is set and __PTHREAD_ONCE_DONE is not.  */
 	  if (val == newval)
 	    {
-	      /* Same generation, some other thread was faster. Wait.  */
-	      lll_futex_wait (once_control, newval, LLL_PRIVATE);
+	      /* Same generation, some other thread was faster.  Wait and
+		 retry.  */
+	      futex_wait_simple ((unsigned int *) once_control,
+				 (unsigned int) newval, FUTEX_PRIVATE);
 	      continue;
 	    }
 	}
@@ -122,7 +124,7 @@ __pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
       atomic_store_release (once_control, __PTHREAD_ONCE_DONE);
 
       /* Wake up all other threads.  */
-      lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE);
+      futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE);
       break;
     }
 
diff --git a/nptl/pthread_rwlock_init.c b/nptl/pthread_rwlock_init.c
index 23ee6c659d..99ab5e35cb 100644
--- a/nptl/pthread_rwlock_init.c
+++ b/nptl/pthread_rwlock_init.c
@@ -58,15 +58,8 @@ __pthread_rwlock_init (rwlock, attr)
 
      If the pshared value is in locking functions XORed with avail
      we get the expected result.  */
-#ifdef __ASSUME_PRIVATE_FUTEX
   rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE
 			     ? 0 : FUTEX_PRIVATE_FLAG);
-#else
-  rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE
-			     ? 0
-			     : THREAD_GETMEM (THREAD_SELF,
-					      header.private_futex));
-#endif
 
   return 0;
 }
diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c
index 004a386fd5..eb7ac8d226 100644
--- a/nptl/pthread_rwlock_rdlock.c
+++ b/nptl/pthread_rwlock_rdlock.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include <sysdep.h>
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <pthread.h>
 #include <pthreadP.h>
 #include <stap-probe.h>
@@ -32,6 +33,8 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock)
 {
   int result = 0;
   bool wake = false;
+  int futex_shared =
+      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
 
   /* Lock is taken in caller.  */
 
@@ -60,9 +63,10 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock)
       /* Free the lock.  */
       lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
 
-      /* Wait for the writer to finish.  */
-      lll_futex_wait (&rwlock->__data.__readers_wakeup, waitval,
-		      rwlock->__data.__shared);
+      /* Wait for the writer to finish.  We do not check the return value
+	 because we decide how to continue based on the state of the rwlock.  */
+      futex_wait_simple (&rwlock->__data.__readers_wakeup, waitval,
+			 futex_shared);
 
       /* Get the lock.  */
       lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
@@ -103,8 +107,7 @@ __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock)
   lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
 
   if (wake)
-    lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
-		    rwlock->__data.__shared);
+    futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
 
   return result;
 }
@@ -117,6 +120,8 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
 {
   int result = 0;
   bool wake = false;
+  int futex_shared =
+      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
 
   LIBC_PROBE (rdlock_entry, 1, rwlock);
 
@@ -164,8 +169,7 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
       lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
 
       if (wake)
-	lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
-			rwlock->__data.__shared);
+	futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
 
       return result;
     }
diff --git a/nptl/pthread_rwlock_timedrdlock.c b/nptl/pthread_rwlock_timedrdlock.c
index 63fb313762..93d235e9e1 100644
--- a/nptl/pthread_rwlock_timedrdlock.c
+++ b/nptl/pthread_rwlock_timedrdlock.c
@@ -19,10 +19,10 @@
 #include <errno.h>
 #include <sysdep.h>
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <pthread.h>
 #include <pthreadP.h>
 #include <sys/time.h>
-#include <kernel-features.h>
 #include <stdbool.h>
 
 
@@ -34,6 +34,8 @@ pthread_rwlock_timedrdlock (rwlock, abstime)
 {
   int result = 0;
   bool wake = false;
+  int futex_shared =
+      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
 
   /* Make sure we are alone.  */
   lll_lock(rwlock->__data.__lock, rwlock->__data.__shared);
@@ -91,38 +93,6 @@ pthread_rwlock_timedrdlock (rwlock, abstime)
 	  break;
 	}
 
-      /* Work around the fact that the kernel rejects negative timeout values
-	 despite them being valid.  */
-      if (__glibc_unlikely (abstime->tv_sec < 0))
-	{
-	  result = ETIMEDOUT;
-	  break;
-	}
-
-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
-     || !defined lll_futex_timed_wait_bitset)
-      /* Get the current time.  So far we support only one clock.  */
-      struct timeval tv;
-      (void) __gettimeofday (&tv, NULL);
-
-      /* Convert the absolute timeout value to a relative timeout.  */
-      struct timespec rt;
-      rt.tv_sec = abstime->tv_sec - tv.tv_sec;
-      rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
-      if (rt.tv_nsec < 0)
-	{
-	  rt.tv_nsec += 1000000000;
-	  --rt.tv_sec;
-	}
-      /* Did we already time out?  */
-      if (rt.tv_sec < 0)
-	{
-	  /* Yep, return with an appropriate error.  */
-	  result = ETIMEDOUT;
-	  break;
-	}
-#endif
-
       /* Remember that we are a reader.  */
       if (++rwlock->__data.__nr_readers_queued == 0)
 	{
@@ -137,17 +107,11 @@ pthread_rwlock_timedrdlock (rwlock, abstime)
       /* Free the lock.  */
       lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
 
-      /* Wait for the writer to finish.  */
-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
-     || !defined lll_futex_timed_wait_bitset)
-      err = lll_futex_timed_wait (&rwlock->__data.__readers_wakeup,
-				  waitval, &rt, rwlock->__data.__shared);
-#else
-      err = lll_futex_timed_wait_bitset (&rwlock->__data.__readers_wakeup,
-					 waitval, abstime,
-					 FUTEX_CLOCK_REALTIME,
-					 rwlock->__data.__shared);
-#endif
+      /* Wait for the writer to finish.  We handle ETIMEDOUT below; on other
+	 return values, we decide how to continue based on the state of the
+	 rwlock.  */
+      err = futex_abstimed_wait (&rwlock->__data.__readers_wakeup, waitval,
+				 abstime, futex_shared);
 
       /* Get the lock.  */
       lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
@@ -155,7 +119,7 @@ pthread_rwlock_timedrdlock (rwlock, abstime)
       --rwlock->__data.__nr_readers_queued;
 
       /* Did the futex call time out?  */
-      if (err == -ETIMEDOUT)
+      if (err == ETIMEDOUT)
 	{
 	  /* Yep, report it.  */
 	  result = ETIMEDOUT;
@@ -167,8 +131,7 @@ pthread_rwlock_timedrdlock (rwlock, abstime)
   lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
 
   if (wake)
-    lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
-		    rwlock->__data.__shared);
+    futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
 
   return result;
 }
diff --git a/nptl/pthread_rwlock_timedwrlock.c b/nptl/pthread_rwlock_timedwrlock.c
index c54253450b..615623a75d 100644
--- a/nptl/pthread_rwlock_timedwrlock.c
+++ b/nptl/pthread_rwlock_timedwrlock.c
@@ -19,10 +19,10 @@
 #include <errno.h>
 #include <sysdep.h>
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <pthread.h>
 #include <pthreadP.h>
 #include <sys/time.h>
-#include <kernel-features.h>
 #include <stdbool.h>
 
 
@@ -34,6 +34,8 @@ pthread_rwlock_timedwrlock (rwlock, abstime)
 {
   int result = 0;
   bool wake_readers = false;
+  int futex_shared =
+      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
 
   /* Make sure we are alone.  */
   lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
@@ -71,37 +73,6 @@ pthread_rwlock_timedwrlock (rwlock, abstime)
 	  break;
 	}
 
-      /* Work around the fact that the kernel rejects negative timeout values
-	 despite them being valid.  */
-      if (__glibc_unlikely (abstime->tv_sec < 0))
-	{
-	  result = ETIMEDOUT;
-	  break;
-	}
-
-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
-     || !defined lll_futex_timed_wait_bitset)
-      /* Get the current time.  So far we support only one clock.  */
-      struct timeval tv;
-      (void) __gettimeofday (&tv, NULL);
-
-      /* Convert the absolute timeout value to a relative timeout.  */
-      struct timespec rt;
-      rt.tv_sec = abstime->tv_sec - tv.tv_sec;
-      rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
-      if (rt.tv_nsec < 0)
-	{
-	  rt.tv_nsec += 1000000000;
-	  --rt.tv_sec;
-	}
-      /* Did we already time out?  */
-      if (rt.tv_sec < 0)
-	{
-	  result = ETIMEDOUT;
-	  break;
-	}
-#endif
-
       /* Remember that we are a writer.  */
       if (++rwlock->__data.__nr_writers_queued == 0)
 	{
@@ -116,17 +87,11 @@ pthread_rwlock_timedwrlock (rwlock, abstime)
       /* Free the lock.  */
       lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
 
-      /* Wait for the writer or reader(s) to finish.  */
-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
-     || !defined lll_futex_timed_wait_bitset)
-      err = lll_futex_timed_wait (&rwlock->__data.__writer_wakeup,
-				  waitval, &rt, rwlock->__data.__shared);
-#else
-      err = lll_futex_timed_wait_bitset (&rwlock->__data.__writer_wakeup,
-					 waitval, abstime,
-					 FUTEX_CLOCK_REALTIME,
-					 rwlock->__data.__shared);
-#endif
+      /* Wait for the writer or reader(s) to finish.  We handle ETIMEDOUT
+	 below; on other return values, we decide how to continue based on
+	 the state of the rwlock.  */
+      err = futex_abstimed_wait (&rwlock->__data.__writer_wakeup, waitval,
+				 abstime, futex_shared);
 
       /* Get the lock.  */
       lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
@@ -135,7 +100,7 @@ pthread_rwlock_timedwrlock (rwlock, abstime)
       --rwlock->__data.__nr_writers_queued;
 
       /* Did the futex call time out?  */
-      if (err == -ETIMEDOUT)
+      if (err == ETIMEDOUT)
 	{
 	  result = ETIMEDOUT;
 	  /* If we prefer writers, it can have happened that readers blocked
@@ -166,8 +131,7 @@ pthread_rwlock_timedwrlock (rwlock, abstime)
 
   /* Might be required after timeouts.  */
   if (wake_readers)
-    lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
-	rwlock->__data.__shared);
+    futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
 
   return result;
 }
diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
index cde123fd5d..256188a14a 100644
--- a/nptl/pthread_rwlock_tryrdlock.c
+++ b/nptl/pthread_rwlock_tryrdlock.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include "pthreadP.h"
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <elide.h>
 #include <stdbool.h>
 
@@ -28,6 +29,8 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
 {
   int result = EBUSY;
   bool wake = false;
+  int futex_shared =
+      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
 
   if (ELIDE_TRYLOCK (rwlock->__data.__rwelision,
 		     rwlock->__data.__lock == 0
@@ -63,8 +66,7 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
   lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
 
   if (wake)
-    lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
-		    rwlock->__data.__shared);
+    futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
 
   return result;
 }
diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c
index d2ad4b0375..bdd115d6bd 100644
--- a/nptl/pthread_rwlock_unlock.c
+++ b/nptl/pthread_rwlock_unlock.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include <sysdep.h>
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <pthread.h>
 #include <pthreadP.h>
 #include <stap-probe.h>
@@ -29,6 +30,9 @@
 int
 __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
 {
+  int futex_shared =
+      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
+
   LIBC_PROBE (rwlock_unlock, 1, rwlock);
 
   if (ELIDE_UNLOCK (rwlock->__data.__writer == 0
@@ -51,16 +55,15 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
 	{
 	  ++rwlock->__data.__writer_wakeup;
 	  lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
-	  lll_futex_wake (&rwlock->__data.__writer_wakeup, 1,
-			  rwlock->__data.__shared);
+	  futex_wake (&rwlock->__data.__writer_wakeup, 1, futex_shared);
 	  return 0;
 	}
       else if (rwlock->__data.__nr_readers_queued)
 	{
 	  ++rwlock->__data.__readers_wakeup;
 	  lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
-	  lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
-			  rwlock->__data.__shared);
+	  futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
+		      futex_shared);
 	  return 0;
 	}
     }
diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c
index 835a62f0eb..60fa909340 100644
--- a/nptl/pthread_rwlock_wrlock.c
+++ b/nptl/pthread_rwlock_wrlock.c
@@ -19,6 +19,7 @@
 #include <errno.h>
 #include <sysdep.h>
 #include <lowlevellock.h>
+#include <futex-internal.h>
 #include <pthread.h>
 #include <pthreadP.h>
 #include <stap-probe.h>
@@ -30,6 +31,8 @@ static int __attribute__((noinline))
 __pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock)
 {
   int result = 0;
+  int futex_shared =
+      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
 
   /* Caller has taken the lock.  */
 
@@ -58,9 +61,11 @@ __pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock)
       /* Free the lock.  */
       lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
 
-      /* Wait for the writer or reader(s) to finish.  */
-      lll_futex_wait (&rwlock->__data.__writer_wakeup, waitval,
-		      rwlock->__data.__shared);
+      /* Wait for the writer or reader(s) to finish.  We do not check the
+	 return value because we decide how to continue based on the state of
+	 the rwlock.  */
+      futex_wait_simple (&rwlock->__data.__writer_wakeup, waitval,
+			 futex_shared);
 
       /* Get the lock.  */
       lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
diff --git a/nptl/pthread_rwlockattr_setpshared.c b/nptl/pthread_rwlockattr_setpshared.c
index 4b4b31093b..0369209a94 100644
--- a/nptl/pthread_rwlockattr_setpshared.c
+++ b/nptl/pthread_rwlockattr_setpshared.c
@@ -18,6 +18,7 @@
 
 #include <errno.h>
 #include "pthreadP.h"
+#include <futex-internal.h>
 
 
 int
@@ -27,9 +28,9 @@ pthread_rwlockattr_setpshared (attr, pshared)
 {
   struct pthread_rwlockattr *iattr;
 
-  if (pshared != PTHREAD_PROCESS_SHARED
-      && __builtin_expect (pshared != PTHREAD_PROCESS_PRIVATE, 0))
-    return EINVAL;
+  int err = futex_supports_pshared (pshared);
+  if (err != 0)
+    return err;
 
   iattr = (struct pthread_rwlockattr *) attr;
 
diff --git a/nptl/sem_init.c b/nptl/sem_init.c
index 575b661f62..bd1b592420 100644
--- a/nptl/sem_init.c
+++ b/nptl/sem_init.c
@@ -21,22 +21,7 @@
 #include <shlib-compat.h>
 #include "semaphoreP.h"
 #include <kernel-features.h>
-
-/* Returns FUTEX_PRIVATE if pshared is zero and private futexes are supported;
-   returns FUTEX_SHARED otherwise.
-   TODO Remove when cleaning up the futex API throughout glibc.  */
-static __always_inline int
-futex_private_if_supported (int pshared)
-{
-  if (pshared != 0)
-    return LLL_SHARED;
-#ifdef __ASSUME_PRIVATE_FUTEX
-  return LLL_PRIVATE;
-#else
-  return THREAD_GETMEM (THREAD_SELF, header.private_futex)
-      ^ FUTEX_PRIVATE_FLAG;
-#endif
-}
+#include <futex-internal.h>
 
 
 int
@@ -48,6 +33,13 @@ __new_sem_init (sem_t *sem, int pshared, unsigned int value)
       __set_errno (EINVAL);
       return -1;
     }
+  pshared = pshared != 0 ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE;
+  int err = futex_supports_pshared (pshared);
+  if (err != 0)
+    {
+      __set_errno (err);
+      return -1;
+    }
 
   /* Map to the internal type.  */
   struct new_sem *isem = (struct new_sem *) sem;
@@ -60,7 +52,8 @@ __new_sem_init (sem_t *sem, int pshared, unsigned int value)
   isem->nwaiters = 0;
 #endif
 
-  isem->private = futex_private_if_supported (pshared);
+  isem->private = (pshared == PTHREAD_PROCESS_PRIVATE
+		   ? FUTEX_PRIVATE : FUTEX_SHARED);
 
   return 0;
 }
diff --git a/nptl/sem_open.c b/nptl/sem_open.c
index ecd051a547..5ee8b71aa5 100644
--- a/nptl/sem_open.c
+++ b/nptl/sem_open.c
@@ -30,6 +30,7 @@
 #include <sys/stat.h>
 #include "semaphoreP.h"
 #include <shm-directory.h>
+#include <futex-internal.h>
 
 
 /* Comparison function for search of existing mapping.  */
@@ -141,6 +142,14 @@ sem_open (const char *name, int oflag, ...)
   int fd;
   sem_t *result;
 
+  /* Check that shared futexes are supported.  */
+  int err = futex_supports_pshared (PTHREAD_PROCESS_SHARED);
+  if (err != 0)
+    {
+      __set_errno (err);
+      return SEM_FAILED;
+    }
+
   /* Create the name of the final file in local variable SHM_NAME.  */
   SHM_GET_NAME (EINVAL, SEM_FAILED, SEM_SHM_PREFIX);
 
@@ -201,7 +210,7 @@ sem_open (const char *name, int oflag, ...)
       sem.newsem.nwaiters = 0;
 #endif
       /* This always is a shared semaphore.  */
-      sem.newsem.private = LLL_SHARED;
+      sem.newsem.private = FUTEX_SHARED;
 
       /* Initialize the remaining bytes as well.  */
       memset ((char *) &sem.initsem + sizeof (struct new_sem), '\0',
diff --git a/nptl/sem_post.c b/nptl/sem_post.c
index b6d30b514f..06d835907e 100644
--- a/nptl/sem_post.c
+++ b/nptl/sem_post.c
@@ -20,37 +20,13 @@
 #include <atomic.h>
 #include <errno.h>
 #include <sysdep.h>
-#include <lowlevellock.h>
+#include <lowlevellock.h>	/* lll_futex* used by the old code.  */
+#include <futex-internal.h>
 #include <internaltypes.h>
 #include <semaphore.h>
 
 #include <shlib-compat.h>
 
-/* Wrapper for lll_futex_wake, with error checking.
-   TODO Remove when cleaning up the futex API throughout glibc.  */
-static __always_inline void
-futex_wake (unsigned int* futex, int processes_to_wake, int private)
-{
-  int res = lll_futex_wake (futex, processes_to_wake, private);
-  /* No error.  Ignore the number of woken processes.  */
-  if (res >= 0)
-    return;
-  switch (res)
-    {
-    case -EFAULT: /* Could have happened due to memory reuse.  */
-    case -EINVAL: /* Could be either due to incorrect alignment (a bug in
-		     glibc or in the application) or due to memory being
-		     reused for a PI futex.  We cannot distinguish between the
-		     two causes, and one of them is correct use, so we do not
-		     act in this case.  */
-      return;
-    case -ENOSYS: /* Must have been caused by a glibc bug.  */
-    /* No other errors are documented at this time.  */
-    default:
-      abort ();
-    }
-}
-
 
 /* See sem_wait for an explanation of the algorithm.  */
 int
diff --git a/nptl/sem_wait.c b/nptl/sem_wait.c
index c1fd10c9d0..fce7ed43ee 100644
--- a/nptl/sem_wait.c
+++ b/nptl/sem_wait.c
@@ -17,6 +17,7 @@
    License along with the GNU C Library; if not, see
    <http://www.gnu.org/licenses/>.  */
 
+#include <lowlevellock.h>	/* lll_futex* used by the old code.  */
 #include "sem_waitcommon.c"
 
 int
diff --git a/nptl/sem_waitcommon.c b/nptl/sem_waitcommon.c
index 772425d33e..d3702c7b4f 100644
--- a/nptl/sem_waitcommon.c
+++ b/nptl/sem_waitcommon.c
@@ -20,7 +20,7 @@
 #include <kernel-features.h>
 #include <errno.h>
 #include <sysdep.h>
-#include <lowlevellock.h>
+#include <futex-internal.h>
 #include <internaltypes.h>
 #include <semaphore.h>
 #include <sys/time.h>
@@ -29,110 +29,6 @@
 #include <shlib-compat.h>
 #include <atomic.h>
 
-/* Wrapper for lll_futex_wait with absolute timeout and error checking.
-   TODO Remove when cleaning up the futex API throughout glibc.  */
-static __always_inline int
-futex_abstimed_wait (unsigned int* futex, unsigned int expected,
-		     const struct timespec* abstime, int private, bool cancel)
-{
-  int err, oldtype;
-  if (abstime == NULL)
-    {
-      if (cancel)
-	oldtype = __pthread_enable_asynccancel ();
-      err = lll_futex_wait (futex, expected, private);
-      if (cancel)
-	__pthread_disable_asynccancel (oldtype);
-    }
-  else
-    {
-#if (defined __ASSUME_FUTEX_CLOCK_REALTIME	\
-     && defined lll_futex_timed_wait_bitset)
-      /* The Linux kernel returns EINVAL for this, but in userspace
-	 such a value is valid.  */
-      if (abstime->tv_sec < 0)
-	return ETIMEDOUT;
-#else
-      struct timeval tv;
-      struct timespec rt;
-      int sec, nsec;
-
-      /* Get the current time.  */
-      __gettimeofday (&tv, NULL);
-
-      /* Compute relative timeout.  */
-      sec = abstime->tv_sec - tv.tv_sec;
-      nsec = abstime->tv_nsec - tv.tv_usec * 1000;
-      if (nsec < 0)
-        {
-          nsec += 1000000000;
-          --sec;
-        }
-
-      /* Already timed out?  */
-      if (sec < 0)
-        return ETIMEDOUT;
-
-      /* Do wait.  */
-      rt.tv_sec = sec;
-      rt.tv_nsec = nsec;
-#endif
-      if (cancel)
-	oldtype = __pthread_enable_asynccancel ();
-#if (defined __ASSUME_FUTEX_CLOCK_REALTIME	\
-     && defined lll_futex_timed_wait_bitset)
-      err = lll_futex_timed_wait_bitset (futex, expected, abstime,
-					 FUTEX_CLOCK_REALTIME, private);
-#else
-      err = lll_futex_timed_wait (futex, expected, &rt, private);
-#endif
-      if (cancel)
-	__pthread_disable_asynccancel (oldtype);
-    }
-  switch (err)
-    {
-    case 0:
-    case -EAGAIN:
-    case -EINTR:
-    case -ETIMEDOUT:
-      return -err;
-
-    case -EFAULT: /* Must have been caused by a glibc or application bug.  */
-    case -EINVAL: /* Either due to wrong alignment or due to the timeout not
-		     being normalized.  Must have been caused by a glibc or
-		     application bug.  */
-    case -ENOSYS: /* Must have been caused by a glibc bug.  */
-    /* No other errors are documented at this time.  */
-    default:
-      abort ();
-    }
-}
-
-/* Wrapper for lll_futex_wake, with error checking.
-   TODO Remove when cleaning up the futex API throughout glibc.  */
-static __always_inline void
-futex_wake (unsigned int* futex, int processes_to_wake, int private)
-{
-  int res = lll_futex_wake (futex, processes_to_wake, private);
-  /* No error.  Ignore the number of woken processes.  */
-  if (res >= 0)
-    return;
-  switch (res)
-    {
-    case -EFAULT: /* Could have happened due to memory reuse.  */
-    case -EINVAL: /* Could be either due to incorrect alignment (a bug in
-		     glibc or in the application) or due to memory being
-		     reused for a PI futex.  We cannot distinguish between the
-		     two causes, and one of them is correct use, so we do not
-		     act in this case.  */
-      return;
-    case -ENOSYS: /* Must have been caused by a glibc bug.  */
-    /* No other errors are documented at this time.  */
-    default:
-      abort ();
-    }
-}
-
 
 /* The semaphore provides two main operations: sem_post adds a token to the
    semaphore; sem_wait grabs a token from the semaphore, potentially waiting
@@ -220,11 +116,12 @@ do_futex_wait (struct new_sem *sem, const struct timespec *abstime)
   int err;
 
 #if __HAVE_64B_ATOMICS
-  err = futex_abstimed_wait ((unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0,
-			     abstime, sem->private, true);
+  err = futex_abstimed_wait_cancelable (
+      (unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0, abstime,
+      sem->private);
 #else
-  err = futex_abstimed_wait (&sem->value, SEM_NWAITERS_MASK, abstime,
-			     sem->private, true);
+  err = futex_abstimed_wait_cancelable (&sem->value, SEM_NWAITERS_MASK,
+					abstime, sem->private);
 #endif
 
   return err;
diff --git a/nptl/unregister-atfork.c b/nptl/unregister-atfork.c
index 3838cb7dee..6d08ed737e 100644
--- a/nptl/unregister-atfork.c
+++ b/nptl/unregister-atfork.c
@@ -20,6 +20,7 @@
 #include <stdlib.h>
 #include <fork.h>
 #include <atomic.h>
+#include <futex-internal.h>
 
 
 void
@@ -114,7 +115,7 @@ __unregister_atfork (dso_handle)
       atomic_decrement (&deleted->handler->refcntr);
       unsigned int val;
       while ((val = deleted->handler->refcntr) != 0)
-	lll_futex_wait (&deleted->handler->refcntr, val, LLL_PRIVATE);
+	futex_wait_simple (&deleted->handler->refcntr, val, FUTEX_PRIVATE);
 
       deleted = deleted->next;
     }