New pthread rwlock that is more scalable.

This replaces the pthread rwlock with a new implementation that uses a
more scalable algorithm (primarily through not using a critical section
anymore to make state changes).  The fast path for rdlock acquisition and
release is now basically a single atomic read-modify write or CAS and a few
branches.  See nptl/pthread_rwlock_common.c for details.

	* nptl/DESIGN-rwlock.txt: Remove.
	* nptl/lowlevelrwlock.sym: Remove.
	* nptl/Makefile: Add new tests.
	* nptl/pthread_rwlock_common.c: New file.  Contains the new rwlock.
	* nptl/pthreadP.h (PTHREAD_RWLOCK_PREFER_READER_P): Remove.
	(PTHREAD_RWLOCK_WRPHASE, PTHREAD_RWLOCK_WRLOCKED,
	PTHREAD_RWLOCK_RWAITING, PTHREAD_RWLOCK_READER_SHIFT,
	PTHREAD_RWLOCK_READER_OVERFLOW, PTHREAD_RWLOCK_WRHANDOVER,
	PTHREAD_RWLOCK_FUTEX_USED): New.
	* nptl/pthread_rwlock_init.c (__pthread_rwlock_init): Adapt to new
	implementation.
	* nptl/pthread_rwlock_rdlock.c (__pthread_rwlock_rdlock_slow): Remove.
	(__pthread_rwlock_rdlock): Adapt.
	* nptl/pthread_rwlock_timedrdlock.c
	(pthread_rwlock_timedrdlock): Adapt.
	* nptl/pthread_rwlock_timedwrlock.c
	(pthread_rwlock_timedwrlock): Adapt.
	* nptl/pthread_rwlock_trywrlock.c (pthread_rwlock_trywrlock): Adapt.
	* nptl/pthread_rwlock_tryrdlock.c (pthread_rwlock_tryrdlock): Adapt.
	* nptl/pthread_rwlock_unlock.c (pthread_rwlock_unlock): Adapt.
	* nptl/pthread_rwlock_wrlock.c (__pthread_rwlock_wrlock_slow): Remove.
	(__pthread_rwlock_wrlock): Adapt.
	* nptl/tst-rwlock10.c: Adapt.
	* nptl/tst-rwlock11.c: Adapt.
	* nptl/tst-rwlock17.c: New file.
	* nptl/tst-rwlock18.c: New file.
	* nptl/tst-rwlock19.c: New file.
	* nptl/tst-rwlock2b.c: New file.
	* nptl/tst-rwlock8.c: Adapt.
	* nptl/tst-rwlock9.c: Adapt.
	* sysdeps/aarch64/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/arm/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/hppa/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/ia64/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/m68k/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/microblaze/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/mips/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/nios2/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/s390/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/sh/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/sparc/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/tile/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
	(pthread_rwlock_t): Adapt.
	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
	(pthread_rwlock_t): Adapt.
	* sysdeps/x86/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* nptl/nptl-printers.py (): Adapt.
	* nptl/nptl_lock_constants.pysym: Adapt.
	* nptl/test-rwlock-printers.py: Adapt.
	* nptl/test-rwlockattr-printers.c: Adapt.
	* nptl/test-rwlockattr-printers.py: Adapt.

1 files changed, 14 insertions, 113 deletions

diff --git a/nptl/pthread_rwlock_timedwrlock.c b/nptl/pthread_rwlock_timedwrlock.c
index 9f024be819..5626505d2c 100644
--- a/nptl/pthread_rwlock_timedwrlock.c
+++ b/nptl/pthread_rwlock_timedwrlock.c
@@ -16,121 +16,22 @@
    License along with the GNU C Library; if not, see
    <http://www.gnu.org/licenses/>.  */
 
-#include <errno.h>
-#include <sysdep.h>
-#include <lowlevellock.h>
-#include <futex-internal.h>
-#include <pthread.h>
-#include <pthreadP.h>
-#include <sys/time.h>
-#include <stdbool.h>
+#include "pthread_rwlock_common.c"
 
-
-/* Try to acquire write lock for RWLOCK or return after specfied time.	*/
+/* See pthread_rwlock_common.c.  */
 int
 pthread_rwlock_timedwrlock (pthread_rwlock_t *rwlock,
-			    const struct timespec *abstime)
+    const struct timespec *abstime)
 {
-  int result = 0;
-  bool wake_readers = false;
-  int futex_shared =
-      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
-
-  /* Make sure we are alone.  */
-  lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
-
-  while (1)
-    {
-      int err;
-
-      /* Get the rwlock if there is no writer and no reader.  */
-      if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0)
-	{
-	  /* Mark self as writer.  */
-	  rwlock->__data.__writer = THREAD_GETMEM (THREAD_SELF, tid);
-	  break;
-	}
-
-      /* Make sure we are not holding the rwlock as a writer.  This is
-	 a deadlock situation we recognize and report.  */
-      if (__builtin_expect (rwlock->__data.__writer
-			    == THREAD_GETMEM (THREAD_SELF, tid), 0))
-	{
-	  result = EDEADLK;
-	  break;
-	}
-
-      /* Make sure the passed in timeout value is valid.  Ideally this
-	 test would be executed once.  But since it must not be
-	 performed if we would not block at all simply moving the test
-	 to the front is no option.  Replicating all the code is
-	 costly while this test is not.  */
-      if (__builtin_expect (abstime->tv_nsec >= 1000000000
-                            || abstime->tv_nsec < 0, 0))
-	{
-	  result = EINVAL;
-	  break;
-	}
-
-      /* Remember that we are a writer.  */
-      if (++rwlock->__data.__nr_writers_queued == 0)
-	{
-	  /* Overflow on number of queued writers.  */
-	  --rwlock->__data.__nr_writers_queued;
-	  result = EAGAIN;
-	  break;
-	}
-
-      int waitval = rwlock->__data.__writer_wakeup;
-
-      /* Free the lock.  */
-      lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
-
-      /* Wait for the writer or reader(s) to finish.  We handle ETIMEDOUT
-	 below; on other return values, we decide how to continue based on
-	 the state of the rwlock.  */
-      err = futex_abstimed_wait (&rwlock->__data.__writer_wakeup, waitval,
-				 abstime, futex_shared);
-
-      /* Get the lock.  */
-      lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
-
-      /* To start over again, remove the thread from the writer list.  */
-      --rwlock->__data.__nr_writers_queued;
-
-      /* Did the futex call time out?  */
-      if (err == ETIMEDOUT)
-	{
-	  result = ETIMEDOUT;
-	  /* If we prefer writers, it can have happened that readers blocked
-	     for us to acquire the lock first.  If we have timed out, we need
-	     to wake such readers if there are any, and if there is no writer
-	     currently (otherwise, the writer will take care of wake-up).
-	     Likewise, even if we prefer readers, we can be responsible for
-	     wake-up (see pthread_rwlock_unlock) if no reader or writer has
-	     acquired the lock.  We have timed out and thus not consumed a
-	     futex wake-up; therefore, if there is no other blocked writer
-	     that would consume the wake-up and thus take over responsibility,
-	     we need to wake blocked readers.  */
-	  if ((!PTHREAD_RWLOCK_PREFER_READER_P (rwlock)
-	       || ((rwlock->__data.__nr_readers == 0)
-		   && (rwlock->__data.__nr_writers_queued == 0)))
-	      && (rwlock->__data.__nr_readers_queued > 0)
-	      && (rwlock->__data.__writer == 0))
-	    {
-	      ++rwlock->__data.__readers_wakeup;
-	      wake_readers = true;
-	    }
-	  break;
-	}
-    }
-
-  /* We are done, free the lock.  */
-  lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
-
-  /* Might be required after timeouts.  */
-  if (wake_readers)
-    futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
-
-  return result;
+  /* Make sure the passed in timeout value is valid.  Note that the previous
+     implementation assumed that this check *must* not be performed if there
+     would in fact be no blocking; however, POSIX only requires that "the
+     validity of the abstime parameter need not be checked if the lock can be
+     immediately acquired" (i.e., we need not but may check it).  */
+  /* ??? Just move this to __pthread_rwlock_wrlock_full?  */
+  if (__glibc_unlikely (abstime->tv_nsec >= 1000000000
+      || abstime->tv_nsec < 0))
+    return EINVAL;
+
+  return __pthread_rwlock_wrlock_full (rwlock, abstime);
 }