New pthread rwlock that is more scalable.

This replaces the pthread rwlock with a new implementation that uses a
more scalable algorithm (primarily through not using a critical section
anymore to make state changes).  The fast path for rdlock acquisition and
release is now basically a single atomic read-modify write or CAS and a few
branches.  See nptl/pthread_rwlock_common.c for details.

	* nptl/DESIGN-rwlock.txt: Remove.
	* nptl/lowlevelrwlock.sym: Remove.
	* nptl/Makefile: Add new tests.
	* nptl/pthread_rwlock_common.c: New file.  Contains the new rwlock.
	* nptl/pthreadP.h (PTHREAD_RWLOCK_PREFER_READER_P): Remove.
	(PTHREAD_RWLOCK_WRPHASE, PTHREAD_RWLOCK_WRLOCKED,
	PTHREAD_RWLOCK_RWAITING, PTHREAD_RWLOCK_READER_SHIFT,
	PTHREAD_RWLOCK_READER_OVERFLOW, PTHREAD_RWLOCK_WRHANDOVER,
	PTHREAD_RWLOCK_FUTEX_USED): New.
	* nptl/pthread_rwlock_init.c (__pthread_rwlock_init): Adapt to new
	implementation.
	* nptl/pthread_rwlock_rdlock.c (__pthread_rwlock_rdlock_slow): Remove.
	(__pthread_rwlock_rdlock): Adapt.
	* nptl/pthread_rwlock_timedrdlock.c
	(pthread_rwlock_timedrdlock): Adapt.
	* nptl/pthread_rwlock_timedwrlock.c
	(pthread_rwlock_timedwrlock): Adapt.
	* nptl/pthread_rwlock_trywrlock.c (pthread_rwlock_trywrlock): Adapt.
	* nptl/pthread_rwlock_tryrdlock.c (pthread_rwlock_tryrdlock): Adapt.
	* nptl/pthread_rwlock_unlock.c (pthread_rwlock_unlock): Adapt.
	* nptl/pthread_rwlock_wrlock.c (__pthread_rwlock_wrlock_slow): Remove.
	(__pthread_rwlock_wrlock): Adapt.
	* nptl/tst-rwlock10.c: Adapt.
	* nptl/tst-rwlock11.c: Adapt.
	* nptl/tst-rwlock17.c: New file.
	* nptl/tst-rwlock18.c: New file.
	* nptl/tst-rwlock19.c: New file.
	* nptl/tst-rwlock2b.c: New file.
	* nptl/tst-rwlock8.c: Adapt.
	* nptl/tst-rwlock9.c: Adapt.
	* sysdeps/aarch64/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/arm/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/hppa/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/ia64/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/m68k/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/microblaze/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/mips/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/nios2/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/s390/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/sh/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/sparc/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/tile/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
	(pthread_rwlock_t): Adapt.
	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
	(pthread_rwlock_t): Adapt.
	* sysdeps/x86/bits/pthreadtypes.h (pthread_rwlock_t): Adapt.
	* nptl/nptl-printers.py (): Adapt.
	* nptl/nptl_lock_constants.pysym: Adapt.
	* nptl/test-rwlock-printers.py: Adapt.
	* nptl/test-rwlockattr-printers.c: Adapt.
	* nptl/test-rwlockattr-printers.py: Adapt.

1 files changed, 11 insertions, 42 deletions

diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c
index 320837d5af..ef46e88263 100644
--- a/nptl/pthread_rwlock_unlock.c
+++ b/nptl/pthread_rwlock_unlock.c
@@ -18,60 +18,29 @@
 
 #include <errno.h>
 #include <sysdep.h>
-#include <lowlevellock.h>
 #include <futex-internal.h>
 #include <pthread.h>
 #include <pthreadP.h>
 #include <stap-probe.h>
-#include <elide.h>
 
+#include "pthread_rwlock_common.c"
 
-/* Unlock RWLOCK.  */
+/* See pthread_rwlock_common.c for an overview.  */
 int
 __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
 {
-  int futex_shared =
-      rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
-
   LIBC_PROBE (rwlock_unlock, 1, rwlock);
 
-  /* Trying to elide an unlocked lock may crash the process.  This
-     is expected and is compatible with POSIX.1-2008: "results are
-     undefined if the read-write lock rwlock is not held by the
-     calling thread".  */
-  if (ELIDE_UNLOCK (rwlock->__data.__writer == 0
-		    && rwlock->__data.__nr_readers == 0))
-    return 0;
-
-  lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
-  if (rwlock->__data.__writer)
-    rwlock->__data.__writer = 0;
+  /* We distinguish between having acquired a read vs. a write lock by looking
+     at the writer TID.  If it's equal to our TID, we must be the writer
+     because nobody else can have stored this value.  Also, if we are a
+     reader, we will read from the wrunlock store with value 0 by the most
+     recent writer because that writer happens-before us.  */
+  if (atomic_load_relaxed (&rwlock->__data.__cur_writer)
+      == THREAD_GETMEM (THREAD_SELF, tid))
+      __pthread_rwlock_wrunlock (rwlock);
   else
-    --rwlock->__data.__nr_readers;
-  /* If there are still readers present, we do not yet need to wake writers
-     nor are responsible to wake any readers.  */
-  if (rwlock->__data.__nr_readers == 0)
-    {
-      /* Note that if there is a blocked writer, we effectively make it
-	 responsible for waking any readers because we don't wake readers in
-	 this case.  */
-      if (rwlock->__data.__nr_writers_queued)
-	{
-	  ++rwlock->__data.__writer_wakeup;
-	  lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
-	  futex_wake (&rwlock->__data.__writer_wakeup, 1, futex_shared);
-	  return 0;
-	}
-      else if (rwlock->__data.__nr_readers_queued)
-	{
-	  ++rwlock->__data.__readers_wakeup;
-	  lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
-	  futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
-		      futex_shared);
-	  return 0;
-	}
-    }
-  lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
+    __pthread_rwlock_rdunlock (rwlock);
   return 0;
 }