From 8491ed6d70b60e4c75cdcfde10ae759898547b08 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Mon, 31 Mar 2014 08:07:46 -0700 Subject: Add adaptive elision to rwlocks This patch relies on the C version of the rwlocks posted earlier. With C rwlocks it is very straight forward to do adaptive elision using TSX. It is based on the infrastructure added earlier for mutexes, but uses its own elision macros. The macros are fairly general purpose and could be used for other elision purposes too. This version is much cleaner than the earlier assembler based version, and in particular implements adaptation which makes it safer. I changed the behavior slightly to not require any changes in the test suite and fully conform to all expected behaviors (generally at the cost of not eliding in various situations). In particular this means the timedlock variants are not elided. Nested trylock aborts. --- nptl/pthread_rwlock_rdlock.c | 7 +++++++ nptl/pthread_rwlock_tryrdlock.c | 7 +++++++ nptl/pthread_rwlock_trywrlock.c | 7 +++++++ nptl/pthread_rwlock_unlock.c | 6 ++++++ nptl/pthread_rwlock_wrlock.c | 7 +++++++ nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h | 2 ++ 6 files changed, 36 insertions(+) (limited to 'nptl') diff --git a/nptl/pthread_rwlock_rdlock.c b/nptl/pthread_rwlock_rdlock.c index 1df0327a22..6eb9e091c4 100644 --- a/nptl/pthread_rwlock_rdlock.c +++ b/nptl/pthread_rwlock_rdlock.c @@ -22,6 +22,7 @@ #include #include #include +#include /* Acquire read lock for RWLOCK. Slow path. */ @@ -102,6 +103,12 @@ __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock) LIBC_PROBE (rdlock_entry, 1, rwlock); + if (ELIDE_LOCK (rwlock->__data.__rwelision, + rwlock->__data.__lock == 0 + && rwlock->__data.__writer == 0 + && rwlock->__data.__nr_readers == 0)) + return 0; + /* Make sure we are alone. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c index f7b1e6b8ac..cf8d68eb3f 100644 --- a/nptl/pthread_rwlock_tryrdlock.c +++ b/nptl/pthread_rwlock_tryrdlock.c @@ -19,6 +19,7 @@ #include #include "pthreadP.h" #include +#include int @@ -26,6 +27,12 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock) { int result = EBUSY; + if (ELIDE_TRYLOCK (rwlock->__data.__rwelision, + rwlock->__data.__lock == 0 + && rwlock->__data.__nr_readers == 0 + && rwlock->__data.__writer, 0)) + return 0; + lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); if (rwlock->__data.__writer == 0 diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c index 106f157c1d..0291fc9e3c 100644 --- a/nptl/pthread_rwlock_trywrlock.c +++ b/nptl/pthread_rwlock_trywrlock.c @@ -19,6 +19,7 @@ #include #include "pthreadP.h" #include +#include int @@ -26,6 +27,12 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock) { int result = EBUSY; + if (ELIDE_TRYLOCK (rwlock->__data.__rwelision, + rwlock->__data.__lock == 0 + && rwlock->__data.__nr_readers == 0 + && rwlock->__data.__writer, 1)) + return 0; + lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0) diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c index d4923838e7..3ebddeb246 100644 --- a/nptl/pthread_rwlock_unlock.c +++ b/nptl/pthread_rwlock_unlock.c @@ -22,6 +22,8 @@ #include #include #include +#include + /* Unlock RWLOCK. */ int @@ -29,6 +31,10 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock) { LIBC_PROBE (rwlock_unlock, 1, rwlock); + if (ELIDE_UNLOCK (rwlock->__data.__writer == 0 + && rwlock->__data.__nr_readers == 0)) + return 0; + lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); if (rwlock->__data.__writer) rwlock->__data.__writer = 0; diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c index de54e51432..91ad82a1be 100644 --- a/nptl/pthread_rwlock_wrlock.c +++ b/nptl/pthread_rwlock_wrlock.c @@ -22,6 +22,7 @@ #include #include #include +#include /* Acquire write lock for RWLOCK. */ @@ -91,6 +92,12 @@ __pthread_rwlock_wrlock (pthread_rwlock_t *rwlock) { LIBC_PROBE (wrlock_entry, 1, rwlock); + if (ELIDE_LOCK (rwlock->__data.__rwelision, + rwlock->__data.__lock == 0 + && rwlock->__data.__writer == 0 + && rwlock->__data.__nr_readers == 0)) + return 0; + /* Make sure we are alone. */ lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h index d70f8b35b1..c9f1c833a8 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h @@ -214,6 +214,8 @@ typedef union long int __align; } pthread_rwlock_t; +#define __PTHREAD_RWLOCK_ELISION_EXTRA 0 + typedef union { char __size[__SIZEOF_PTHREAD_RWLOCKATTR_T]; -- cgit 1.4.1