Add elision to pthread_mutex_{try,timed,un}lock

Add elision paths to the basic mutex locks.

The normal path has a check for RTM and upgrades the lock
to RTM when available. Trylocks cannot automatically upgrade,
so they check for elision every time.

We use a 4 byte value in the mutex to store the lock
elision adaptation state. This is separate from the adaptive
spin state and uses a separate field.

Condition variables currently do not support elision.

Recursive mutexes and condition variables may be supported at some point,
but are not in the current implementation. Also "trylock" will
not automatically enable elision unless some other lock call
has been already called on the lock.

This version does not use IFUNC, so it means every lock has one
additional check for elision. Benchmarking showed the overhead
to be negligible.

8 files changed, 134 insertions, 5 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
index b417da5e22..7b6fbc18aa 100644
--- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
+++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
@@ -2,8 +2,15 @@
 
 #define LLL_MUTEX_LOCK(mutex) \
   lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
+
+/* Not actually elided so far. Needed? */
+#define LLL_MUTEX_LOCK_ELISION(mutex)  \
+  ({ lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)); 0; })
+
 #define LLL_MUTEX_TRYLOCK(mutex) \
   lll_cond_trylock ((mutex)->__data.__lock)
+#define LLL_MUTEX_TRYLOCK_ELISION(mutex) LLL_MUTEX_TRYLOCK(mutex)
+
 #define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
   lll_robust_cond_lock ((mutex)->__data.__lock, id, \
 			PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
index ccd896ca03..1852e07842 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
@@ -101,14 +101,23 @@ typedef union
        binary compatibility.  */
     int __kind;
 #ifdef __x86_64__
-    int __spins;
+    short __spins;
+    short __elision;
     __pthread_list_t __list;
 # define __PTHREAD_MUTEX_HAVE_PREV	1
+# define __PTHREAD_MUTEX_HAVE_ELISION   1
 #else
     unsigned int __nusers;
     __extension__ union
     {
-      int __spins;
+      struct
+      {
+        short __espins;
+	short __elision;
+# define __spins d.__espins
+# define __elision d.__elision
+# define __PTHREAD_MUTEX_HAVE_ELISION   2
+      } d;
       __pthread_slist_t __list;
     };
 #endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c b/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c
index 2939fc9640..689a6fbe2a 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c
+++ b/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c
@@ -24,9 +24,8 @@
 
 #define aconf __elision_aconf
 
-/* Try to elide a futex trylock. FUTEX is the futex variable. TRY_LOCK is the
-   adaptation counter in the mutex. UPGRADED is != 0 when this is for an
-   automatically upgraded lock.  */
+/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is the
+   adaptation counter in the mutex.  */
 
 int
 __lll_trylock_elision (int *futex, short *adapt_count)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h b/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h
new file mode 100644
index 0000000000..703e930be1
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h
@@ -0,0 +1,31 @@
+/* force-elision.h: Automatic enabling of elision for mutexes
+   Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>. */
+
+/* Check for elision on this lock without upgrading.  */
+#define DO_ELISION(m)							\
+  (__pthread_force_elision						\
+   && (m->__data.__kind & PTHREAD_MUTEX_NO_ELISION_NP) == 0)		\
+
+/* Automatically enable elision for existing user lock kinds.  */
+#define FORCE_ELISION(m, s)						\
+  if (__pthread_force_elision						\
+      && (m->__data.__kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0)	\
+    {									\
+      mutex->__data.__kind |= PTHREAD_MUTEX_ELISION_NP;			\
+      s;								\
+    }
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c
new file mode 100644
index 0000000000..962f101073
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c
@@ -0,0 +1,21 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>. */
+
+/* The cond lock is not actually elided yet, but we still need to handle
+   already elided locks.  */
+#include <elision-conf.h>
+#include "sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c
new file mode 100644
index 0000000000..e639460530
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c
@@ -0,0 +1,21 @@
+/* Elided version of pthread_mutex_lock.
+   Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>. */
+#include <elision-conf.h>
+#include "force-elision.h"
+
+#include "nptl/pthread_mutex_lock.c"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c
new file mode 100644
index 0000000000..5115d0304a
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c
@@ -0,0 +1,20 @@
+/* Elided version of pthread_mutex_timedlock.
+   Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>. */
+#include <elision-conf.h>
+#include "force-elision.h"
+#include "nptl/pthread_mutex_timedlock.c"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c
new file mode 100644
index 0000000000..9bb4a659d4
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c
@@ -0,0 +1,21 @@
+/* Elided version of pthread_mutex_trylock.
+   Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>. */
+#include <elision-conf.h>
+#include "force-elision.h"
+
+#include "nptl/pthread_mutex_trylock.c"