htl: Reimplement GSCOPE

This is a new implementation of GSCOPE which largely mirrors its NPTL
counterpart. Same as in NPTL, instead of a global flag shared between
threads, there is now a per-thread GSCOPE flag stored in each thread's
TCB. This makes entering and exiting a GSCOPE faster at the expense of
making THREAD_GSCOPE_WAIT () slower.

The largest win is the elimination of many redundant gsync_wake () RPC
calls; previously, even simplest programs would make dozens of fully
redundant gsync_wake () calls.

Signed-off-by: Sergey Bugaev <bugaevc@gmail.com>
Message-Id: <20210915171110.226187-3-bugaevc@gmail.com>
Reviewed-by: Samuel Thibault <samuel.thibault@ens-lyon.org>

3 files changed, 76 insertions, 20 deletions

diff --git a/sysdeps/htl/dl-thread_gscope_wait.c b/sysdeps/htl/dl-thread_gscope_wait.c
new file mode 100644
index 0000000000..7557e77aab
--- /dev/null
+++ b/sysdeps/htl/dl-thread_gscope_wait.c
@@ -0,0 +1,55 @@
+/* Out-of-line notification function for the GSCOPE locking mechanism.
+   Copyright (C) 2007-2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <ldsodefs.h>
+#include <pthread.h>
+#include <htl/pt-internal.h>
+
+void
+__thread_gscope_wait (void)
+{
+  size_t i;
+  struct __pthread *t;
+  int *gscope_flagp;
+
+  __libc_rwlock_rdlock (GL (dl_pthread_threads_lock));
+
+  /* Iterate over the list of threads.  */
+  for (i = 0; i < GL (dl_pthread_num_threads); ++i)
+    {
+      t = GL (dl_pthread_threads[i]);
+      if (t == NULL || t->tcb->gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
+        continue;
+
+      gscope_flagp = &t->tcb->gscope_flag;
+
+      /* We have to wait until this thread is done with the global
+         scope.  First tell the thread that we are waiting and
+         possibly have to be woken.  */
+      if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
+                                                THREAD_GSCOPE_FLAG_WAIT,
+                                                THREAD_GSCOPE_FLAG_USED))
+        continue;
+
+      do
+        lll_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
+      while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
+    }
+
+  __libc_rwlock_unlock (GL (dl_pthread_threads_lock));
+}
diff --git a/sysdeps/mach/hurd/i386/tls.h b/sysdeps/mach/hurd/i386/tls.h
index 057b2613f3..2ac65df921 100644
--- a/sysdeps/mach/hurd/i386/tls.h
+++ b/sysdeps/mach/hurd/i386/tls.h
@@ -369,6 +369,27 @@ _hurd_tls_new (thread_t child, struct i386_thread_state *state, tcbhead_t *tcb)
   return err;
 }
 
+/* Global scope switch support.  */
+# define THREAD_GSCOPE_IN_TCB      1
+
+# define THREAD_GSCOPE_FLAG_UNUSED 0
+# define THREAD_GSCOPE_FLAG_USED   1
+# define THREAD_GSCOPE_FLAG_WAIT   2
+
+# define THREAD_GSCOPE_SET_FLAG() \
+  THREAD_SETMEM (THREAD_SELF, gscope_flag, THREAD_GSCOPE_FLAG_USED)
+
+# define THREAD_GSCOPE_RESET_FLAG() \
+  ({                                                                         \
+    int __flag;                                                              \
+    asm volatile ("xchgl %0, %%gs:%P1"                                       \
+                  : "=r" (__flag)                                            \
+                  : "i" (offsetof (tcbhead_t, gscope_flag)),                 \
+                    "0" (THREAD_GSCOPE_FLAG_UNUSED));                        \
+    if (__flag == THREAD_GSCOPE_FLAG_WAIT)                                   \
+      lll_wake (THREAD_SELF->gscope_flag, LLL_PRIVATE);                      \
+  })
+
 #endif	/* !__ASSEMBLER__ */
 
 #endif	/* i386/tls.h */
diff --git a/sysdeps/mach/hurd/tls.h b/sysdeps/mach/hurd/tls.h
index f83956d3d7..8e66d5ff53 100644
--- a/sysdeps/mach/hurd/tls.h
+++ b/sysdeps/mach/hurd/tls.h
@@ -52,26 +52,6 @@
 # define GET_DTV(descr) \
   (((tcbhead_t *) (descr))->dtv)
 
-/* Global scope switch support.  */
-#define THREAD_GSCOPE_IN_TCB      0
-#define THREAD_GSCOPE_GLOBAL
-#define THREAD_GSCOPE_SET_FLAG() \
-  atomic_exchange_and_add_acq (&GL(dl_thread_gscope_count), 1)
-#define THREAD_GSCOPE_RESET_FLAG() \
-  do 									      \
-    if (atomic_exchange_and_add_rel (&GL(dl_thread_gscope_count), -1) == 1)   \
-      lll_wake (GL(dl_thread_gscope_count), 0);				      \
-  while (0)
-#define THREAD_GSCOPE_WAIT() \
-  do 									      \
-    {									      \
-      int count;							      \
-      atomic_write_barrier ();						      \
-      while ((count = GL(dl_thread_gscope_count)))			      \
-        lll_wait (GL(dl_thread_gscope_count), count, 0);		      \
-    }									      \
-  while (0)
-
 #endif /* !ASSEMBLER */