New pthread_barrier algorithm to fulfill barrier destruction requirements.

The previous barrier implementation did not fulfill the POSIX requirements
for when a barrier can be destroyed.  Specifically, it was possible that
threads that haven't noticed yet that their round is complete still access
the barrier's memory, and that those accesses can happen after the barrier
has been legally destroyed.
The new algorithm does not have this issue, and it avoids using a lock
internally.

8 files changed, 11 insertions, 629 deletions

diff --git a/sysdeps/nptl/internaltypes.h b/sysdeps/nptl/internaltypes.h
index 74e848f74e..203c548550 100644
--- a/sysdeps/nptl/internaltypes.h
+++ b/sysdeps/nptl/internaltypes.h
@@ -92,15 +92,18 @@ struct pthread_rwlockattr
 };
 
 
-/* Barrier data structure.  */
+/* Barrier data structure.  See pthread_barrier_wait for a description
+   of how these fields are used.  */
 struct pthread_barrier
 {
-  unsigned int curr_event;
-  int lock;
-  unsigned int left;
-  unsigned int init_count;
-  int private;
+  unsigned int in;
+  unsigned int current_round;
+  unsigned int count;
+  int shared;
+  unsigned int out;
 };
+/* See pthread_barrier_wait for a description.  */
+#define BARRIER_IN_THRESHOLD (UINT_MAX/2)
 
 
 /* Barrier variable attribute data structure.  */
diff --git a/sysdeps/sparc/nptl/pthread_barrier_destroy.c b/sysdeps/sparc/nptl/pthread_barrier_destroy.c
deleted file mode 100644
index 1a61b357ed..0000000000
--- a/sysdeps/sparc/nptl/pthread_barrier_destroy.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <errno.h>
-#include "pthreadP.h"
-#include <lowlevellock.h>
-#include <sparc-nptl.h>
-
-int
-pthread_barrier_destroy (pthread_barrier_t *barrier)
-{
-  union sparc_pthread_barrier *ibarrier;
-  int result = EBUSY;
-
-  ibarrier = (union sparc_pthread_barrier *) barrier;
-
-  int private = ibarrier->s.pshared ? LLL_SHARED : LLL_PRIVATE;
-
-  lll_lock (ibarrier->b.lock, private);
-
-  if (__glibc_likely (ibarrier->b.left == ibarrier->b.init_count))
-    /* The barrier is not used anymore.  */
-    result = 0;
-  else
-    /* Still used, return with an error.  */
-    lll_unlock (ibarrier->b.lock, private);
-
-  return result;
-}
diff --git a/sysdeps/sparc/nptl/pthread_barrier_init.c b/sysdeps/sparc/nptl/pthread_barrier_init.c
deleted file mode 100644
index 1bbeb1eaef..0000000000
--- a/sysdeps/sparc/nptl/pthread_barrier_init.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <errno.h>
-#include "pthreadP.h"
-#include <lowlevellock.h>
-#include <futex-internal.h>
-#include <sparc-nptl.h>
-
-int
-__pthread_barrier_init (pthread_barrier_t *barrier,
-			const pthread_barrierattr_t *attr, unsigned int count)
-{
-  union sparc_pthread_barrier *ibarrier;
-
-  if (__glibc_unlikely (count == 0))
-    return EINVAL;
-
-  struct pthread_barrierattr *iattr = (struct pthread_barrierattr *) attr;
-  if (iattr != NULL)
-    {
-      int err = futex_supports_pshared (iattr->pshared);
-      if (err != 0)
-	return err;
-    }
-
-  ibarrier = (union sparc_pthread_barrier *) barrier;
-
-  /* Initialize the individual fields.  */
-  ibarrier->b.lock = LLL_LOCK_INITIALIZER;
-  ibarrier->b.left = count;
-  ibarrier->b.init_count = count;
-  ibarrier->b.curr_event = 0;
-  ibarrier->s.left_lock = 0;
-  ibarrier->s.pshared = (iattr && iattr->pshared == PTHREAD_PROCESS_SHARED);
-
-  return 0;
-}
-weak_alias (__pthread_barrier_init, pthread_barrier_init)
diff --git a/sysdeps/sparc/nptl/pthread_barrier_wait.c b/sysdeps/sparc/nptl/pthread_barrier_wait.c
deleted file mode 100644
index 55cb1e76b2..0000000000
--- a/sysdeps/sparc/nptl/pthread_barrier_wait.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/* Copyright (C) 2003-2016 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <errno.h>
-#include <sysdep.h>
-#include <lowlevellock.h>
-#include <pthreadP.h>
-#include <sparc-nptl.h>
-#include <futex-internal.h>
-
-/* Wait on barrier.  */
-int
-__pthread_barrier_wait (pthread_barrier_t *barrier)
-{
-  union sparc_pthread_barrier *ibarrier
-    = (union sparc_pthread_barrier *) barrier;
-  int result = 0;
-  int private = ibarrier->s.pshared ? LLL_SHARED : LLL_PRIVATE;
-  int futex_private = ibarrier->s.pshared ? FUTEX_SHARED : FUTEX_PRIVATE;
-
-  /* Make sure we are alone.  */
-  lll_lock (ibarrier->b.lock, private);
-
-  /* One more arrival.  */
-  --ibarrier->b.left;
-
-  /* Are these all?  */
-  if (ibarrier->b.left == 0)
-    {
-      /* Yes. Increment the event counter to avoid invalid wake-ups and
-	 tell the current waiters that it is their turn.  */
-      ++ibarrier->b.curr_event;
-
-      /* Wake up everybody.  */
-      futex_wake (&ibarrier->b.curr_event, INT_MAX, futex_private);
-
-      /* This is the thread which finished the serialization.  */
-      result = PTHREAD_BARRIER_SERIAL_THREAD;
-    }
-  else
-    {
-      /* The number of the event we are waiting for.  The barrier's event
-	 number must be bumped before we continue.  */
-      unsigned int event = ibarrier->b.curr_event;
-
-      /* Before suspending, make the barrier available to others.  */
-      lll_unlock (ibarrier->b.lock, private);
-
-      /* Wait for the event counter of the barrier to change.  */
-      do
-	futex_wait_simple (&ibarrier->b.curr_event, event, futex_private);
-      while (event == ibarrier->b.curr_event);
-    }
-
-  /* Make sure the init_count is stored locally or in a register.  */
-  unsigned int init_count = ibarrier->b.init_count;
-
-  /* If this was the last woken thread, unlock.  */
-  if (atomic_increment_val (&ibarrier->b.left) == init_count)
-    /* We are done.  */
-    lll_unlock (ibarrier->b.lock, private);
-
-  return result;
-}
-weak_alias (__pthread_barrier_wait, pthread_barrier_wait)
diff --git a/sysdeps/sparc/sparc32/pthread_barrier_wait.c b/sysdeps/sparc/sparc32/pthread_barrier_wait.c
index c61de5f4bd..e5ef911f62 100644
--- a/sysdeps/sparc/sparc32/pthread_barrier_wait.c
+++ b/sysdeps/sparc/sparc32/pthread_barrier_wait.c
@@ -1,96 +1 @@
-/* Copyright (C) 2003-2016 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <errno.h>
-#include <sysdep.h>
-#include <lowlevellock.h>
-#include <pthreadP.h>
-#include <sparc-nptl.h>
-#include <futex-internal.h>
-
-/* Wait on barrier.  */
-int
-__pthread_barrier_wait (pthread_barrier_t *barrier)
-{
-  union sparc_pthread_barrier *ibarrier
-    = (union sparc_pthread_barrier *) barrier;
-  int result = 0;
-  int private = ibarrier->s.pshared ? LLL_SHARED : LLL_PRIVATE;
-  int futex_private = ibarrier->s.pshared ? FUTEX_SHARED : FUTEX_PRIVATE;
-
-  /* Make sure we are alone.  */
-  lll_lock (ibarrier->b.lock, private);
-
-  /* One more arrival.  */
-  --ibarrier->b.left;
-
-  /* Are these all?  */
-  if (ibarrier->b.left == 0)
-    {
-      /* Yes. Increment the event counter to avoid invalid wake-ups and
-	 tell the current waiters that it is their turn.  */
-      ++ibarrier->b.curr_event;
-
-      /* Wake up everybody.  */
-      futex_wake (&ibarrier->b.curr_event, INT_MAX, futex_private);
-
-      /* This is the thread which finished the serialization.  */
-      result = PTHREAD_BARRIER_SERIAL_THREAD;
-    }
-  else
-    {
-      /* The number of the event we are waiting for.  The barrier's event
-	 number must be bumped before we continue.  */
-      unsigned int event = ibarrier->b.curr_event;
-
-      /* Before suspending, make the barrier available to others.  */
-      lll_unlock (ibarrier->b.lock, private);
-
-      /* Wait for the event counter of the barrier to change.  */
-      do
-	futex_wait_simple (&ibarrier->b.curr_event, event, futex_private);
-      while (event == ibarrier->b.curr_event);
-    }
-
-  /* Make sure the init_count is stored locally or in a register.  */
-  unsigned int init_count = ibarrier->b.init_count;
-
-  /* If this was the last woken thread, unlock.  */
-  if (__atomic_is_v9 || ibarrier->s.pshared == 0)
-    {
-      if (atomic_increment_val (&ibarrier->b.left) == init_count)
-	/* We are done.  */
-	lll_unlock (ibarrier->b.lock, private);
-    }
-  else
-    {
-      unsigned int left;
-      /* Slightly more complicated.  On pre-v9 CPUs, atomic_increment_val
-	 is only atomic for threads within the same process, not for
-	 multiple processes.  */
-      __sparc32_atomic_do_lock24 (&ibarrier->s.left_lock);
-      left = ++ibarrier->b.left;
-      __sparc32_atomic_do_unlock24 (&ibarrier->s.left_lock);
-      if (left == init_count)
-        /* We are done.  */
-	lll_unlock (ibarrier->b.lock, private);
-    }
-
-  return result;
-}
-weak_alias (__pthread_barrier_wait, pthread_barrier_wait)
+#error No support for pthread barriers on pre-v9 sparc.
diff --git a/sysdeps/sparc/sparc32/sparcv9/pthread_barrier_wait.c b/sysdeps/sparc/sparc32/sparcv9/pthread_barrier_wait.c
index 8336f5e00c..246c8d49de 100644
--- a/sysdeps/sparc/sparc32/sparcv9/pthread_barrier_wait.c
+++ b/sysdeps/sparc/sparc32/sparcv9/pthread_barrier_wait.c
@@ -1 +1 @@
-#include <sysdeps/sparc/nptl/pthread_barrier_wait.c>
+#include <nptl/pthread_barrier_wait.c>
diff --git a/sysdeps/unix/sysv/linux/i386/pthread_barrier_wait.S b/sysdeps/unix/sysv/linux/i386/pthread_barrier_wait.S
deleted file mode 100644
index 1f5910c472..0000000000
--- a/sysdeps/unix/sysv/linux/i386/pthread_barrier_wait.S
+++ /dev/null
@@ -1,187 +0,0 @@
-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <lowlevellock.h>
-#include <lowlevelbarrier.h>
-
-	.text
-
-	.globl	__pthread_barrier_wait
-	.type	__pthread_barrier_wait,@function
-	.align	16
-__pthread_barrier_wait:
-	cfi_startproc
-	pushl	%ebx
-	cfi_adjust_cfa_offset(4)
-	cfi_offset(%ebx, -8)
-
-	movl	8(%esp), %ebx
-
-	/* Get the mutex.  */
-	movl	$1, %edx
-	xorl	%eax, %eax
-	LOCK
-	cmpxchgl %edx, MUTEX(%ebx)
-	jnz	1f
-
-	/* One less waiter.  If this was the last one needed wake
-	   everybody.  */
-2:	subl	$1, LEFT(%ebx)
-	je	3f
-
-	/* There are more threads to come.  */
-	pushl	%esi
-	cfi_adjust_cfa_offset(4)
-	cfi_offset(%esi, -12)
-
-#if CURR_EVENT == 0
-	movl	(%ebx), %edx
-#else
-	movl	CURR_EVENT(%ebx), %edx
-#endif
-
-	/* Release the mutex.  */
-	LOCK
-	subl	$1, MUTEX(%ebx)
-	jne	6f
-
-	/* Wait for the remaining threads.  The call will return immediately
-	   if the CURR_EVENT memory has meanwhile been changed.  */
-7:
-#if FUTEX_WAIT == 0
-	movl	PRIVATE(%ebx), %ecx
-#else
-	movl	$FUTEX_WAIT, %ecx
-	orl	PRIVATE(%ebx), %ecx
-#endif
-	xorl	%esi, %esi
-8:	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-
-	/* Don't return on spurious wakeups.  The syscall does not change
-	   any register except %eax so there is no need to reload any of
-	   them.  */
-#if CURR_EVENT == 0
-	cmpl	%edx, (%ebx)
-#else
-	cmpl	%edx, CURR_EVENT(%ebx)
-#endif
-	je	8b
-
-	/* Increment LEFT.  If this brings the count back to the
-	   initial count unlock the object.  */
-	movl	$1, %edx
-	movl	INIT_COUNT(%ebx), %ecx
-	LOCK
-	xaddl	%edx, LEFT(%ebx)
-	subl	$1, %ecx
-	cmpl	%ecx, %edx
-	jne	10f
-
-	/* Release the mutex.  We cannot release the lock before
-	   waking the waiting threads since otherwise a new thread might
-	   arrive and gets waken up, too.  */
-	LOCK
-	subl	$1, MUTEX(%ebx)
-	jne	9f
-
-	/* Note: %esi is still zero.  */
-10:	movl	%esi, %eax		/* != PTHREAD_BARRIER_SERIAL_THREAD */
-
-	popl	%esi
-	cfi_adjust_cfa_offset(-4)
-	cfi_restore(%esi)
-	popl	%ebx
-	cfi_adjust_cfa_offset(-4)
-	cfi_restore(%ebx)
-	ret
-
-	cfi_adjust_cfa_offset(4)
-	cfi_offset(%ebx, -8)
-
-	/* The necessary number of threads arrived.  */
-3:
-#if CURR_EVENT == 0
-	addl	$1, (%ebx)
-#else
-	addl	$1, CURR_EVENT(%ebx)
-#endif
-
-	/* Wake up all waiters.  The count is a signed number in the kernel
-	   so 0x7fffffff is the highest value.  */
-	movl	$0x7fffffff, %edx
-	movl	$FUTEX_WAKE, %ecx
-	orl	PRIVATE(%ebx), %ecx
-	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-
-	/* Increment LEFT.  If this brings the count back to the
-	   initial count unlock the object.  */
-	movl	$1, %edx
-	movl	INIT_COUNT(%ebx), %ecx
-	LOCK
-	xaddl	%edx, LEFT(%ebx)
-	subl	$1, %ecx
-	cmpl	%ecx, %edx
-	jne	5f
-
-	/* Release the mutex.  We cannot release the lock before
-	   waking the waiting threads since otherwise a new thread might
-	   arrive and gets waken up, too.  */
-	LOCK
-	subl	$1, MUTEX(%ebx)
-	jne	4f
-
-5:	orl	$-1, %eax		/* == PTHREAD_BARRIER_SERIAL_THREAD */
-
-	popl	%ebx
-	cfi_adjust_cfa_offset(-4)
-	cfi_restore(%ebx)
-	ret
-
-	cfi_adjust_cfa_offset(4)
-	cfi_offset(%ebx, -8)
-1:	movl	PRIVATE(%ebx), %ecx
-	leal	MUTEX(%ebx), %edx
-	xorl	$LLL_SHARED, %ecx
-	call	__lll_lock_wait
-	jmp	2b
-
-4:	movl	PRIVATE(%ebx), %ecx
-	leal	MUTEX(%ebx), %eax
-	xorl	$LLL_SHARED, %ecx
-	call	__lll_unlock_wake
-	jmp	5b
-
-	cfi_adjust_cfa_offset(4)
-	cfi_offset(%esi, -12)
-6:	movl	PRIVATE(%ebx), %ecx
-	leal	MUTEX(%ebx), %eax
-	xorl	$LLL_SHARED, %ecx
-	call	__lll_unlock_wake
-	jmp	7b
-
-9:	movl	PRIVATE(%ebx), %ecx
-	leal	MUTEX(%ebx), %eax
-	xorl	$LLL_SHARED, %ecx
-	call	__lll_unlock_wake
-	jmp	10b
-	cfi_endproc
-	.size	__pthread_barrier_wait,.-__pthread_barrier_wait
-weak_alias (__pthread_barrier_wait, pthread_barrier_wait)
diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S b/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
deleted file mode 100644
index 650f567b59..0000000000
--- a/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
+++ /dev/null
@@ -1,161 +0,0 @@
-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <lowlevellock.h>
-#include <lowlevelbarrier.h>
-
-
-	.text
-
-	.globl	__pthread_barrier_wait
-	.type	__pthread_barrier_wait,@function
-	.align	16
-__pthread_barrier_wait:
-	/* Get the mutex.  */
-	xorl	%eax, %eax
-	movl	$1, %esi
-	LOCK
-	cmpxchgl %esi, MUTEX(%rdi)
-	jnz	1f
-
-	/* One less waiter.  If this was the last one needed wake
-	   everybody.  */
-2:	decl	LEFT(%rdi)
-	je	3f
-
-	/* There are more threads to come.  */
-#if CURR_EVENT == 0
-	movl	(%rdi), %edx
-#else
-	movl	CURR_EVENT(%rdi), %edx
-#endif
-
-	/* Release the mutex.  */
-	LOCK
-	decl	MUTEX(%rdi)
-	jne	6f
-
-	/* Wait for the remaining threads.  The call will return immediately
-	   if the CURR_EVENT memory has meanwhile been changed.  */
-7:
-#if FUTEX_WAIT == 0
-	movl	PRIVATE(%rdi), %esi
-#else
-	movl	$FUTEX_WAIT, %esi
-	orl	PRIVATE(%rdi), %esi
-#endif
-	xorq	%r10, %r10
-8:	movl	$SYS_futex, %eax
-	syscall
-
-	/* Don't return on spurious wakeups.  The syscall does not change
-	   any register except %eax so there is no need to reload any of
-	   them.  */
-#if CURR_EVENT == 0
-	cmpl	%edx, (%rdi)
-#else
-	cmpl	%edx, CURR_EVENT(%rdi)
-#endif
-	je	8b
-
-	/* Increment LEFT.  If this brings the count back to the
-	   initial count unlock the object.  */
-	movl	$1, %edx
-	movl	INIT_COUNT(%rdi), %eax
-	LOCK
-	xaddl	%edx, LEFT(%rdi)
-	subl	$1, %eax
-	cmpl	%eax, %edx
-	jne,pt	10f
-
-	/* Release the mutex.  We cannot release the lock before
-	   waking the waiting threads since otherwise a new thread might
-	   arrive and gets waken up, too.  */
-	LOCK
-	decl	MUTEX(%rdi)
-	jne	9f
-
-10:	xorl	%eax, %eax		/* != PTHREAD_BARRIER_SERIAL_THREAD */
-
-	retq
-
-	/* The necessary number of threads arrived.  */
-3:
-#if CURR_EVENT == 0
-	incl	(%rdi)
-#else
-	incl	CURR_EVENT(%rdi)
-#endif
-
-	/* Wake up all waiters.  The count is a signed number in the kernel
-	   so 0x7fffffff is the highest value.  */
-	movl	$0x7fffffff, %edx
-	movl	$FUTEX_WAKE, %esi
-	orl	PRIVATE(%rdi), %esi
-	movl	$SYS_futex, %eax
-	syscall
-
-	/* Increment LEFT.  If this brings the count back to the
-	   initial count unlock the object.  */
-	movl	$1, %edx
-	movl	INIT_COUNT(%rdi), %eax
-	LOCK
-	xaddl	%edx, LEFT(%rdi)
-	subl	$1, %eax
-	cmpl	%eax, %edx
-	jne,pt	5f
-
-	/* Release the mutex.  We cannot release the lock before
-	   waking the waiting threads since otherwise a new thread might
-	   arrive and gets waken up, too.  */
-	LOCK
-	decl	MUTEX(%rdi)
-	jne	4f
-
-5:	orl	$-1, %eax		/* == PTHREAD_BARRIER_SERIAL_THREAD */
-
-	retq
-
-1:	movl	PRIVATE(%rdi), %esi
-	addq	$MUTEX, %rdi
-	xorl	$LLL_SHARED, %esi
-	callq	__lll_lock_wait
-	subq	$MUTEX, %rdi
-	jmp	2b
-
-4:	movl	PRIVATE(%rdi), %esi
-	addq	$MUTEX, %rdi
-	xorl	$LLL_SHARED, %esi
-	callq	__lll_unlock_wake
-	jmp	5b
-
-6:	movl	PRIVATE(%rdi), %esi
-	addq	$MUTEX, %rdi
-	xorl	$LLL_SHARED, %esi
-	callq	__lll_unlock_wake
-	subq	$MUTEX, %rdi
-	jmp	7b
-
-9:	movl	PRIVATE(%rdi), %esi
-	addq	$MUTEX, %rdi
-	xorl	$LLL_SHARED, %esi
-	callq	__lll_unlock_wake
-	jmp	10b
-	.size	__pthread_barrier_wait,.-__pthread_barrier_wait
-weak_alias (__pthread_barrier_wait, pthread_barrier_wait)