Update.

	* sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
	(pthread_barrier_wait): After wakeup, release lock only when the
	last thread stopped using the barrier object.
	* sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
	(pthread_barrier_wait): Likewise.
	* sysdeps/pthread/pthread_barrier_wait.c (pthread_barrier_wait):
	Likewise.
	* Makefile (tests): Add tst-barrier4.
	* tst-barrier4.c: New file.

3 files changed, 96 insertions, 36 deletions

diff --git a/nptl/sysdeps/pthread/pthread_barrier_wait.c b/nptl/sysdeps/pthread/pthread_barrier_wait.c
index 69274d6d64..f6dc3e5f16 100644
--- a/nptl/sysdeps/pthread/pthread_barrier_wait.c
+++ b/nptl/sysdeps/pthread/pthread_barrier_wait.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -29,6 +29,7 @@ pthread_barrier_wait (barrier)
      pthread_barrier_t *barrier;
 {
   struct pthread_barrier *ibarrier = (struct pthread_barrier *) barrier;
+  int result = 0;
 
   /* Make sure we are alone.  */
   lll_lock (ibarrier->lock);
@@ -39,41 +40,42 @@ pthread_barrier_wait (barrier)
   /* Are these all?  */
   if (ibarrier->left == 0)
     {
-      /* Yes.  Restore the barrier to be empty.  */
-      ibarrier->left = ibarrier->init_count;
-
-      /* Increment the event counter to avoid invalid wake-ups and
+      /* Yes. Increment the event counter to avoid invalid wake-ups and
 	 tell the current waiters that it is their turn.  */
       ++ibarrier->curr_event;
 
       /* Wake up everybody.  */
       lll_futex_wake (&ibarrier->curr_event, INT_MAX);
 
-      /* The barrier is open for business again.  */
-      lll_unlock (ibarrier->lock);
-
       /* This is the thread which finished the serialization.  */
-      return PTHREAD_BARRIER_SERIAL_THREAD;
+      result = PTHREAD_BARRIER_SERIAL_THREAD;
     }
-
-  /* The number of the event we are waiting for.  The barrier's event
-     number must be bumped before we continue.  */
-  unsigned int event = ibarrier->curr_event;
-  do
+  else
     {
-      /* Before suspending, make the barrier available to others.  */
-      lll_unlock (ibarrier->lock);
-
-      /* Wait for the event counter of the barrier to change.  */
-      lll_futex_wait (&ibarrier->curr_event, event);
-
-      /* We are going to access shared data.  */
-      lll_lock (ibarrier->lock);
+      /* The number of the event we are waiting for.  The barrier's event
+	 number must be bumped before we continue.  */
+      unsigned int event = ibarrier->curr_event;
+      do
+	{
+	  /* Before suspending, make the barrier available to others.  */
+	  lll_unlock (ibarrier->lock);
+
+	  /* Wait for the event counter of the barrier to change.  */
+	  lll_futex_wait (&ibarrier->curr_event, event);
+
+	  /* We are going to access shared data.  */
+	  lll_lock (ibarrier->lock);
+	}
+      while (event == ibarrier->curr_event);
     }
-  while (event == ibarrier->curr_event);
 
-  /* We are done, let others use the barrier.  */
-  lll_unlock (ibarrier->lock);
+  /* Make sure the init_count is stored locally or in a register.  */
+  unsigned int init_count = ibarrier->init_count;
+
+  /* If this was the last woken thread, unlock.  */
+  if (atomic_exchange_and_add (ibarrier->left, 1) == init_count - 1)
+    /* We are done.  */
+    lll_unlock (ibarrier->lock);
 
-  return 0;
+  return result;
 }
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
index d4b61a10c1..114284c44c 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -84,16 +84,32 @@ pthread_barrier_wait:
 #endif
 	je,pn	8b
 
+	/* Increment LEFT.  If this brings the count back to the
+	   initial count unlock the object.  */
+	movl	$1, %edx
+	movl	INIT_COUNT(%ebx), %ecx
+	LOCK
+	xaddl	%edx, LEFT(%ebx)
+	subl	$1, %ecx
+	cmpl	%ecx, %edx
+	jne,pt	10f
+
+	/* Release the mutex.  We cannot release the lock before
+	   waking the waiting threads since otherwise a new thread might
+	   arrive and gets waken up, too.  */
+	LOCK
+	subl	$1, MUTEX(%ebx)
+	jne	9f
+
 	/* Note: %esi is still zero.  */
-	movl	%esi, %eax		/* != PTHREAD_BARRIER_SERIAL_THREAD */
+10:	movl	%esi, %eax		/* != PTHREAD_BARRIER_SERIAL_THREAD */
 
 	popl	%esi
 	popl	%ebx
 	ret
 
 	/* The necessary number of threads arrived.  */
-3:	movl	INIT_COUNT(%ebx), %eax
-	movl	%eax, LEFT(%ebx)
+3:
 #if CURR_EVENT == 0
 	addl	$1, (%ebx)
 #else
@@ -107,6 +123,16 @@ pthread_barrier_wait:
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 
+	/* Increment LEFT.  If this brings the count back to the
+	   initial count unlock the object.  */
+	movl	$1, %edx
+	movl	INIT_COUNT(%ebx), %ecx
+	LOCK
+	xaddl	%edx, LEFT(%ebx)
+	subl	$1, %ecx
+	cmpl	%ecx, %edx
+	jne,pt	5f
+
 	/* Release the mutex.  We cannot release the lock before
 	   waking the waiting threads since otherwise a new thread might
 	   arrive and gets waken up, too.  */
@@ -130,4 +156,8 @@ pthread_barrier_wait:
 6:	leal	MUTEX(%ebx), %eax
 	call	__lll_mutex_unlock_wake
 	jmp	7b
+
+9:	leal	MUTEX(%ebx), %eax
+	call	__lll_mutex_unlock_wake
+	jmp	10b
 	.size	pthread_barrier_wait,.-pthread_barrier_wait
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
index 672f2cac0e..e1593f32ff 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -78,14 +78,29 @@ pthread_barrier_wait:
 #endif
 	je	8b
 
-	/* Note: %esi is still zero.  */
-	movl	%esi, %eax		/* != PTHREAD_BARRIER_SERIAL_THREAD */
+	/* Increment LEFT.  If this brings the count back to the
+	   initial count unlock the object.  */
+	movl	$1, %edx
+	movl	INIT_COUNT(%rdi), %eax
+	LOCK
+	xaddl	%edx, LEFT(%rdi)
+	subl	$1, %eax
+	cmpl	%eax, %edx
+	jne,pt	10f
+
+	/* Release the mutex.  We cannot release the lock before
+	   waking the waiting threads since otherwise a new thread might
+	   arrive and gets waken up, too.  */
+	LOCK
+	decl	MUTEX(%rdi)
+	jne	9f
+
+10:	xorl	%eax, %eax		/* != PTHREAD_BARRIER_SERIAL_THREAD */
 
 	retq
 
 	/* The necessary number of threads arrived.  */
-3:	movl	INIT_COUNT(%rdi), %eax
-	movl	%eax, LEFT(%rdi)
+3:
 #if CURR_EVENT == 0
 	incl	(%rdi)
 #else
@@ -99,6 +114,16 @@ pthread_barrier_wait:
 	movq	$SYS_futex, %rax
 	syscall
 
+	/* Increment LEFT.  If this brings the count back to the
+	   initial count unlock the object.  */
+	movl	$1, %edx
+	movl	INIT_COUNT(%rdi), %eax
+	LOCK
+	xaddl	%edx, LEFT(%rdi)
+	subl	$1, %eax
+	cmpl	%eax, %edx
+	jne,pt	5f
+
 	/* Release the mutex.  We cannot release the lock before
 	   waking the waiting threads since otherwise a new thread might
 	   arrive and gets waken up, too.  */
@@ -117,11 +142,14 @@ pthread_barrier_wait:
 
 4:	addq	$MUTEX, %rdi
 	callq	__lll_mutex_unlock_wake
-	subq	$MUTEX, %rdi
 	jmp	5b
 
 6:	addq	$MUTEX, %rdi
 	callq	__lll_mutex_unlock_wake
 	subq	$MUTEX, %rdi
 	jmp	7b
+
+9:	addq	$MUTEX, %rdi
+	callq	__lll_mutex_unlock_wake
+	jmp	10b
 	.size	pthread_barrier_wait,.-pthread_barrier_wait