Update.

2003-04-04  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/pthread/createthread.c (create_thread): Add some more
	comments explaining when to set multiple_threads and when not.

	* pthreadP.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
	THREAD_ATOMIC_BIT_SET if not already defined.
	* sysdeps/i386/tls.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
	THREAD_ATOMIC_BIT_SET:
	* sysdeps/x86_64/tls.h: Likewise.
	* cleanup_defer.c (_pthread_cleanup_push_defer): Rewrite to use
	THREAD_ATOMIC_CMPXCHG_VAL.
	(_pthread_cleanup_pop_restore): Likewise.
	* cancellation.c (__pthread_enable_asynccancel): Likewise.
	(__pthread_enable_asynccancel_2): Likewise.
	(__pthread_disable_asynccancel): Likewise.
	* libc-cancellation.c (__libc_enable_asynccancel): Likewise.
	(__libc_disable_asynccancel): Likewise.
	* init.c (sigcancel_handler): Likewise.
	* pthread_setcancelstate.c (__pthread_setcancelstate): Likewise.
	* pthread_setcanceltype.c (__pthread_setcanceltype): Likewise.

10 files changed, 188 insertions, 56 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index d8467e8cda..7273ece303 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,25 @@
+2003-04-04  Ulrich Drepper  <drepper@redhat.com>
+
+	* sysdeps/pthread/createthread.c (create_thread): Add some more
+	comments explaining when to set multiple_threads and when not.
+
+	* pthreadP.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
+	THREAD_ATOMIC_BIT_SET if not already defined.
+	* sysdeps/i386/tls.h: Define THREAD_ATOMIC_CMPXCHG_VAL and
+	THREAD_ATOMIC_BIT_SET:
+	* sysdeps/x86_64/tls.h: Likewise.
+	* cleanup_defer.c (_pthread_cleanup_push_defer): Rewrite to use
+	THREAD_ATOMIC_CMPXCHG_VAL.
+	(_pthread_cleanup_pop_restore): Likewise.
+	* cancellation.c (__pthread_enable_asynccancel): Likewise.
+	(__pthread_enable_asynccancel_2): Likewise.
+	(__pthread_disable_asynccancel): Likewise.
+	* libc-cancellation.c (__libc_enable_asynccancel): Likewise.
+	(__libc_disable_asynccancel): Likewise.
+	* init.c (sigcancel_handler): Likewise.
+	* pthread_setcancelstate.c (__pthread_setcancelstate): Likewise.
+	* pthread_setcanceltype.c (__pthread_setcanceltype): Likewise.
+
 2003-04-03  Ulrich Drepper  <drepper@redhat.com>
 
 	* init.c (sigcancel_handler): Don't set EXITING_BIT here.
diff --git a/nptl/cancellation.c b/nptl/cancellation.c
index d88cae3aa7..c7477d87df 100644
--- a/nptl/cancellation.c
+++ b/nptl/cancellation.c
@@ -20,7 +20,6 @@
 #include <setjmp.h>
 #include <stdlib.h>
 #include "pthreadP.h"
-#include "atomic.h"
 
 
 /* The next two functions are similar to pthread_setcanceltype() but
@@ -31,18 +30,18 @@ attribute_hidden
 __pthread_enable_asynccancel (void)
 {
   struct pthread *self = THREAD_SELF;
-  int oldval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -52,6 +51,9 @@ __pthread_enable_asynccancel (void)
 
 	  break;
 	}
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
 
   return oldval;
@@ -63,17 +65,22 @@ internal_function attribute_hidden
 __pthread_enable_asynccancel_2 (int *oldvalp)
 {
   struct pthread *self = THREAD_SELF;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      int oldval = *oldvalp = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      /* We have to store the value before enablying asynchronous
+	 cancellation.  */
+      *oldvalp = oldval;
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -97,17 +104,21 @@ __pthread_disable_asynccancel (int oldtype)
     return;
 
   struct pthread *self = THREAD_SELF;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval & ~CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
 }
diff --git a/nptl/cleanup_defer.c b/nptl/cleanup_defer.c
index 5d6ed0360b..7945a0ddbb 100644
--- a/nptl/cleanup_defer.c
+++ b/nptl/cleanup_defer.c
@@ -18,7 +18,6 @@
    02111-1307 USA.  */
 
 #include "pthreadP.h"
-#include <atomic.h>
 
 
 void
@@ -37,13 +36,19 @@ _pthread_cleanup_push_defer (buffer, routine, arg)
 
   /* Disable asynchronous cancellation for now.  */
   if (__builtin_expect (cancelhandling & CANCELTYPE_BITMASK, 0))
-    {
-      while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						   cancelhandling
-						   & ~CANCELTYPE_BITMASK,
-						   cancelhandling))
-	cancelhandling = self->cancelhandling;
-    }
+    while (1)
+      {
+	int newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						cancelhandling
+						& ~CANCELTYPE_BITMASK,
+						cancelhandling);
+	if (__builtin_expect (newval == cancelhandling, 1))
+	  /* Successfully replaced the value.  */
+	  break;
+
+	/* Prepare for the next round.  */
+	cancelhandling = newval;
+      }
 
   buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
 			  ? PTHREAD_CANCEL_ASYNCHRONOUS
@@ -53,6 +58,7 @@ _pthread_cleanup_push_defer (buffer, routine, arg)
 }
 strong_alias (_pthread_cleanup_push_defer, __pthread_cleanup_push_defer)
 
+
 void
 _pthread_cleanup_pop_restore (buffer, execute)
      struct _pthread_cleanup_buffer *buffer;
@@ -67,11 +73,19 @@ _pthread_cleanup_pop_restore (buffer, execute)
       && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
 	  & CANCELTYPE_BITMASK) == 0)
     {
-      while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						   cancelhandling
-						   | CANCELTYPE_BITMASK,
-						   cancelhandling))
-	cancelhandling = self->cancelhandling;
+      while (1)
+	{
+	  int newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						  cancelhandling
+						  | CANCELTYPE_BITMASK,
+						  cancelhandling);
+	  if (__builtin_expect (newval == cancelhandling, 1))
+	    /* Successfully replaced the value.  */
+	    break;
+
+	  /* Prepare for the next round.  */
+	  cancelhandling = newval;
+	}
 
       CANCELLATION_P (self);
     }
diff --git a/nptl/init.c b/nptl/init.c
index 9d557cc938..7a6883f4cb 100644
--- a/nptl/init.c
+++ b/nptl/init.c
@@ -135,20 +135,21 @@ sigcancel_handler (int sig __attribute ((unused)))
 {
   struct pthread *self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
       /* We are canceled now.  When canceled by another thread this flag
 	 is already set but if the signal is directly send (internally or
 	 from another process) is has to be done here.  */
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = oldval | CANCELING_BITMASK | CANCELED_BITMASK;
 
       if (oldval == newval || (oldval & EXITING_BITMASK) != 0)
 	/* Already canceled or exiting.  */
 	break;
 
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (curval == oldval)
 	{
 	  /* Set the return value.  */
 	  THREAD_SETMEM (self, result, PTHREAD_CANCELED);
@@ -160,6 +161,8 @@ sigcancel_handler (int sig __attribute ((unused)))
 
 	  break;
 	}
+
+      oldval = curval;
     }
 }
 
diff --git a/nptl/libc-cancellation.c b/nptl/libc-cancellation.c
index 344a2181c8..72a6d108fd 100644
--- a/nptl/libc-cancellation.c
+++ b/nptl/libc-cancellation.c
@@ -33,13 +33,11 @@ attribute_hidden
 __libc_enable_asynccancel (void)
 {
   struct pthread *self = THREAD_SELF;
-  int oldval;
-  int newval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
-  do
+  while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
-      newval = oldval | CANCELTYPE_BITMASK;
+      int newval = oldval | CANCELTYPE_BITMASK;
 
       if (__builtin_expect ((oldval & CANCELED_BITMASK) != 0, 0))
 	{
@@ -47,10 +45,14 @@ __libc_enable_asynccancel (void)
 	  if ((oldval & EXITING_BITMASK) != 0)
 	    break;
 
-	  if (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						    newval, oldval))
-	    /* Somebody else modified the word, try again.  */
-	    continue;
+	  int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						  newval, oldval);
+	  if (__builtin_expect (curval != oldval, 0))
+	    {
+	      /* Somebody else modified the word, try again.  */
+	      oldval = curval;
+	      continue;
+	    }
 
 	  THREAD_SETMEM (self, result, PTHREAD_CANCELED);
 
@@ -58,9 +60,15 @@ __libc_enable_asynccancel (void)
 
 	  /* NOTREACHED */
 	}
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
+	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
-  while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-					       newval, oldval));
 
   return oldval;
 }
@@ -76,19 +84,23 @@ __libc_disable_asynccancel (int oldtype)
     return;
 
   struct pthread *self = THREAD_SELF;
-  int oldval;
-  int newval;
+  int oldval = THREAD_GETMEM (self, cancelhandling);
 
-  do
+  while (1)
     {
-      oldval = THREAD_GETMEM (self, cancelhandling);
-      newval = oldval & ~CANCELTYPE_BITMASK;
+      int newval = oldval & ~CANCELTYPE_BITMASK;
 
       if (newval == oldval)
 	break;
+
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
+	break;
+
+      /* Prepare the next round.  */
+      oldval = curval;
     }
-  while (atomic_compare_and_exchange_bool_acq (&self->cancelhandling, newval,
-					       oldval));
 }
 
 #endif
diff --git a/nptl/pthread_setcancelstate.c b/nptl/pthread_setcancelstate.c
index a6af063d67..a452c2ec46 100644
--- a/nptl/pthread_setcancelstate.c
+++ b/nptl/pthread_setcancelstate.c
@@ -34,9 +34,9 @@ __pthread_setcancelstate (state, oldstate)
 
   self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = (state == PTHREAD_CANCEL_DISABLE
 		    ? oldval | CANCELSTATE_BITMASK
 		    : oldval & ~CANCELSTATE_BITMASK);
@@ -54,14 +54,18 @@ __pthread_setcancelstate (state, oldstate)
 
       /* Update the cancel handling word.  This has to be done
 	 atomically since other bits could be modified as well.  */
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    __do_cancel ();
 
 	  break;
 	}
+
+      /* Prepare for the next round.  */
+      oldval = curval;
     }
 
   return 0;
diff --git a/nptl/pthread_setcanceltype.c b/nptl/pthread_setcanceltype.c
index 5a046351b9..bbe87ba64b 100644
--- a/nptl/pthread_setcanceltype.c
+++ b/nptl/pthread_setcanceltype.c
@@ -34,9 +34,9 @@ __pthread_setcanceltype (type, oldtype)
 
   self = THREAD_SELF;
 
+  int oldval = THREAD_GETMEM (self, cancelhandling);
   while (1)
     {
-      int oldval = THREAD_GETMEM (self, cancelhandling);
       int newval = (type == PTHREAD_CANCEL_ASYNCHRONOUS
 		    ? oldval | CANCELTYPE_BITMASK
 		    : oldval & ~CANCELTYPE_BITMASK);
@@ -54,8 +54,9 @@ __pthread_setcanceltype (type, oldtype)
 
       /* Update the cancel handling word.  This has to be done
 	 atomically since other bits could be modified as well.  */
-      if (! atomic_compare_and_exchange_bool_acq (&self->cancelhandling,
-						  newval, oldval))
+      int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
+					      oldval);
+      if (__builtin_expect (curval == oldval, 1))
 	{
 	  if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	    {
@@ -65,6 +66,9 @@ __pthread_setcanceltype (type, oldtype)
 
 	  break;
 	}
+
+      /* Prepare for the next round.  */
+      oldval = curval;
     }
 
   return 0;
diff --git a/nptl/sysdeps/i386/tls.h b/nptl/sysdeps/i386/tls.h
index 7fe2222c68..4f2e8180d3 100644
--- a/nptl/sysdeps/i386/tls.h
+++ b/nptl/sysdeps/i386/tls.h
@@ -169,6 +169,14 @@ union user_desc_init
 # define INIT_SYSINFO
 #endif
 
+#ifndef LOCK
+# ifdef UP
+#  define LOCK  /* nothing */
+# else
+#  define LOCK "lock;"
+# endif
+#endif
+
 /* Code to initially initialize the thread pointer.  This might need
    special attention since 'errno' is not yet available and if the
    operation can cause a failure 'errno' must not be touched.  */
@@ -352,6 +360,32 @@ union user_desc_init
        }})
 
 
+/* Atomic compare and exchange on TLS, returning old value.  */
+#define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \
+  ({ __typeof (descr->member) __ret;					      \
+     __typeof (oldval) __old = (oldval);				      \
+     if (sizeof (descr->member) == 4)					      \
+       asm volatile (LOCK "cmpxchgl %2, %%gs:%P3"			      \
+		     : "=a" (__ret)					      \
+		     : "0" (__old), "r" (newval),			      \
+		       "i" (offsetof (struct pthread, member)));	      \
+     else								      \
+       /* Not necessary for other sizes in the moment.  */		      \
+       abort ();							      \
+     __ret; })
+
+
+/* Atomic set bit.  */
+#define THREAD_ATOMIC_BIT_SET(descr, member, bit) \
+  (void) ({ if (sizeof ((descr)->member) == 4)				      \
+	      asm volatile (LOCK "orl %1, %%gs:%P0"			      \
+			    :: "i" (offsetof (struct pthread, member)),	      \
+			       "ir" (1 << (bit)));			      \
+	    else							      \
+	      /* Not necessary for other sizes in the moment.  */	      \
+	      abort (); })
+
+
 /* Call the user-provided thread function.  */
 #define CALL_THREAD_FCT(descr) \
   ({ void *__res;							      \
diff --git a/nptl/sysdeps/pthread/createthread.c b/nptl/sysdeps/pthread/createthread.c
index fae744f298..9d00e4e135 100644
--- a/nptl/sysdeps/pthread/createthread.c
+++ b/nptl/sysdeps/pthread/createthread.c
@@ -83,11 +83,11 @@ create_thread (struct pthread *pd, STACK_VARIABLES_PARMS)
 	    /* Failed.  */
 	    return errno;
 
+	  /* We now have for sure more than one thread.  The main
+	     thread might not yet have the flag set.  No need to set
+	     the global variable again if this is what we use.  */
 #ifdef TLS_MULTIPLE_THREADS_IN_TCB
-	  /* We now have for sure more than one thread.  */
-	  pd->header.multiple_threads = 1;
-#else
-	  __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
+	  THREAD_SETMEM (THREAD_SELF, header.multiple_threads, 1);
 #endif
 
 	  /* Now fill in the information about the new thread in
@@ -155,8 +155,10 @@ create_thread (struct pthread *pd, STACK_VARIABLES_PARMS)
     /* Failed.  */
     return errno;
 
+  /* We now have for sure more than one thread.  The main thread might
+     not yet have the flag set.  No need to set the global variable
+     again if this is what we use.  */
 #ifdef TLS_MULTIPLE_THREADS_IN_TCB
-  /* We now have for sure more than one thread.  */
   THREAD_SETMEM (THREAD_SELF, header.multiple_threads, 1);
 #endif
 
diff --git a/nptl/sysdeps/x86_64/tls.h b/nptl/sysdeps/x86_64/tls.h
index f382db2861..dec1b5d1b7 100644
--- a/nptl/sysdeps/x86_64/tls.h
+++ b/nptl/sysdeps/x86_64/tls.h
@@ -253,6 +253,32 @@ typedef struct
        }})
 
 
+/* Atomic compare and exchange on TLS, returning old value.  */
+#define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \
+  ({ __typeof (descr->member) __ret;					      \
+     __typeof (oldval) __old = (oldval);				      \
+     if (sizeof (descr->member) == 4)					      \
+       asm volatile (LOCK "cmpxchgl %2, %%fs:%P3"			      \
+		     : "=a" (__ret)					      \
+		     : "0" (__old), "r" (newval),			      \
+		       "i" (offsetof (struct pthread, member)));	      \
+     else								      \
+       /* Not necessary for other sizes in the moment.  */		      \
+       abort ();							      \
+     __ret; })
+
+
+/* Atomic set bit.  */
+#define THREAD_ATOMIC_BIT_SET(descr, member, bit) \
+  (void) ({ if (sizeof ((descr)->member) == 4)				      \
+	      asm volatile (LOCK "orl %1, %%fs:%P0"			      \
+			    :: "i" (offsetof (struct pthread, member)),	      \
+			       "ir" (1 << (bit)));			      \
+	    else							      \
+	      /* Not necessary for other sizes in the moment.  */	      \
+	      abort (); })
+
+
 #define CALL_THREAD_FCT(descr) \
   ({ void *__res;							      \
      asm volatile ("movq %%fs:%P2, %%rdi\n\t"				      \