[BZ #4586]

2007-06-06  Jakub Jelinek  <jakub@redhat.com>
	BZ #4586
	* sysdeps/i386/ldbl2mpn.c (__mpn_extract_long_double): Treat
	pseudo-zeros as zero.
	* sysdeps/x86_64/ldbl2mpn.c: New file.
	* sysdeps/ia64/ldbl2mpn.c: New file.

13 files changed, 197 insertions, 59 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/fork.c b/nptl/sysdeps/unix/sysv/linux/fork.c
index 98bb237c06..c6dadb5683 100644
--- a/nptl/sysdeps/unix/sysv/linux/fork.c
+++ b/nptl/sysdeps/unix/sysv/linux/fork.c
@@ -203,7 +203,7 @@ __libc_fork (void)
 
 	  if (atomic_decrement_and_test (&allp->handler->refcntr)
 	      && allp->handler->need_signal)
-	    lll_futex_wake (allp->handler->refcntr, 1);
+	    lll_private_futex_wake (allp->handler->refcntr, 1);
 
 	  allp = allp->next;
 	}
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index aa963f77e7..474b39ae18 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -23,6 +23,8 @@
 #include <time.h>
 #include <sys/param.h>
 #include <bits/pthreadtypes.h>
+#include <kernel-features.h>
+#include <tcb-offsets.h>
 
 #ifndef LOCK_INSTR
 # ifdef UP
@@ -41,6 +43,13 @@
 #define FUTEX_PRIVATE_FLAG	128
 
 
+/* Values for 'private' parameter of locking macros.  Yes, the
+   definition seems to be backwards.  But it is not.  The bit will be
+   reversed before passing to the system call.  */
+#define LLL_PRIVATE	0
+#define LLL_SHARED	FUTEX_PRIVATE_FLAG
+
+
 /* Initializer for compatibility lock.  */
 #define LLL_MUTEX_LOCK_INITIALIZER		(0)
 #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED	(1)
@@ -145,7 +154,11 @@ LLL_STUB_UNWIND_INFO_START					\
 LLL_STUB_UNWIND_INFO_END
 
 
-#define lll_futex_wait(futex, val) \
+#define lll_futex_wait(futex, val, private) \
+  lll_futex_timed_wait (futex, val, NULL, private)
+
+
+#define lll_futex_timed_wait(futex, val, timeout, private) \
   ({									      \
     int __status;							      \
     register __typeof (val) _val asm ("edx") = (val);			      \
@@ -153,7 +166,7 @@ LLL_STUB_UNWIND_INFO_END
 		      LLL_ENTER_KERNEL					      \
 		      LLL_EBX_LOAD					      \
 		      : "=a" (__status)					      \
-		      : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (0),	      \
+		      : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
 			"c" (FUTEX_WAIT), "d" (_val),			      \
 			"i" (offsetof (tcbhead_t, sysinfo))		      \
 		      : "memory");					      \
@@ -161,7 +174,27 @@ LLL_STUB_UNWIND_INFO_END
   })
 
 
-#define lll_futex_timed_wait(futex, val, timeout)			      \
+#define lll_futex_wake(futex, nr, private) \
+  do {									      \
+    int __ignore;							      \
+    register __typeof (nr) _nr asm ("edx") = (nr);			      \
+    __asm __volatile (LLL_EBX_LOAD					      \
+		      LLL_ENTER_KERNEL					      \
+		      LLL_EBX_LOAD					      \
+		      : "=a" (__ignore)					      \
+		      : "0" (SYS_futex), LLL_EBX_REG (futex),		      \
+			"c" (FUTEX_WAKE), "d" (_nr),			      \
+			"i" (0) /* phony, to align next arg's number */,      \
+			"i" (offsetof (tcbhead_t, sysinfo)));		      \
+  } while (0)
+
+
+#define lll_private_futex_wait(futex, val) \
+  lll_private_futex_timed_wait (futex, val, NULL)
+
+
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define lll_private_futex_timed_wait(futex, val, timeout) \
   ({									      \
     int __status;							      \
     register __typeof (val) _val asm ("edx") = (val);			      \
@@ -170,14 +203,14 @@ LLL_STUB_UNWIND_INFO_END
 		      LLL_EBX_LOAD					      \
 		      : "=a" (__status)					      \
 		      : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
-			"c" (FUTEX_WAIT), "d" (_val),			      \
+			"c" (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)), "d" (_val),   \
 			"i" (offsetof (tcbhead_t, sysinfo))		      \
 		      : "memory");					      \
     __status;								      \
   })
 
 
-#define lll_futex_wake(futex, nr) \
+# define lll_private_futex_wake(futex, nr) \
   do {									      \
     int __ignore;							      \
     register __typeof (nr) _nr asm ("edx") = (nr);			      \
@@ -186,10 +219,45 @@ LLL_STUB_UNWIND_INFO_END
 		      LLL_EBX_LOAD					      \
 		      : "=a" (__ignore)					      \
 		      : "0" (SYS_futex), LLL_EBX_REG (futex),		      \
-			"c" (FUTEX_WAKE), "d" (_nr),			      \
+			"c" (FUTEX_WAKE | FUTEX_PRIVATE_FLAG), "d" (_nr),     \
 			"i" (0) /* phony, to align next arg's number */,      \
 			"i" (offsetof (tcbhead_t, sysinfo)));		      \
   } while (0)
+#else
+# define lll_private_futex_timed_wait(futex, val, timeout) \
+  ({									      \
+    int __status;							      \
+    int __ignore;							      \
+    register __typeof (val) _val asm ("edx") = (val);			      \
+    __asm __volatile ("movl %%gs:%P7, %%ecx\n\t"			      \
+		      LLL_EBX_LOAD					      \
+		      LLL_ENTER_KERNEL					      \
+		      LLL_EBX_LOAD					      \
+		      : "=a" (__status), "=c" (__ignore)		      \
+		      : LLL_EBX_REG (futex), "0" (SYS_futex), "S" (timeout),  \
+			"d" (_val), "i" (offsetof (tcbhead_t, sysinfo)),      \
+			"i" (PRIVATE_FUTEX)				      \
+		      : "memory");					      \
+    __status;								      \
+  })
+
+
+# define lll_private_futex_wake(futex, nr) \
+  do {									      \
+    int __ignore;							      \
+    int __ignore2;							      \
+    register __typeof (nr) _nr asm ("edx") = (nr);			      \
+    __asm __volatile ("orl %%gs:%P7, %%ecx\n\t"				      \
+		      LLL_EBX_LOAD					      \
+		      LLL_ENTER_KERNEL					      \
+		      LLL_EBX_LOAD					      \
+		      : "=a" (__ignore), "=c" (__ignore2)		      \
+		      : LLL_EBX_REG (futex), "0" (SYS_futex),		      \
+			"1" (FUTEX_WAKE), "d" (_nr),			      \
+			"i" (offsetof (tcbhead_t, sysinfo)),		      \
+			"i" (PRIVATE_FUTEX));				      \
+  } while (0)
+#endif
 
 
 /* Does not preserve %eax and %ecx.  */
@@ -415,21 +483,6 @@ extern int __lll_mutex_unlock_wake (int *__futex)
 				"i" (offsetof (tcbhead_t, sysinfo))); })
 
 
-#define lll_futex_wake(futex, nr) \
-  do {									      \
-    int __ignore;							      \
-    register __typeof (nr) _nr asm ("edx") = (nr);			      \
-    __asm __volatile (LLL_EBX_LOAD					      \
-		      LLL_ENTER_KERNEL					      \
-		      LLL_EBX_LOAD					      \
-		      : "=a" (__ignore)					      \
-		      : "0" (SYS_futex), LLL_EBX_REG (futex),		      \
-			"c" (FUTEX_WAKE), "d" (_nr),			      \
-			"i" (0) /* phony, to align next arg's number */,      \
-			"i" (offsetof (tcbhead_t, sysinfo)));		      \
-  } while (0)
-
-
 #define lll_mutex_islocked(futex) \
   (futex != 0)
 
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
index 38d78884d4..ab7f605f0c 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
@@ -31,7 +31,9 @@ __lll_lock_wait (int *futex)
     {
       int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1);
       if (oldval != 0)
-	lll_futex_wait (futex, 2);
+	lll_futex_wait (futex, 2,
+			// XYZ check mutex flag
+			LLL_SHARED);
     }
   while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0);
 }
@@ -68,7 +70,9 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime)
       /* Wait.  */
       int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1);
       if (oldval != 0)
-	lll_futex_timed_wait (futex, 2, &rt);
+	lll_futex_timed_wait (futex, 2, &rt,
+			      // XYZ check mutex flag
+			      LLL_SHARED);
     }
   while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0);
 
@@ -108,8 +112,9 @@ __lll_timedwait_tid (int *tidp, const struct timespec *abstime)
       if (rt.tv_sec < 0)
 	return ETIMEDOUT;
 
-      /* Wait until thread terminates.  */
-      if (lll_futex_timed_wait (tidp, tid, &rt) == -ETIMEDOUT)
+      /* Wait until thread terminates.  The kernel so far does not use
+	 the private futex operations for this.  */
+      if (lll_futex_timed_wait (tidp, tid, &rt, LLL_SHARED) == -ETIMEDOUT)
 	return ETIMEDOUT;
     }
 
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c
index 30ef991bd0..54cee0859b 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c
@@ -44,7 +44,9 @@ __lll_robust_lock_wait (int *futex)
 	  && atomic_compare_and_exchange_bool_acq (futex, newval, oldval))
 	continue;
 
-      lll_futex_wait (futex, newval);
+      lll_futex_wait (futex, newval,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
 
     try:
       ;
@@ -100,7 +102,9 @@ __lll_robust_timedlock_wait (int *futex, const struct timespec *abstime)
 	  && atomic_compare_and_exchange_bool_acq (futex, newval, oldval))
 	continue;
 
-      lll_futex_timed_wait (futex, newval, &rt);
+      lll_futex_timed_wait (futex, newval, &rt,
+			    // XYZ check mutex flag
+			    LLL_SHARED);
 
     try:
       ;
diff --git a/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h b/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h
index 6b3d3682da..e805b63001 100644
--- a/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h
+++ b/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h
@@ -1,5 +1,5 @@
 /* Defintions for lowlevel handling in ld.so.
-   Copyright (C) 2006 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -76,7 +76,7 @@ typedef int __rtld_mrlock_t;
 	    atomic_or (&(lock), __RTLD_MRLOCK_RWAIT);			      \
 	    oldval |= __RTLD_MRLOCK_RWAIT;				      \
 	  }								      \
-	lll_futex_wait (lock, oldval);					      \
+	lll_private_futex_wait (lock, oldval);				      \
       }									      \
   out:;									      \
   } while (0)
@@ -90,7 +90,7 @@ typedef int __rtld_mrlock_t;
 			  == (__RTLD_MRLOCK_INC | __RTLD_MRLOCK_WWAIT), 0))   \
       /* We have to wake all threads since there might be some queued	      \
 	 readers already.  */						      \
-      lll_futex_wake (&(lock), 0x7fffffff);				      \
+      lll_private_futex_wake (&(lock), 0x7fffffff);			      \
   } while (0)
 
 
@@ -119,7 +119,7 @@ typedef int __rtld_mrlock_t;
 	  }								      \
 	atomic_or (&(lock), __RTLD_MRLOCK_WWAIT);			      \
 	oldval |= __RTLD_MRLOCK_WWAIT;					      \
-	lll_futex_wait (lock, oldval);					      \
+	lll_private_futex_wait (lock, oldval);				      \
       }									      \
   out:;									      \
   } while (0)
@@ -129,7 +129,7 @@ typedef int __rtld_mrlock_t;
   do {				 \
     int oldval = atomic_exchange_and_add (&(lock), -__RTLD_MRLOCK_WRITER);    \
     if (__builtin_expect ((oldval & __RTLD_MRLOCK_RWAIT) != 0, 0))	      \
-      lll_futex_wake (&(lock), 0x7fffffff);				      \
+      lll_private_futex_wake (&(lock), 0x7fffffff);			      \
   } while (0)
 
 
@@ -142,12 +142,12 @@ typedef int __rtld_mrlock_t;
 	int val = word;							      \
 	if (val == 0)							      \
 	  break;							      \
-	lll_futex_wait (&(word), val);					      \
+	lll_private_futex_wait (&(word), val);				      \
       }									      \
   } while (0)
 
 
 #define __rtld_notify(word) \
-  lll_futex_wake (&(word), 1)
+  lll_private_futex_wake (&(word), 1)
 
 #endif
diff --git a/nptl/sysdeps/unix/sysv/linux/sem_post.c b/nptl/sysdeps/unix/sysv/linux/sem_post.c
index f8bc695292..7f90325585 100644
--- a/nptl/sysdeps/unix/sysv/linux/sem_post.c
+++ b/nptl/sysdeps/unix/sysv/linux/sem_post.c
@@ -35,7 +35,9 @@ __new_sem_post (sem_t *sem)
   atomic_full_barrier ();
   if (isem->nwaiters > 0)
     {
-      int err = lll_futex_wake (&isem->value, 1);
+      int err = lll_futex_wake (&isem->value, 1,
+				// XYZ check mutex flag
+				LLL_SHARED);
       if (__builtin_expect (err, 0) < 0)
 	{
 	  __set_errno (-err);
@@ -55,7 +57,8 @@ __old_sem_post (sem_t *sem)
   int *futex = (int *) sem;
 
   int nr = atomic_increment_val (futex);
-  int err = lll_futex_wake (futex, 1);
+  /* We always have to assume it is a shared semaphore.  */
+  int err = lll_futex_wake (futex, 1, LLL_SHARED);
   if (__builtin_expect (err, 0) < 0)
     {
       __set_errno (-err);
diff --git a/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c b/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c
index 4c8acd0938..8f92d78abe 100644
--- a/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c
+++ b/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c
@@ -84,7 +84,9 @@ sem_timedwait (sem_t *sem, const struct timespec *abstime)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       int oldtype = __pthread_enable_asynccancel ();
 
-      err = lll_futex_timed_wait (&isem->value, 0, &rt);
+      err = lll_futex_timed_wait (&isem->value, 0, &rt,
+				  // XYZ check mutex flag
+				  LLL_SHARED);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (oldtype);
diff --git a/nptl/sysdeps/unix/sysv/linux/sem_wait.c b/nptl/sysdeps/unix/sysv/linux/sem_wait.c
index 05114c9704..12f3f16c2d 100644
--- a/nptl/sysdeps/unix/sysv/linux/sem_wait.c
+++ b/nptl/sysdeps/unix/sysv/linux/sem_wait.c
@@ -56,7 +56,9 @@ __new_sem_wait (sem_t *sem)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       int oldtype = __pthread_enable_asynccancel ();
 
-      err = lll_futex_wait (&isem->value, 0);
+      err = lll_futex_wait (&isem->value, 0,
+			    // XYZ check mutex flag
+			    LLL_SHARED);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (oldtype);
@@ -100,7 +102,8 @@ __old_sem_wait (sem_t *sem)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       int oldtype = __pthread_enable_asynccancel ();
 
-      err = lll_futex_wait (futex, 0);
+      /* Always assume the semaphore is shared.  */
+      err = lll_futex_wait (futex, 0, LLL_SHARED);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (oldtype);
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c b/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c
index 3b07cc127d..83fedef8e8 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jakub Jelinek <jakub@redhat.com>, 2003.
 
@@ -30,7 +30,7 @@ clear_once_control (void *arg)
   pthread_once_t *once_control = (pthread_once_t *) arg;
 
   *once_control = 0;
-  lll_futex_wake (once_control, INT_MAX);
+  lll_private_futex_wake (once_control, INT_MAX);
 }
 
 
@@ -84,7 +84,7 @@ __pthread_once (once_control, init_routine)
       atomic_increment (once_control);
 
       /* Wake up all other threads.  */
-      lll_futex_wake (once_control, INT_MAX);
+      lll_private_futex_wake (once_control, INT_MAX);
       break;
     }
 
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c
index 4dfd11dcbe..868e0d2819 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -52,7 +52,9 @@ pthread_barrier_wait (barrier)
       ++ibarrier->b.curr_event;
 
       /* Wake up everybody.  */
-      lll_futex_wake (&ibarrier->b.curr_event, INT_MAX);
+      lll_futex_wake (&ibarrier->b.curr_event, INT_MAX,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
 
       /* This is the thread which finished the serialization.  */
       result = PTHREAD_BARRIER_SERIAL_THREAD;
@@ -68,7 +70,9 @@ pthread_barrier_wait (barrier)
 
       /* Wait for the event counter of the barrier to change.  */
       do
-	lll_futex_wait (&ibarrier->b.curr_event, event);
+	lll_futex_wait (&ibarrier->b.curr_event, event,
+			// XYZ check mutex flag
+			LLL_SHARED);
       while (event == ibarrier->b.curr_event);
     }
 
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c
index be1cc60b11..527aedfdc7 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c
@@ -1,5 +1,5 @@
 /* sem_post -- post to a POSIX semaphore.  SPARC version.
-   Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc.
+   Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jakub Jelinek <jakub@redhat.com>, 2003.
 
@@ -39,7 +39,9 @@ __new_sem_post (sem_t *sem)
       nr = ++*futex;
       __sparc32_atomic_do_unlock24 (futex + 1);
     }
-  int err = lll_futex_wake (futex, nr);
+  int err = lll_futex_wake (futex, nr,
+			    // XYZ check mutex flag
+			    LLL_SHARED);
   if (__builtin_expect (err, 0) < 0)
     {
       __set_errno (-err);
diff --git a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c
index 964f5b7094..240ce597f9 100644
--- a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c
+++ b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -104,7 +104,7 @@ __unregister_atfork (dso_handle)
       atomic_decrement (&deleted->handler->refcntr);
       unsigned int val;
       while ((val = deleted->handler->refcntr) != 0)
-	lll_futex_wait (&deleted->handler->refcntr, val);
+	lll_private_futex_wait (&deleted->handler->refcntr, val);
 
       deleted = deleted->next;
     }
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index d3055cbb23..b86d95e93d 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -23,6 +23,8 @@
 #include <time.h>
 #include <sys/param.h>
 #include <bits/pthreadtypes.h>
+#include <kernel-features.h>
+#include <tcb-offsets.h>
 
 #ifndef LOCK_INSTR
 # ifdef UP
@@ -42,6 +44,13 @@
 #define FUTEX_PRIVATE_FLAG	128
 
 
+/* Values for 'private' parameter of locking macros.  Yes, the
+   definition seems to be backwards.  But it is not.  The bit will be
+   reversed before passing to the system call.  */
+#define LLL_PRIVATE	0
+#define LLL_SHARED	FUTEX_PRIVATE_FLAG
+
+
 /* Initializer for compatibility lock.  */
 #define LLL_MUTEX_LOCK_INITIALIZER		(0)
 #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED	(1)
@@ -149,44 +158,97 @@ LLL_STUB_UNWIND_INFO_START					\
 LLL_STUB_UNWIND_INFO_END
 
 
-#define lll_futex_wait(futex, val) \
+#define lll_futex_wait(futex, val, private) \
+  lll_futex_timed_wait(futex, val, NULL, private)
+
+
+#define lll_futex_timed_wait(futex, val, timeout, private) \
   ({									      \
+    register const struct timespec *__to __asm ("r10") = timeout;	      \
     int __status;							      \
     register __typeof (val) _val __asm ("edx") = (val);			      \
-    __asm __volatile ("xorq %%r10, %%r10\n\t"				      \
-		      "syscall"						      \
+    __asm __volatile ("syscall"						      \
 		      : "=a" (__status)					      \
 		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT),	      \
-			"d" (_val)					      \
-		      : "memory", "cc", "r10", "r11", "cx");		      \
+		        "d" (_val), "r" (__to)				      \
+		      : "memory", "cc", "r11", "cx");			      \
     __status;								      \
   })
 
 
-#define lll_futex_timed_wait(futex, val, timeout)			      \
+#define lll_futex_wake(futex, nr, private) \
+  do {									      \
+    int __ignore;							      \
+    register __typeof (nr) _nr __asm ("edx") = (nr);			      \
+    __asm __volatile ("syscall"						      \
+		      : "=a" (__ignore)					      \
+		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE),	      \
+			"d" (_nr)					      \
+		      : "memory", "cc", "r10", "r11", "cx");		      \
+  } while (0)
+
+
+#define lll_private_futex_wait(futex, val) \
+  lll_private_futex_timed_wait (futex, val, NULL)
+
+
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define lll_private_futex_timed_wait(futex, val, timeout) \
   ({									      \
     register const struct timespec *__to __asm ("r10") = timeout;	      \
     int __status;							      \
     register __typeof (val) _val __asm ("edx") = (val);			      \
     __asm __volatile ("syscall"						      \
 		      : "=a" (__status)					      \
-		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT),	      \
+		      : "0" (SYS_futex), "D" (futex),			      \
+			"S" (FUTEX_WAIT | FUTEX_PRIVATE_FLAG),		      \
 		        "d" (_val), "r" (__to)				      \
 		      : "memory", "cc", "r11", "cx");			      \
     __status;								      \
   })
 
 
-#define lll_futex_wake(futex, nr) \
+# define lll_private_futex_wake(futex, nr) \
   do {									      \
     int __ignore;							      \
     register __typeof (nr) _nr __asm ("edx") = (nr);			      \
     __asm __volatile ("syscall"						      \
 		      : "=a" (__ignore)					      \
-		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE),	      \
+		      : "0" (SYS_futex), "D" (futex),			      \
+			"S" (FUTEX_WAKE | FUTEX_PRIVATE_FLAG),		      \
 			"d" (_nr)					      \
 		      : "memory", "cc", "r10", "r11", "cx");		      \
   } while (0)
+#else
+# define lll_private_futex_timed_wait(futex, val, timeout) \
+  ({									      \
+    register const struct timespec *__to __asm ("r10") = timeout;	      \
+    int __status;							      \
+    int __ignore;							      \
+    register __typeof (val) _val __asm ("edx") = (val);			      \
+    __asm __volatile ("movl %%fs:%P3, %%esi\n\t"			      \
+		      "syscall"						      \
+		      : "=a" (__status), "=S" (__ignore)		      \
+		      : "0" (SYS_futex), "i" (PRIVATE_FUTEX), "D" (futex),    \
+		        "d" (_val), "r" (__to)				      \
+		      : "memory", "cc", "r11", "cx");			      \
+    __status;								      \
+  })
+
+
+# define lll_private_futex_wake(futex, nr) \
+  do {									      \
+    int __ignore;							      \
+    int __ignore2;							      \
+    register __typeof (nr) _nr __asm ("edx") = (nr);			      \
+    __asm __volatile ("orl %%fs:%P3, %%esi\n\t"				      \
+		      "syscall"						      \
+		      : "=a" (__ignore), "=S" (__ignore2)		      \
+		      : "0" (SYS_futex), "i" (PRIVATE_FUTEX), "D" (futex),    \
+			"1" (FUTEX_WAKE), "d" (_nr)			      \
+		      : "memory", "cc", "r10", "r11", "cx");		      \
+  } while (0)
+#endif
 
 
 /* Does not preserve %eax and %ecx.  */