diff options
Diffstat (limited to 'nptl/sysdeps/unix/sysv')
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/fork.c | 2 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h | 95 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/lowlevellock.c | 13 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c | 8 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h | 14 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/sem_post.c | 7 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/sem_timedwait.c | 4 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/sem_wait.c | 7 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c | 6 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c | 10 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c | 6 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/unregister-atfork.c | 4 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h | 80 |
13 files changed, 197 insertions, 59 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/fork.c b/nptl/sysdeps/unix/sysv/linux/fork.c index 98bb237c06..c6dadb5683 100644 --- a/nptl/sysdeps/unix/sysv/linux/fork.c +++ b/nptl/sysdeps/unix/sysv/linux/fork.c @@ -203,7 +203,7 @@ __libc_fork (void) if (atomic_decrement_and_test (&allp->handler->refcntr) && allp->handler->need_signal) - lll_futex_wake (allp->handler->refcntr, 1); + lll_private_futex_wake (allp->handler->refcntr, 1); allp = allp->next; } diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index aa963f77e7..474b39ae18 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -23,6 +23,8 @@ #include <time.h> #include <sys/param.h> #include <bits/pthreadtypes.h> +#include <kernel-features.h> +#include <tcb-offsets.h> #ifndef LOCK_INSTR # ifdef UP @@ -41,6 +43,13 @@ #define FUTEX_PRIVATE_FLAG 128 +/* Values for 'private' parameter of locking macros. Yes, the + definition seems to be backwards. But it is not. The bit will be + reversed before passing to the system call. */ +#define LLL_PRIVATE 0 +#define LLL_SHARED FUTEX_PRIVATE_FLAG + + /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED (1) @@ -145,7 +154,11 @@ LLL_STUB_UNWIND_INFO_START \ LLL_STUB_UNWIND_INFO_END -#define lll_futex_wait(futex, val) \ +#define lll_futex_wait(futex, val, private) \ + lll_futex_timed_wait (futex, val, NULL, private) + + +#define lll_futex_timed_wait(futex, val, timeout, private) \ ({ \ int __status; \ register __typeof (val) _val asm ("edx") = (val); \ @@ -153,7 +166,7 @@ LLL_STUB_UNWIND_INFO_END LLL_ENTER_KERNEL \ LLL_EBX_LOAD \ : "=a" (__status) \ - : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (0), \ + : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout), \ "c" (FUTEX_WAIT), "d" (_val), \ "i" (offsetof (tcbhead_t, sysinfo)) \ : "memory"); \ @@ -161,7 +174,27 @@ LLL_STUB_UNWIND_INFO_END }) -#define lll_futex_timed_wait(futex, val, timeout) \ +#define lll_futex_wake(futex, nr, private) \ + do { \ + int __ignore; \ + register __typeof (nr) _nr asm ("edx") = (nr); \ + __asm __volatile (LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__ignore) \ + : "0" (SYS_futex), LLL_EBX_REG (futex), \ + "c" (FUTEX_WAKE), "d" (_nr), \ + "i" (0) /* phony, to align next arg's number */, \ + "i" (offsetof (tcbhead_t, sysinfo))); \ + } while (0) + + +#define lll_private_futex_wait(futex, val) \ + lll_private_futex_timed_wait (futex, val, NULL) + + +#ifdef __ASSUME_PRIVATE_FUTEX +# define lll_private_futex_timed_wait(futex, val, timeout) \ ({ \ int __status; \ register __typeof (val) _val asm ("edx") = (val); \ @@ -170,14 +203,14 @@ LLL_STUB_UNWIND_INFO_END LLL_EBX_LOAD \ : "=a" (__status) \ : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout), \ - "c" (FUTEX_WAIT), "d" (_val), \ + "c" (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)), "d" (_val), \ "i" (offsetof (tcbhead_t, sysinfo)) \ : "memory"); \ __status; \ }) -#define lll_futex_wake(futex, nr) \ +# define lll_private_futex_wake(futex, nr) \ do { \ int __ignore; \ register __typeof (nr) _nr asm ("edx") = (nr); \ @@ -186,10 +219,45 @@ LLL_STUB_UNWIND_INFO_END LLL_EBX_LOAD \ : "=a" (__ignore) \ : "0" (SYS_futex), LLL_EBX_REG (futex), \ - "c" (FUTEX_WAKE), "d" (_nr), \ + "c" (FUTEX_WAKE | FUTEX_PRIVATE_FLAG), "d" (_nr), \ "i" (0) /* phony, to align next arg's number */, \ "i" (offsetof (tcbhead_t, sysinfo))); \ } while (0) +#else +# define lll_private_futex_timed_wait(futex, val, timeout) \ + ({ \ + int __status; \ + int __ignore; \ + register __typeof (val) _val asm ("edx") = (val); \ + __asm __volatile ("movl %%gs:%P7, %%ecx\n\t" \ + LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__status), "=c" (__ignore) \ + : LLL_EBX_REG (futex), "0" (SYS_futex), "S" (timeout), \ + "d" (_val), "i" (offsetof (tcbhead_t, sysinfo)), \ + "i" (PRIVATE_FUTEX) \ + : "memory"); \ + __status; \ + }) + + +# define lll_private_futex_wake(futex, nr) \ + do { \ + int __ignore; \ + int __ignore2; \ + register __typeof (nr) _nr asm ("edx") = (nr); \ + __asm __volatile ("orl %%gs:%P7, %%ecx\n\t" \ + LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__ignore), "=c" (__ignore2) \ + : LLL_EBX_REG (futex), "0" (SYS_futex), \ + "1" (FUTEX_WAKE), "d" (_nr), \ + "i" (offsetof (tcbhead_t, sysinfo)), \ + "i" (PRIVATE_FUTEX)); \ + } while (0) +#endif /* Does not preserve %eax and %ecx. */ @@ -415,21 +483,6 @@ extern int __lll_mutex_unlock_wake (int *__futex) "i" (offsetof (tcbhead_t, sysinfo))); }) -#define lll_futex_wake(futex, nr) \ - do { \ - int __ignore; \ - register __typeof (nr) _nr asm ("edx") = (nr); \ - __asm __volatile (LLL_EBX_LOAD \ - LLL_ENTER_KERNEL \ - LLL_EBX_LOAD \ - : "=a" (__ignore) \ - : "0" (SYS_futex), LLL_EBX_REG (futex), \ - "c" (FUTEX_WAKE), "d" (_nr), \ - "i" (0) /* phony, to align next arg's number */, \ - "i" (offsetof (tcbhead_t, sysinfo))); \ - } while (0) - - #define lll_mutex_islocked(futex) \ (futex != 0) diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c index 38d78884d4..ab7f605f0c 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c +++ b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c @@ -31,7 +31,9 @@ __lll_lock_wait (int *futex) { int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1); if (oldval != 0) - lll_futex_wait (futex, 2); + lll_futex_wait (futex, 2, + // XYZ check mutex flag + LLL_SHARED); } while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0); } @@ -68,7 +70,9 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime) /* Wait. */ int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1); if (oldval != 0) - lll_futex_timed_wait (futex, 2, &rt); + lll_futex_timed_wait (futex, 2, &rt, + // XYZ check mutex flag + LLL_SHARED); } while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0); @@ -108,8 +112,9 @@ __lll_timedwait_tid (int *tidp, const struct timespec *abstime) if (rt.tv_sec < 0) return ETIMEDOUT; - /* Wait until thread terminates. */ - if (lll_futex_timed_wait (tidp, tid, &rt) == -ETIMEDOUT) + /* Wait until thread terminates. The kernel so far does not use + the private futex operations for this. */ + if (lll_futex_timed_wait (tidp, tid, &rt, LLL_SHARED) == -ETIMEDOUT) return ETIMEDOUT; } diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c index 30ef991bd0..54cee0859b 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c @@ -44,7 +44,9 @@ __lll_robust_lock_wait (int *futex) && atomic_compare_and_exchange_bool_acq (futex, newval, oldval)) continue; - lll_futex_wait (futex, newval); + lll_futex_wait (futex, newval, + // XYZ check mutex flag + LLL_SHARED); try: ; @@ -100,7 +102,9 @@ __lll_robust_timedlock_wait (int *futex, const struct timespec *abstime) && atomic_compare_and_exchange_bool_acq (futex, newval, oldval)) continue; - lll_futex_timed_wait (futex, newval, &rt); + lll_futex_timed_wait (futex, newval, &rt, + // XYZ check mutex flag + LLL_SHARED); try: ; diff --git a/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h b/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h index 6b3d3682da..e805b63001 100644 --- a/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h +++ b/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h @@ -1,5 +1,5 @@ /* Defintions for lowlevel handling in ld.so. - Copyright (C) 2006 Free Software Foundation, Inc. + Copyright (C) 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -76,7 +76,7 @@ typedef int __rtld_mrlock_t; atomic_or (&(lock), __RTLD_MRLOCK_RWAIT); \ oldval |= __RTLD_MRLOCK_RWAIT; \ } \ - lll_futex_wait (lock, oldval); \ + lll_private_futex_wait (lock, oldval); \ } \ out:; \ } while (0) @@ -90,7 +90,7 @@ typedef int __rtld_mrlock_t; == (__RTLD_MRLOCK_INC | __RTLD_MRLOCK_WWAIT), 0)) \ /* We have to wake all threads since there might be some queued \ readers already. */ \ - lll_futex_wake (&(lock), 0x7fffffff); \ + lll_private_futex_wake (&(lock), 0x7fffffff); \ } while (0) @@ -119,7 +119,7 @@ typedef int __rtld_mrlock_t; } \ atomic_or (&(lock), __RTLD_MRLOCK_WWAIT); \ oldval |= __RTLD_MRLOCK_WWAIT; \ - lll_futex_wait (lock, oldval); \ + lll_private_futex_wait (lock, oldval); \ } \ out:; \ } while (0) @@ -129,7 +129,7 @@ typedef int __rtld_mrlock_t; do { \ int oldval = atomic_exchange_and_add (&(lock), -__RTLD_MRLOCK_WRITER); \ if (__builtin_expect ((oldval & __RTLD_MRLOCK_RWAIT) != 0, 0)) \ - lll_futex_wake (&(lock), 0x7fffffff); \ + lll_private_futex_wake (&(lock), 0x7fffffff); \ } while (0) @@ -142,12 +142,12 @@ typedef int __rtld_mrlock_t; int val = word; \ if (val == 0) \ break; \ - lll_futex_wait (&(word), val); \ + lll_private_futex_wait (&(word), val); \ } \ } while (0) #define __rtld_notify(word) \ - lll_futex_wake (&(word), 1) + lll_private_futex_wake (&(word), 1) #endif diff --git a/nptl/sysdeps/unix/sysv/linux/sem_post.c b/nptl/sysdeps/unix/sysv/linux/sem_post.c index f8bc695292..7f90325585 100644 --- a/nptl/sysdeps/unix/sysv/linux/sem_post.c +++ b/nptl/sysdeps/unix/sysv/linux/sem_post.c @@ -35,7 +35,9 @@ __new_sem_post (sem_t *sem) atomic_full_barrier (); if (isem->nwaiters > 0) { - int err = lll_futex_wake (&isem->value, 1); + int err = lll_futex_wake (&isem->value, 1, + // XYZ check mutex flag + LLL_SHARED); if (__builtin_expect (err, 0) < 0) { __set_errno (-err); @@ -55,7 +57,8 @@ __old_sem_post (sem_t *sem) int *futex = (int *) sem; int nr = atomic_increment_val (futex); - int err = lll_futex_wake (futex, 1); + /* We always have to assume it is a shared semaphore. */ + int err = lll_futex_wake (futex, 1, LLL_SHARED); if (__builtin_expect (err, 0) < 0) { __set_errno (-err); diff --git a/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c b/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c index 4c8acd0938..8f92d78abe 100644 --- a/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c +++ b/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c @@ -84,7 +84,9 @@ sem_timedwait (sem_t *sem, const struct timespec *abstime) /* Enable asynchronous cancellation. Required by the standard. */ int oldtype = __pthread_enable_asynccancel (); - err = lll_futex_timed_wait (&isem->value, 0, &rt); + err = lll_futex_timed_wait (&isem->value, 0, &rt, + // XYZ check mutex flag + LLL_SHARED); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (oldtype); diff --git a/nptl/sysdeps/unix/sysv/linux/sem_wait.c b/nptl/sysdeps/unix/sysv/linux/sem_wait.c index 05114c9704..12f3f16c2d 100644 --- a/nptl/sysdeps/unix/sysv/linux/sem_wait.c +++ b/nptl/sysdeps/unix/sysv/linux/sem_wait.c @@ -56,7 +56,9 @@ __new_sem_wait (sem_t *sem) /* Enable asynchronous cancellation. Required by the standard. */ int oldtype = __pthread_enable_asynccancel (); - err = lll_futex_wait (&isem->value, 0); + err = lll_futex_wait (&isem->value, 0, + // XYZ check mutex flag + LLL_SHARED); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (oldtype); @@ -100,7 +102,8 @@ __old_sem_wait (sem_t *sem) /* Enable asynchronous cancellation. Required by the standard. */ int oldtype = __pthread_enable_asynccancel (); - err = lll_futex_wait (futex, 0); + /* Always assume the semaphore is shared. */ + err = lll_futex_wait (futex, 0, LLL_SHARED); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (oldtype); diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c b/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c index 3b07cc127d..83fedef8e8 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c +++ b/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2003. @@ -30,7 +30,7 @@ clear_once_control (void *arg) pthread_once_t *once_control = (pthread_once_t *) arg; *once_control = 0; - lll_futex_wake (once_control, INT_MAX); + lll_private_futex_wake (once_control, INT_MAX); } @@ -84,7 +84,7 @@ __pthread_once (once_control, init_routine) atomic_increment (once_control); /* Wake up all other threads. */ - lll_futex_wake (once_control, INT_MAX); + lll_private_futex_wake (once_control, INT_MAX); break; } diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c index 4dfd11dcbe..868e0d2819 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c +++ b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -52,7 +52,9 @@ pthread_barrier_wait (barrier) ++ibarrier->b.curr_event; /* Wake up everybody. */ - lll_futex_wake (&ibarrier->b.curr_event, INT_MAX); + lll_futex_wake (&ibarrier->b.curr_event, INT_MAX, + // XYZ check mutex flag + LLL_SHARED); /* This is the thread which finished the serialization. */ result = PTHREAD_BARRIER_SERIAL_THREAD; @@ -68,7 +70,9 @@ pthread_barrier_wait (barrier) /* Wait for the event counter of the barrier to change. */ do - lll_futex_wait (&ibarrier->b.curr_event, event); + lll_futex_wait (&ibarrier->b.curr_event, event, + // XYZ check mutex flag + LLL_SHARED); while (event == ibarrier->b.curr_event); } diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c index be1cc60b11..527aedfdc7 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c +++ b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c @@ -1,5 +1,5 @@ /* sem_post -- post to a POSIX semaphore. SPARC version. - Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc. + Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2003. @@ -39,7 +39,9 @@ __new_sem_post (sem_t *sem) nr = ++*futex; __sparc32_atomic_do_unlock24 (futex + 1); } - int err = lll_futex_wake (futex, nr); + int err = lll_futex_wake (futex, nr, + // XYZ check mutex flag + LLL_SHARED); if (__builtin_expect (err, 0) < 0) { __set_errno (-err); diff --git a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c index 964f5b7094..240ce597f9 100644 --- a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c +++ b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -104,7 +104,7 @@ __unregister_atfork (dso_handle) atomic_decrement (&deleted->handler->refcntr); unsigned int val; while ((val = deleted->handler->refcntr) != 0) - lll_futex_wait (&deleted->handler->refcntr, val); + lll_private_futex_wait (&deleted->handler->refcntr, val); deleted = deleted->next; } diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index d3055cbb23..b86d95e93d 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -23,6 +23,8 @@ #include <time.h> #include <sys/param.h> #include <bits/pthreadtypes.h> +#include <kernel-features.h> +#include <tcb-offsets.h> #ifndef LOCK_INSTR # ifdef UP @@ -42,6 +44,13 @@ #define FUTEX_PRIVATE_FLAG 128 +/* Values for 'private' parameter of locking macros. Yes, the + definition seems to be backwards. But it is not. The bit will be + reversed before passing to the system call. */ +#define LLL_PRIVATE 0 +#define LLL_SHARED FUTEX_PRIVATE_FLAG + + /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED (1) @@ -149,44 +158,97 @@ LLL_STUB_UNWIND_INFO_START \ LLL_STUB_UNWIND_INFO_END -#define lll_futex_wait(futex, val) \ +#define lll_futex_wait(futex, val, private) \ + lll_futex_timed_wait(futex, val, NULL, private) + + +#define lll_futex_timed_wait(futex, val, timeout, private) \ ({ \ + register const struct timespec *__to __asm ("r10") = timeout; \ int __status; \ register __typeof (val) _val __asm ("edx") = (val); \ - __asm __volatile ("xorq %%r10, %%r10\n\t" \ - "syscall" \ + __asm __volatile ("syscall" \ : "=a" (__status) \ : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT), \ - "d" (_val) \ - : "memory", "cc", "r10", "r11", "cx"); \ + "d" (_val), "r" (__to) \ + : "memory", "cc", "r11", "cx"); \ __status; \ }) -#define lll_futex_timed_wait(futex, val, timeout) \ +#define lll_futex_wake(futex, nr, private) \ + do { \ + int __ignore; \ + register __typeof (nr) _nr __asm ("edx") = (nr); \ + __asm __volatile ("syscall" \ + : "=a" (__ignore) \ + : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE), \ + "d" (_nr) \ + : "memory", "cc", "r10", "r11", "cx"); \ + } while (0) + + +#define lll_private_futex_wait(futex, val) \ + lll_private_futex_timed_wait (futex, val, NULL) + + +#ifdef __ASSUME_PRIVATE_FUTEX +# define lll_private_futex_timed_wait(futex, val, timeout) \ ({ \ register const struct timespec *__to __asm ("r10") = timeout; \ int __status; \ register __typeof (val) _val __asm ("edx") = (val); \ __asm __volatile ("syscall" \ : "=a" (__status) \ - : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT), \ + : "0" (SYS_futex), "D" (futex), \ + "S" (FUTEX_WAIT | FUTEX_PRIVATE_FLAG), \ "d" (_val), "r" (__to) \ : "memory", "cc", "r11", "cx"); \ __status; \ }) -#define lll_futex_wake(futex, nr) \ +# define lll_private_futex_wake(futex, nr) \ do { \ int __ignore; \ register __typeof (nr) _nr __asm ("edx") = (nr); \ __asm __volatile ("syscall" \ : "=a" (__ignore) \ - : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE), \ + : "0" (SYS_futex), "D" (futex), \ + "S" (FUTEX_WAKE | FUTEX_PRIVATE_FLAG), \ "d" (_nr) \ : "memory", "cc", "r10", "r11", "cx"); \ } while (0) +#else +# define lll_private_futex_timed_wait(futex, val, timeout) \ + ({ \ + register const struct timespec *__to __asm ("r10") = timeout; \ + int __status; \ + int __ignore; \ + register __typeof (val) _val __asm ("edx") = (val); \ + __asm __volatile ("movl %%fs:%P3, %%esi\n\t" \ + "syscall" \ + : "=a" (__status), "=S" (__ignore) \ + : "0" (SYS_futex), "i" (PRIVATE_FUTEX), "D" (futex), \ + "d" (_val), "r" (__to) \ + : "memory", "cc", "r11", "cx"); \ + __status; \ + }) + + +# define lll_private_futex_wake(futex, nr) \ + do { \ + int __ignore; \ + int __ignore2; \ + register __typeof (nr) _nr __asm ("edx") = (nr); \ + __asm __volatile ("orl %%fs:%P3, %%esi\n\t" \ + "syscall" \ + : "=a" (__ignore), "=S" (__ignore2) \ + : "0" (SYS_futex), "i" (PRIVATE_FUTEX), "D" (futex), \ + "1" (FUTEX_WAKE), "d" (_nr) \ + : "memory", "cc", "r10", "r11", "cx"); \ + } while (0) +#endif /* Does not preserve %eax and %ecx. */ |