diff options
Diffstat (limited to 'nptl')
40 files changed, 300 insertions, 122 deletions
diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c index c31ca808bf..76d75fef2c 100644 --- a/nptl/allocatestack.c +++ b/nptl/allocatestack.c @@ -949,7 +949,7 @@ __nptl_setxid (struct xid_command *cmdp) int cur = cmdp->cntr; while (cur != 0) { - lll_futex_wait (&cmdp->cntr, cur); + lll_private_futex_wait (&cmdp->cntr, cur); cur = cmdp->cntr; } @@ -1035,7 +1035,7 @@ __wait_lookup_done (void) continue; do - lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT); + lll_private_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT); while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); } @@ -1057,7 +1057,7 @@ __wait_lookup_done (void) continue; do - lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT); + lll_private_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT); while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); } diff --git a/nptl/init.c b/nptl/init.c index 82adeae6a6..be8359ea21 100644 --- a/nptl/init.c +++ b/nptl/init.c @@ -216,7 +216,7 @@ sighandler_setxid (int sig, siginfo_t *si, void *ctx) __xidcmd->id[1], __xidcmd->id[2]); if (atomic_decrement_val (&__xidcmd->cntr) == 0) - lll_futex_wake (&__xidcmd->cntr, 1); + lll_private_futex_wake (&__xidcmd->cntr, 1); /* Reset the SETXID flag. */ struct pthread *self = THREAD_SELF; @@ -225,7 +225,7 @@ sighandler_setxid (int sig, siginfo_t *si, void *ctx) /* And release the futex. */ self->setxid_futex = 1; - lll_futex_wake (&self->setxid_futex, 1); + lll_private_futex_wake (&self->setxid_futex, 1); } diff --git a/nptl/lowlevellock.h b/nptl/lowlevellock.h index 338da39990..0600e1794d 100644 --- a/nptl/lowlevellock.h +++ b/nptl/lowlevellock.h @@ -1,5 +1,5 @@ /* Low level locking macros used in NPTL implementation. Stub version. - Copyright (C) 2002 Free Software Foundation, Inc. + Copyright (C) 2002, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -21,16 +21,6 @@ #include <atomic.h> -/* Implement generic mutex. Basic futex syscall support is required: - - lll_futex_wait(futex, value) - call sys_futex with FUTEX_WAIT - and third parameter VALUE - - lll_futex_wake(futex, value) - call sys_futex with FUTEX_WAKE - and third parameter VALUE -*/ - - /* Mutex lock counter: bit 31 clear means unlocked; bit 31 set means locked. @@ -66,7 +56,9 @@ __generic_mutex_lock (int *mutex) if (v >= 0) continue; - lll_futex_wait (mutex, v); + lll_futex_wait (mutex, v, + // XYZ check mutex flag + LLL_SHARED); } } @@ -82,7 +74,9 @@ __generic_mutex_unlock (int *mutex) /* There are other threads waiting for this mutex, wake one of them up. */ - lll_futex_wake (mutex, 1); + lll_futex_wake (mutex, 1, + // XYZ check mutex flag + LLL_SHARED); } diff --git a/nptl/pthread_barrier_wait.c b/nptl/pthread_barrier_wait.c index c6b563f242..e96a3e5473 100644 --- a/nptl/pthread_barrier_wait.c +++ b/nptl/pthread_barrier_wait.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -45,7 +45,9 @@ pthread_barrier_wait (barrier) ++ibarrier->curr_event; /* Wake up everybody. */ - lll_futex_wake (&ibarrier->curr_event, INT_MAX); + lll_futex_wake (&ibarrier->curr_event, INT_MAX, + // XYZ check mutex flag + LLL_SHARED); /* This is the thread which finished the serialization. */ result = PTHREAD_BARRIER_SERIAL_THREAD; @@ -61,7 +63,9 @@ pthread_barrier_wait (barrier) /* Wait for the event counter of the barrier to change. */ do - lll_futex_wait (&ibarrier->curr_event, event); + lll_futex_wait (&ibarrier->curr_event, event, + // XYZ check mutex flag + LLL_SHARED); while (event == ibarrier->curr_event); } diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c index 2b8b5460f4..aec33f3bd8 100644 --- a/nptl/pthread_cond_broadcast.c +++ b/nptl/pthread_cond_broadcast.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -69,7 +69,9 @@ __pthread_cond_broadcast (cond) { /* The requeue functionality is not available. */ wake_all: - lll_futex_wake (&cond->__data.__futex, INT_MAX); + lll_futex_wake (&cond->__data.__futex, INT_MAX, + // XYZ check mutex flag + LLL_SHARED); } /* That's all. */ diff --git a/nptl/pthread_cond_destroy.c b/nptl/pthread_cond_destroy.c index 0b8d4119d1..8574b6118f 100644 --- a/nptl/pthread_cond_destroy.c +++ b/nptl/pthread_cond_destroy.c @@ -59,14 +59,18 @@ __pthread_cond_destroy (cond) && cond->__data.__mutex != (void *) ~0l) { pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; - lll_futex_wake (&mut->__data.__lock, INT_MAX); + lll_futex_wake (&mut->__data.__lock, INT_MAX, + // XYZ check mutex flag + LLL_SHARED); } do { lll_mutex_unlock (cond->__data.__lock); - lll_futex_wait (&cond->__data.__nwaiters, nwaiters); + lll_futex_wait (&cond->__data.__nwaiters, nwaiters, + // XYZ check mutex flag + LLL_SHARED); lll_mutex_lock (cond->__data.__lock); diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c index 5a9bbcad91..a4faf41854 100644 --- a/nptl/pthread_cond_signal.c +++ b/nptl/pthread_cond_signal.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -44,11 +44,15 @@ __pthread_cond_signal (cond) /* Wake one. */ if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1, - 1, &cond->__data.__lock), + 1, &cond->__data.__lock, + // XYZ check mutex flag + LLL_SHARED), 0)) return 0; - lll_futex_wake (&cond->__data.__futex, 1); + lll_futex_wake (&cond->__data.__futex, 1, + // XYZ check mutex flag + LLL_SHARED); } /* We are done. */ diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c index 261b4484ba..d1c29d2377 100644 --- a/nptl/pthread_cond_timedwait.c +++ b/nptl/pthread_cond_timedwait.c @@ -153,7 +153,9 @@ __pthread_cond_timedwait (cond, mutex, abstime) /* Wait until woken by signal or broadcast. */ err = lll_futex_timed_wait (&cond->__data.__futex, - futex_val, &rt); + futex_val, &rt, + // XYZ check mutex flag + LLL_SHARED); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (cbuffer.oldtype); @@ -196,7 +198,9 @@ __pthread_cond_timedwait (cond, mutex, abstime) and it can be successfully destroyed. */ if (cond->__data.__total_seq == -1ULL && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) - lll_futex_wake (&cond->__data.__nwaiters, 1); + lll_futex_wake (&cond->__data.__nwaiters, 1, + // XYZ check mutex flag + LLL_SHARED); /* We are done with the condvar. */ lll_mutex_unlock (cond->__data.__lock); diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c index 6e00a28f6a..e524aa6c94 100644 --- a/nptl/pthread_cond_wait.c +++ b/nptl/pthread_cond_wait.c @@ -71,7 +71,9 @@ __condvar_cleanup (void *arg) if (cbuffer->cond->__data.__total_seq == -1ULL && cbuffer->cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) { - lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1); + lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1, + // XYZ check mutex flag + LLL_SHARED); destroying = 1; } @@ -80,7 +82,9 @@ __condvar_cleanup (void *arg) /* Wake everybody to make sure no condvar signal gets lost. */ if (! destroying) - lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX); + lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, + // XYZ check mutex flag + LLL_SHARED); /* Get the mutex before returning unless asynchronous cancellation is in effect. */ @@ -146,7 +150,9 @@ __pthread_cond_wait (cond, mutex) cbuffer.oldtype = __pthread_enable_asynccancel (); /* Wait until woken by signal or broadcast. */ - lll_futex_wait (&cond->__data.__futex, futex_val); + lll_futex_wait (&cond->__data.__futex, futex_val, + // XYZ check mutex flag + LLL_SHARED); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (cbuffer.oldtype); @@ -175,7 +181,9 @@ __pthread_cond_wait (cond, mutex) and it can be successfully destroyed. */ if (cond->__data.__total_seq == -1ULL && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) - lll_futex_wake (&cond->__data.__nwaiters, 1); + lll_futex_wake (&cond->__data.__nwaiters, 1, + // XYZ check mutex flag + LLL_SHARED); /* We are done with the condvar. */ lll_mutex_unlock (cond->__data.__lock); diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c index 79729ced03..b1f852dc95 100644 --- a/nptl/pthread_create.c +++ b/nptl/pthread_create.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -385,7 +385,7 @@ start_thread (void *arg) /* Some other thread might call any of the setXid functions and expect us to reply. In this case wait until we did that. */ do - lll_futex_wait (&pd->setxid_futex, 0); + lll_private_futex_wait (&pd->setxid_futex, 0); while (pd->cancelhandling & SETXID_BITMASK); /* Reset the value so that the stack can be reused. */ diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c index 1c3ee4fe25..a17013bb76 100644 --- a/nptl/pthread_mutex_lock.c +++ b/nptl/pthread_mutex_lock.c @@ -408,7 +408,9 @@ __pthread_mutex_lock (mutex) break; if (oldval != ceilval) - lll_futex_wait (&mutex->__data.__lock, ceilval | 2); + lll_futex_wait (&mutex->__data.__lock, ceilval | 2, + // XYZ check mutex flag + LLL_SHARED); } while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, ceilval | 2, ceilval) diff --git a/nptl/pthread_mutex_setprioceiling.c b/nptl/pthread_mutex_setprioceiling.c index cd13d1c14c..301fb63d21 100644 --- a/nptl/pthread_mutex_setprioceiling.c +++ b/nptl/pthread_mutex_setprioceiling.c @@ -1,5 +1,5 @@ /* Set current priority ceiling of pthread_mutex_t. - Copyright (C) 2006 Free Software Foundation, Inc. + Copyright (C) 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2006. @@ -80,7 +80,9 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling) break; if (oldval != ceilval) - lll_futex_wait (&mutex->__data.__lock, ceilval | 2); + lll_futex_wait (&mutex->__data.__lock, ceilval | 2, + // XYZ check mutex flag + LLL_SHARED); } while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, ceilval | 2, ceilval) @@ -110,7 +112,9 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling) | (prioceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT); atomic_full_barrier (); - lll_futex_wake (&mutex->__data.__lock, INT_MAX); + lll_futex_wake (&mutex->__data.__lock, INT_MAX, + // XYZ check mutex flag + LLL_SHARED); return 0; } diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c index 8fd681c6ef..74504e44a1 100644 --- a/nptl/pthread_mutex_timedlock.c +++ b/nptl/pthread_mutex_timedlock.c @@ -441,7 +441,9 @@ pthread_mutex_timedlock (mutex, abstime) } lll_futex_timed_wait (&mutex->__data.__lock, - ceilval | 2, &rt); + ceilval | 2, &rt, + // XYZ check mutex flag + LLL_SHARED); } } while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c index 33919d60af..1e6b276580 100644 --- a/nptl/pthread_mutex_unlock.c +++ b/nptl/pthread_mutex_unlock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -240,7 +240,9 @@ __pthread_mutex_unlock_usercnt (mutex, decr) newval, oldval)); if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1) - lll_futex_wake (&mutex->__data.__lock, 1); + lll_futex_wake (&mutex->__data.__lock, 1, + // XYZ check mutex flag + LLL_SHARED); int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT; return __pthread_tpp_change_priority (oldprio, -1); diff --git a/nptl/pthread_rwlock_timedrdlock.c b/nptl/pthread_rwlock_timedrdlock.c index caff5894fe..654d628b2f 100644 --- a/nptl/pthread_rwlock_timedrdlock.c +++ b/nptl/pthread_rwlock_timedrdlock.c @@ -114,7 +114,9 @@ pthread_rwlock_timedrdlock (rwlock, abstime) /* Wait for the writer to finish. */ err = lll_futex_timed_wait (&rwlock->__data.__readers_wakeup, - waitval, &rt); + waitval, &rt, + // XYZ check mutex flag + LLL_SHARED); /* Get the lock. */ lll_mutex_lock (rwlock->__data.__lock); diff --git a/nptl/pthread_rwlock_timedwrlock.c b/nptl/pthread_rwlock_timedwrlock.c index 97c0598f96..354beb0846 100644 --- a/nptl/pthread_rwlock_timedwrlock.c +++ b/nptl/pthread_rwlock_timedwrlock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -104,7 +104,9 @@ pthread_rwlock_timedwrlock (rwlock, abstime) /* Wait for the writer or reader(s) to finish. */ err = lll_futex_timed_wait (&rwlock->__data.__writer_wakeup, - waitval, &rt); + waitval, &rt, + // XYZ check mutex flag + LLL_SHARED); /* Get the lock. */ lll_mutex_lock (rwlock->__data.__lock); diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c index 9cae8b6c22..87a77a94ab 100644 --- a/nptl/pthread_rwlock_unlock.c +++ b/nptl/pthread_rwlock_unlock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -38,14 +38,18 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock) { ++rwlock->__data.__writer_wakeup; lll_mutex_unlock (rwlock->__data.__lock); - lll_futex_wake (&rwlock->__data.__writer_wakeup, 1); + lll_futex_wake (&rwlock->__data.__writer_wakeup, 1, + // XYZ check mutex flag + LLL_SHARED); return 0; } else if (rwlock->__data.__nr_readers_queued) { ++rwlock->__data.__readers_wakeup; lll_mutex_unlock (rwlock->__data.__lock); - lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX); + lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, + // XYZ check mutex flag + LLL_SHARED); return 0; } } diff --git a/nptl/sysdeps/alpha/tls.h b/nptl/sysdeps/alpha/tls.h index 64ddcd5c01..388a399c73 100644 --- a/nptl/sysdeps/alpha/tls.h +++ b/nptl/sysdeps/alpha/tls.h @@ -131,7 +131,7 @@ typedef struct = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ THREAD_GSCOPE_FLAG_UNUSED); \ if (__res == THREAD_GSCOPE_FLAG_WAIT) \ - lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ } \ while (0) #define THREAD_GSCOPE_SET_FLAG() \ diff --git a/nptl/sysdeps/i386/tls.h b/nptl/sysdeps/i386/tls.h index 7d3c5f97ad..1ffd4cf494 100644 --- a/nptl/sysdeps/i386/tls.h +++ b/nptl/sysdeps/i386/tls.h @@ -449,7 +449,7 @@ union user_desc_init : "i" (offsetof (struct pthread, header.gscope_flag)), \ "0" (THREAD_GSCOPE_FLAG_UNUSED)); \ if (__res == THREAD_GSCOPE_FLAG_WAIT) \ - lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ } \ while (0) #define THREAD_GSCOPE_SET_FLAG() \ diff --git a/nptl/sysdeps/ia64/tls.h b/nptl/sysdeps/ia64/tls.h index a144c28b09..a5b7b76e2a 100644 --- a/nptl/sysdeps/ia64/tls.h +++ b/nptl/sysdeps/ia64/tls.h @@ -173,7 +173,7 @@ register struct pthread *__thread_self __asm__("r13"); = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ THREAD_GSCOPE_FLAG_UNUSED); \ if (__res == THREAD_GSCOPE_FLAG_WAIT) \ - lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ } \ while (0) #define THREAD_GSCOPE_SET_FLAG() \ diff --git a/nptl/sysdeps/powerpc/tls.h b/nptl/sysdeps/powerpc/tls.h index bd9c99e06e..06c60e8ba2 100644 --- a/nptl/sysdeps/powerpc/tls.h +++ b/nptl/sysdeps/powerpc/tls.h @@ -190,7 +190,7 @@ register void *__thread_register __asm__ ("r13"); = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ THREAD_GSCOPE_FLAG_UNUSED); \ if (__res == THREAD_GSCOPE_FLAG_WAIT) \ - lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ } \ while (0) #define THREAD_GSCOPE_SET_FLAG() \ diff --git a/nptl/sysdeps/pthread/aio_misc.h b/nptl/sysdeps/pthread/aio_misc.h index c5a11f4550..25ab74e272 100644 --- a/nptl/sysdeps/pthread/aio_misc.h +++ b/nptl/sysdeps/pthread/aio_misc.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2006 Free Software Foundation, Inc. +/* Copyright (C) 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -30,7 +30,7 @@ #define AIO_MISC_NOTIFY(waitlist) \ do { \ if (*waitlist->counterp > 0 && --*waitlist->counterp == 0) \ - lll_futex_wake (waitlist->counterp, 1); \ + lll_private_futex_wake (waitlist->counterp, 1); \ } while (0) #define AIO_MISC_WAIT(result, futex, timeout, cancel) \ @@ -49,7 +49,8 @@ int status; \ do \ { \ - status = lll_futex_timed_wait (futexaddr, oldval, timeout); \ + status = lll_private_futex_timed_wait (futexaddr, oldval, \ + timeout); \ if (status != -EWOULDBLOCK) \ break; \ \ diff --git a/nptl/sysdeps/pthread/gai_misc.h b/nptl/sysdeps/pthread/gai_misc.h index 9f6a73dad1..5f6990040a 100644 --- a/nptl/sysdeps/pthread/gai_misc.h +++ b/nptl/sysdeps/pthread/gai_misc.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2006 Free Software Foundation, Inc. +/* Copyright (C) 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -31,7 +31,7 @@ #define GAI_MISC_NOTIFY(waitlist) \ do { \ if (*waitlist->counterp > 0 && --*waitlist->counterp == 0) \ - lll_futex_wake (waitlist->counterp, 1); \ + lll_private_futex_wake (waitlist->counterp, 1); \ } while (0) #define GAI_MISC_WAIT(result, futex, timeout, cancel) \ @@ -50,7 +50,8 @@ int status; \ do \ { \ - status = lll_futex_timed_wait (futexaddr, oldval, timeout); \ + status = lll_private_futex_timed_wait (futexaddr, oldval, \ + timeout); \ if (status != -EWOULDBLOCK) \ break; \ \ diff --git a/nptl/sysdeps/s390/tls.h b/nptl/sysdeps/s390/tls.h index de8a81bd09..f10db8f227 100644 --- a/nptl/sysdeps/s390/tls.h +++ b/nptl/sysdeps/s390/tls.h @@ -183,7 +183,7 @@ typedef struct = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ THREAD_GSCOPE_FLAG_UNUSED); \ if (__res == THREAD_GSCOPE_FLAG_WAIT) \ - lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ } \ while (0) #define THREAD_GSCOPE_SET_FLAG() \ diff --git a/nptl/sysdeps/sh/tls.h b/nptl/sysdeps/sh/tls.h index 6d6eff665f..07bab4f576 100644 --- a/nptl/sysdeps/sh/tls.h +++ b/nptl/sysdeps/sh/tls.h @@ -160,7 +160,7 @@ typedef struct = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ THREAD_GSCOPE_FLAG_UNUSED); \ if (__res == THREAD_GSCOPE_FLAG_WAIT) \ - lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ } \ while (0) #define THREAD_GSCOPE_SET_FLAG() \ diff --git a/nptl/sysdeps/sparc/tls.h b/nptl/sysdeps/sparc/tls.h index 60e52fd4ea..788c974c1c 100644 --- a/nptl/sysdeps/sparc/tls.h +++ b/nptl/sysdeps/sparc/tls.h @@ -151,7 +151,7 @@ register struct pthread *__thread_self __asm__("%g7"); = atomic_exchange_rel (&THREAD_SELF->header.gscope_flag, \ THREAD_GSCOPE_FLAG_UNUSED); \ if (__res == THREAD_GSCOPE_FLAG_WAIT) \ - lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ } \ while (0) #define THREAD_GSCOPE_SET_FLAG() \ diff --git a/nptl/sysdeps/unix/sysv/linux/fork.c b/nptl/sysdeps/unix/sysv/linux/fork.c index 98bb237c06..c6dadb5683 100644 --- a/nptl/sysdeps/unix/sysv/linux/fork.c +++ b/nptl/sysdeps/unix/sysv/linux/fork.c @@ -203,7 +203,7 @@ __libc_fork (void) if (atomic_decrement_and_test (&allp->handler->refcntr) && allp->handler->need_signal) - lll_futex_wake (allp->handler->refcntr, 1); + lll_private_futex_wake (allp->handler->refcntr, 1); allp = allp->next; } diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index aa963f77e7..474b39ae18 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -23,6 +23,8 @@ #include <time.h> #include <sys/param.h> #include <bits/pthreadtypes.h> +#include <kernel-features.h> +#include <tcb-offsets.h> #ifndef LOCK_INSTR # ifdef UP @@ -41,6 +43,13 @@ #define FUTEX_PRIVATE_FLAG 128 +/* Values for 'private' parameter of locking macros. Yes, the + definition seems to be backwards. But it is not. The bit will be + reversed before passing to the system call. */ +#define LLL_PRIVATE 0 +#define LLL_SHARED FUTEX_PRIVATE_FLAG + + /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED (1) @@ -145,7 +154,11 @@ LLL_STUB_UNWIND_INFO_START \ LLL_STUB_UNWIND_INFO_END -#define lll_futex_wait(futex, val) \ +#define lll_futex_wait(futex, val, private) \ + lll_futex_timed_wait (futex, val, NULL, private) + + +#define lll_futex_timed_wait(futex, val, timeout, private) \ ({ \ int __status; \ register __typeof (val) _val asm ("edx") = (val); \ @@ -153,7 +166,7 @@ LLL_STUB_UNWIND_INFO_END LLL_ENTER_KERNEL \ LLL_EBX_LOAD \ : "=a" (__status) \ - : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (0), \ + : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout), \ "c" (FUTEX_WAIT), "d" (_val), \ "i" (offsetof (tcbhead_t, sysinfo)) \ : "memory"); \ @@ -161,7 +174,27 @@ LLL_STUB_UNWIND_INFO_END }) -#define lll_futex_timed_wait(futex, val, timeout) \ +#define lll_futex_wake(futex, nr, private) \ + do { \ + int __ignore; \ + register __typeof (nr) _nr asm ("edx") = (nr); \ + __asm __volatile (LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__ignore) \ + : "0" (SYS_futex), LLL_EBX_REG (futex), \ + "c" (FUTEX_WAKE), "d" (_nr), \ + "i" (0) /* phony, to align next arg's number */, \ + "i" (offsetof (tcbhead_t, sysinfo))); \ + } while (0) + + +#define lll_private_futex_wait(futex, val) \ + lll_private_futex_timed_wait (futex, val, NULL) + + +#ifdef __ASSUME_PRIVATE_FUTEX +# define lll_private_futex_timed_wait(futex, val, timeout) \ ({ \ int __status; \ register __typeof (val) _val asm ("edx") = (val); \ @@ -170,14 +203,14 @@ LLL_STUB_UNWIND_INFO_END LLL_EBX_LOAD \ : "=a" (__status) \ : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout), \ - "c" (FUTEX_WAIT), "d" (_val), \ + "c" (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)), "d" (_val), \ "i" (offsetof (tcbhead_t, sysinfo)) \ : "memory"); \ __status; \ }) -#define lll_futex_wake(futex, nr) \ +# define lll_private_futex_wake(futex, nr) \ do { \ int __ignore; \ register __typeof (nr) _nr asm ("edx") = (nr); \ @@ -186,10 +219,45 @@ LLL_STUB_UNWIND_INFO_END LLL_EBX_LOAD \ : "=a" (__ignore) \ : "0" (SYS_futex), LLL_EBX_REG (futex), \ - "c" (FUTEX_WAKE), "d" (_nr), \ + "c" (FUTEX_WAKE | FUTEX_PRIVATE_FLAG), "d" (_nr), \ "i" (0) /* phony, to align next arg's number */, \ "i" (offsetof (tcbhead_t, sysinfo))); \ } while (0) +#else +# define lll_private_futex_timed_wait(futex, val, timeout) \ + ({ \ + int __status; \ + int __ignore; \ + register __typeof (val) _val asm ("edx") = (val); \ + __asm __volatile ("movl %%gs:%P7, %%ecx\n\t" \ + LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__status), "=c" (__ignore) \ + : LLL_EBX_REG (futex), "0" (SYS_futex), "S" (timeout), \ + "d" (_val), "i" (offsetof (tcbhead_t, sysinfo)), \ + "i" (PRIVATE_FUTEX) \ + : "memory"); \ + __status; \ + }) + + +# define lll_private_futex_wake(futex, nr) \ + do { \ + int __ignore; \ + int __ignore2; \ + register __typeof (nr) _nr asm ("edx") = (nr); \ + __asm __volatile ("orl %%gs:%P7, %%ecx\n\t" \ + LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__ignore), "=c" (__ignore2) \ + : LLL_EBX_REG (futex), "0" (SYS_futex), \ + "1" (FUTEX_WAKE), "d" (_nr), \ + "i" (offsetof (tcbhead_t, sysinfo)), \ + "i" (PRIVATE_FUTEX)); \ + } while (0) +#endif /* Does not preserve %eax and %ecx. */ @@ -415,21 +483,6 @@ extern int __lll_mutex_unlock_wake (int *__futex) "i" (offsetof (tcbhead_t, sysinfo))); }) -#define lll_futex_wake(futex, nr) \ - do { \ - int __ignore; \ - register __typeof (nr) _nr asm ("edx") = (nr); \ - __asm __volatile (LLL_EBX_LOAD \ - LLL_ENTER_KERNEL \ - LLL_EBX_LOAD \ - : "=a" (__ignore) \ - : "0" (SYS_futex), LLL_EBX_REG (futex), \ - "c" (FUTEX_WAKE), "d" (_nr), \ - "i" (0) /* phony, to align next arg's number */, \ - "i" (offsetof (tcbhead_t, sysinfo))); \ - } while (0) - - #define lll_mutex_islocked(futex) \ (futex != 0) diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c index 38d78884d4..ab7f605f0c 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c +++ b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c @@ -31,7 +31,9 @@ __lll_lock_wait (int *futex) { int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1); if (oldval != 0) - lll_futex_wait (futex, 2); + lll_futex_wait (futex, 2, + // XYZ check mutex flag + LLL_SHARED); } while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0); } @@ -68,7 +70,9 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime) /* Wait. */ int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1); if (oldval != 0) - lll_futex_timed_wait (futex, 2, &rt); + lll_futex_timed_wait (futex, 2, &rt, + // XYZ check mutex flag + LLL_SHARED); } while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0); @@ -108,8 +112,9 @@ __lll_timedwait_tid (int *tidp, const struct timespec *abstime) if (rt.tv_sec < 0) return ETIMEDOUT; - /* Wait until thread terminates. */ - if (lll_futex_timed_wait (tidp, tid, &rt) == -ETIMEDOUT) + /* Wait until thread terminates. The kernel so far does not use + the private futex operations for this. */ + if (lll_futex_timed_wait (tidp, tid, &rt, LLL_SHARED) == -ETIMEDOUT) return ETIMEDOUT; } diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c index 30ef991bd0..54cee0859b 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c @@ -44,7 +44,9 @@ __lll_robust_lock_wait (int *futex) && atomic_compare_and_exchange_bool_acq (futex, newval, oldval)) continue; - lll_futex_wait (futex, newval); + lll_futex_wait (futex, newval, + // XYZ check mutex flag + LLL_SHARED); try: ; @@ -100,7 +102,9 @@ __lll_robust_timedlock_wait (int *futex, const struct timespec *abstime) && atomic_compare_and_exchange_bool_acq (futex, newval, oldval)) continue; - lll_futex_timed_wait (futex, newval, &rt); + lll_futex_timed_wait (futex, newval, &rt, + // XYZ check mutex flag + LLL_SHARED); try: ; diff --git a/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h b/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h index 6b3d3682da..e805b63001 100644 --- a/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h +++ b/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h @@ -1,5 +1,5 @@ /* Defintions for lowlevel handling in ld.so. - Copyright (C) 2006 Free Software Foundation, Inc. + Copyright (C) 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -76,7 +76,7 @@ typedef int __rtld_mrlock_t; atomic_or (&(lock), __RTLD_MRLOCK_RWAIT); \ oldval |= __RTLD_MRLOCK_RWAIT; \ } \ - lll_futex_wait (lock, oldval); \ + lll_private_futex_wait (lock, oldval); \ } \ out:; \ } while (0) @@ -90,7 +90,7 @@ typedef int __rtld_mrlock_t; == (__RTLD_MRLOCK_INC | __RTLD_MRLOCK_WWAIT), 0)) \ /* We have to wake all threads since there might be some queued \ readers already. */ \ - lll_futex_wake (&(lock), 0x7fffffff); \ + lll_private_futex_wake (&(lock), 0x7fffffff); \ } while (0) @@ -119,7 +119,7 @@ typedef int __rtld_mrlock_t; } \ atomic_or (&(lock), __RTLD_MRLOCK_WWAIT); \ oldval |= __RTLD_MRLOCK_WWAIT; \ - lll_futex_wait (lock, oldval); \ + lll_private_futex_wait (lock, oldval); \ } \ out:; \ } while (0) @@ -129,7 +129,7 @@ typedef int __rtld_mrlock_t; do { \ int oldval = atomic_exchange_and_add (&(lock), -__RTLD_MRLOCK_WRITER); \ if (__builtin_expect ((oldval & __RTLD_MRLOCK_RWAIT) != 0, 0)) \ - lll_futex_wake (&(lock), 0x7fffffff); \ + lll_private_futex_wake (&(lock), 0x7fffffff); \ } while (0) @@ -142,12 +142,12 @@ typedef int __rtld_mrlock_t; int val = word; \ if (val == 0) \ break; \ - lll_futex_wait (&(word), val); \ + lll_private_futex_wait (&(word), val); \ } \ } while (0) #define __rtld_notify(word) \ - lll_futex_wake (&(word), 1) + lll_private_futex_wake (&(word), 1) #endif diff --git a/nptl/sysdeps/unix/sysv/linux/sem_post.c b/nptl/sysdeps/unix/sysv/linux/sem_post.c index f8bc695292..7f90325585 100644 --- a/nptl/sysdeps/unix/sysv/linux/sem_post.c +++ b/nptl/sysdeps/unix/sysv/linux/sem_post.c @@ -35,7 +35,9 @@ __new_sem_post (sem_t *sem) atomic_full_barrier (); if (isem->nwaiters > 0) { - int err = lll_futex_wake (&isem->value, 1); + int err = lll_futex_wake (&isem->value, 1, + // XYZ check mutex flag + LLL_SHARED); if (__builtin_expect (err, 0) < 0) { __set_errno (-err); @@ -55,7 +57,8 @@ __old_sem_post (sem_t *sem) int *futex = (int *) sem; int nr = atomic_increment_val (futex); - int err = lll_futex_wake (futex, 1); + /* We always have to assume it is a shared semaphore. */ + int err = lll_futex_wake (futex, 1, LLL_SHARED); if (__builtin_expect (err, 0) < 0) { __set_errno (-err); diff --git a/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c b/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c index 4c8acd0938..8f92d78abe 100644 --- a/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c +++ b/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c @@ -84,7 +84,9 @@ sem_timedwait (sem_t *sem, const struct timespec *abstime) /* Enable asynchronous cancellation. Required by the standard. */ int oldtype = __pthread_enable_asynccancel (); - err = lll_futex_timed_wait (&isem->value, 0, &rt); + err = lll_futex_timed_wait (&isem->value, 0, &rt, + // XYZ check mutex flag + LLL_SHARED); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (oldtype); diff --git a/nptl/sysdeps/unix/sysv/linux/sem_wait.c b/nptl/sysdeps/unix/sysv/linux/sem_wait.c index 05114c9704..12f3f16c2d 100644 --- a/nptl/sysdeps/unix/sysv/linux/sem_wait.c +++ b/nptl/sysdeps/unix/sysv/linux/sem_wait.c @@ -56,7 +56,9 @@ __new_sem_wait (sem_t *sem) /* Enable asynchronous cancellation. Required by the standard. */ int oldtype = __pthread_enable_asynccancel (); - err = lll_futex_wait (&isem->value, 0); + err = lll_futex_wait (&isem->value, 0, + // XYZ check mutex flag + LLL_SHARED); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (oldtype); @@ -100,7 +102,8 @@ __old_sem_wait (sem_t *sem) /* Enable asynchronous cancellation. Required by the standard. */ int oldtype = __pthread_enable_asynccancel (); - err = lll_futex_wait (futex, 0); + /* Always assume the semaphore is shared. */ + err = lll_futex_wait (futex, 0, LLL_SHARED); /* Disable asynchronous cancellation. */ __pthread_disable_asynccancel (oldtype); diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c b/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c index 3b07cc127d..83fedef8e8 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c +++ b/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2003. @@ -30,7 +30,7 @@ clear_once_control (void *arg) pthread_once_t *once_control = (pthread_once_t *) arg; *once_control = 0; - lll_futex_wake (once_control, INT_MAX); + lll_private_futex_wake (once_control, INT_MAX); } @@ -84,7 +84,7 @@ __pthread_once (once_control, init_routine) atomic_increment (once_control); /* Wake up all other threads. */ - lll_futex_wake (once_control, INT_MAX); + lll_private_futex_wake (once_control, INT_MAX); break; } diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c index 4dfd11dcbe..868e0d2819 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c +++ b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. @@ -52,7 +52,9 @@ pthread_barrier_wait (barrier) ++ibarrier->b.curr_event; /* Wake up everybody. */ - lll_futex_wake (&ibarrier->b.curr_event, INT_MAX); + lll_futex_wake (&ibarrier->b.curr_event, INT_MAX, + // XYZ check mutex flag + LLL_SHARED); /* This is the thread which finished the serialization. */ result = PTHREAD_BARRIER_SERIAL_THREAD; @@ -68,7 +70,9 @@ pthread_barrier_wait (barrier) /* Wait for the event counter of the barrier to change. */ do - lll_futex_wait (&ibarrier->b.curr_event, event); + lll_futex_wait (&ibarrier->b.curr_event, event, + // XYZ check mutex flag + LLL_SHARED); while (event == ibarrier->b.curr_event); } diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c index be1cc60b11..527aedfdc7 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c +++ b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c @@ -1,5 +1,5 @@ /* sem_post -- post to a POSIX semaphore. SPARC version. - Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc. + Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Jakub Jelinek <jakub@redhat.com>, 2003. @@ -39,7 +39,9 @@ __new_sem_post (sem_t *sem) nr = ++*futex; __sparc32_atomic_do_unlock24 (futex + 1); } - int err = lll_futex_wake (futex, nr); + int err = lll_futex_wake (futex, nr, + // XYZ check mutex flag + LLL_SHARED); if (__builtin_expect (err, 0) < 0) { __set_errno (-err); diff --git a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c index 964f5b7094..240ce597f9 100644 --- a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c +++ b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -104,7 +104,7 @@ __unregister_atfork (dso_handle) atomic_decrement (&deleted->handler->refcntr); unsigned int val; while ((val = deleted->handler->refcntr) != 0) - lll_futex_wait (&deleted->handler->refcntr, val); + lll_private_futex_wait (&deleted->handler->refcntr, val); deleted = deleted->next; } diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index d3055cbb23..b86d95e93d 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -23,6 +23,8 @@ #include <time.h> #include <sys/param.h> #include <bits/pthreadtypes.h> +#include <kernel-features.h> +#include <tcb-offsets.h> #ifndef LOCK_INSTR # ifdef UP @@ -42,6 +44,13 @@ #define FUTEX_PRIVATE_FLAG 128 +/* Values for 'private' parameter of locking macros. Yes, the + definition seems to be backwards. But it is not. The bit will be + reversed before passing to the system call. */ +#define LLL_PRIVATE 0 +#define LLL_SHARED FUTEX_PRIVATE_FLAG + + /* Initializer for compatibility lock. */ #define LLL_MUTEX_LOCK_INITIALIZER (0) #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED (1) @@ -149,44 +158,97 @@ LLL_STUB_UNWIND_INFO_START \ LLL_STUB_UNWIND_INFO_END -#define lll_futex_wait(futex, val) \ +#define lll_futex_wait(futex, val, private) \ + lll_futex_timed_wait(futex, val, NULL, private) + + +#define lll_futex_timed_wait(futex, val, timeout, private) \ ({ \ + register const struct timespec *__to __asm ("r10") = timeout; \ int __status; \ register __typeof (val) _val __asm ("edx") = (val); \ - __asm __volatile ("xorq %%r10, %%r10\n\t" \ - "syscall" \ + __asm __volatile ("syscall" \ : "=a" (__status) \ : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT), \ - "d" (_val) \ - : "memory", "cc", "r10", "r11", "cx"); \ + "d" (_val), "r" (__to) \ + : "memory", "cc", "r11", "cx"); \ __status; \ }) -#define lll_futex_timed_wait(futex, val, timeout) \ +#define lll_futex_wake(futex, nr, private) \ + do { \ + int __ignore; \ + register __typeof (nr) _nr __asm ("edx") = (nr); \ + __asm __volatile ("syscall" \ + : "=a" (__ignore) \ + : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE), \ + "d" (_nr) \ + : "memory", "cc", "r10", "r11", "cx"); \ + } while (0) + + +#define lll_private_futex_wait(futex, val) \ + lll_private_futex_timed_wait (futex, val, NULL) + + +#ifdef __ASSUME_PRIVATE_FUTEX +# define lll_private_futex_timed_wait(futex, val, timeout) \ ({ \ register const struct timespec *__to __asm ("r10") = timeout; \ int __status; \ register __typeof (val) _val __asm ("edx") = (val); \ __asm __volatile ("syscall" \ : "=a" (__status) \ - : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT), \ + : "0" (SYS_futex), "D" (futex), \ + "S" (FUTEX_WAIT | FUTEX_PRIVATE_FLAG), \ "d" (_val), "r" (__to) \ : "memory", "cc", "r11", "cx"); \ __status; \ }) -#define lll_futex_wake(futex, nr) \ +# define lll_private_futex_wake(futex, nr) \ do { \ int __ignore; \ register __typeof (nr) _nr __asm ("edx") = (nr); \ __asm __volatile ("syscall" \ : "=a" (__ignore) \ - : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE), \ + : "0" (SYS_futex), "D" (futex), \ + "S" (FUTEX_WAKE | FUTEX_PRIVATE_FLAG), \ "d" (_nr) \ : "memory", "cc", "r10", "r11", "cx"); \ } while (0) +#else +# define lll_private_futex_timed_wait(futex, val, timeout) \ + ({ \ + register const struct timespec *__to __asm ("r10") = timeout; \ + int __status; \ + int __ignore; \ + register __typeof (val) _val __asm ("edx") = (val); \ + __asm __volatile ("movl %%fs:%P3, %%esi\n\t" \ + "syscall" \ + : "=a" (__status), "=S" (__ignore) \ + : "0" (SYS_futex), "i" (PRIVATE_FUTEX), "D" (futex), \ + "d" (_val), "r" (__to) \ + : "memory", "cc", "r11", "cx"); \ + __status; \ + }) + + +# define lll_private_futex_wake(futex, nr) \ + do { \ + int __ignore; \ + int __ignore2; \ + register __typeof (nr) _nr __asm ("edx") = (nr); \ + __asm __volatile ("orl %%fs:%P3, %%esi\n\t" \ + "syscall" \ + : "=a" (__ignore), "=S" (__ignore2) \ + : "0" (SYS_futex), "i" (PRIVATE_FUTEX), "D" (futex), \ + "1" (FUTEX_WAKE), "d" (_nr) \ + : "memory", "cc", "r10", "r11", "cx"); \ + } while (0) +#endif /* Does not preserve %eax and %ecx. */ diff --git a/nptl/sysdeps/x86_64/tls.h b/nptl/sysdeps/x86_64/tls.h index f3532529bb..029848a8fa 100644 --- a/nptl/sysdeps/x86_64/tls.h +++ b/nptl/sysdeps/x86_64/tls.h @@ -355,7 +355,7 @@ typedef struct : "i" (offsetof (struct pthread, header.gscope_flag)), \ "0" (THREAD_GSCOPE_FLAG_UNUSED)); \ if (__res == THREAD_GSCOPE_FLAG_WAIT) \ - lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ + lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1); \ } \ while (0) #define THREAD_GSCOPE_SET_FLAG() \ |