diff options
Diffstat (limited to 'nptl/sysdeps/unix')
8 files changed, 131 insertions, 31 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S index 88885b735d..f246711489 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -17,6 +17,8 @@ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. */ +#include <kernel-features.h> + /* In libc.so we do not unconditionally use the lock prefix. Only if the application is using threads. */ #ifndef UP @@ -27,4 +29,17 @@ 0: #endif +/* All locks in libc are private. Use the kernel feature if possible. */ +#define FUTEX_PRIVATE_FLAG 128 +#ifdef __ASSUME_PRIVATE_FUTEX +# define FUTEX_WAIT (0 | FUTEX_PRIVATE_FLAG) +# define FUTEX_WAKE (1 | FUTEX_PRIVATE_FLAG) +#else +# define LOAD_FUTEX_WAIT(reg) \ + movl %gs:PRIVATE_FUTEX, reg +# define LOAD_FUTEX_WAKE(reg) \ + movl %gs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAKE, reg +#endif + #include "lowlevellock.S" diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S index e2da5b04cf..1c47b37bb0 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -32,8 +32,22 @@ #define SYS_gettimeofday __NR_gettimeofday #define SYS_futex 240 -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 +#ifndef FUTEX_WAIT +# define FUTEX_WAIT 0 +# define FUTEX_WAKE 1 +#endif + +#ifndef LOAD_FUTEX_WAIT +# if FUTEX_WAIT == 0 +# define LOAD_FUTEX_WAIT(reg) \ + xorl reg, reg +# else +# define LOAD_FUTEX_WAIT(reg) \ + movl $FUTEX_WAIT, reg +# endif +# define LOAD_FUTEX_WAKE(reg) \ + movl $FUTEX_WAKE, reg +#endif .globl __lll_mutex_lock_wait @@ -55,7 +69,7 @@ __lll_mutex_lock_wait: movl $2, %edx movl %ecx, %ebx xorl %esi, %esi /* No timeout. */ - xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ + LOAD_FUTEX_WAIT (%ecx) cmpl %edx, %eax /* NB: %edx == 2 */ jne 2f @@ -151,7 +165,7 @@ __lll_mutex_timedlock_wait: /* Futex call. */ movl %esp, %esi - xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ + LOAD_FUTEX_WAIT (%ecx) movl $SYS_futex, %eax ENTER_KERNEL movl %eax, %ecx @@ -252,7 +266,7 @@ __lll_mutex_unlock_wake: movl %eax, %ebx movl $0, (%eax) - movl $FUTEX_WAKE, %ecx + LOAD_FUTEX_WAKE (%ecx) movl $1, %edx /* Wake one thread. */ movl $SYS_futex, %eax ENTER_KERNEL @@ -314,6 +328,8 @@ __lll_timedwait_tid: jz 4f movl %esp, %esi + /* XXX The kernel so far uses global futex for the wakeup at + all times. */ xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ movl %ebp, %ebx movl $SYS_futex, %eax diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 21de09fe91..b89d5dd7d1 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -38,6 +38,7 @@ #define FUTEX_LOCK_PI 6 #define FUTEX_UNLOCK_PI 7 #define FUTEX_TRYLOCK_PI 8 +#define FUTEX_PRIVATE_FLAG 128 /* Initializer for compatibility lock. */ diff --git a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S index 312933c5e4..8ff0dad27f 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -19,6 +19,8 @@ #include <unwindbuf.h> #include <sysdep.h> +#include <kernel-features.h> + #ifndef UP # define LOCK lock @@ -26,8 +28,10 @@ # define LOCK #endif -#define SYS_futex 240 -#define FUTEX_WAKE 1 +#define SYS_futex 240 +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_PRIVATE_FLAG 128 .comm __fork_generation, 4, 4 @@ -90,7 +94,16 @@ __pthread_once: jnz 3f /* Different for generation -> run initializer. */ /* Somebody else got here first. Wait. */ - movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */ +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx +#else +# if FUTEX_WAIT == 0 + movl %gs:PRIVATE_FUTEX, %ecx +# else + movl $FUTEX_WAIT, %ecx + orl %gs:PRIVATE_FUTEX, %ecx +# endif +#endif movl $SYS_futex, %eax ENTER_KERNEL jmp 6b @@ -131,7 +144,12 @@ __pthread_once: /* Wake up all other threads. */ movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx +#else movl $FUTEX_WAKE, %ecx + orl %gs:PRIVATE_FUTEX, %ecx +#endif movl $SYS_futex, %eax ENTER_KERNEL @@ -152,7 +170,12 @@ __pthread_once: movl $0, (%ebx) movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx +#else movl $FUTEX_WAKE, %ecx + orl %gs:PRIVATE_FUTEX, %ecx +#endif movl $SYS_futex, %eax ENTER_KERNEL diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S index 3621efa4fc..21bf58cbbd 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -17,6 +17,8 @@ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. */ +#include <kernel-features.h> + /* In libc.so we do not unconditionally use the lock prefix. Only if the application is using threads. */ #ifndef UP @@ -27,4 +29,17 @@ 0: #endif +/* All locks in libc are private. Use the kernel feature if possible. */ +#define FUTEX_PRIVATE_FLAG 128 +#ifdef __ASSUME_PRIVATE_FUTEX +# define FUTEX_WAIT (0 | FUTEX_PRIVATE_FLAG) +# define FUTEX_WAKE (1 | FUTEX_PRIVATE_FLAG) +#else +# define LOAD_FUTEX_WAIT(reg) \ + movl %fs:PRIVATE_FUTEX, reg +# define LOAD_FUTEX_WAKE(reg) \ + movl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAKE, reg +#endif + #include "lowlevellock.S" diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S index 6724ded762..de70b1852c 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S @@ -31,8 +31,23 @@ #endif #define SYS_futex 202 -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 +#ifndef FUTEX_WAIT +# define FUTEX_WAIT 0 +# define FUTEX_WAKE 1 +#endif + +#ifndef LOAD_FUTEX_WAIT +# if FUTEX_WAIT == 0 +# define LOAD_FUTEX_WAIT(reg) \ + xorl reg, reg +# else +# define LOAD_FUTEX_WAIT(reg) \ + movl $FUTEX_WAIT, reg +# endif +# define LOAD_FUTEX_WAKE(reg) \ + movl $FUTEX_WAKE, reg +#endif + /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 @@ -52,11 +67,7 @@ __lll_mutex_lock_wait: cfi_offset(%rdx, -24) xorq %r10, %r10 /* No timeout. */ movl $2, %edx -#if FUTEX_WAIT == 0 - xorl %esi, %esi -#else - movl $FUTEX_WAIT, %esi -#endif + LOAD_FUTEX_WAIT (%esi) cmpl %edx, %eax /* NB: %edx == 2 */ jne 2f @@ -151,11 +162,7 @@ __lll_mutex_timedlock_wait: je 8f movq %rsp, %r10 -#if FUTEX_WAIT == 0 - xorl %esi, %esi -#else - movl $FUTEX_WAIT, %esi -#endif + LOAD_FUTEX_WAIT (%esi) movq %r12, %rdi movl $SYS_futex, %eax syscall @@ -247,7 +254,7 @@ __lll_mutex_unlock_wake: cfi_offset(%rdx, -24) movl $0, (%rdi) - movl $FUTEX_WAKE, %esi + LOAD_FUTEX_WAKE (%esi) movl $1, %edx /* Wake one thread. */ movl $SYS_futex, %eax syscall @@ -311,6 +318,8 @@ __lll_timedwait_tid: jz 4f movq %rsp, %r10 + /* XXX The kernel so far uses global futex for the wakeup at + all times. */ #if FUTEX_WAIT == 0 xorl %esi, %esi #else diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index bb988f3b2b..fbe48b37aa 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -39,6 +39,7 @@ #define FUTEX_LOCK_PI 6 #define FUTEX_UNLOCK_PI 7 #define FUTEX_TRYLOCK_PI 8 +#define FUTEX_PRIVATE_FLAG 128 /* Initializer for compatibility lock. */ diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S index 9db5516913..7740c599d1 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -17,14 +17,19 @@ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. */ +#include <kernel-features.h> +#include <tcb-offsets.h> + #ifndef UP # define LOCK lock #else # define LOCK #endif -#define SYS_futex 202 -#define FUTEX_WAKE 1 +#define SYS_futex 202 +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_PRIVATE_FLAG 128 .comm __fork_generation, 4, 4 @@ -74,10 +79,15 @@ __pthread_once: jnz 3f /* Different for generation -> run initializer. */ /* Somebody else got here first. Wait. */ -#if FUTEX_WAIT == 0 - xorl %esi, %esi +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi #else +# if FUTEX_WAIT == 0 + movl %fs:PRIVATE_FUTEX, %esi +# else movl $FUTEX_WAIT, %esi + orl %fs:PRIVATE_FUTEX, %esi +# endif #endif movl $SYS_futex, %eax syscall @@ -106,7 +116,12 @@ __pthread_once: /* Wake up all other threads. */ movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi +#else movl $FUTEX_WAKE, %esi + orl %fs:PRIVATE_FUTEX, %esi +#endif movl $SYS_futex, %eax syscall @@ -133,7 +148,12 @@ clear_once_control: movl $0, (%rdi) movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi +#else movl $FUTEX_WAKE, %esi + orl %fs:PRIVATE_FUTEX, %esi +#endif movl $SYS_futex, %eax syscall |