diff options
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux')
15 files changed, 714 insertions, 19 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/Makefile b/nptl/sysdeps/unix/sysv/linux/Makefile index 14f513c127..88dce1a886 100644 --- a/nptl/sysdeps/unix/sysv/linux/Makefile +++ b/nptl/sysdeps/unix/sysv/linux/Makefile @@ -1,4 +1,4 @@ -# Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +# Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. # This file is part of the GNU C Library. # Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -24,7 +24,8 @@ sysdep_routines += register-atfork unregister-atfork libc_pthread_init \ libpthread-sysdep_routines += pt-fork pthread_mutex_cond_lock gen-as-const-headers += lowlevelcond.sym lowlevelrwlock.sym \ - lowlevelbarrier.sym unwindbuf.sym + lowlevelbarrier.sym unwindbuf.sym \ + lowlevelrobustlock.sym endif ifeq ($(subdir),posix) diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h index cb916917e5..41a54d4b0d 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h @@ -43,6 +43,13 @@ typedef union } pthread_attr_t; +typedef struct __pthread_internal_list +{ + struct __pthread_internal_list *__prev; + struct __pthread_internal_list *__next; +} __pthread_list_t; + + /* Data structures for mutex handling. The structure of the attribute type is deliberately not exposed. */ typedef union @@ -57,8 +64,7 @@ typedef union binary compatibility. */ int __kind; int __spins; - struct __pthread_mutex_s *__next; - struct __pthread_mutex_s *__prev; + __pthread_list_t __list; #define __PTHREAD_MUTEX_HAVE_PREV 1 } __data; char __size[__SIZEOF_PTHREAD_MUTEX_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S new file mode 100644 index 0000000000..1c516c7424 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S @@ -0,0 +1,186 @@ +/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include <sysdep.h> +#include <pthread-errnos.h> +#include <lowlevelrobustlock.h> + + .text + +#ifndef LOCK +# ifdef UP +# define LOCK +# else +# define LOCK lock +# endif +#endif + +#define SYS_gettimeofday __NR_gettimeofday +#define SYS_futex 240 +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_WAITERS 0x80000000 +#define FUTEX_OWNER_DIED 0x40000000 + + + .globl __lll_robust_mutex_lock_wait + .type __lll_robust_mutex_lock_wait,@function + .hidden __lll_robust_mutex_lock_wait + .align 16 +__lll_robust_mutex_lock_wait: + pushl %edx + pushl %ebx + pushl %esi + + movl %ecx, %ebx + xorl %esi, %esi /* No timeout. */ + xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ + +4: movl %eax, %edx + orl $FUTEX_WAITERS, %edx + + testl $FUTEX_OWNER_DIED, %eax + jnz 3f + + cmpl %edx, %eax /* NB: %edx == 2 */ + je 1f + + LOCK + cmpxchgl %edx, (%ebx) + jnz 2f + +1: movl $SYS_futex, %eax + ENTER_KERNEL + + movl (%ebx), %eax + +2: test %eax, %eax + jne 4b + + movl %gs:TID, %edx + LOCK + cmpxchgl %edx, (%ebx) + jnz 4b + /* NB: %eax == 0 */ + +3: popl %esi + popl %ebx + popl %edx + ret + .size __lll_robust_mutex_lock_wait,.-__lll_robust_mutex_lock_wait + + + .globl __lll_robust_mutex_timedlock_wait + .type __lll_robust_mutex_timedlock_wait,@function + .hidden __lll_robust_mutex_timedlock_wait + .align 16 +__lll_robust_mutex_timedlock_wait: + /* Check for a valid timeout value. */ + cmpl $1000000000, 4(%edx) + jae 3f + + pushl %edi + pushl %esi + pushl %ebx + pushl %ebp + + /* Stack frame for the timespec and timeval structs. */ + subl $12, %esp + + movl %ecx, %ebp + movl %edx, %edi + +1: movl %eax, 8(%esp) + + /* Get current time. */ + movl %esp, %ebx + xorl %ecx, %ecx + movl $SYS_gettimeofday, %eax + ENTER_KERNEL + + /* Compute relative timeout. */ + movl 4(%esp), %eax + movl $1000, %edx + mul %edx /* Milli seconds to nano seconds. */ + movl (%edi), %ecx + movl 4(%edi), %edx + subl (%esp), %ecx + subl %eax, %edx + jns 4f + addl $1000000000, %edx + subl $1, %ecx +4: testl %ecx, %ecx + js 8f /* Time is already up. */ + + /* Store relative timeout. */ + movl %ecx, (%esp) + movl %edx, 4(%esp) + + movl %ebp, %ebx + + movl 8(%esp), %edx + movl %edx, %eax + orl $FUTEX_WAITERS, %edx + + testl $FUTEX_OWNER_DIED, %eax + jnz 6f + + cmpl %eax, %edx + je 2f + + LOCK + cmpxchgl %edx, (%ebx) + movl $0, %ecx /* Must use mov to avoid changing cc. */ + jnz 5f + +2: + /* Futex call. */ + movl %esp, %esi + xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ + movl $SYS_futex, %eax + ENTER_KERNEL + movl %eax, %ecx + + movl (%ebx), %eax + +5: testl %eax, %eax + jne 7f + + movl %gs:TID, %edx + LOCK + cmpxchgl %edx, (%ebx) + jnz 7f + +6: addl $12, %esp + popl %ebp + popl %ebx + popl %esi + popl %edi + ret + + /* Check whether the time expired. */ +7: cmpl $-ETIMEDOUT, %ecx + jne 1b + +8: movl $ETIMEDOUT, %eax + jmp 6b + +3: movl $EINVAL, %eax + ret + .size __lll_robust_mutex_timedlock_wait,.-__lll_robust_mutex_timedlock_wait diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S new file mode 100644 index 0000000000..f768e16a7d --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S @@ -0,0 +1,20 @@ +/* Copyright (C) 2002, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include "../i486/lowlevelrobustlock.S" diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S new file mode 100644 index 0000000000..f768e16a7d --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S @@ -0,0 +1,20 @@ +/* Copyright (C) 2002, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include "../i486/lowlevelrobustlock.S" diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index b233d9747f..f3b97143c4 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -139,6 +139,16 @@ extern int __lll_mutex_unlock_wake (int *__futex) ret; }) +#define lll_robust_mutex_trylock(futex, id) \ + ({ int ret; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ + : "=a" (ret), "=m" (futex) \ + : "r" (id), "m" (futex), \ + "0" (LLL_MUTEX_LOCK_INITIALIZER) \ + : "memory"); \ + ret; }) + + #define lll_mutex_cond_trylock(futex) \ ({ int ret; \ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ @@ -167,6 +177,25 @@ extern int __lll_mutex_unlock_wake (int *__futex) : "memory"); }) +#define lll_robust_mutex_lock(futex, id) \ + ({ int result, ignore; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ + "jnz _L_mutex_lock_%=\n\t" \ + ".subsection 1\n\t" \ + ".type _L_mutex_lock_%=,@function\n" \ + "_L_mutex_lock_%=:\n\t" \ + "leal %2, %%ecx\n\t" \ + "call __lll_robust_mutex_lock_wait\n\t" \ + "jmp 1f\n\t" \ + ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n" \ + ".previous\n" \ + "1:" \ + : "=a" (result), "=c" (ignore), "=m" (futex) \ + : "0" (0), "1" (id), "m" (futex) \ + : "memory"); \ + result; }) + + /* Special version of lll_mutex_lock which causes the unlock function to always wakeup waiters. */ #define lll_mutex_cond_lock(futex) \ @@ -187,6 +216,25 @@ extern int __lll_mutex_unlock_wake (int *__futex) : "memory"); }) +#define lll_robust_mutex_cond_lock(futex, id) \ + ({ int result, ignore; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ + "jnz _L_mutex_cond_lock_%=\n\t" \ + ".subsection 1\n\t" \ + ".type _L_mutex_cond_lock_%=,@function\n" \ + "_L_mutex_cond_lock_%=:\n\t" \ + "leal %2, %%ecx\n\t" \ + "call __lll_robust_mutex_lock_wait\n\t" \ + "jmp 1f\n\t" \ + ".size _L_mutex_cond_lock_%=,.-_L_mutex_cond_lock_%=\n"\ + ".previous\n" \ + "1:" \ + : "=a" (result), "=c" (ignore), "=m" (futex) \ + : "0" (0), "1" (id | FUTEX_WAITERS), "m" (futex) \ + : "memory"); \ + result; }) + + #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2; \ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ @@ -208,9 +256,30 @@ extern int __lll_mutex_unlock_wake (int *__futex) result; }) +#define lll_robust_mutex_timedlock(futex, timeout, id) \ + ({ int result, ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ + "jnz _L_mutex_timedlock_%=\n\t" \ + ".subsection 1\n\t" \ + ".type _L_mutex_timedlock_%=,@function\n" \ + "_L_mutex_timedlock_%=:\n\t" \ + "leal %3, %%ecx\n\t" \ + "movl %7, %%edx\n\t" \ + "call __lll_robust_mutex_timedlock_wait\n\t" \ + "jmp 1f\n\t" \ + ".size _L_mutex_timedlock_%=,.-_L_mutex_timedlock_%=\n"\ + ".previous\n" \ + "1:" \ + : "=a" (result), "=c" (ignore1), "=&d" (ignore2), \ + "=m" (futex) \ + : "0" (0), "1" (id), "m" (futex), "m" (timeout) \ + : "memory"); \ + result; }) + + #define lll_mutex_unlock(futex) \ (void) ({ int ignore; \ - __asm __volatile (LOCK_INSTR "subl $1,%0\n\t" \ + __asm __volatile (LOCK_INSTR "subl $1, %0\n\t" \ "jne _L_mutex_unlock_%=\n\t" \ ".subsection 1\n\t" \ ".type _L_mutex_unlock_%=,@function\n" \ @@ -226,6 +295,53 @@ extern int __lll_mutex_unlock_wake (int *__futex) : "memory"); }) +#define lll_robust_mutex_unlock(futex) \ + (void) ({ int ignore; \ + __asm __volatile (LOCK_INSTR "andl %2, %0\n\t" \ + "jne _L_mutex_unlock_%=\n\t" \ + ".subsection 1\n\t" \ + ".type _L_mutex_unlock_%=,@function\n" \ + "_L_mutex_unlock_%=:\n\t" \ + "leal %0, %%eax\n\t" \ + "call __lll_mutex_unlock_wake\n\t" \ + "jmp 1f\n\t" \ + ".size _L_mutex_unlock_%=,.-_L_mutex_unlock_%=\n" \ + ".previous\n" \ + "1:" \ + : "=m" (futex), "=&a" (ignore) \ + : "i" (FUTEX_TID_MASK), "m" (futex) \ + : "memory"); }) + + +#define lll_robust_mutex_dead(futex) \ + (void) ({ int __ignore; \ + register int _nr asm ("edx") = 1; \ + __asm __volatile (LOCK_INSTR "orl %5, (%2)\n\t" \ + LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__ignore) \ + : "0" (SYS_futex), LLL_EBX_REG (&(futex)), \ + "c" (FUTEX_WAKE), "d" (_nr), \ + "i" (FUTEX_OWNER_DIED), \ + "i" (offsetof (tcbhead_t, sysinfo))); }) + + +#define lll_futex_wake(futex, nr) \ + do { \ + int __ignore; \ + register __typeof (nr) _nr asm ("edx") = (nr); \ + __asm __volatile (LLL_EBX_LOAD \ + LLL_ENTER_KERNEL \ + LLL_EBX_LOAD \ + : "=a" (__ignore) \ + : "0" (SYS_futex), LLL_EBX_REG (futex), \ + "c" (FUTEX_WAKE), "d" (_nr), \ + "i" (0) /* phony, to align next arg's number */, \ + "i" (offsetof (tcbhead_t, sysinfo))); \ + } while (0) + + #define lll_mutex_islocked(futex) \ (futex != 0) diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h index a13bb080a4..892769dca4 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h @@ -43,6 +43,13 @@ typedef union } pthread_attr_t; +typedef struct __pthread_internal_list +{ + struct __pthread_internal_list *__prev; + struct __pthread_internal_list *__next; +} __pthread_list_t; + + /* Data structures for mutex handling. The structure of the attribute type is not exposed on purpose. */ typedef union @@ -57,8 +64,7 @@ typedef union binary compatibility. */ int __kind; int __spins; - struct __pthread_mutex_s *__next; - struct __pthread_mutex_s *__prev; + __pthread_list_t __list; #define __PTHREAD_MUTEX_HAVE_PREV 1 } __data; char __size[__SIZEOF_PTHREAD_MUTEX_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym new file mode 100644 index 0000000000..2f1e9da52b --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym @@ -0,0 +1,6 @@ +#include <stddef.h> +#include <pthreadP.h> + +-- + +TID offsetof (struct pthread, tid) diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h index 493c2ab557..a7150f6aef 100644 --- a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h @@ -58,6 +58,20 @@ typedef union } pthread_attr_t; +#if __WORDSIZE == 64 +typedef struct __pthread_internal_list +{ + struct __pthread_internal_list *__prev; + struct __pthread_internal_list *__next; +} __pthread_list_t; +#else +typedef struct __pthread_internal_slist +{ + struct __pthread_internal_slist *__next; +} __pthread_slist_t; +#endif + + /* Data structures for mutex handling. The structure of the attribute type is deliberately not exposed. */ typedef union @@ -75,15 +89,14 @@ typedef union int __kind; #if __WORDSIZE == 64 int __spins; - struct __pthread_mutex_s *__next; - struct __pthread_mutex_s *__prev; + __pthread_list_t __list; # define __PTHREAD_MUTEX_HAVE_PREV 1 #else unsigned int __nusers; __extension__ union { int __spins; - struct __pthread_mutex_s *__next; + __pthread_slist_t __list; }; #endif } __data; diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c index 990db87416..a97351f880 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c +++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c @@ -1,7 +1,8 @@ #include <pthreadP.h> -#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock(mutex) -#define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_cond_trylock(mutex) +#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock (mutex) +#define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_cond_trylock (mutex) +#define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_mutex_cond_lock (mutex, id) #define __pthread_mutex_lock __pthread_mutex_cond_lock #define NO_INCR diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h index df8beeb685..c77031d7bb 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h @@ -57,6 +57,20 @@ typedef union } pthread_attr_t; +#if __WORDSIZE == 64 +typedef struct __pthread_internal_list +{ + struct __pthread_internal_list *__prev; + struct __pthread_internal_list *__next; +} __pthread_list_t; +#else +typedef struct __pthread_internal_slist +{ + struct __pthread_internal_slist *__next; +} __pthread_slist_t; +#endif + + /* Data structures for mutex handling. The structure of the attribute type is not exposed on purpose. */ typedef union @@ -74,15 +88,14 @@ typedef union int __kind; #if __WORDSIZE == 64 int __spins; - struct __pthread_mutex_s *__next; - struct __pthread_mutex_s *__prev; + __pthread_list_t __list; # define __PTHREAD_MUTEX_HAVE_PREV 1 #else unsigned int __nusers; __extension__ union { int __spins; - struct __pthread_mutex_s *__next; + __pthread_slist_t __list; }; #endif } __data; diff --git a/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h index 7642eccab0..969686dd5a 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h @@ -44,6 +44,12 @@ typedef union } pthread_attr_t; +typedef struct __pthread_internal_slist +{ + struct __pthread_internal_slist *__next; +} __pthread_slist_t; + + /* Data structures for mutex handling. The structure of the attribute type is not exposed on purpose. */ typedef union @@ -60,7 +66,7 @@ typedef union __extension__ union { int __spins; - struct __pthread_mutex_s *__next; + __pthread_slist_t __list; }; } __data; char __size[__SIZEOF_PTHREAD_MUTEX_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h index 7dc61c12e1..e734c1205e 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h @@ -58,6 +58,20 @@ typedef union } pthread_attr_t; +#if __WORDSIZE == 64 +typedef struct __pthread_internal_list +{ + struct __pthread_internal_list *__prev; + struct __pthread_internal_list *__next; +} __pthread_list_t; +#else +typedef struct __pthread_internal_slist +{ + struct __pthread_internal_slist *__next; +} __pthread_slist_t; +#endif + + /* Data structures for mutex handling. The structure of the attribute type is deliberately not exposed. */ typedef union @@ -75,15 +89,14 @@ typedef union int __kind; #if __WORDSIZE == 64 int __spins; - struct __pthread_mutex_s *__next; - struct __pthread_mutex_s *__prev; + __pthread_list_t __list; # define __PTHREAD_MUTEX_HAVE_PREV 1 #else unsigned int __nusers; __extension__ union { int __spins; - struct __pthread_mutex_s *__next; + __pthread_slist_t __list; }; #endif } __data; diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index ebcfe6eef1..f91ce9b8d9 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -111,6 +111,16 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; ret; }) +#define lll_robust_mutex_trylock(futex, id) \ + ({ int ret; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ + : "=a" (ret), "=m" (futex) \ + : "r" (id), "m" (futex), \ + "0" (LLL_MUTEX_LOCK_INITIALIZER) \ + : "memory"); \ + ret; }) + + #define lll_mutex_cond_trylock(futex) \ ({ int ret; \ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ @@ -139,6 +149,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; : "cx", "r11", "cc", "memory"); }) +#define lll_robust_mutex_lock(futex, id) \ + ({ int result, ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %2, %%rdi\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_robust_mutex_lock_wait\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ + "=a" (result) \ + : "0" (id), "m" (futex), "3" (0) \ + : "cx", "r11", "cc", "memory"); \ + result; }) + + #define lll_mutex_cond_lock(futex) \ (void) ({ int ignore1, ignore2, ignore3; \ __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ @@ -157,6 +186,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; : "cx", "r11", "cc", "memory"); }) +#define lll_robust_mutex_cond_lock(futex, id) \ + ({ int result, ignore1, ignore2; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %2, %%rdi\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_robust_mutex_lock_wait\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ + "=a" (result) \ + : "0" (id | FUTEX_WAITERS), "m" (futex), "3" (0) \ + : "cx", "r11", "cc", "memory"); \ + result; }) + + #define lll_mutex_timedlock(futex, timeout) \ ({ int result, ignore1, ignore2, ignore3; \ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t" \ @@ -177,6 +225,26 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; result; }) +#define lll_robust_mutex_timedlock(futex, timeout, id) \ + ({ int result, ignore1, ignore2, ignore3; \ + __asm __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %4, %%rdi\n\t" \ + "movq %8, %%rdx\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_robust_mutex_timedlock_wait\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=a" (result), "=&D" (ignore1), "=S" (ignore2), \ + "=&d" (ignore3), "=m" (futex) \ + : "0" (0), "2" (id), "m" (futex), "m" (timeout) \ + : "memory", "cx", "cc", "r10", "r11"); \ + result; }) + + #define lll_mutex_unlock(futex) \ (void) ({ int ignore; \ __asm __volatile (LOCK_INSTR "decl %0\n\t" \ @@ -194,6 +262,34 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; : "ax", "cx", "r11", "cc", "memory"); }) +#define lll_robust_mutex_unlock(futex) \ + (void) ({ int ignore; \ + __asm __volatile (LOCK_INSTR "andl %2, %0\n\t" \ + "jne 1f\n\t" \ + ".subsection 1\n" \ + "1:\tleaq %0, %%rdi\n\t" \ + "subq $128, %%rsp\n\t" \ + "callq __lll_mutex_unlock_wake\n\t" \ + "addq $128, %%rsp\n\t" \ + "jmp 2f\n\t" \ + ".previous\n" \ + "2:" \ + : "=m" (futex), "=&D" (ignore) \ + : "i" (FUTEX_TID_MASK), "m" (futex) \ + : "ax", "cx", "r11", "cc", "memory"); }) + + +#define lll_robust_mutex_dead(futex) \ + (void) ({ int ignore; \ + __asm __volatile (LOCK_INSTR "orl %3, (%2)\n\t" \ + "syscall" \ + : "=m" (futex), "=a" (ignore) \ + : "D" (&(futex)), "i" (FUTEX_OWNER_DIED), \ + "S" (FUTEX_WAKE), "1" (__NR_futex), \ + "d" (1) \ + : "cx", "r11", "cc", "memory"); }) + + #define lll_mutex_islocked(futex) \ (futex != LLL_MUTEX_LOCK_INITIALIZER) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S new file mode 100644 index 0000000000..7bb9191691 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S @@ -0,0 +1,192 @@ +/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include <sysdep.h> +#include <pthread-errnos.h> +#include <lowlevelrobustlock.h> + + .text + +#ifndef LOCK +# ifdef UP +# define LOCK +# else +# define LOCK lock +# endif +#endif + +#define SYS_futex 202 +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_WAITERS 0x80000000 +#define FUTEX_OWNER_DIED 0x40000000 + +/* For the calculation see asm/vsyscall.h. */ +#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 + + + .globl __lll_robust_mutex_lock_wait + .type __lll_robust_mutex_lock_wait,@function + .hidden __lll_robust_mutex_lock_wait + .align 16 +__lll_robust_mutex_lock_wait: + pushq %r10 + pushq %rdx + + xorq %r10, %r10 /* No timeout. */ +#if FUTEX_WAIT == 0 + xorl %esi, %esi +#else + movl $FUTEX_WAIT, %esi +#endif + +4: movl %eax, %edx + orl $FUTEX_WAITERS, %edx + + testl $FUTEX_OWNER_DIED, %eax + jnz 3f + + cmpl %edx, %eax + je 1f + + LOCK + cmpxchgl %edx, (%rdi) + jnz 2f + +1: movl $SYS_futex, %eax + syscall + + movl (%rdi), %eax + +2: testl %eax, %eax + jne 4b + + movl %fs:TID, %edx + LOCK + cmpxchgl %edx, (%rdi) + jnz 4b + /* NB: %rax == 0 */ + +3: popq %rdx + popq %r10 + retq + .size __lll_robust_mutex_lock_wait,.-__lll_robust_mutex_lock_wait + + + .globl __lll_robust_mutex_timedlock_wait + .type __lll_robust_mutex_timedlock_wait,@function + .hidden __lll_robust_mutex_timedlock_wait + .align 16 +__lll_robust_mutex_timedlock_wait: + /* Check for a valid timeout value. */ + cmpq $1000000000, 8(%rdx) + jae 3f + + pushq %r8 + pushq %r9 + pushq %r12 + pushq %r13 + + /* Stack frame for the timespec and timeval structs. */ + subq $24, %rsp + + movq %rdi, %r12 + movq %rdx, %r13 + +1: movq %rax, 16(%rsp) + + /* Get current time. */ + movq %rsp, %rdi + xorl %esi, %esi + movq $VSYSCALL_ADDR_vgettimeofday, %rax + /* This is a regular function call, all caller-save registers + might be clobbered. */ + callq *%rax + + /* Compute relative timeout. */ + movq 8(%rsp), %rax + movl $1000, %edi + mul %rdi /* Milli seconds to nano seconds. */ + movq (%r13), %rdi + movq 8(%r13), %rsi + subq (%rsp), %rdi + subq %rax, %rsi + jns 4f + addq $1000000000, %rsi + decq %rdi +4: testq %rdi, %rdi + js 8f /* Time is already up. */ + + /* Futex call. */ + movq %rdi, (%rsp) /* Store relative timeout. */ + movq %rsi, 8(%rsp) + + movq 16(%rsp), %rdx + movl %edx, %eax + orl $FUTEX_WAITERS, %edx + + testl $FUTEX_OWNER_DIED, %eax + jnz 6f + + cmpl %eax, %edx + je 2f + + LOCK + cmpxchgl %edx, (%r12) + movq $0, %rcx /* Must use mov to avoid changing cc. */ + jnz 5f + +2: movq %rsp, %r10 +#if FUTEX_WAIT == 0 + xorl %esi, %esi +#else + movl $FUTEX_WAIT, %esi +#endif + movq %r12, %rdi + movl $SYS_futex, %eax + syscall + movq %rax, %rcx + + movl (%r12), %eax + +5: testl %eax, %eax + jne 7f + + movl %fs:TID, %edx + LOCK + cmpxchgl %edx, (%r12) + jnz 7f + +6: addq $24, %rsp + popq %r13 + popq %r12 + popq %r9 + popq %r8 + retq + + /* Check whether the time expired. */ +7: cmpq $-ETIMEDOUT, %rcx + jne 1b + +8: movl $ETIMEDOUT, %eax + jmp 6b + +3: movl $EINVAL, %eax + retq + .size __lll_robust_mutex_timedlock_wait,.-__lll_robust_mutex_timedlock_wait |