diff options
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux/x86_64')
5 files changed, 53 insertions, 43 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h index e29c77529e..661e476aa7 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h @@ -76,11 +76,12 @@ typedef union struct { int __lock; - int __clock; + unsigned int __futex; unsigned long long int __total_seq; unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; + int __clock; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index 6d29d761fa..e8d7bd9bb6 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -32,6 +32,7 @@ #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 #define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 #define EINVAL 22 @@ -55,39 +56,42 @@ __pthread_cond_broadcast: #endif jnz 1f -2: addq $wakeup_seq, %rdi - movq total_seq-wakeup_seq(%rdi), %rcx - cmpq (%rdi), %rcx +2: addq $cond_futex, %rdi + movq total_seq-cond_futex(%rdi), %r9 + cmpq wakeup_seq-cond_futex(%rdi), %r9 jna 4f /* Cause all currently waiting threads to recognize they are woken up. */ - movq %rcx, (%rdi) - movq %rcx, woken_seq-wakeup_seq(%rdi) - incl broadcast_seq-wakeup_seq(%rdi) + movq %r9, wakeup_seq-cond_futex(%rdi) + movq %r9, woken_seq-cond_futex(%rdi) + addq %r9, %r9 + movl %r9d, (%rdi) + incl broadcast_seq-cond_futex(%rdi) /* Get the address of the mutex used. */ - movq dep_mutex-wakeup_seq(%rdi), %r8 + movq dep_mutex-cond_futex(%rdi), %r8 /* Unlock. */ LOCK - decl cond_lock-wakeup_seq(%rdi) + decl cond_lock-cond_futex(%rdi) jne 7f 8: cmpq $-1, %r8 je 9f /* Wake up all threads. */ - movq $FUTEX_REQUEUE, %rsi + movq $FUTEX_CMP_REQUEUE, %rsi movq $SYS_futex, %rax movl $1, %edx movq $0x7fffffff, %r10 syscall -#ifndef __ASSUME_FUTEX_REQUEUE - cmpq $-EINVAL, %rax - je 9f -#endif + /* For any kind of error, which mainly is EAGAIN, we try again + with WAKE. The general test also covers running on old + kernels. */ + cmpq $-4095, %rax + jae 9f 10: xorl %eax, %eax retq @@ -95,7 +99,7 @@ __pthread_cond_broadcast: .align 16 /* Unlock. */ 4: LOCK - decl cond_lock-wakeup_seq(%rdi) + decl cond_lock-cond_futex(%rdi) jne 5f 6: xorl %eax, %eax @@ -113,14 +117,14 @@ __pthread_cond_broadcast: jmp 2b /* Unlock in loop requires wakeup. */ -5: addq $cond_lock-wakeup_seq, %rdi +5: addq $cond_lock-cond_futex, %rdi callq __lll_mutex_unlock_wake jmp 6b /* Unlock in loop requires wakeup. */ -7: addq $cond_lock-wakeup_seq, %rdi +7: addq $cond_lock-cond_futex, %rdi callq __lll_mutex_unlock_wake - subq $cond_lock-wakeup_seq, %rdi + subq $cond_lock-cond_futex, %rdi jmp 8b 9: /* The futex requeue functionality is not available. */ diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index 92eadfc433..62bb74cc1a 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -56,13 +56,14 @@ __pthread_cond_signal: #endif jnz 1f -2: addq $wakeup_seq, %rdi +2: addq $cond_futex, %rdi movq total_seq(%r8), %rcx - cmpq (%rdi), %rcx + cmpq wakeup_seq(%r8), %rcx jbe 4f /* Bump the wakeup number. */ - addq $1, (%rdi) + addq $1, wakeup_seq(%r8) + addl $1, (%rdi) /* Wake up one thread. */ movq $FUTEX_WAKE, %rsi @@ -95,11 +96,7 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: -#if cond_lock != 0 - addq $cond_lock-wakeup_seq, %rdi -#else movq %r8, %rdi -#endif callq __lll_mutex_unlock_wake jmp 6b .size __pthread_cond_signal, .-__pthread_cond_signal diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 80cbf7e430..e75f05e07f 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -105,6 +105,7 @@ __pthread_cond_timedwait: movq 8(%rsp), %rdi incq total_seq(%rdi) + incl cond_futex(%rdi) /* Install cancellation handler. */ #ifdef PIC @@ -118,9 +119,9 @@ __pthread_cond_timedwait: /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi - movq wakeup_seq(%rdi), %r12 + movq wakeup_seq(%rdi), %r9 movl broadcast_seq(%rdi), %edx - movq %r12, 40(%rsp) + movq %r9, 40(%rsp) movl %edx, 4(%rsp) /* Get the current time. */ @@ -172,6 +173,8 @@ __pthread_cond_timedwait: 21: movq %rcx, 24(%rsp) movq %rdx, 32(%rsp) + movl cond_futex(%rdi), %r12d + /* Unlock. */ LOCK #if cond_lock == 0 @@ -187,7 +190,7 @@ __pthread_cond_timedwait: leaq 24(%rsp), %r10 xorq %rsi, %rsi /* movq $FUTEX_WAIT, %rsi */ movq %r12, %rdx - addq $wakeup_seq-cond_lock, %rdi + addq $cond_futex, %rdi movq $SYS_futex, %rax syscall movq %rax, %r14 @@ -211,21 +214,22 @@ __pthread_cond_timedwait: movq woken_seq(%rdi), %rax - movq wakeup_seq(%rdi), %r12 + movq wakeup_seq(%rdi), %r9 cmpl 4(%rsp), %edx jne 23f - cmpq 40(%rsp), %r12 + cmpq 40(%rsp), %r9 jbe 15f - cmpq %rax, %r12 + cmpq %rax, %r9 ja 9f 15: cmpq $-ETIMEDOUT, %r14 jne 8b 13: incq wakeup_seq(%rdi) + incl cond_futex(%rdi) movq $ETIMEDOUT, %r14 jmp 14f diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 065eb11813..9e7da301d3 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -69,6 +69,8 @@ __condvar_cleanup: incq woken_seq(%rdi) + incl cond_futex(%rdi) + 3: LOCK #if cond_lock == 0 decl (%rdi) @@ -82,7 +84,7 @@ __condvar_cleanup: callq __lll_mutex_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ -2: addq $wakeup_seq, %rdi +2: addq $cond_futex, %rdi movq $FUTEX_WAKE, %rsi movl $0x7fffffff, %edx movq $SYS_futex, %rax @@ -154,6 +156,7 @@ __pthread_cond_wait: movq 8(%rsp), %rdi incq total_seq(%rdi) + incl cond_futex(%rdi) /* Install cancellation handler. */ #ifdef PIC @@ -167,13 +170,14 @@ __pthread_cond_wait: /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi - movq wakeup_seq(%rdi), %r12 + movq wakeup_seq(%rdi), %r9 movl broadcast_seq(%rdi), %edx - movq %r12, 24(%rsp) + movq %r9, 24(%rsp) movl %edx, 4(%rsp) /* Unlock. */ -8: LOCK +8: movl cond_futex(%rdi), %r12d + LOCK #if cond_lock == 0 decl (%rdi) #else @@ -187,7 +191,7 @@ __pthread_cond_wait: movq 8(%rsp), %rdi xorq %r10, %r10 movq %r12, %rdx - addq $wakeup_seq-cond_lock, %rdi + addq $cond_futex-cond_lock, %rdi movq $SYS_futex, %rax movq %r10, %rsi /* movq $FUTEX_WAIT, %rsi */ syscall @@ -211,15 +215,15 @@ __pthread_cond_wait: movq woken_seq(%rdi), %rax - movq wakeup_seq(%rdi), %r12 + movq wakeup_seq(%rdi), %r9 cmpl 4(%rsp), %edx jne 16f - cmpq 24(%rsp), %r12 + cmpq 24(%rsp), %r9 jbe 8b - cmpq %rax, %r12 + cmpq %rax, %r9 jna 8b incq woken_seq(%rdi) @@ -359,8 +363,8 @@ versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, .byte 0x40+.Lsubq-.Lpush_r12 # DW_CFA_advance_loc+N .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 2 # DW_CFA_advance_loc1 - .byte .Laddq-.Lsubq + .byte 3 # DW_CFA_advance_loc2 + .2byte .Laddq-.Lsubq .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 .byte 0x40+.Lpop_r12-.Laddq # DW_CFA_advance_loc+N |