diff options
Diffstat (limited to 'nptl/sysdeps')
16 files changed, 217 insertions, 30 deletions
diff --git a/nptl/sysdeps/pthread/pthread_cond_timedwait.c b/nptl/sysdeps/pthread/pthread_cond_timedwait.c index 7de2b2936f..c6606c9bf9 100644 --- a/nptl/sysdeps/pthread/pthread_cond_timedwait.c +++ b/nptl/sysdeps/pthread/pthread_cond_timedwait.c @@ -67,6 +67,7 @@ __pthread_cond_timedwait (cond, mutex, abstime) /* We have one new user of the condvar. */ ++cond->__data.__total_seq; ++cond->__data.__futex; + cond->__data.__nwaiters += 1 << COND_CLOCK_BITS; /* Remember the mutex we are using here. If there is already a different address store this is a bad user bug. Do not store @@ -98,7 +99,8 @@ __pthread_cond_timedwait (cond, mutex, abstime) INTERNAL_SYSCALL_DECL (err); int ret; ret = INTERNAL_SYSCALL (clock_gettime, err, 2, - cond->__data.__clock, &rt); + cond->__data.__nwaiters & COND_CLOCK_BITS, + &rt); # ifndef __ASSUME_POSIX_TIMERS if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (ret, err), 0)) { @@ -185,6 +187,16 @@ __pthread_cond_timedwait (cond, mutex, abstime) ++cond->__data.__woken_seq; bc_out: + + cond->__data.__nwaiters -= 1 << COND_CLOCK_BITS; + + /* If pthread_cond_destroy was called on this variable already, + notify the pthread_cond_destroy caller all waiters have left + and it can be successfully destroyed. */ + if (cond->__data.__total_seq == -1ULL + && cond->__data.__nwaiters < (1 << COND_CLOCK_BITS)) + lll_futex_wake (&cond->__data.__nwaiters, 1); + /* We are done with the condvar. */ lll_mutex_unlock (cond->__data.__lock); diff --git a/nptl/sysdeps/pthread/pthread_cond_wait.c b/nptl/sysdeps/pthread/pthread_cond_wait.c index 45187b5240..86669458a0 100644 --- a/nptl/sysdeps/pthread/pthread_cond_wait.c +++ b/nptl/sysdeps/pthread/pthread_cond_wait.c @@ -42,6 +42,7 @@ __condvar_cleanup (void *arg) { struct _condvar_cleanup_buffer *cbuffer = (struct _condvar_cleanup_buffer *) arg; + unsigned int destroying; /* We are going to modify shared data. */ lll_mutex_lock (cbuffer->cond->__data.__lock); @@ -55,11 +56,25 @@ __condvar_cleanup (void *arg) ++cbuffer->cond->__data.__futex; } + cbuffer->cond->__data.__nwaiters -= 1 << COND_CLOCK_BITS; + + /* If pthread_cond_destroy was called on this variable already, + notify the pthread_cond_destroy caller all waiters have left + and it can be successfully destroyed. */ + destroying = 0; + if (cbuffer->cond->__data.__total_seq == -1ULL + && cbuffer->cond->__data.__nwaiters < (1 << COND_CLOCK_BITS)) + { + lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1); + destroying = 1; + } + /* We are done. */ lll_mutex_unlock (cbuffer->cond->__data.__lock); /* Wake everybody to make sure no condvar signal gets lost. */ - lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX); + if (! destroying) + lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX); /* Get the mutex before returning unless asynchronous cancellation is in effect. */ @@ -90,6 +105,7 @@ __pthread_cond_wait (cond, mutex) /* We have one new user of the condvar. */ ++cond->__data.__total_seq; ++cond->__data.__futex; + cond->__data.__nwaiters += 1 << COND_CLOCK_BITS; /* Remember the mutex we are using here. If there is already a different address store this is a bad user bug. Do not store @@ -145,6 +161,16 @@ __pthread_cond_wait (cond, mutex) ++cond->__data.__woken_seq; bc_out: + + cond->__data.__nwaiters -= 1 << COND_CLOCK_BITS; + + /* If pthread_cond_destroy was called on this varaible already, + notify the pthread_cond_destroy caller all waiters have left + and it can be successfully destroyed. */ + if (cond->__data.__total_seq == -1ULL + && cond->__data.__nwaiters < (1 << COND_CLOCK_BITS)) + lll_futex_wake (&cond->__data.__nwaiters, 1); + /* We are done with the condvar. */ lll_mutex_unlock (cond->__data.__lock); diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h index 62c853cee9..fd20d572af 100644 --- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h @@ -81,7 +81,7 @@ typedef union unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; - int __clock; + unsigned int __nwaiters; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h index 8d7858a071..fb62c0d99b 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h @@ -81,7 +81,7 @@ typedef union unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; - int __clock; + unsigned int __nwaiters; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S index eecec8aee3..699c2cb227 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S @@ -87,6 +87,7 @@ __pthread_cond_timedwait: addl $1, total_seq(%ebx) adcl $0, total_seq+4(%ebx) addl $1, cond_futex(%ebx) + addl $(1 << clock_bits), cond_nwaiters(%ebx) #define FRAME_SIZE 24 subl $FRAME_SIZE, %esp @@ -104,8 +105,9 @@ __pthread_cond_timedwait: 8: movl %ebx, %edx #ifdef __NR_clock_gettime /* Get the clock number. */ - movl cond_clock(%ebx), %ebx - /* Only clocks 0 and 1 are allowed. Both are handled in the + movl cond_nwaiters(%ebx), %ebx + andl $((1 << clock_bits) - 1), %ebx + /* Only clocks 0 and 1 are allowed so far. Both are handled in the kernel. */ leal 4(%esp), %ecx movl $__NR_clock_gettime, %eax @@ -226,7 +228,25 @@ __pthread_cond_timedwait: 14: addl $1, woken_seq(%ebx) adcl $0, woken_seq+4(%ebx) -24: LOCK +24: subl $(1 << clock_bits), cond_nwaiters(%ebx) + + /* Wake up a thread which wants to destroy the condvar object. */ + movl total_seq(%ebx), %eax + andl total_seq+4(%ebx), %eax + cmpl $0xffffffff, %eax + jne 25f + movl cond_nwaiters(%ebx), %eax + andl $~((1 << clock_bits) - 1), %eax + jne 25f + + addl $cond_nwaiters, %ebx + movl $SYS_futex, %eax + movl $FUTEX_WAKE, %ecx + movl $1, %edx + ENTER_KERNEL + subl $cond_nwaiters, %ebx + +25: LOCK #if cond_lock == 0 subl $1, (%ebx) #else @@ -394,7 +414,27 @@ __condvar_tw_cleanup: addl $1, woken_seq(%ebx) adcl $0, woken_seq+4(%ebx) -3: LOCK +3: subl $(1 << clock_bits), cond_nwaiters(%ebx) + + /* Wake up a thread which wants to destroy the condvar object. */ + xorl %edi, %edi + movl total_seq(%ebx), %eax + andl total_seq+4(%ebx), %eax + cmpl $0xffffffff, %eax + jne 4f + movl cond_nwaiters(%ebx), %eax + andl $~((1 << clock_bits) - 1), %eax + jne 4f + + addl $cond_nwaiters, %ebx + movl $SYS_futex, %eax + movl $FUTEX_WAKE, %ecx + movl $1, %edx + ENTER_KERNEL + subl $cond_nwaiters, %ebx + movl $1, %edi + +4: LOCK #if cond_lock == 0 subl $1, (%ebx) #else @@ -410,13 +450,15 @@ __condvar_tw_cleanup: call __lll_mutex_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ -2: addl $cond_futex, %ebx +2: testl %edi, %edi + jnz 5f + addl $cond_futex, %ebx movl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL - movl 24+FRAME_SIZE(%esp), %eax +5: movl 24+FRAME_SIZE(%esp), %eax call __pthread_mutex_cond_lock movl %esi, (%esp) diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index 3fe7f8c17a..d282785151 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -80,6 +80,7 @@ __pthread_cond_wait: addl $1, total_seq(%ebx) adcl $0, total_seq+4(%ebx) addl $1, cond_futex(%ebx) + addl $(1 << clock_bits), cond_nwaiters(%ebx) #define FRAME_SIZE 16 subl $FRAME_SIZE, %esp @@ -156,7 +157,25 @@ __pthread_cond_wait: adcl $0, woken_seq+4(%ebx) /* Unlock */ -16: LOCK +16: subl $(1 << clock_bits), cond_nwaiters(%ebx) + + /* Wake up a thread which wants to destroy the condvar object. */ + movl total_seq(%ebx), %eax + andl total_seq+4(%ebx), %eax + cmpl $0xffffffff, %eax + jne 17f + movl cond_nwaiters(%ebx), %eax + andl $~((1 << clock_bits) - 1), %eax + jne 17f + + addl $cond_nwaiters, %ebx + movl $SYS_futex, %eax + movl $FUTEX_WAKE, %ecx + movl $1, %edx + ENTER_KERNEL + subl $cond_nwaiters, %ebx + +17: LOCK #if cond_lock == 0 subl $1, (%ebx) #else @@ -286,7 +305,27 @@ __condvar_w_cleanup: addl $1, woken_seq(%ebx) adcl $0, woken_seq+4(%ebx) -3: LOCK +3: subl $(1 << clock_bits), cond_nwaiters(%ebx) + + /* Wake up a thread which wants to destroy the condvar object. */ + xorl %edi, %edi + movl total_seq(%ebx), %eax + andl total_seq+4(%ebx), %eax + cmpl $0xffffffff, %eax + jne 4f + movl cond_nwaiters(%ebx), %eax + andl $~((1 << clock_bits) - 1), %eax + jne 4f + + addl $cond_nwaiters, %ebx + movl $SYS_futex, %eax + movl $FUTEX_WAKE, %ecx + movl $1, %edx + ENTER_KERNEL + subl $cond_nwaiters, %ebx + movl $1, %edi + +4: LOCK #if cond_lock == 0 subl $1, (%ebx) #else @@ -302,13 +341,15 @@ __condvar_w_cleanup: call __lll_mutex_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ -2: addl $cond_futex, %ebx +2: testl %edi, %edi + jnz 5f + addl $cond_futex, %ebx movl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL - movl 20+FRAME_SIZE(%esp), %eax +5: movl 20+FRAME_SIZE(%esp), %eax call __pthread_mutex_cond_lock movl %esi, (%esp) diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h index 958af47162..5b442cbca7 100644 --- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h @@ -81,7 +81,7 @@ typedef union unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; - int __clock; + unsigned int __nwaiters; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h index 43b6856bbf..1dec19e57d 100644 --- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h +++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h @@ -75,6 +75,12 @@ struct pthread_condattr }; +/* The __NWAITERS field is used as a counter and to house the number + of bits which represent the clock. COND_CLOCK_BITS is the number + of bits reserved for the clock. */ +#define COND_CLOCK_BITS 1 + + /* Read-write lock variable attribute data structure. */ struct pthread_rwlockattr { diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym index 2e193e6d2e..c5e7978069 100644 --- a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym @@ -1,13 +1,16 @@ #include <stddef.h> +#include <sched.h> #include <bits/pthreadtypes.h> +#include <internaltypes.h> -- cond_lock offsetof (pthread_cond_t, __data.__lock) cond_futex offsetof (pthread_cond_t, __data.__futex) -cond_clock offsetof (pthread_cond_t, __data.__clock) +cond_nwaiters offsetof (pthread_cond_t, __data.__nwaiters) total_seq offsetof (pthread_cond_t, __data.__total_seq) wakeup_seq offsetof (pthread_cond_t, __data.__wakeup_seq) woken_seq offsetof (pthread_cond_t, __data.__woken_seq) dep_mutex offsetof (pthread_cond_t, __data.__mutex) broadcast_seq offsetof (pthread_cond_t, __data.__broadcast_seq) +clock_bits COND_CLOCK_BITS diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h index 5a000d30af..a493821c07 100644 --- a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h @@ -101,7 +101,7 @@ typedef union unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; - int __clock; + unsigned int __nwaiters; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h index 142286451c..17cfaa981d 100644 --- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h @@ -100,7 +100,7 @@ typedef union unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; - int __clock; + unsigned int __nwaiters; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h index b8afbc50b5..5bf3aff46a 100644 --- a/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h @@ -82,7 +82,7 @@ typedef union unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; - int __clock; + unsigned int __nwaiters; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h index e86dd000a1..34d3df7abb 100644 --- a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h @@ -101,7 +101,7 @@ typedef union unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; - int __clock; + unsigned int __nwaiters; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h index a7bc5816cc..e5cc605f24 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h @@ -100,7 +100,7 @@ typedef union unsigned long long int __wakeup_seq; unsigned long long int __woken_seq; void *__mutex; - int __clock; + int __nwaiters; unsigned int __broadcast_seq; } __data; char __size[__SIZEOF_PTHREAD_COND_T]; diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 95f8aabd11..67bec6caa7 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -111,6 +111,7 @@ __pthread_cond_timedwait: movq 8(%rsp), %rdi incq total_seq(%rdi) incl cond_futex(%rdi) + addl $(1 << clock_bits), cond_nwaiters(%rdi) /* Install cancellation handler. */ #ifdef PIC @@ -135,8 +136,9 @@ __pthread_cond_timedwait: /* Get the clock number. Note that the field in the condvar structure stores the number minus 1. */ movq 8(%rsp), %rdi - movl cond_clock(%rdi), %edi - /* Only clocks 0 and 1 are allowed. Both are handled in the + movl cond_nwaiters(%rdi), %edi + andl $((1 << clock_bits) - 1), %edi + /* Only clocks 0 and 1 are allowed so far. Both are handled in the kernel. */ leaq 24(%rsp), %rsi movq $__NR_clock_gettime, %rax @@ -244,7 +246,23 @@ __pthread_cond_timedwait: 9: xorq %r14, %r14 14: incq woken_seq(%rdi) -24: LOCK +24: subl $(1 << clock_bits), cond_nwaiters(%rdi) + + /* Wake up a thread which wants to destroy the condvar object. */ + cmpq $0xffffffffffffffff, total_seq(%rdi) + jne 25f + movl cond_nwaiters(%rdi), %eax + andl $~((1 << clock_bits) - 1), %eax + jne 25f + + addq $cond_nwaiters, %rdi + movq $SYS_futex, %rax + movq $FUTEX_WAKE, %rsi + movl $1, %edx + syscall + subq $cond_nwaiters, %rdi + +25: LOCK #if cond_lock == 0 decl (%rdi) #else diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 9e7da301d3..f5de0a280c 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -40,6 +40,8 @@ .globl __condvar_cleanup .hidden __condvar_cleanup __condvar_cleanup: + pushq %r12 + /* Get internal lock. */ movq %rdi, %r8 movq 8(%rdi), %rdi @@ -66,12 +68,28 @@ __condvar_cleanup: jne 3f incq wakeup_seq(%rdi) - incq woken_seq(%rdi) - incl cond_futex(%rdi) -3: LOCK +3: subl $(1 << clock_bits), cond_nwaiters(%rdi) + + /* Wake up a thread which wants to destroy the condvar object. */ + xorq %r12, %r12 + cmpq $0xffffffffffffffff, total_seq(%rdi) + jne 4f + movl cond_nwaiters(%rdi), %eax + andl $~((1 << clock_bits) - 1), %eax + jne 4f + + addq $cond_nwaiters, %rdi + movq $SYS_futex, %rax + movq $FUTEX_WAKE, %rsi + movl $1, %edx + syscall + subq $cond_nwaiters, %rdi + movq $1, %r12 + +4: LOCK #if cond_lock == 0 decl (%rdi) #else @@ -84,15 +102,19 @@ __condvar_cleanup: callq __lll_mutex_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ -2: addq $cond_futex, %rdi +2: testq %r12, %r12 + jnz 5f + addq $cond_futex, %rdi movq $FUTEX_WAKE, %rsi movl $0x7fffffff, %edx movq $SYS_futex, %rax syscall - movq 16(%r8), %rdi +5: movq 16(%r8), %rdi callq __pthread_mutex_cond_lock + popq %r12 + retq .size __condvar_cleanup, .-__condvar_cleanup @@ -157,6 +179,7 @@ __pthread_cond_wait: movq 8(%rsp), %rdi incq total_seq(%rdi) incl cond_futex(%rdi) + addl $(1 << clock_bits), cond_nwaiters(%rdi) /* Install cancellation handler. */ #ifdef PIC @@ -229,7 +252,23 @@ __pthread_cond_wait: incq woken_seq(%rdi) /* Unlock */ -16: LOCK +16: subl $(1 << clock_bits), cond_nwaiters(%rdi) + + /* Wake up a thread which wants to destroy the condvar object. */ + cmpq $0xffffffffffffffff, total_seq(%rdi) + jne 17f + movl cond_nwaiters(%rdi), %eax + andl $~((1 << clock_bits) - 1), %eax + jne 17f + + addq $cond_nwaiters, %rdi + movq $SYS_futex, %rax + movq $FUTEX_WAKE, %rsi + movl $1, %edx + syscall + subq $cond_nwaiters, %rdi + +17: LOCK #if cond_lock == 0 decl (%rdi) #else |