From 22502ea2a70d56057159b72d75f152ce2ccc1aec Mon Sep 17 00:00:00 2001 From: Ulrich Drepper Date: Tue, 14 Aug 2007 03:21:23 +0000 Subject: * sysdeps/unix/sysv/linux/kernel-features.h (__ASSUME_PSELECT, __ASSUME_PPOLL, __ASSUME_ATFCTS, __ASSUME_SET_ROBUST_LIST, __ASSUME_UTIMENSAT, __ASSUME_FALLOCATE): Update per-arch conditions when each feature was introduced. * sysdeps/unix/sysv/linux/dl-vdso.c: Don't include dl-hash.h. * sysdeps/unix/sysv/linux/dl-vdso.h: Don't include dl-hash.h if NDEBUG. (CHECK_HASH): New macro. (PREPARE_VERSION): Use it. * sysdeps/unix/sysv/linux/pselect.c (__generic_pselect): Only provide prototype if not __ASSUME_PSELECT. * sysdeps/unix/sysv/linux/ppoll.c (__generic_ppoll): Only provide prototype if not __ASSUME_PPOLL. * sysdeps/unix/sysv/linux/dl-osinfo.h (ROUND): #undef after use. * sysdeps/unix/clock_settime.c (freq, __pthread_clock_settime, hp_timing_settime): Don't define or declare if HANDLED_CPUTIME is defined. --- nptl/ChangeLog | 37 +++++ .../sysv/linux/i386/i486/pthread_cond_broadcast.S | 60 ++++++-- .../sysv/linux/i386/i486/pthread_cond_signal.S | 41 +++++- .../sysv/linux/i386/i486/pthread_cond_timedwait.S | 158 +++++++++++++++++---- .../unix/sysv/linux/i386/i486/pthread_cond_wait.S | 156 ++++++++++++++++---- .../sysv/linux/i386/i486/pthread_rwlock_rdlock.S | 2 +- .../linux/i386/i486/pthread_rwlock_timedrdlock.S | 2 +- .../linux/i386/i486/pthread_rwlock_timedwrlock.S | 2 +- .../sysv/linux/i386/i486/pthread_rwlock_unlock.S | 2 +- .../sysv/linux/i386/i486/pthread_rwlock_wrlock.S | 2 +- nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h | 2 +- .../sysv/linux/x86_64/pthread_cond_broadcast.S | 4 +- .../unix/sysv/linux/x86_64/pthread_cond_signal.S | 2 +- 13 files changed, 394 insertions(+), 76 deletions(-) (limited to 'nptl') diff --git a/nptl/ChangeLog b/nptl/ChangeLog index 345829cf49..e56ece02bd 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -1,3 +1,40 @@ +2007-08-13 Jakub Jelinek + + * sysdeps/unix/sysv/linux/i386/lowlevellock.h (__lll_private_flag): + Fix a pasto. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S + (__pthread_cond_broadcast): Pass LLL_PRIVATE to lll_* and or + FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. + Don't use FUTEX_CMP_REQUEUE if dep_mutex is not process private. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S + (__pthread_cond_signal): Pass LLL_PRIVATE to lll_* and or + FUTEX_PRIVATE_FLAG into SYS_futex op if cv is process private. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Include + kernel-features.h. + (__pthread_cond_wait, __condvar_w_cleanup): Pass LLL_PRIVATE to + lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is + process private. Switch DW_CFA_advance_loc1 and some + DW_CFA_advance_loc .eh_frame opcodes to DW_CFA_advance_loc4. + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S + (__pthread_cond_timedwait, __condvar_tw_cleanup): Pass LLL_PRIVATE to + lll_* and or FUTEX_PRIVATE_FLAG into SYS_futex op if cv is + process private. Switch DW_CFA_advance_loc{1,2} and some + DW_CFA_advance_loc .eh_frame opcodes to DW_CFA_advance_loc4. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Use + #ifdef __ASSUME_PRIVATE_FUTEX instead of #if __ASSUME_PRIVATE_FUTEX. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: + Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise. + * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: + Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S + (__pthread_cond_broadcast): Compare %r8 instead of + dep_mutex-cond_*(%rdi) with $-1. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S + (__pthread_cond_signal): Xor FUTEX_WAKE_OP with FUTEX_WAKE instead + of oring. + 2007-08-13 Ulrich Drepper * sysdeps/unix/sysv/linux/i386/i786/Implies: New file. diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S index 122d83afee..776c47f6cc 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S @@ -83,11 +83,18 @@ __pthread_cond_broadcast: je 9f /* XXX: The kernel so far doesn't support requeue to PI futex. */ - testl $PI_BIT, MUTEX_KIND(%edi) + /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same + type of futex (private resp. shared). */ + testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi) jne 9f /* Wake up all threads. */ - movl $FUTEX_CMP_REQUEUE, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx +#else + movl %gs:PRIVATE_FUTEX, %ecx + orl $FUTEX_CMP_REQUEUE, %ecx +#endif movl $SYS_futex, %eax movl $0x7fffffff, %esi movl $1, %edx @@ -132,28 +139,63 @@ __pthread_cond_broadcast: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b /* Unlock in loop requires waekup. */ 5: leal cond_lock-cond_futex(%ebx), %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 6b /* Unlock in loop requires waekup. */ 7: leal cond_lock-cond_futex(%ebx), %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 8b 9: /* The futex requeue functionality is not available. */ movl $0x7fffffff, %edx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax ENTER_KERNEL jmp 10b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S index e3510c8ab1..36a18036c5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S @@ -70,7 +70,18 @@ __pthread_cond_signal: /* Wake up one thread. */ pushl %esi pushl %ebp - movl $FUTEX_WAKE_OP, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE_OP, %ecx movl $SYS_futex, %eax movl $1, %edx movl $1, %esi @@ -92,7 +103,9 @@ __pthread_cond_signal: popl %ebx ret -7: movl $FUTEX_WAKE, %ecx +7: /* %ecx should be either FUTEX_WAKE_OP or + FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx movl $SYS_futex, %eax /* %edx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ @@ -106,8 +119,16 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: movl %edi, %eax - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 6b @@ -118,8 +139,16 @@ __pthread_cond_signal: #else leal cond_lock(%edi), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%edi) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S index ff56c7ae4a..83f8db25bb 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S @@ -158,7 +158,20 @@ __pthread_cond_timedwait: movl %eax, (%esp) leal 4(%esp), %esi - xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */ +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif +#if FUTEX_WAIT != 0 + addl $FUTEX_WAIT, %ecx +#endif movl %edi, %edx addl $cond_futex, %ebx .Ladd_cond_futex: @@ -232,7 +245,18 @@ __pthread_cond_timedwait: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -280,8 +304,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b @@ -293,8 +325,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 4b @@ -305,8 +345,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 6b @@ -317,8 +365,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 11b @@ -339,8 +395,16 @@ __pthread_cond_timedwait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake movl %esi, %eax @@ -401,8 +465,16 @@ __condvar_tw_cleanup: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait 1: movl broadcast_seq(%ebx), %eax @@ -441,7 +513,18 @@ __condvar_tw_cleanup: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -460,15 +543,34 @@ __condvar_tw_cleanup: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testl %edi, %edi jnz 5f addl $cond_futex, %ebx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL @@ -588,12 +690,12 @@ __condvar_tw_cleanup: .uleb128 20 .byte 0x83 # DW_CFA_offset %ebx .uleb128 5 - .byte 2 # DW_CFA_advance_loc1 - .byte .Lsubl-.Lpush_ebx + .byte 4 # DW_CFA_advance_loc4 + .4byte .Lsubl-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE - .byte 3 # DW_CFA_advance_loc2 - .2byte .Laddl-.Lsubl + .byte 4 # DW_CFA_advance_loc4 + .4byte .Laddl-.Lsubl .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 .byte 0x40+.Lpop_ebx-.Laddl # DW_CFA_advance_loc+N @@ -615,7 +717,8 @@ __condvar_tw_cleanup: .byte 0x40+.LSbl1-.Lpop_edi # DW_CFA_advance_loc+N .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 - .byte 0x40+.LSbl2-.LSbl1 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl2-.LSbl1 .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE .byte 0x85 # DW_CFA_offset %ebp @@ -626,14 +729,15 @@ __condvar_tw_cleanup: .uleb128 4 .byte 0x83 # DW_CFA_offset %ebx .uleb128 5 - .byte 0x40+.LSbl3-.LSbl2 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl3-.LSbl2 .byte 14 # DW_CFA_def_cfa_offset .uleb128 20 + .byte 4 # DW_CFA_advance_loc4 #if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS - .byte 0x40+.LSbl4-.LSbl3 # DW_CFA_advance_loc+N + .4byte .LSbl4-.LSbl3 #else - .byte 4 # DW_CFA_advance_loc4 - .long .LSbl5-.LSbl3 + .4byte .LSbl5-.LSbl3 #endif .byte 14 # DW_CFA_def_cfa_offset .uleb128 20+FRAME_SIZE diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S index 68741bffe8..5b301979b5 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S @@ -22,6 +22,7 @@ #include #include #include +#include .text @@ -100,7 +101,20 @@ __pthread_cond_wait: 4: call __pthread_enable_asynccancel movl %eax, (%esp) - movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */ +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif +#if FUTEX_WAIT != 0 + addl $FUTEX_WAIT, %ecx +#endif movl %edi, %edx addl $cond_futex, %ebx .Ladd_cond_futex: @@ -161,7 +175,18 @@ __pthread_cond_wait: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -197,8 +222,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 2b @@ -210,8 +243,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 4b @@ -222,8 +263,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait jmp 6b @@ -234,8 +283,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake jmp 11b @@ -256,8 +313,16 @@ __pthread_cond_wait: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake movl %esi, %eax @@ -292,8 +357,16 @@ __condvar_w_cleanup: #else leal cond_lock(%ebx), %edx #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_lock_wait 1: movl broadcast_seq(%ebx), %eax @@ -332,7 +405,18 @@ __condvar_w_cleanup: addl $cond_nwaiters, %ebx movl $SYS_futex, %eax - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_nwaiters(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $1, %edx ENTER_KERNEL subl $cond_nwaiters, %ebx @@ -351,15 +435,34 @@ __condvar_w_cleanup: #else leal cond_lock(%ebx), %eax #endif - /* XYZ */ - movl $LLL_SHARED, %ecx +#if (LLL_SHARED-LLL_PRIVATE) > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex(%ebx) + setne %cl + subl $1, %ecx + andl $(LLL_SHARED-LLL_PRIVATE), %ecx +#if LLL_PRIVATE != 0 + addl $LLL_PRIVATE, %ecx +#endif call __lll_unlock_wake /* Wake up all waiters to make sure no signal gets lost. */ 2: testl %edi, %edi jnz 5f addl $cond_futex, %ebx - movl $FUTEX_WAKE, %ecx +#if FUTEX_PRIVATE_FLAG > 255 + xorl %ecx, %ecx +#endif + cmpl $-1, dep_mutex-cond_futex(%ebx) + sete %cl + subl $1, %ecx +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %ecx +#else + andl %gs:PRIVATE_FUTEX, %ecx +#endif + addl $FUTEX_WAKE, %ecx movl $SYS_futex, %eax movl $0x7fffffff, %edx ENTER_KERNEL @@ -473,12 +576,12 @@ __condvar_w_cleanup: .uleb128 16 .byte 0x83 # DW_CFA_offset %ebx .uleb128 4 - .byte 2 # DW_CFA_advance_loc1 - .byte .Lsubl-.Lpush_ebx + .byte 4 # DW_CFA_advance_loc4 + .4byte .Lsubl-.Lpush_ebx .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 2 # DW_CFA_advance_loc1 - .byte .Laddl-.Lsubl + .byte 4 # DW_CFA_advance_loc4 + .4byte .Laddl-.Lsubl .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 .byte 0x40+ .Lpop_ebx-.Laddl # DW_CFA_advance_loc+N @@ -502,13 +605,16 @@ __condvar_w_cleanup: .uleb128 3 .byte 0x83 # DW_CFA_offset %ebx .uleb128 4 - .byte 0x40+.LSbl2-.LSbl1 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl2-.LSbl1 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE - .byte 0x40+.LSbl3-.LSbl2 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl3-.LSbl2 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16 - .byte 0x40+.LSbl4-.LSbl3 # DW_CFA_advance_loc+N + .byte 4 # DW_CFA_advance_loc4 + .4byte .LSbl4-.LSbl3 .byte 14 # DW_CFA_def_cfa_offset .uleb128 16+FRAME_SIZE .align 4 diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S index d30bc459bb..d8f1bd54a2 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S @@ -69,7 +69,7 @@ __pthread_rwlock_rdlock: jne 10f 11: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebx), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S index 86c9873ec7..0d96e03252 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S @@ -100,7 +100,7 @@ pthread_rwlock_timedrdlock: movl %edx, 4(%esp) movl %esi, %edx -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebp), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S index 5f6c80f8e5..e78fdf6dda 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S @@ -98,7 +98,7 @@ pthread_rwlock_timedwrlock: movl %edx, 4(%esp) movl %esi, %edx -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebp), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S index 3f19cd1802..a23e1b50a8 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S @@ -74,7 +74,7 @@ __pthread_rwlock_unlock: jne 7f 8: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%edi), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S index 70d0fcfde8..65b99fe7d3 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S @@ -67,7 +67,7 @@ __pthread_rwlock_wrlock: jne 10f 11: -#if __ASSUME_PRIVATE_FUTEX +#ifdef __ASSUME_PRIVATE_FUTEX movzbl PSHARED(%ebx), %ecx xorl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx #else diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 2f663aa68b..8f67616af7 100644 --- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -83,7 +83,7 @@ ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) \ : (fl)) \ : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG); \ - asm ("andl %%fs:%P1, %0" : "+r" (__fl) \ + asm ("andl %%gs:%P1, %0" : "+r" (__fl) \ : "i" (offsetof (struct pthread, header.private_futex))); \ __fl | (fl); })) # endif diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index 6b8a29e768..6155255eb0 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -132,7 +132,7 @@ __pthread_cond_broadcast: /* Unlock in loop requires wakeup. */ 7: addq $cond_lock-cond_futex, %rdi - cmpq $-1, dep_mutex-cond_lock(%rdi) + cmpq $-1, %r8 movl $LLL_PRIVATE, %eax movl $LLL_SHARED, %esi cmovne %eax, %esi @@ -141,7 +141,7 @@ __pthread_cond_broadcast: jmp 8b 9: /* The futex requeue functionality is not available. */ - cmpq $-1, dep_mutex-cond_futex(%rdi) + cmpq $-1, %r8 movl $0x7fffffff, %edx #ifdef __ASSUME_PRIVATE_FUTEX movl $FUTEX_WAKE, %eax diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index 8be6d4a21b..8f65f2cd69 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -87,7 +87,7 @@ __pthread_cond_signal: 7: /* %esi should be either FUTEX_WAKE_OP or FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ - xorl $(FUTEX_WAKE | FUTEX_WAKE_OP), %esi + xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %esi movl $SYS_futex, %eax /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ -- cgit 1.4.1