diff options
Diffstat (limited to 'nptl')
6 files changed, 547 insertions, 11 deletions
diff --git a/nptl/pthread_rwlock_init.c b/nptl/pthread_rwlock_init.c index 27f25ac2ab..95d9da4005 100644 --- a/nptl/pthread_rwlock_init.c +++ b/nptl/pthread_rwlock_init.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -37,13 +37,7 @@ __pthread_rwlock_init (rwlock, attr) iattr = ((const struct pthread_rwlockattr *) attr) ?: &default_attr; - rwlock->__data.__lock = 0; - rwlock->__data.__nr_readers = 0; - rwlock->__data.__readers_wakeup = 0; - rwlock->__data.__writer_wakeup = 0; - rwlock->__data.__nr_readers_queued = 0; - rwlock->__data.__nr_writers_queued = 0; - rwlock->__data.__writer = 0; + memset (rwlock, '\0', sizeof (*rwlock)); rwlock->__data.__flags = iattr->lockkind == PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP; @@ -74,9 +68,6 @@ __pthread_rwlock_init (rwlock, attr) header.private_futex)); #endif - rwlock->__data.__pad1 = 0; - rwlock->__data.__pad2 = 0; - return 0; } strong_alias (__pthread_rwlock_init, pthread_rwlock_init) diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S index 35eb09cd0c..54a47e111f 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S @@ -31,6 +31,127 @@ .align 16 __pthread_rwlock_rdlock: cfi_startproc + movq $NRW_RC, %rax + LOCK + xaddq %rax, NRW_WORD(%rdi) + js .Lundo + +.Lagain1: + movq $(NRW_WL|NRW_WP), %rcx + movq $(NRW_WL|NRW_WW_MASK), %rdx + testq %rax, %rcx + setnz %cl + testq %rax, %rdx + setnz %dl + orb %cl, %dl + jnz .Lwait + + xorl %eax, %eax + ret + +.Lwait: movq $NRW_WL, %rcx + testq %rcx, %rax + jz 4f + + movl %fs:TID, %ecx + cmpl %ecx, WRITER(%rdi) + je .Ldeadlk + +4: xorq %r8, %r8 + movq $NRW_RC, %rdx + addq %rdx, %rax + movq $(NRW_WL|NRW_AR), %rcx + movq $(NRW_WP|NRW_WW_MASK), %rdx + testq %rax, %rcx + setz %cl + testq %rax, %rdx + setnz %dl + testb %dl, %cl + jz 1f + + movq $NRW_RW-NRW_RC, %rdx + addq %rax, %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lagain1 + + movq $NRW_RC_MASK, %rax + testq %rdx, %rax + movl $1, %r8d + jz .Lwake_waiter + +1: xorq %r10, %r10 + movl $NRW_R_WAKEUP, %r9d + movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi + xorl PSHARED(%rdi), %esi + leaq NRW_WORD+4(%rdi), %rdi +2: shldq $32, %rax, %rdx + movl $__NR_futex, %eax + syscall + movq -4(%rdi), %rax + +5: movq $(NRW_WL|NRW_WP), %rcx + movq $(NRW_WL|NRW_WW_MASK), %rdx + testq %rax, %rcx + setnz %cl + testq %rax, %rdx + setnz %dl + orb %cl, %dl + jnz 2b + + testl %r8d, %r8d + jz 3f + + movq $NRW_RC-NRW_RW, %rcx + addq %rcx, %rdx + LOCK + cmpxchgq %rdx, -4(%rdi) + jnz 5b + +3: xorl %eax, %eax + ret + +.Lwake_waiter: + movq %rdx, %r10 + movl $__NR_futex, %eax + movl $(FUTEX_WAKE_BITSET|FUTEX_PRIVATE_FLAG), %esi + xorl PSHARED(%rdi), %esi + leaq NRW_WORD(%rdi), %rdi + movl $1, %edx + movl $NRW_W_WAKEUP, %r9d + syscall + leaq -NRW_WORD(%rdi), %rdi + movq %r10, %rdx + jmp 1b + +.Lundo: movq $-NRW_RC, %rcx + movq %rax, %rdx + subq %rcx, %rax + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jz .Lret + + movq $NRW_RC_OVFL, %r8 +.Lagain2: + testq %r8, %rax + jz .Lagain1 + leaq (%rax,%rcx), %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lagain2 + +.Lret: movl $EAGAIN, %eax + ret + +.Ldeadlk: + movq $NRW_RC, %rdx + LOCK + subq %rdx, NRW_WORD(%rdi) + + movl $EDEADLK, %eax + ret + +#if 0 xorq %r10, %r10 /* Get the lock. */ @@ -168,6 +289,7 @@ __pthread_rwlock_rdlock: subq $MUTEX, %rdi #endif jmp 13b +#endif cfi_endproc .size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S index 23b218af34..0fb925ceec 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S @@ -34,6 +34,143 @@ .align 16 pthread_rwlock_timedrdlock: cfi_startproc + movq %rsi, %r10 + movq $NRW_RC, %rax + LOCK + xaddq %rax, NRW_WORD(%rdi) + js .Lundo + +.Lagain1: + movq $(NRW_WL|NRW_WP), %rcx + movq $(NRW_WL|NRW_WW_MASK), %rdx + testq %rax, %rcx + setnz %cl + testq %rax, %rdx + setnz %dl + orb %cl, %dl + jnz .Lwait + + xorl %eax, %eax + ret + +.Lwait: movq $NRW_WL, %rcx + testq %rcx, %rax + jz 4f + + movl %fs:TID, %ecx + cmpl %ecx, WRITER(%rdi) + je .Ldeadlk + +4: xorq %r8, %r8 + movq $NRW_RC, %rdx + addq %rdx, %rax + movq $(NRW_WL|NRW_AR), %rcx + movq $(NRW_WP|NRW_WW_MASK), %rdx + testq %rax, %rcx + setz %cl + testq %rax, %rdx + setnz %dl + testb %dl, %cl + jz 1f + + movq $NRW_RW-NRW_RC, %rdx + addq %rax, %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lagain1 + + movq $NRW_RC_MASK, %rax + testq %rdx, %rax + movq $NRW_RW-NRW_RC, %r8 + jz .Lwake_waiter + +1: movl $NRW_R_WAKEUP, %r9d + leaq NRW_WORD+4(%rdi), %rdi +2: movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG|FUTEX_CLOCK_REALTIME), %esi + xorl PSHARED-(NRW_WORD+4)(%rdi), %esi + shldq $32, %rax, %rdx + movl $__NR_futex, %eax + syscall + movl %eax, %esi + movq -4(%rdi), %rax + +6: movq $(NRW_WL|NRW_WP), %rcx + movq $(NRW_WL|NRW_WW_MASK), %rdx + testq %rax, %rcx + setnz %cl + testq %rax, %rdx + setnz %dl + orb %cl, %dl + jz 5f + + cmpl $-EWOULDBLOCK, %esi + je 2b + + movq $-NRW_RC, %rdx + subq %r8, %rdx + addq %rax, %rdx + LOCK + cmpxchgq %rdx, -4(%rdi) + jnz 6b + + negl %esi + movl %esi, %eax + ret + +5: testl %r8d, %r8d + jz 3f + + movq $NRW_RC-NRW_RW, %rcx + addq %rcx, %rdx + LOCK + cmpxchgq %rdx, -4(%rdi) + jnz 6b + +3: xorl %eax, %eax + ret + +.Lwake_waiter: + movq %rdx, %r10 + movl $__NR_futex, %eax + movl $(FUTEX_WAKE_BITSET|FUTEX_PRIVATE_FLAG), %esi + xorl PSHARED(%rdi), %esi + leaq NRW_WORD(%rdi), %rdi + movl $1, %edx + movl $NRW_W_WAKEUP, %r9d + syscall + leaq -NRW_WORD(%rdi), %rdi + movq %r10, %rdx + jmp 1b + +.Lundo: movq $-NRW_RC, %rcx + movq %rax, %rdx + subq %rcx, %rax + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jz .Lret + + movq $NRW_RC_OVFL, %r8 +.Lagain2: + testq %r8, %rax + jz .Lagain1 + leaq (%rax,%rcx), %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lagain2 + +.Lret: movl $EAGAIN, %eax + ret + +.Ldeadlk: + movq $NRW_RC, %rdx + LOCK + subq %rdx, NRW_WORD(%rdi) + + movl $EDEADLK, %eax + ret + + +#if 0 pushq %r12 cfi_adjust_cfa_offset(8) cfi_rel_offset(%r12, 0) @@ -271,5 +408,6 @@ pthread_rwlock_timedrdlock: 19: movl $EINVAL, %edx jmp 9b +#endif cfi_endproc .size pthread_rwlock_timedrdlock,.-pthread_rwlock_timedrdlock diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S index cd867b60dc..14d7637429 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S @@ -34,6 +34,109 @@ .align 16 pthread_rwlock_timedwrlock: cfi_startproc + movq %rsi, %r10 + movq NRW_WORD(%rdi), %rax +.Lagain: + movq $(NRW_WW_MASK|NRW_WL|NRW_RC_MASK), %rdx + testq %rdx, %rax + jnz .Lwait + + // XXX Probably make AR a don't-care for !WP. Unconditionally set it + movq $(NRW_AR|NRW_WL), %rdx + movq $NRW_WL, %rcx + testq $NRW_WP, %rax + cmovz %rcx, %rdx + orq %rax, %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lagain + + movl %fs:TID, %eax + movl %eax, WRITER(%rdi) + + xorl %eax, %eax + ret + +.Lwait: movq $NRW_WL, %rcx + testq %rcx, %rax + jz 1f + + movl %fs:TID, %eax + cmpl %eax, WRITER(%rdi) + je .Ldeadlk + +1: leaq NRW_WORD+4(%rdi), %rdi + + movq $NRW_WW, %rdx + movq $NRW_WW_MASK, %rcx + addq %rax, %rdx + testq %rcx, %rdx + jz .Lovfl + + LOCK + cmpxchgq %rdx, -4(%rdi) + jnz .Lagain + + movl $NRW_W_WAKEUP, %r9d +.Lwait2: + movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG|FUTEX_CLOCK_REALTIME), %esi + movl $__NR_futex, %eax + xorl PSHARED-(NRW_WORD+4)(%rdi), %esi + syscall + movl %eax, %esi + movq -4(%rdi), %rax + +.Lagain2: + movq $(NRW_WL|NRW_RC_MASK), %rdx + movq $(NRW_WL|NRW_AR), %rcx + testq %rdx, %rax + movq $NRW_AR, %rsi + setz %cl + andq %rax, %rcx + cmpq %rsi, %rcx + sete %cl + orb %dl, %cl + jnz 2f + + cmpl $-EWOULDBLOCK, %esi + jne .Lwait2 + + movq $-NRW_WW, %rdx + addq %rax, %rdx + LOCK + cmpxchgq %rdx, -4(%rdi) + jnz .Lagain2 + + negl %esi + movl %esi, %eax + ret + +2: movq $-NRW_WW, %rdx + addq %rax, %rdx + xorl %ecx, %ecx + testq $NRW_WP, %rax + cmovz %rcx, %rsi + orq %rsi, %rdx + + LOCK + cmpxchgq %rdx, -4(%rdi) + jnz .Lagain2 + + movl %fs:TID, %eax + movl %eax, WRITER(%rdi) + + xorl %eax, %eax + ret + +.Lovfl: movl $EAGAIN, %eax + ret + +.Ldeadlk: + movl $EDEADLK, %eax + ret + + +#if 0 pushq %r12 cfi_adjust_cfa_offset(8) cfi_rel_offset(%r12, 0) @@ -263,5 +366,6 @@ pthread_rwlock_timedwrlock: 19: movl $EINVAL, %edx jmp 9b +#endif cfi_endproc .size pthread_rwlock_timedwrlock,.-pthread_rwlock_timedwrlock diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S index 03391d0fc2..1953637055 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S @@ -30,6 +30,97 @@ .align 16 __pthread_rwlock_unlock: cfi_startproc + movq NRW_WORD(%rdi), %rax + movq $NRW_WL, %rdx + testq %rdx, %rax + jnz .Lunlock_writer + +.Lagain: + movq $-NRW_RC, %rdx + addq %rax, %rdx + movq $NRW_RC_MASK, %rcx + testq %rcx, %rdx + jz .Llast_reader + +1: LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lagain + + xorl %eax, %eax + ret + +.Llast_reader: + movq $NRW_WW_MASK, %rcx + testq %rax, %rcx + jz 1b + + movq $NRW_AR, %rcx + xorl %esi, %esi + testq $NRW_WP, %rax + cmovz %esi, %ecx + orq %rcx, %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lagain + +.Lwake_writer: + movl $1, %edx + movl $NRW_W_WAKEUP, %r9d +.Lwake: movl $(FUTEX_WAKE_BITSET|FUTEX_PRIVATE_FLAG), %esi + xorl PSHARED(%rdi), %esi + leaq NRW_WORD(%rdi), %rdi + movl $__NR_futex, %eax + syscall + +.Lout: xorl %eax, %eax + ret + +.Lunlock_writer: + movq %rax, %rdx + movq $NRW_WW_MASK, %rcx + testq %rcx, %rax + jz .Lno_writers + movq $NRW_RC_MASK, %rcx + testq %rcx, %rax + jz 2f + testq $NRW_WP, %rax + jz .Lwake_readers + +2: movq $~NRW_WL, %rcx + andq %rcx, %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lunlock_writer + jmp .Lwake_writer + +.Lno_writers: + movq $~(NRW_WL|NRW_AR), %rcx + andq %rcx, %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lunlock_writer + + movq $(NRW_RW_MASK|NRW_RC_MASK), %rcx + testq %rcx, %rax + jz .Lout + + movl $0x7fffffff, %edx + movl $NRW_R_WAKEUP, %r9d + jmp .Lwake + +.Lwake_readers: + movq $~NRW_WL, %rcx + andq %rcx, %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lunlock_writer + + movl $0x7fffffff, %edx + movl $NRW_R_WAKEUP, %r9d + jmp .Lwake + + +#if 0 /* Get the lock. */ movl $1, %esi xorl %eax, %eax @@ -120,6 +211,7 @@ __pthread_rwlock_unlock: #endif callq __lll_unlock_wake jmp 8b +#endif cfi_endproc .size __pthread_rwlock_unlock,.-__pthread_rwlock_unlock diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S index be6b8d8e20..63fc2dfd91 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S @@ -31,6 +31,94 @@ .align 16 __pthread_rwlock_wrlock: cfi_startproc + movq NRW_WORD(%rdi), %rax +.Lagain: + movq $(NRW_WW_MASK|NRW_WL|NRW_RC_MASK), %rdx + testq %rdx, %rax + jnz .Lwait + + // XXX Probably make AR a don't-care for !WP. Unconditionally set it + movq $(NRW_AR|NRW_WL), %rdx + movq $NRW_WL, %rcx + testq $NRW_WP, %rax + cmovz %rcx, %rdx + orq %rax, %rdx + LOCK + cmpxchgq %rdx, NRW_WORD(%rdi) + jnz .Lagain + + movl %fs:TID, %eax + movl %eax, WRITER(%rdi) + + xorl %eax, %eax + ret + +.Lwait: movq $NRW_WL, %rcx + testq %rcx, %rax + jz 1f + + movl %fs:TID, %eax + cmpl %eax, WRITER(%rdi) + je .Ldeadlk + +1: leaq NRW_WORD+4(%rdi), %rdi + + movq $NRW_WW, %rdx + movq $NRW_WW_MASK, %rcx + addq %rax, %rdx + testq %rcx, %rdx + jz .Lovfl + + LOCK + cmpxchgq %rdx, -4(%rdi) + jnz .Lagain + + xorq %r10, %r10 + movl $NRW_W_WAKEUP, %r9d +.Lwait2: + movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi + movl $__NR_futex, %eax + xorl PSHARED-(NRW_WORD+4)(%rdi), %esi + syscall + movq -4(%rdi), %rax + +.Lagain2: + movq $(NRW_WL|NRW_RC_MASK), %rdx + movq $(NRW_WL|NRW_AR), %rcx + testq %rdx, %rax + movq $NRW_AR, %rsi + setz %cl + andq %rax, %rcx + cmpq %rsi, %rcx + sete %cl + orb %dl, %cl + jz .Lwait2 + + movq $-NRW_WW, %rdx + addq %rax, %rdx + xorl %ecx, %ecx + testq $NRW_WP, %rax + cmovz %rcx, %rsi + orq %rsi, %rdx + + LOCK + cmpxchgq %rdx, -4(%rdi) + jnz .Lagain2 + + movl %fs:TID, %eax + movl %eax, WRITER(%rdi) + + xorl %eax, %eax + ret + +.Lovfl: movl $EAGAIN, %eax + ret + +.Ldeadlk: + movl $EDEADLK, %eax + ret + +#if 0 xorq %r10, %r10 /* Get the lock. */ @@ -156,6 +244,7 @@ __pthread_rwlock_wrlock: subq $MUTEX, %rdi #endif jmp 13b +#endif cfi_endproc .size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock |