Update.

	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Fix asm syntax.
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/sem_post.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/sem_trywait.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/sem_wait.S: Likewise.

	* sysdeps/x86_64/tls.h (THREAD_SELF, THREAD_GETMEM, THREAD_GETMEM_NC,
	THREAD_SETMEM, THREAD_SETMEM_NC): Correct asm syntax.

	* allocatestack.c [! TLS_MULTIPLE_THREADS_IN_TCB] (allocate_stack):
	Initialize *__libc_multiple_threads_ptr not __libc_multiple_threads.
	* sysdeps/pthread/createthread.c [! TLS_MULTIPLE_THREADS_IN_TCB]
	(create_thread): Likewise.
	Define __pthread_multiple_threads and __libc_multiple_threads_ptr.
	* init.c (__pthread_initialize_minimal_internal): Initialize
	__libc_multiple_threads_ptr if necessary.
	* pthreadP.h: Adjust prototype for __libc_pthread_init.  Declare
	__pthread_multiple_threads and __libc_multiple_threads_ptr.
	* sysdeps/unix/sysv/linux/libc_pthread_init.c: Define
	__libc_multiple_threads.
	(__libc_pthread_init): Return pointer to __libc_pthread_init if
	necessary.

	* sysdeps/i386/tls.h (THREAD_SETMEM): Fix one-byte variant.
	(THREAD_SETMEM_NC): Likewise.

	* sysdeps/x86_64/pthread_spin_trylock.c: Removed.
	* sysdeps/x86_64/pthread_spin_trylock.S: New file.
	* sysdeps/x86_64/pthread_spin_unlock.c: Removed.
	* sysdeps/x86_64/pthread_spin_unlock.S: New file.

	* sysdeps/i386/i486/pthread_spin_trylock.S (pthread_spin_trylock):
	Eliminate one entire instruction.

19 files changed, 135 insertions, 48 deletions

diff --git a/nptl/sysdeps/i386/i486/pthread_spin_trylock.S b/nptl/sysdeps/i386/i486/pthread_spin_trylock.S
index abda929c81..0a27312c72 100644
--- a/nptl/sysdeps/i386/i486/pthread_spin_trylock.S
+++ b/nptl/sysdeps/i386/i486/pthread_spin_trylock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -35,13 +35,12 @@ pthread_spin_trylock:
 	xorl	%ecx, %ecx
 	LOCK
 	cmpxchgl %ecx, (%edx)
-	movl	$EBUSY, %edx
-	movl	%ecx, %eax
+	movl	$EBUSY, %eax
 #ifdef HAVE_CMOV
-	cmovnel	%edx, %eax
+	cmovel	%ecx, %eax
 #else
-	je	0f
-	movl	%edx, %eax
+	jne	0f
+	movl	%ecx, %eax
 0:
 #endif
 	ret
diff --git a/nptl/sysdeps/i386/tls.h b/nptl/sysdeps/i386/tls.h
index 5b70586f27..6fb6adca30 100644
--- a/nptl/sysdeps/i386/tls.h
+++ b/nptl/sysdeps/i386/tls.h
@@ -303,7 +303,7 @@ union user_desc_init
 /* Same as THREAD_SETMEM, but the member offset can be non-constant.  */
 # define THREAD_SETMEM(descr, member, value) \
   ({ if (sizeof (descr->member) == 1)					      \
-       asm volatile ("movb %0,%%gs:%P1" :				      \
+       asm volatile ("movb %b0,%%gs:%P1" :				      \
 		     : "iq" (value),					      \
 		       "i" (offsetof (struct pthread, member)));	      \
      else if (sizeof (descr->member) == 4)				      \
@@ -328,7 +328,7 @@ union user_desc_init
 /* Set member of the thread descriptor directly.  */
 # define THREAD_SETMEM_NC(descr, member, idx, value) \
   ({ if (sizeof (descr->member[0]) == 1)				      \
-       asm volatile ("movb %0,%%gs:%P1(%2)" :				      \
+       asm volatile ("movb %b0,%%gs:%P1(%2)" :				      \
 		     : "iq" (value),					      \
 		       "i" (offsetof (struct pthread, member)),		      \
 		       "r" (idx));					      \
diff --git a/nptl/sysdeps/pthread/createthread.c b/nptl/sysdeps/pthread/createthread.c
index 0223fa868f..4ee92afec4 100644
--- a/nptl/sysdeps/pthread/createthread.c
+++ b/nptl/sysdeps/pthread/createthread.c
@@ -39,6 +39,14 @@
 #endif
 
 
+#ifndef TLS_MULTIPLE_THREADS_IN_TCB
+/* Variable set to a nonzero value if more than one thread runs or ran.  */
+int __pthread_multiple_threads attribute_hidden;
+/* Pointer to the corresponding variable in libc.  */
+int *__libc_multiple_threads_ptr attribute_hidden;
+#endif
+
+
 static int
 create_thread (struct pthread *pd, STACK_VARIABLES_PARMS)
 {
@@ -79,7 +87,7 @@ create_thread (struct pthread *pd, STACK_VARIABLES_PARMS)
 	  /* We now have for sure more than one thread.  */
 	  pd->multiple_threads = 1;
 #else
-	  __pthread_multiple_threads = __libc_multiple_threads = 1;
+	  __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 #endif
 
 	  /* Now fill in the information about the new thread in
diff --git a/nptl/sysdeps/unix/sysv/linux/libc_pthread_init.c b/nptl/sysdeps/unix/sysv/linux/libc_pthread_init.c
index 06cca367b3..f8c79a22be 100644
--- a/nptl/sysdeps/unix/sysv/linux/libc_pthread_init.c
+++ b/nptl/sysdeps/unix/sysv/linux/libc_pthread_init.c
@@ -30,7 +30,13 @@
 struct fork_handler __pthread_child_handler attribute_hidden;
 
 
+#ifdef TLS_MULTIPLE_THREADS_IN_TCB
 void
+#else
+int __libc_multiple_threads attribute_hidden;
+
+int *
+#endif
 __libc_pthread_init (ptr, reclaim, functions)
      unsigned long int *ptr;
      void (*reclaim) (void);
@@ -52,4 +58,8 @@ __libc_pthread_init (ptr, reclaim, functions)
   memcpy (&__libc_pthread_functions, functions,
 	  sizeof (__libc_pthread_functions));
 #endif
+
+#ifndef TLS_MULTIPLE_THREADS_IN_TCB
+  return &__libc_multiple_threads;
+#endif
 }
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
index 1585921ff0..39215280d5 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
@@ -113,8 +113,8 @@ __lll_unlock_wake:
 	.hidden	__lll_timedwait_tid
 	.align	16
 __lll_timedwait_tid:
-	movl	%rdi, %r8
-	movl	%rsi, %r9
+	movq	%rdi, %r8
+	movq	%rsi, %r9
 
 	subq	$16, %rsp
 
@@ -125,7 +125,7 @@ __lll_timedwait_tid:
 	syscall
 
 	/* Compute relative timeout.  */
-	movq	8(%esp), %rax
+	movq	8(%rsp), %rax
 	movq	$1000, %rdi
 	mul	%rdi		/* Milli seconds to nano seconds.  */
 	movq	(%r9), %rdi
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index f7e9450a1a..122b1c7ab0 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -90,7 +90,7 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
 		       ".previous\n"					      \
 		       "2:"						      \
 		       : "=a" (result), "=&D" (ignore1), "=&S" (ignore2),     \
-			 "=&d" (ignore2), "=m" (futex)			      \
+			 "=&d" (ignore3), "=m" (futex)			      \
 		       : "0" (1), "4" (futex), "m" (timeout)		      \
 		       : "memory", "cx", "cc", "r10");			      \
      result; })
@@ -245,14 +245,15 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
     int __ignore;							      \
     register __typeof (tid) _tid asm ("edx") = (tid);			      \
     if (_tid != 0)							      \
-      __asm __volatile ("1:\tmovq %1, %%rax\n\t"			      \
+      __asm __volatile ("xorq %%r10, %%r10\n\t"				      \
+			"1:\tmovq %3, %%rax\n\t"			      \
 			"syscall\n\t"					      \
 			"cmpl $0, (%%rdi)\n\t"				      \
 			"jne 1b"					      \
 			: "=&a" (__ignore)				      \
-			: "i" (SYS_futex), "D" (&tid), "r10" (0),	      \
-			  "S" (FUTEX_WAIT), "d" (_tid)			      \
-			: "memory", "cc", "r11", "cx");			      \
+			: "S" (FUTEX_WAIT), "i" (SYS_futex), "D" (&tid),      \
+			  "d" (_tid)					      \
+			: "memory", "cc", "r10", "r11", "cx");		      \
   } while (0)
 
 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S
index 55e4ec4682..2e36b9a72d 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S
@@ -84,7 +84,7 @@ __lll_mutex_timedlock_wait:
 
 	movq	%rdi, %r8
 	movq	%rdx, %r9
-	leal	1(%eax), %edx
+	leaq	1(%rax), %rdx
 
 	/* Get current time.  */
 1:
@@ -94,7 +94,7 @@ __lll_mutex_timedlock_wait:
 	syscall
 
 	/* Compute relative timeout.  */
-	movq	8(%esp), %rax
+	movq	8(%rsp), %rax
 	movq	$1000, %rdi
 	mul	%rdi		/* Milli seconds to nano seconds.  */
 	movq	(%r9), %rdi
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 64458628f5..11635baa22 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -62,7 +62,7 @@ __pthread_cond_signal:
 	/* Wake up one thread.  */
 	movq	$FUTEX_WAKE, %rsi
 	movq	$SYS_futex, %rax
-	movq	%rsi, %edx	/* movl $1, %edx */
+	movq	%rsi, %rdx	/* movl $1, %edx */
 	syscall
 
 	/* Unlock.  */
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 193cc081e1..831d4f9c90 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -42,7 +42,7 @@
 __condvar_cleanup:
 	/* Get internal lock.  */
 	movq	%rdi, %r8
-	movl	8(%rdi), %rdi
+	movq	8(%rdi), %rdi
 	movl	$1, %esi
 	LOCK
 #if cond_lock == 0
@@ -89,7 +89,7 @@ __condvar_cleanup:
 2:	testq	$2, (%r8)
 	jne	3f
 
-	movq	16(%r8), %edi
+	movq	16(%r8), %rdi
 	callq	__pthread_mutex_lock_internal
 
 3:	retq
@@ -101,7 +101,7 @@ __condvar_cleanup:
 	.type	__pthread_cond_wait, @function
 	.align	16
 __pthread_cond_wait:
-	pushq	%12
+	pushq	%r12
 	subq	$64, %rsp
 
 	/* Prepare structure passed to cancellation handler.  */
@@ -131,7 +131,7 @@ __pthread_cond_wait:
 
 	/* Install cancellation handler.  */
 #ifdef PIC
-	leaq	__condvar_cleanup@GOTOFF(%rip), %rsi
+	leaq	__condvar_cleanup(%rip), %rsi
 #else
 	leaq	__condvar_cleanup, %rsi
 #endif
@@ -153,7 +153,7 @@ __pthread_cond_wait:
 #endif
 	jne	3f
 
-4:	movq	%rsp, %edi
+4:	movq	%rsp, %rdi
 	callq	__pthread_enable_asynccancel_2
 
 	movq	8(%rsp), %rdi
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
index 8db9d9413b..97a21d0a9f 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
@@ -73,7 +73,7 @@ __pthread_once:
 
 	/* Somebody else got here first.  Wait.  */
 	movq	%r10, %rsi		/* movl $FUTEX_WAIT, %ecx */
-	movq	$SYS_futex, %eax
+	movq	$SYS_futex, %rax
 	syscall
 	jmp	6b
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S
index 945de2d01e..6189a9ac6e 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S
@@ -40,7 +40,7 @@
 __new_sem_post:
 	movl	$1, %edx
 	LOCK
-	xaddl	%edx, (%ebx)
+	xaddl	%edx, (%rdi)
 
 	movq	$SYS_futex, %rax
 	movq	$FUTEX_WAKE, %rsi
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
index afdec13e50..8f73d6a208 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S
@@ -44,7 +44,7 @@ sem_timedwait:
 2:	testl	%eax, %eax
 	je	1f
 
-	leal	-1(%eax), %edx
+	leaq	-1(%rax), %rdx
 	LOCK
 	cmpxchgl %edx, (%rdi)
 	jne	2b
@@ -64,13 +64,13 @@ sem_timedwait:
 	movl	$EINVAL, %eax
 	jae	6f
 
-7:	xorl	%rsi, %rsi
+7:	xorq	%rsi, %rsi
 	movq	%rsp, %rdi
 	movq	$SYS_gettimeofday, %rax
 	syscall
 
 	/* Compute relative timeout.  */
-	movq	8(%rsp), %eax
+	movq	8(%rsp), %rax
 	movq	$1000, %rdi
 	mul	%rdi		/* Milli seconds to nano seconds.  */
 	movq	(%r9), %rdi
@@ -84,8 +84,8 @@ sem_timedwait:
 	movl	$ETIMEDOUT, %eax
 	js	6f		/* Time is already up.  */
 
-	movl	%rdi, (%rsp)	/* Store relative timeout.  */
-	movl	%rsi, 8(%rsp)
+	movq	%rdi, (%rsp)	/* Store relative timeout.  */
+	movq	%rsi, 8(%rsp)
 
 	movq	%r8, %rdi
 	xorq	%rsi, %rsi
@@ -102,7 +102,7 @@ sem_timedwait:
 8:	testl	%eax, %eax
 	je	7b
 
-	leal	-1(%eax), %ecx
+	leaq	-1(%rax), %rcx
 	LOCK
 	cmpxchgl %ecx, (%rdi)
 	jne	8b
@@ -111,7 +111,7 @@ sem_timedwait:
 	xorl	%eax, %eax
 	retq
 
-3:	negq	%eax
+3:	negq	%rax
 6:
 #if USE___THREAD
 	movq	errno@gottpoff(%rip), %rdx
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S
index 663868febb..5785acff5a 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S
@@ -42,7 +42,7 @@ __new_sem_trywait:
 2:	testl	%eax, %eax
 	jz	1f
 
-	leal	-1(%eax), %edx
+	leaq	-1(%rax), %rdx
 	LOCK
 	cmpxchgl %edx, (%rdi)
 	jne	2b
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S
index d71e6b4100..69e6658d67 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S
@@ -41,7 +41,7 @@ __new_sem_wait:
 2:	testl	%eax, %eax
 	je	1f
 
-	leal	-1(%eax), %edx
+	leaq	-1(%rax), %rdx
 	LOCK
 	cmpxchgl %edx, (%rdi)
 	jne	2b
diff --git a/nptl/sysdeps/x86_64/pthread_spin_trylock.S b/nptl/sysdeps/x86_64/pthread_spin_trylock.S
new file mode 100644
index 0000000000..175714d6b0
--- /dev/null
+++ b/nptl/sysdeps/x86_64/pthread_spin_trylock.S
@@ -0,0 +1,40 @@
+/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#define EBUSY	16
+
+
+#ifdef UP
+# define LOCK
+#else
+# define LOCK lock
+#endif
+
+	.globl	pthread_spin_trylock
+	.type	pthread_spin_trylock,@function
+	.align	16
+pthread_spin_trylock:
+	movl	$1, %eax
+	xorl	%ecx, %ecx
+	LOCK
+	cmpxchgl %ecx, (%rdi)
+	movl	$EBUSY, %eax
+	cmovel	%ecx, %eax
+	retq
+	.size	pthread_spin_trylock,.-pthread_spin_trylock
diff --git a/nptl/sysdeps/x86_64/pthread_spin_trylock.c b/nptl/sysdeps/x86_64/pthread_spin_trylock.c
deleted file mode 100644
index 7ddb37154b..0000000000
--- a/nptl/sysdeps/x86_64/pthread_spin_trylock.c
+++ /dev/null
@@ -1 +0,0 @@
-#include "../i386/i686/pthread_spin_trylock.c"
diff --git a/nptl/sysdeps/x86_64/pthread_spin_unlock.S b/nptl/sysdeps/x86_64/pthread_spin_unlock.S
new file mode 100644
index 0000000000..d3e13bde98
--- /dev/null
+++ b/nptl/sysdeps/x86_64/pthread_spin_unlock.S
@@ -0,0 +1,31 @@
+/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+	.globl	pthread_spin_unlock
+	.type	pthread_spin_unlock,@function
+	.align	16
+pthread_spin_unlock:
+	movl	$1, (%rdi)
+	xorl	%eax, %eax
+	retq
+	.size	pthread_spin_unlock,.-pthread_spin_unlock
+
+	/* The implementation of pthread_spin_init is identical.  */
+	.globl	pthread_spin_init
+pthread_spin_init = pthread_spin_unlock
diff --git a/nptl/sysdeps/x86_64/pthread_spin_unlock.c b/nptl/sysdeps/x86_64/pthread_spin_unlock.c
deleted file mode 100644
index b287dc1f5c..0000000000
--- a/nptl/sysdeps/x86_64/pthread_spin_unlock.c
+++ /dev/null
@@ -1 +0,0 @@
-#include "../i386/pthread_spin_unlock.c"
diff --git a/nptl/sysdeps/x86_64/tls.h b/nptl/sysdeps/x86_64/tls.h
index 45646e4485..66ae0f3f07 100644
--- a/nptl/sysdeps/x86_64/tls.h
+++ b/nptl/sysdeps/x86_64/tls.h
@@ -145,7 +145,7 @@ typedef struct
    do not get optimized away.  */
 # define THREAD_SELF \
   ({ struct pthread *__self;						      \
-     asm ("movq %%fs:%c1,%0" : "=r" (__self)				      \
+     asm ("movq %%fs:%c1,%q0" : "=r" (__self)				      \
 	  : "i" (offsetof (struct pthread, self)));		 	      \
      __self;})
 
@@ -168,7 +168,7 @@ typedef struct
 	      4 or 8.  */						      \
 	   abort ();							      \
 									      \
-	 asm ("movq %%fs:%P1,%0"					      \
+	 asm ("movq %%fs:%P1,%q0"					      \
 	      : "=r" (__value)						      \
 	      : "i" (offsetof (struct pthread, member)));		      \
        }								      \
@@ -179,12 +179,12 @@ typedef struct
 # define THREAD_GETMEM_NC(descr, member, idx) \
   ({ __typeof (descr->member[0]) __value;				      \
      if (sizeof (__value) == 1)						      \
-       asm ("movb %%fs:%P2(%3),%b0"					      \
+       asm ("movb %%fs:%P2(%q3),%b0"					      \
 	    : "=q" (__value)						      \
 	    : "0" (0), "i" (offsetof (struct pthread, member[0])),	      \
 	      "r" (idx));						      \
      else if (sizeof (__value) == 4)					      \
-       asm ("movl %%fs:%P1(,%2,4),%0"					      \
+       asm ("movl %%fs:%P1(,%q2,4),%0"					      \
 	    : "=r" (__value)						      \
 	    : "i" (offsetof (struct pthread, member[0])), "r" (idx));	      \
      else								      \
@@ -194,7 +194,7 @@ typedef struct
 	      4 or 8.  */						      \
 	   abort ();							      \
 									      \
-	 asm ("movq %%fs:%P1(,%2,8),%0"					      \
+	 asm ("movq %%fs:%P1(,%q2,8),%q0"				      \
 	      : "=r" (__value)						      \
 	      : "i" (offsetof (struct pthread, member[0])), "r" (idx));	      \
        }								      \
@@ -204,7 +204,7 @@ typedef struct
 /* Same as THREAD_SETMEM, but the member offset can be non-constant.  */
 # define THREAD_SETMEM(descr, member, value) \
   ({ if (sizeof (descr->member) == 1)					      \
-       asm volatile ("movb %0,%%fs:%P1" :				      \
+       asm volatile ("movb %b0,%%fs:%P1" :				      \
 		     : "iq" (value),					      \
 		       "i" (offsetof (struct pthread, member)));	      \
      else if (sizeof (descr->member) == 4)				      \
@@ -218,7 +218,7 @@ typedef struct
 	      4 or 8.  */						      \
 	   abort ();							      \
 									      \
-	 asm volatile ("movq %0,%%fs:%P1" :				      \
+	 asm volatile ("movq %q0,%%fs:%P1" :				      \
 		       : "ir" ((unsigned long int) value),		      \
 			 "i" (offsetof (struct pthread, member)));	      \
        }})
@@ -227,12 +227,12 @@ typedef struct
 /* Set member of the thread descriptor directly.  */
 # define THREAD_SETMEM_NC(descr, member, idx, value) \
   ({ if (sizeof (descr->member[0]) == 1)				      \
-       asm volatile ("movb %0,%%fs:%P1(%2)" :				      \
+       asm volatile ("movb %b0,%%fs:%P1(%q2)" :				      \
 		     : "iq" (value),					      \
 		       "i" (offsetof (struct pthread, member[0])),	      \
 		       "r" (idx));					      \
      else if (sizeof (descr->member[0]) == 4)				      \
-       asm volatile ("movl %0,%%fs:%P1(,%2,4)" :			      \
+       asm volatile ("movl %0,%%fs:%P1(,%q2,4)" :			      \
 		     : "ir" (value),					      \
 		       "i" (offsetof (struct pthread, member[0])),	      \
 		       "r" (idx));					      \
@@ -243,7 +243,7 @@ typedef struct
 	      4 or 8.  */						      \
 	   abort ();							      \
 									      \
-	 asm volatile ("movq %0,%%fs:%P1(,%2,8)" :			      \
+	 asm volatile ("movq %q0,%%fs:%P1(,%q2,8)" :			      \
 		       : "r" ((unsigned long int) value),		      \
 			 "i" (offsetof (struct pthread, member[0])),	      \
 			 "r" (idx));					      \