sparc: Add multiarch support for memset/bzero/memcpy. davem/sparc

2010-02-25  David S. Miller  <davem@davemloft.net>

	* sysdeps/sparc/elf/rtld-global-offsets.sym: New file.
	* sysdeps/sparc/Makefile (csu): Add rtld-global-offsets.sym to
	gen-as-const-headers.
	* sysdeps/sparc/sparc32/sparcv9/multiarch/Makefile: New file.
	* sysdeps/sparc/sparc32/sparcv9/multiarch/memcpy.S: New file.
	* sysdeps/sparc/sparc32/sparcv9/multiarch/memset.S: New file.
	* sysdeps/sparc/sparc32/sparcv9/sparcv9b/memcpy.S: Move to...
	* sysdeps/sparc/sparc32/sparcv9/multiarch/memcpy-ultra3.S: ...here.
	* sysdeps/sparc/sparc32/sparcv9/sparcv9v/memcpy.S: Move to...
	* sysdeps/sparc/sparc32/sparcv9/multiarch/memcpy-niagara1.S: ...here.
	* sysdeps/sparc/sparc32/sparcv9/sparcv9v/memset.S: Move to...
	* sysdeps/sparc/sparc32/sparcv9/multiarch/memset-niagara1.S: ...here.
	* sysdeps/sparc/sparc32/sparcv9/sparcv9v2/memcpy.S: Move to...
	* sysdeps/sparc/sparc32/sparcv9/multiarch/memcpy-niagara2.S: ...here.
	* sysdeps/sparc/sparc32/sparcv9/sparcv9v2/memset.S: Removed.
	* sysdeps/sparc/sparc64/multiarch/Makefile: New file.
	* sysdeps/sparc/sparc64/sparcv9v/memcpy.S: Move to...
	* sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S: ...here.
	* sysdeps/sparc/sparc64/sparcv9v2/memcpy.S: Move to...
	* sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S: ...here.
	* sysdeps/sparc/sparc64/sparcv9b/memcpy.S: Move to...
	* sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S: ...here.
	* sysdeps/sparc/sparc64/sparcv9v/memset.S: Move to...
	* sysdeps/sparc/sparc64/multiarch/memset-niagara1.S: ...here.
	* sysdeps/sparc/sparc64/sparcv9v2/memset.S: Removed.
	* sysdeps/sparc/sparc64/multiarch/memcpy.S: New file.
	* sysdeps/sparc/sparc64/multiarch/memset.S: New file.

8 files changed, 284 insertions, 22 deletions

diff --git a/sysdeps/sparc/sparc64/multiarch/Makefile b/sysdeps/sparc/sparc64/multiarch/Makefile
new file mode 100644
index 0000000000..4d45042a95
--- /dev/null
+++ b/sysdeps/sparc/sparc64/multiarch/Makefile
@@ -0,0 +1,4 @@
+ifeq ($(subdir),string)
+sysdep_routines += memcpy-ultra3 memcpy-niagara1 memcpy-niagara2 \
+		   memset-niagara1
+endif
diff --git a/sysdeps/sparc/sparc64/sparcv9v/memcpy.S b/sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S
index 116c7113ff..6a78295e81 100644
--- a/sysdeps/sparc/sparc64/sparcv9v/memcpy.S
+++ b/sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S
@@ -36,6 +36,8 @@
 #define XCC xcc
 #endif
 
+#if !defined NOT_IN_libc
+
 	.register	%g2,#scratch
 	.register	%g3,#scratch
 	.register	%g6,#scratch
@@ -43,10 +45,10 @@
 	.text
 
 	.align		32
-ENTRY(memcpy)
-#ifndef USE_BPR
+ENTRY(__memcpy_niagara1)
+# ifndef USE_BPR
 	srl		%o2, 0, %o2
-#endif
+# endif
 100:	/* %o0=dst, %o1=src, %o2=len */
 	mov		%o0, %g5
 	cmp		%o2, 0
@@ -335,6 +337,6 @@ ENTRY(memcpy)
 	retl
 	 mov		%g5, %o0
 
-END(memcpy)
+END(__memcpy_niagara1)
 
-libc_hidden_builtin_def (memcpy)
+#endif
diff --git a/sysdeps/sparc/sparc64/sparcv9v2/memcpy.S b/sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S
index 300b12f427..35f6989aca 100644
--- a/sysdeps/sparc/sparc64/sparcv9v2/memcpy.S
+++ b/sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S
@@ -138,6 +138,8 @@
 	LOAD(ldd, base + 0x28, %x5); \
 	LOAD(ldd, base + 0x30, %x6);
 
+#if !defined NOT_IN_libc
+
 	.register	%g2,#scratch
 	.register	%g3,#scratch
 	.register	%g6,#scratch
@@ -145,10 +147,10 @@
 	.text
 
 	.align		32
-ENTRY(memcpy)
-#ifndef USE_BPR
+ENTRY(__memcpy_niagara2)
+# ifndef USE_BPR
 	srl		%o2, 0, %o2
-#endif
+# endif
 100:	/* %o0=dst, %o1=src, %o2=len */
 	mov		%o0, %g5
 	cmp		%o2, 0
@@ -485,6 +487,6 @@ ENTRY(memcpy)
 	retl
 	 mov		%g5, %o0
 
-END(memcpy)
+END(__memcpy_niagara2)
 
-libc_hidden_builtin_def (memcpy)
+#endif
diff --git a/sysdeps/sparc/sparc64/sparcv9b/memcpy.S b/sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S
index 389e09d820..34ca089f93 100644
--- a/sysdeps/sparc/sparc64/sparcv9b/memcpy.S
+++ b/sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S
@@ -31,6 +31,8 @@
 #define XCC xcc
 #endif
 
+#if !defined NOT_IN_libc
+
 	.register	%g2,#scratch
 	.register	%g3,#scratch
 	.register	%g6,#scratch
@@ -52,7 +54,7 @@
 	 * of up to 2.4GB per second.
 	 */
 	.align		32
-ENTRY(memcpy)
+ENTRY(__memcpy_ultra3)
 
 100: /* %o0=dst, %o1=src, %o2=len */
 	mov		%o0, %g5
@@ -313,6 +315,6 @@ small_copy_unaligned:
 	retl
 	 mov		%g5, %o0
 
-END(memcpy)
+END(__memcpy_ultra3)
 
-libc_hidden_builtin_def (memcpy)
+#endif
\ No newline at end of file
diff --git a/sysdeps/sparc/sparc64/multiarch/memcpy.S b/sysdeps/sparc/sparc64/multiarch/memcpy.S
new file mode 100644
index 0000000000..a708de10e2
--- /dev/null
+++ b/sysdeps/sparc/sparc64/multiarch/memcpy.S
@@ -0,0 +1,107 @@
+/* Multiple versions of memcpy
+   Copyright (C) 2010 Free Software Foundation, Inc.
+   Contributed by David S. Miller (davem@davemloft.net)
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <sysdep.h>
+#include <rtld-global-offsets.h>
+
+#if !defined NOT_IN_libc
+	.text
+ENTRY(memcpy)
+	.type	memcpy, @gnu_indirect_function
+# ifdef SHARED
+	mov	%o7, %o5
+	sethi	%hi(_GLOBAL_OFFSET_TABLE_-4), %o3
+	call	1f
+	 or	%o3, %lo(_GLOBAL_OFFSET_TABLE_+4), %o3
+1:	add	%o7, %o3, %o3
+	mov	%o5, %o7
+	sethi	%hi(_rtld_global_ro), %o2
+	or	%o2, %lo(_rtld_global_ro), %o2
+#  ifdef __arch64__
+	ldx	[%o3 + %o2], %o2
+	ldx	[%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET], %o2
+#  else
+	ld	[%o3 + %o2], %o2
+	ld	[%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET + 4], %o2
+#  endif
+# else
+	set	_dl_hwcap, %o3
+#  ifdef __arch64__
+	ldx	[%o3], %o2
+#  else
+	ld	[%o3 + 4], %o2
+#  endif
+# endif
+	andcc	%o2, 0x80, %g0	! HWCAP_SPARC_N2
+	be	1f
+	 andcc	%o2, 0x40, %g0	! HWCAP_SPARC_BLKINIT
+# ifdef SHARED
+	sethi	%gdop_hix22(__memcpy_niagara2), %o1
+	xor	%o1, %gdop_lox10(__memcpy_niagara2), %o1
+# else
+	set	__memcpy_niagara2, %o1
+# endif
+	ba	10f
+	 nop
+1:	be	1f
+	 andcc	%o2, 0x20, %g0	! HWCAP_SPARC_ULTRA3
+# ifdef SHARED
+	sethi	%gdop_hix22(__memcpy_niagara1), %o1
+	xor	%o1, %gdop_lox10(__memcpy_niagara1), %o1
+# else
+	set	__memcpy_niagara1, %o1
+# endif
+	ba	10f
+	 nop
+1:	be	9f
+	 nop
+# ifdef SHARED
+	sethi	%gdop_hix22(__memcpy_ultra3), %o1
+	xor	%o1, %gdop_lox10(__memcpy_ultra3), %o1
+# else
+	set	__memcpy_ultra3, %o1
+# endif
+	ba	10f
+	 nop
+9:
+# ifdef SHARED
+	sethi	%gdop_hix22(__memcpy_ultra1), %o1
+	xor	%o1, %gdop_lox10(__memcpy_ultra1), %o1
+# else
+	set	__memcpy_ultra1, %o1
+# endif
+10:
+# ifdef SHARED
+	add	%o3, %o1, %o1
+# endif
+	retl
+	 mov	%o1, %o0
+END(memcpy)
+
+# undef libc_hidden_builtin_def
+/* IFUNC doesn't work with the hidden functions in a shared library.  */
+# define libc_hidden_builtin_def(name) \
+	.globl __GI_memcpy; __GI_memcpy = __memcpy_ultra1
+
+#define memcpy __memcpy_ultra1
+
+#endif
+
+#include "../memcpy.S"
diff --git a/sysdeps/sparc/sparc64/sparcv9v/memset.S b/sysdeps/sparc/sparc64/multiarch/memset-niagara1.S
index 64817b8871..20ea056216 100644
--- a/sysdeps/sparc/sparc64/sparcv9v/memset.S
+++ b/sysdeps/sparc/sparc64/multiarch/memset-niagara1.S
@@ -29,12 +29,14 @@
 #define XCC xcc
 #endif
 
+#if !defined NOT_IN_libc
+
 	.register	%g2,#scratch
 
 	.text
 	.align		32
 
-ENTRY(memset)
+ENTRY(__memset_niagara1)
 	/* %o0=buf, %o1=pat, %o2=len */
 	and		%o1, 0xff, %o3
 	mov		%o2, %o1
@@ -45,14 +47,14 @@ ENTRY(memset)
 	sllx		%o2, 32, %g1
 	ba,pt		%XCC, 1f
 	 or		%g1, %o2, %o2
-END(memset)
+END(__memset_niagara1)
 
-ENTRY(__bzero)
+ENTRY(__bzero_niagara1)
 	clr		%o2
 1:
-#ifndef USE_BRP
+# ifndef USE_BRP
 	srl		%o1, 0, %o1
-#endif
+# endif
 	brz,pn		%o1, 90f
 	 mov		%o0, %o3
 
@@ -125,7 +127,6 @@ ENTRY(__bzero)
 90:
 	retl
 	 mov		%o3, %o0
-END(__bzero)
+END(__bzero_niagara1)
 
-libc_hidden_builtin_def (memset)
-weak_alias (__bzero, bzero)
+#endif
diff --git a/sysdeps/sparc/sparc64/multiarch/memset.S b/sysdeps/sparc/sparc64/multiarch/memset.S
new file mode 100644
index 0000000000..23e513f18f
--- /dev/null
+++ b/sysdeps/sparc/sparc64/multiarch/memset.S
@@ -0,0 +1,145 @@
+/* Multiple versions of memset and bzero
+   Copyright (C) 2010 Free Software Foundation, Inc.
+   Contributed by David S. Miller (davem@davemloft.net)
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <sysdep.h>
+#include <rtld-global-offsets.h>
+
+#if !defined NOT_IN_libc
+	.text
+ENTRY(memset)
+	.type	memset, @gnu_indirect_function
+# ifdef SHARED
+	mov	%o7, %o5
+	sethi	%hi(_GLOBAL_OFFSET_TABLE_-4), %o3
+	call	1f
+	 or	%o3, %lo(_GLOBAL_OFFSET_TABLE_+4), %o3
+1:	add	%o7, %o3, %o3
+	mov	%o5, %o7
+	sethi	%hi(_rtld_global_ro), %o2
+	or	%o2, %lo(_rtld_global_ro), %o2
+#  ifdef __arch64__
+	ldx	[%o3 + %o2], %o2
+	ldx	[%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET], %o2
+#  else
+	ld	[%o3 + %o2], %o2
+	ld	[%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET + 4], %o2
+#  endif
+# else
+	set	_dl_hwcap, %o3
+#  ifdef __arch64__
+	ldx	[%o3], %o2
+#  else
+	ld	[%o3 + 4], %o2
+#  endif
+# endif
+	andcc	%o2, 0x40, %g0	! HWCAP_SPARC_BLKINIT
+	be	9f
+	 nop
+# ifdef SHARED
+	sethi	%gdop_hix22(__memset_niagara1), %o1
+	xor	%o1, %gdop_lox10(__memset_niagara1), %o1
+# else
+	set	__memset_niagara1, %o1
+# endif
+	ba	10f
+	 nop
+9:
+# ifdef SHARED
+	sethi	%gdop_hix22(__memset_ultra1), %o1
+	xor	%o1, %gdop_lox10(__memset_ultra1), %o1
+# else
+	set	__memset_ultra1, %o1
+# endif
+10:
+# ifdef SHARED
+	add	%o3, %o1, %o1
+# endif
+	retl
+	 mov	%o1, %o0
+END(memset)
+
+ENTRY(__bzero)
+	.type	bzero, @gnu_indirect_function
+# ifdef SHARED
+	mov	%o7, %o5
+	sethi	%hi(_GLOBAL_OFFSET_TABLE_-4), %o3
+	call	1f
+	 or	%o3, %lo(_GLOBAL_OFFSET_TABLE_+4), %o3
+1:	add	%o7, %o3, %o3
+	mov	%o5, %o7
+	sethi	%hi(_rtld_global_ro), %o2
+	or	%o2, %lo(_rtld_global_ro), %o2
+#  ifdef __arch64__
+	ldx	[%o3 + %o2], %o2
+	ldx	[%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET], %o2
+#  else
+	ld	[%o3 + %o2], %o2
+	ld	[%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET + 4], %o2
+#  endif
+# else
+	set	_dl_hwcap, %o3
+#  ifdef __arch64__
+	ldx	[%o3], %o2
+#  else
+	ld	[%o3 + 4], %o2
+#  endif
+# endif
+	andcc	%o2, 0x40, %g0	! HWCAP_SPARC_BLKINIT
+	be	9f
+	 nop
+# ifdef SHARED
+	sethi	%gdop_hix22(__bzero_niagara1), %o1
+	xor	%o1, %gdop_lox10(__bzero_niagara1), %o1
+# else
+	set	__bzero_niagara1, %o1
+# endif
+	ba	10f
+	 nop
+9:
+# ifdef SHARED
+	sethi	%gdop_hix22(__memset_ultra1), %o1
+	xor	%o1, %gdop_lox10(__memset_ultra1), %o1
+# else
+	set	__bzero_ultra1, %o1
+# endif
+10:
+# ifdef SHARED
+	add	%o3, %o1, %o1
+# endif
+	retl
+	 mov	%o1, %o0
+END(__bzero)
+
+weak_alias (__bzero, bzero)
+
+# undef weak_alias
+# define weak_alias(a, b)
+
+# undef libc_hidden_builtin_def
+/* IFUNC doesn't work with the hidden functions in a shared library.  */
+# define libc_hidden_builtin_def(name) \
+	.globl __GI_memset; __GI_memset = __memset_ultra1
+
+#define memset __memset_ultra1
+#define __bzero __bzero_ultra1
+
+#endif
+
+#include "../memset.S"
diff --git a/sysdeps/sparc/sparc64/sparcv9v2/memset.S b/sysdeps/sparc/sparc64/sparcv9v2/memset.S
deleted file mode 100644
index 809d3ed9c6..0000000000
--- a/sysdeps/sparc/sparc64/sparcv9v2/memset.S
+++ /dev/null
@@ -1 +0,0 @@
-#include <sparc64/sparcv9v/memset.S>