Update.

1999-05-18  Jakub Jelinek  <jj@ultra.linux.cz>

	* sysdeps/sparc/sparc64/rawmemchr.S: New file.
	* sysdeps/sparc/sparc32/sparcv9/rawmemchr.S: New file.

	* sysdeps/sparc/sparc64/memchr.S: Fix if the second argument
	contains garbage in bits 8-63.
	* sysdeps/sparc/sparc64/memset.S: Ditto.
	* sysdeps/sparc/sparc64/strchr.S: Ditto.

4 files changed, 250 insertions, 74 deletions

diff --git a/sysdeps/sparc/sparc64/memchr.S b/sysdeps/sparc/sparc64/memchr.S
index 9adf33ffc6..87e9022779 100644
--- a/sysdeps/sparc/sparc64/memchr.S
+++ b/sysdeps/sparc/sparc64/memchr.S
@@ -56,37 +56,37 @@
 	.text
 	.align		32
 ENTRY(memchr)
-	sll		%o1, 8, %g3			/* IEU0		Group		*/
+	and		%o1, 0xff, %o1			/* IEU0		Group		*/
 #ifdef USE_BPR
 	brz,pn		%o2, 12f			/* CTI+IEU1			*/
 #else
 	tst		%o2				/* IEU1				*/
 	be,pn		%XCC, 12f			/* CTI				*/
-	 srl		%o1, 0, %o1			/* IEU0				*/
 #endif
-	 add		%o0, %o2, %o2			/* IEU0		Group		*/
-	sethi		%hi(0x01010101), %g1		/* IEU1				*/
+	 sll		%o1, 8, %g3			/* IEU0		Group		*/
+	add		%o0, %o2, %o2			/* IEU1				*/
 
-	or		%g3, %o1, %g3			/* IEU0		Group		*/
+	sethi		%hi(0x01010101), %g1		/* IEU0		Group		*/
+	or		%g3, %o1, %g3			/* IEU1				*/
 	ldub		[%o0], %o3			/* Load				*/
 	sllx		%g3, 16, %g5			/* IEU0		Group		*/
-	or		%g1, %lo(0x01010101), %g1	/* IEU1				*/
 
+	or		%g1, %lo(0x01010101), %g1	/* IEU1				*/
 	sllx		%g1, 32, %g2			/* IEU0		Group		*/
 	or		%g3, %g5, %g3			/* IEU1				*/
 	sllx		%g3, 32, %g5			/* IEU0		Group		*/
-	cmp		%o3, %o1			/* IEU1				*/
 
+	cmp		%o3, %o1			/* IEU1				*/
 	be,pn		%xcc, 13f			/* CTI				*/
 	 or		%g1, %g2, %g1			/* IEU0		Group		*/
 	andcc		%o0, 7, %g0			/* IEU1				*/
-	bne,a,pn	%icc, 21f			/* CTI				*/
 
+	bne,a,pn	%icc, 21f			/* CTI				*/
 	 add		%o0, 1, %o0			/* IEU0		Group		*/
 	ldx		[%o0], %o3			/* Load		Group		*/
 	sllx		%g1, 7, %g2			/* IEU0				*/
-	or		%g3, %g5, %g3			/* IEU1				*/
 
+	or		%g3, %g5, %g3			/* IEU1				*/
 1:	add		%o0, 8, %o0			/* IEU0		Group		*/
 	xor		%o3, %g3, %o4			/* IEU1				*/
 							/* %g1 = 0101010101010101	*
@@ -95,49 +95,50 @@ ENTRY(memchr)
 							 * %o3 =      value		*
 							 * %o4 =   value XOR c		*/
 2:	cmp		%o0, %o2			/* IEU1		Group		*/
-	bg,pn		%XCC, 11f			/* CTI				*/
 
+	bg,pn		%XCC, 11f			/* CTI				*/
 	 ldxa		[%o0] ASI_PNF, %o3		/* Load				*/
 	sub		%o4, %g1, %o5			/* IEU0		Group		*/
 	add		%o0, 8, %o0			/* IEU1				*/
 #ifdef EIGHTBIT_NOT_RARE
 	andn		%o5, %o4, %o5			/* IEU0		Group		*/
 #endif
-	andcc		%o5, %g2, %g0			/* IEU1		Group		*/
 
+	andcc		%o5, %g2, %g0			/* IEU1		Group		*/
 	be,a,pt		%xcc, 2b			/* CTI				*/
 	 xor		%o3, %g3, %o4			/* IEU0				*/
 	srlx		%o4, 56, %g5			/* IEU0				*/
-	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 3f			/* CTI				*/
 	 srlx		%o4, 48, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	be,pn		%icc, 4f			/* CTI				*/
 
+	be,pn		%icc, 4f			/* CTI				*/
 	 srlx		%o4, 40, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
-	 srlx		%o4, 32, %g5			/* IEU0				*/
 
+	 srlx		%o4, 32, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 6f			/* CTI				*/
 	 srlx		%o4, 24, %g5			/* IEU0				*/
-	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 7f			/* CTI				*/
 	 srlx		%o4, 16, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	be,pn		%icc, 8f			/* CTI				*/
 
+	be,pn		%icc, 8f			/* CTI				*/
 	 srlx		%o4, 8, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 9f			/* CTI				*/
-	 andcc		%o4, 0xff, %g0			/* IEU1		Group		*/
 
+	 andcc		%o4, 0xff, %g0			/* IEU1		Group		*/
 	bne,pt		%icc, 2b			/* CTI				*/
 	 xor		%o3, %g3, %o4			/* IEU0				*/
 	retl						/* CTI+IEU1	Group		*/
+
 	 add		%o0, -9, %o0			/* IEU0				*/
 
 	.align		16
diff --git a/sysdeps/sparc/sparc64/memset.S b/sysdeps/sparc/sparc64/memset.S
index 8106a5c999..c2b23647f5 100644
--- a/sysdeps/sparc/sparc64/memset.S
+++ b/sysdeps/sparc/sparc64/memset.S
@@ -38,11 +38,9 @@
 	.text
 	.align		32
 ENTRY(memset)
-#ifndef USE_BPR
-	srl		%o1, 0, %o1
+	andcc		%o1, 0xff, %o1
 	mov		%o0, %g3
-#endif
-	brz,a,pt	%o1, 50f
+	be,a,pt		%icc, 50f
 #ifndef USE_BPR
 	 srl		%o2, 0, %o1
 #else
diff --git a/sysdeps/sparc/sparc64/rawmemchr.S b/sysdeps/sparc/sparc64/rawmemchr.S
new file mode 100644
index 0000000000..8f48ed940a
--- /dev/null
+++ b/sysdeps/sparc/sparc64/rawmemchr.S
@@ -0,0 +1,178 @@
+/* rawmemchr (str, ch) -- Return pointer to first occurrence of CH in STR.
+   For SPARC v9.
+   Copyright (C) 1999 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Jakub Jelinek <jj@ultra.linux.cz>.
+
+   This version is developed using the same algorithm as the fast C
+   version which carries the following introduction:
+
+   Based on strlen implementation by Torbjorn Granlund (tege@sics.se),
+   with help from Dan Sahlin (dan@sics.se) and
+   commentary by Jim Blandy (jimb@ai.mit.edu);
+   adaptation to memchr suggested by Dick Karpinski (dick@cca.ucsf.edu),
+   and implemented by Roland McGrath (roland@ai.mit.edu).
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Library General Public License as
+   published by the Free Software Foundation; either version 2 of the
+   License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Library General Public License for more details.
+
+   You should have received a copy of the GNU Library General Public
+   License along with the GNU C Library; see the file COPYING.LIB.  If not,
+   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+   Boston, MA 02111-1307, USA.  */
+
+#include <sysdep.h>
+#include <asm/asi.h>
+#ifndef XCC
+#define XCC xcc
+#define USE_BPR
+#endif
+
+	/* Normally, this uses
+	   ((xword - 0x0101010101010101) & 0x8080808080808080) test
+	   to find out if any byte in xword could be zero. This is fast, but
+	   also gives false alarm for any byte in range 0x81-0xff. It does
+	   not matter for correctness, as if this test tells us there could
+	   be some zero byte, we check it byte by byte, but if bytes with
+	   high bits set are common in the strings, then this will give poor
+	   performance. You can #define EIGHTBIT_NOT_RARE and the algorithm
+	   will use one tick slower, but more precise test
+	   ((xword - 0x0101010101010101) & (~xword) & 0x8080808080808080),
+	   which does not give any false alarms (but if some bits are set,
+	   one cannot assume from it which bytes are zero and which are not).
+	   It is yet to be measured, what is the correct default for glibc
+	   in these days for an average user.
+	 */
+
+	.text
+	.align		32
+ENTRY(__rawmemchr)
+	and		%o1, 0xff, %o1			/* IEU0		Group		*/
+	sethi		%hi(0x01010101), %g1		/* IEU1				*/
+	ldub		[%o0], %o3			/* Load				*/
+	sll		%o1, 8, %o4			/* IEU0		Group		*/
+	
+	or		%g1, %lo(0x01010101), %g1	/* IEU1				*/
+	sllx		%g1, 32, %g2			/* IEU0		Group		*/
+	or		%o4, %o1, %o4			/* IEU1				*/
+	andcc		%o0, 7, %g0			/* IEU1		Group		*/
+
+	sll		%o4, 16, %g5			/* IEU0				*/
+	or		%o4, %g5, %o4			/* IEU0		Group		*/
+	or		%g1, %g2, %g1			/* IEU1				*/
+	bne,pn		%icc, 32f			/* CTI				*/
+
+	 sllx		%o4, 32, %g5			/* IEU0		Group		*/
+	cmp		%o3, %o1			/* IEU1				*/
+	be,pn		%icc, 30f			/* CTI				*/
+	 sllx		%g1, 7, %g2			/* IEU0		Group		*/
+
+18:	ldx		[%o0], %o3			/* Load				*/
+	or		%o4, %g5, %o4			/* IEU1				*/
+	add		%o0, 8, %o0			/* IEU0		Group		*/
+19:	xor		%o3, %o4, %o3			/* IEU0		Group		*/
+
+	sub		%o3, %g1, %o2			/* IEU0		Group		*/
+#ifdef EIGHTBIT_NOT_RARE
+	andn		%o2, %o3, %g7			/* IEU0		Group		*/
+	ldxa		[%o0] ASI_PNF, %o3		/* Load				*/
+	andcc		%g7, %g2, %g0			/* IEU1		Group		*/
+#else
+	ldxa		[%o0] ASI_PNF, %o3		/* Load				*/
+	andcc		%o2, %g2, %g0			/* IEU1		Group		*/
+#endif
+	be,pt		%xcc, 19b			/* CTI				*/
+
+	 add		%o0, 8, %o0			/* IEU0				*/
+ 	addcc		%o2, %g1, %g3			/* IEU1		Group		*/
+	srlx		%o2, 32, %o2			/* IEU0				*/
+20:	andcc		%o2, %g2, %g0			/* IEU1		Group		*/
+
+	be,pn		%xcc, 21f			/* CTI				*/
+	 srlx		%g3, 56, %o2			/* IEU0				*/
+	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
+	be,pn		%icc, 29f			/* CTI				*/
+
+	 srlx		%g3, 48, %o2			/* IEU0				*/
+	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
+	be,pn		%icc, 28f			/* CTI				*/
+	 srlx		%g3, 40, %o2			/* IEU0				*/
+
+	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
+	be,pn		%icc, 27f			/* CTI				*/
+	 srlx		%g3, 32, %o2			/* IEU0				*/
+	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
+
+	be,pn		%icc, 26f			/* CTI				*/
+21:	 srlx		%g3, 24, %o2			/* IEU0				*/
+	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
+	be,pn		%icc, 25f			/* CTI				*/
+
+	 srlx		%g3, 16, %o2			/* IEU0				*/
+	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
+	be,pn		%icc, 24f			/* CTI				*/
+	 srlx		%g3, 8, %o2			/* IEU0				*/
+
+	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
+	be,pn		%icc, 23f			/* CTI				*/
+	 xor		%o3, %o4, %o3			/* IEU0				*/
+	andcc		%g3, 0xff, %g0			/* IEU1		Group		*/
+
+	be,pn		%icc, 22f			/* CTI				*/
+	 sub		%o3, %g1, %o2			/* IEU0				*/
+	ldxa		[%o0] ASI_PNF, %o3		/* Load				*/
+	andcc		%o2, %g2, %g0			/* IEU1		Group		*/
+
+	be,pt		%xcc, 19b			/* CTI				*/
+	 add		%o0, 8, %o0			/* IEU0				*/
+	addcc		%o2, %g1, %g3			/* IEU1		Group		*/
+	ba,pt		%xcc, 20b			/* CTI				*/
+
+	 srlx		%o2, 32, %o2			/* IEU0				*/
+
+	.align		16
+22:	retl						/* CTI+IEU1	Group		*/
+	 add		%o0, -9, %o0			/* IEU0				*/
+23:	retl						/* CTI+IEU1	Group		*/
+	 add		%o0, -10, %o0			/* IEU0				*/
+
+24:	retl						/* CTI+IEU1	Group		*/
+	 add		%o0, -11, %o0			/* IEU0				*/
+25:	retl						/* CTI+IEU1	Group		*/
+	 add		%o0, -12, %o0			/* IEU0				*/
+
+26:	retl						/* CTI+IEU1	Group		*/
+	 add		%o0, -13, %o0			/* IEU0				*/
+27:	retl						/* CTI+IEU1	Group		*/
+	 add		%o0, -14, %o0			/* IEU0				*/
+
+28:	retl						/* CTI+IEU1	Group		*/
+	 add		%o0, -15, %o0			/* IEU0				*/
+29:	retl						/* CTI+IEU1	Group		*/
+	 add		%o0, -16, %o0			/* IEU0				*/
+
+30:	retl						/* CTI+IEU1	Group		*/
+	 nop						/* IEU0				*/
+
+	.align		16
+32:	andcc		%o0, 7, %g0			/* IEU1		Group		*/
+	be,a,pn		%icc, 18b			/* CTI				*/
+	 sllx		%g1, 7, %g2			/* IEU0				*/
+	add		%o0, 1, %o0			/* IEU0		Group		*/
+
+	cmp		%o3, %o1			/* IEU1				*/
+	bne,a,pt	%icc, 32b			/* CTI				*/
+	 lduba		[%o0] ASI_PNF, %o3		/* Load				*/
+	retl						/* CTI+IEU1	Group		*/
+
+	 add		%o0, -1, %o0			/* IEU0				*/
+END(__rawmemchr)
+
+weak_alias (__rawmemchr, rawmemchr)
diff --git a/sysdeps/sparc/sparc64/strchr.S b/sysdeps/sparc/sparc64/strchr.S
index 8324c4c8d6..b017640914 100644
--- a/sysdeps/sparc/sparc64/strchr.S
+++ b/sysdeps/sparc/sparc64/strchr.S
@@ -46,34 +46,32 @@
 	.text
 	.align		32
 ENTRY(strchr)
-#ifndef USE_BPR
-	srl		%o1, 0, %o1			/* IEU0		Group		*/
-#endif
-	brz,pn		%o1, 17f			/* CTI+IEU1	Group		*/
-	 sllx		%o1, 8, %g3			/* IEU0				*/
-	sethi		%hi(0x01010101), %g1		/* IEU0		Group		*/
-	or		%g3, %o1, %g3			/* IEU1				*/
+	andcc		%o1, 0xff, %o1			/* IEU1		Group		*/
+	be,pn		%icc, 17f			/* CTI				*/
+	 sllx		%o1, 8, %g3			/* IEU0		Group		*/
+	sethi		%hi(0x01010101), %g1		/* IEU1				*/
 
+	or		%g3, %o1, %g3			/* IEU0		Group		*/
 	ldub		[%o0], %o3			/* Load				*/
 	sllx		%g3, 16, %g5			/* IEU0		Group		*/
 	or		%g1, %lo(0x01010101), %g1	/* IEU1				*/
-	sllx		%g1, 32, %g2			/* IEU0		Group		*/
 
+	sllx		%g1, 32, %g2			/* IEU0		Group		*/
 	brz,pn		%o3, 5f				/* CTI+IEU1			*/
 	 orcc		%g3, %g5, %g3			/* IEU1		Group		*/
 	sllx		%g3, 32, %g5			/* IEU0				*/
-	cmp		%o3, %o1			/* IEU1		Group		*/
 
+	cmp		%o3, %o1			/* IEU1		Group		*/
 	be,pn		%xcc, 14f			/* CTI				*/
 	 or		%g1, %g2, %g1			/* IEU0				*/
 	andcc		%o0, 7, %g0			/* IEU1		Group		*/
-	bne,a,pn	%icc, 15f			/* CTI				*/
 
+	bne,a,pn	%icc, 15f			/* CTI				*/
 	 add		%o0, 1, %o0			/* IEU0				*/
 	ldx		[%o0], %o3			/* Load		Group		*/
 1:	sllx		%g1, 7, %g2			/* IEU0				*/
-	or		%g3, %g5, %g3			/* IEU1				*/
 
+	or		%g3, %g5, %g3			/* IEU1				*/
 	add		%o0, 8, %o0			/* IEU0		Group		*/
 	xor		%o3, %g3, %o4			/* IEU1				*/
 							/* %g1 = 0101010101010101	*
@@ -82,8 +80,8 @@ ENTRY(strchr)
 							 * %o3 =      value		*
 							 * %o4 =   value XOR c		*/
 2:	sub		%o3, %g1, %o2			/* IEU0		Group		*/
-	sub		%o4, %g1, %o5			/* IEU1				*/
 
+	sub		%o4, %g1, %o5			/* IEU1				*/
 #ifdef EIGHTBIT_NOT_RARE
 	andn		%o2, %o3, %g7			/* IEU0		Group		*/
 	andn		%o5, %o4, %o5			/* IEU1				*/
@@ -94,86 +92,87 @@ ENTRY(strchr)
 	or		%o5, %o2, %o5			/* IEU0		Group		*/
 #endif
 	add		%o0, 8, %o0			/* IEU1				*/
-	andcc		%o5, %g2, %g0			/* IEU1		Group		*/
 
+	andcc		%o5, %g2, %g0			/* IEU1		Group		*/
 	be,a,pt		%xcc, 2b			/* CTI				*/
 	 xor		%o3, %g3, %o4			/* IEU0				*/
 	srlx		%o5, 32, %g5			/* IEU0		Group		*/
-	add		%o2, %g1, %o2			/* IEU1				*/
 
+	add		%o2, %g1, %o2			/* IEU1				*/
 3:	andcc		%g5, %g2, %g0			/* IEU1		Group		*/
 	be,pn		%xcc, 4f			/* CTI				*/
 	 srlx		%o2, 56, %g5			/* IEU0				*/
-	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
 	 srlx		%o4, 56, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	be,pn		%icc, 6f			/* CTI				*/
 
+	be,pn		%icc, 6f			/* CTI				*/
 	 srlx		%o2, 48, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
-	 srlx		%o4, 48, %g5			/* IEU0				*/
 
+	 srlx		%o4, 48, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 7f			/* CTI				*/
 	 srlx		%o2, 40, %g5			/* IEU0				*/
-	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
 	 srlx		%o4, 40, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	be,pn		%icc, 8f			/* CTI				*/
 
+	be,pn		%icc, 8f			/* CTI				*/
 	 srlx		%o2, 32, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
-	 srlx		%o4, 32, %g5			/* IEU0				*/
 
+	 srlx		%o4, 32, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 9f			/* CTI				*/
 4:	 srlx		%o2, 24, %g5			/* IEU0				*/
-	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
 	 srlx		%o4, 24, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	be,pn		%icc, 10f			/* CTI				*/
 
+	be,pn		%icc, 10f			/* CTI				*/
 	 srlx		%o2, 16, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
-	 srlx		%o4, 16, %g5			/* IEU0				*/
 
+	 srlx		%o4, 16, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 11f			/* CTI				*/
 	 srlx		%o2, 8, %g5			/* IEU0				*/
-	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
 	 srlx		%o4, 8, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	be,pn		%icc, 12f			/* CTI				*/
 
+	be,pn		%icc, 12f			/* CTI				*/
 	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 5f			/* CTI				*/
 	 sub		%o3, %g1, %o2			/* IEU0				*/
-	andcc		%o4, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%o4, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 13f			/* CTI				*/
 	 xor		%o3, %g3, %o4			/* IEU0				*/
 	ldxa		[%o0] ASI_PNF, %o3		/* Load		Group		*/
-	sub		%o4, %g1, %o5			/* IEU0				*/
 
+	sub		%o4, %g1, %o5			/* IEU0				*/
 	or		%o5, %o2, %o5			/* IEU1				*/
 	add		%o0, 8, %o0			/* IEU0		Group		*/
 	andcc		%o5, %g2, %g0			/* IEU1				*/
-	be,a,pt		%xcc, 2b			/* CTI				*/
 
+	be,a,pt		%xcc, 2b			/* CTI				*/
 	 xor		%o3, %g3, %o4			/* IEU0		Group		*/
 	srlx		%o5, 32, %g5			/* IEU0		Group		*/
 	ba,pt		%xcc, 3b			/* CTI				*/
+
 	 add		%o2, %g1, %o2			/* IEU1				*/
 
 	.align		16
@@ -330,29 +329,27 @@ END(strchr)
 
 	.align		32
 ENTRY(strrchr)
-#ifndef USE_BPR
-	srl		%o1, 0, %o1			/* IEU0		Group		*/
-#endif
-	brz,pn		%o1, 17b			/* CTI+IEU1	Group		*/
+	andcc		%o1, 0xff, %o1			/* IEU1		Group		*/
+	be,pn		%icc, 17b			/* CTI				*/
 	 clr		%g4				/* IEU0				*/
 	andcc		%o0, 7, %g0			/* IEU1		Group		*/
-	bne,pn		%icc, 13f			/* CTI				*/
 
+	bne,pn		%icc, 13f			/* CTI				*/
 	 sllx		%o1, 8, %g3			/* IEU0				*/
 	ldx		[%o0], %o3			/* Load		Group		*/
 1:	sethi		%hi(0x01010101), %g1		/* IEU0				*/
-	or		%g3, %o1, %g3			/* IEU1				*/
 
+	or		%g3, %o1, %g3			/* IEU1				*/
 	sllx		%g3, 16, %g5			/* IEU0		Group		*/
 	or		%g1, %lo(0x01010101), %g1	/* IEU1				*/
 	sllx		%g1, 32, %g2			/* IEU0		Group		*/
-	or		%g3, %g5, %g3			/* IEU1				*/
 
+	or		%g3, %g5, %g3			/* IEU1				*/
 	sllx		%g3, 32, %g5			/* IEU0		Group		*/
 	or		%g1, %g2, %g1			/* IEU1				*/
 	sllx		%g1, 7, %g2			/* IEU0		Group		*/
-	or		%g3, %g5, %g3			/* IEU1				*/
 
+	or		%g3, %g5, %g3			/* IEU1				*/
 	add		%o0, 8, %o0			/* IEU0		Group		*/
 	xor		%o3, %g3, %o4			/* IEU1				*/
 							/* %g1 = 0101010101010101	*
@@ -361,97 +358,99 @@ ENTRY(strrchr)
 							 * %o3 =   value		*
 							 * %o4 =   value XOR c		*/
 2:	sub		%o3, %g1, %o2			/* IEU0		Group		*/
+
 3:	sub		%o4, %g1, %o5			/* IEU1				*/
 #ifdef EIGHTBIT_NOT_RARE
 	andn		%o2, %o3, %g7			/* IEU0		Group		*/
 	andn		%o5, %o4, %o5			/* IEU1				*/
 	ldxa		[%o0] ASI_PNF, %o3		/* Load				*/
+
 	or		%o5, %g7, %o5			/* IEU0		Group		*/
 #else
 	ldxa		[%o0] ASI_PNF, %o3		/* Load				*/
+
 	or		%o5, %o2, %o5			/* IEU0		Group		*/
 #endif
-
 	add		%o0, 8, %o0			/* IEU1				*/
 	andcc		%o5, %g2, %g0			/* IEU1		Group		*/
 	be,a,pt		%xcc, 2b			/* CTI				*/
-	 xor		%o3, %g3, %o4			/* IEU0				*/
 
+	 xor		%o3, %g3, %o4			/* IEU0				*/
 	srlx		%o5, 32, %g5			/* IEU0		Group		*/
 	add		%o2, %g1, %o2			/* IEU1				*/
 	andcc		%g5, %g2, %g0			/* IEU1		Group		*/
-	be,pn		%xcc, 7f			/* CTI				*/
 
+	be,pn		%xcc, 7f			/* CTI				*/
 	 srlx		%o2, 56, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 12f			/* CTI				*/
-	 srlx		%o4, 56, %g5			/* IEU0				*/
 
+	 srlx		%o4, 56, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	srlx		%o2, 48, %g5			/* IEU0				*/
 	be,a,pn		%icc, 4f			/* CTI				*/
-	 add		%o0, -16, %g4			/* IEU0		Group		*/
 
+	 add		%o0, -16, %g4			/* IEU0		Group		*/
 4:	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 12f			/* CTI				*/
 	 srlx		%o4, 48, %g5			/* IEU0				*/
-	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	srlx		%o2, 40, %g5			/* IEU0				*/
 	be,a,pn		%icc, 5f			/* CTI				*/
 	 add		%o0, -15, %g4			/* IEU0		Group		*/
-5:	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+5:	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 12f			/* CTI				*/
 	 srlx		%o4, 40, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	srlx		%o2, 32, %g5			/* IEU0				*/
 
+	srlx		%o2, 32, %g5			/* IEU0				*/
 	be,a,pn		%icc, 6f			/* CTI				*/
 	 add		%o0, -14, %g4			/* IEU0		Group		*/
 6:	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	be,pn		%icc, 12f			/* CTI				*/
 
+	be,pn		%icc, 12f			/* CTI				*/
 	 srlx		%o4, 32, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,a,pn		%icc, 7f			/* CTI				*/
-	 add		%o0, -13, %g4			/* IEU0				*/
 
+	 add		%o0, -13, %g4			/* IEU0				*/
 7:	srlx		%o2, 24, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 12f			/* CTI				*/
-	 srlx		%o4, 24, %g5			/* IEU0				*/
 
+	 srlx		%o4, 24, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	srlx		%o2, 16, %g5			/* IEU0				*/
 	be,a,pn		%icc, 8f			/* CTI				*/
-	 add		%o0, -12, %g4			/* IEU0		Group		*/
 
+	 add		%o0, -12, %g4			/* IEU0		Group		*/
 8:	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 12f			/* CTI				*/
 	 srlx		%o4, 16, %g5			/* IEU0				*/
-	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	srlx		%o2, 8, %g5			/* IEU0				*/
 	be,a,pn		%icc, 9f			/* CTI				*/
 	 add		%o0, -11, %g4			/* IEU0		Group		*/
-9:	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 
+9:	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 12f			/* CTI				*/
 	 srlx		%o4, 8, %g5			/* IEU0				*/
 	andcc		%g5, 0xff, %g0			/* IEU1		Group		*/
-	be,a,pn		%icc, 10f			/* CTI				*/
 
+	be,a,pn		%icc, 10f			/* CTI				*/
 	 add		%o0, -10, %g4			/* IEU0				*/
 10:	andcc		%o2, 0xff, %g0			/* IEU1		Group		*/
 	be,pn		%icc, 12f			/* CTI				*/
-	 sub		%o3, %g1, %o2			/* IEU0				*/
 
+	 sub		%o3, %g1, %o2			/* IEU0				*/
 	andcc		%o4, 0xff, %g0			/* IEU1		Group		*/
 	be,a,pn		%icc, 11f			/* CTI				*/
 	 add		%o0, -9, %g4			/* IEU0				*/
-11:	ba,pt		%xcc, 3b			/* CTI		Group		*/
 
+11:	ba,pt		%xcc, 3b			/* CTI		Group		*/
 	 xor		%o3, %g3, %o4			/* IEU0		Group		*/
 12:	retl						/* CTI+IEU1	Group		*/
 	 mov		%g4, %o0			/* IEU0				*/