x86: Move strrchr SSE2 implementation to multiarch/strrchr-sse2.S

This commit doesn't affect libc.so.6, its just housekeeping to prepare
for adding explicit ISA level support.

Tested build on x86_64 and x86_32 with/without multiarch.

4 files changed, 366 insertions, 377 deletions

diff --git a/sysdeps/x86_64/multiarch/strrchr-sse2.S b/sysdeps/x86_64/multiarch/strrchr-sse2.S
index 866396e947..6ee7a5e33a 100644
--- a/sysdeps/x86_64/multiarch/strrchr-sse2.S
+++ b/sysdeps/x86_64/multiarch/strrchr-sse2.S
@@ -17,12 +17,358 @@
    <https://www.gnu.org/licenses/>.  */
 
 #if IS_IN (libc)
-# define STRRCHR __strrchr_sse2
+# ifndef STRRCHR
+#  define STRRCHR __strrchr_sse2
+# endif
+#endif
+
+#include <sysdep.h>
+
+#ifdef USE_AS_WCSRCHR
+# define PCMPEQ	pcmpeqd
+# define CHAR_SIZE	4
+# define PMINU	pminud
+#else
+# define PCMPEQ	pcmpeqb
+# define CHAR_SIZE	1
+# define PMINU	pminub
+#endif
+
+#define PAGE_SIZE	4096
+#define VEC_SIZE	16
+
+	.text
+ENTRY(STRRCHR)
+	movd	%esi, %xmm0
+	movq	%rdi, %rax
+	andl	$(PAGE_SIZE - 1), %eax
+#ifndef USE_AS_WCSRCHR
+	punpcklbw %xmm0, %xmm0
+	punpcklwd %xmm0, %xmm0
+#endif
+	pshufd	$0, %xmm0, %xmm0
+	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
+	ja	L(cross_page)
+
+L(cross_page_continue):
+	movups	(%rdi), %xmm1
+	pxor	%xmm2, %xmm2
+	PCMPEQ	%xmm1, %xmm2
+	pmovmskb %xmm2, %ecx
+	testl	%ecx, %ecx
+	jz	L(aligned_more)
+
+	PCMPEQ	%xmm0, %xmm1
+	pmovmskb %xmm1, %eax
+	leal	-1(%rcx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(ret0)
+	bsrl	%eax, %eax
+	addq	%rdi, %rax
+	/* We are off by 3 for wcsrchr if search CHAR is non-zero. If
+	   search CHAR is zero we are correct. Either way `andq
+	   -CHAR_SIZE, %rax` gets the correct result.  */
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+L(ret0):
+	ret
+
+	/* Returns for first vec x1/x2 have hard coded backward search
+	   path for earlier matches.  */
+	.p2align 4
+L(first_vec_x0_test):
+	PCMPEQ	%xmm0, %xmm1
+	pmovmskb %xmm1, %eax
+	testl	%eax, %eax
+	jz	L(ret0)
+	bsrl	%eax, %eax
+	addq	%r8, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(first_vec_x1):
+	PCMPEQ	%xmm0, %xmm2
+	pmovmskb %xmm2, %eax
+	leal	-1(%rcx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(first_vec_x0_test)
+	bsrl	%eax, %eax
+	leaq	(VEC_SIZE)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(first_vec_x1_test):
+	PCMPEQ	%xmm0, %xmm2
+	pmovmskb %xmm2, %eax
+	testl	%eax, %eax
+	jz	L(first_vec_x0_test)
+	bsrl	%eax, %eax
+	leaq	(VEC_SIZE)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(first_vec_x2):
+	PCMPEQ	%xmm0, %xmm3
+	pmovmskb %xmm3, %eax
+	leal	-1(%rcx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(first_vec_x1_test)
+	bsrl	%eax, %eax
+	leaq	(VEC_SIZE * 2)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(aligned_more):
+	/* Save original pointer if match was in VEC 0.  */
+	movq	%rdi, %r8
+	andq	$-VEC_SIZE, %rdi
+
+	movaps	VEC_SIZE(%rdi), %xmm2
+	pxor	%xmm3, %xmm3
+	PCMPEQ	%xmm2, %xmm3
+	pmovmskb %xmm3, %ecx
+	testl	%ecx, %ecx
+	jnz	L(first_vec_x1)
+
+	movaps	(VEC_SIZE * 2)(%rdi), %xmm3
+	pxor	%xmm4, %xmm4
+	PCMPEQ	%xmm3, %xmm4
+	pmovmskb %xmm4, %ecx
+	testl	%ecx, %ecx
+	jnz	L(first_vec_x2)
+
+	addq	$VEC_SIZE, %rdi
+	/* Save pointer again before realigning.  */
+	movq	%rdi, %rsi
+	andq	$-(VEC_SIZE * 2), %rdi
+	.p2align 4
+L(first_loop):
+	/* Do 2x VEC at a time.  */
+	movaps	(VEC_SIZE * 2)(%rdi), %xmm4
+	movaps	(VEC_SIZE * 3)(%rdi), %xmm5
+	/* Since SSE2 no pminud so wcsrchr needs seperate logic for
+	   detecting zero. Note if this is found to be a bottleneck it
+	   may be worth adding an SSE4.1 wcsrchr implementation.  */
+#ifdef USE_AS_WCSRCHR
+	movaps	%xmm5, %xmm6
+	pxor	%xmm8, %xmm8
+
+	PCMPEQ	%xmm8, %xmm5
+	PCMPEQ	%xmm4, %xmm8
+	por	%xmm5, %xmm8
+#else
+	movaps	%xmm5, %xmm6
+	PMINU	%xmm4, %xmm5
+#endif
+
+	movaps	%xmm4, %xmm9
+	PCMPEQ	%xmm0, %xmm4
+	PCMPEQ	%xmm0, %xmm6
+	movaps	%xmm6, %xmm7
+	por	%xmm4, %xmm6
+#ifndef USE_AS_WCSRCHR
+	pxor	%xmm8, %xmm8
+	PCMPEQ	%xmm5, %xmm8
+#endif
+	pmovmskb %xmm8, %ecx
+	pmovmskb %xmm6, %eax
 
-# undef weak_alias
-# define weak_alias(strrchr, rindex)
-# undef libc_hidden_builtin_def
-# define libc_hidden_builtin_def(strrchr)
+	addq	$(VEC_SIZE * 2), %rdi
+	/* Use `addl` 1) so we can undo it with `subl` and 2) it can
+	   macro-fuse with `jz`.  */
+	addl	%ecx, %eax
+	jz	L(first_loop)
+
+	/* Check if there is zero match.  */
+	testl	%ecx, %ecx
+	jz	L(second_loop_match)
+
+	/* Check if there was a match in last iteration.  */
+	subl	%ecx, %eax
+	jnz	L(new_match)
+
+L(first_loop_old_match):
+	PCMPEQ	%xmm0, %xmm2
+	PCMPEQ	%xmm0, %xmm3
+	pmovmskb %xmm2, %ecx
+	pmovmskb %xmm3, %eax
+	addl	%eax, %ecx
+	jz	L(first_vec_x0_test)
+	/* NB: We could move this shift to before the branch and save a
+	   bit of code size / performance on the fall through. The
+	   branch leads to the null case which generally seems hotter
+	   than char in first 3x VEC.  */
+	sall	$16, %eax
+	orl	%ecx, %eax
+
+	bsrl	%eax, %eax
+	addq	%rsi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(new_match):
+	pxor	%xmm6, %xmm6
+	PCMPEQ	%xmm9, %xmm6
+	pmovmskb %xmm6, %eax
+	sall	$16, %ecx
+	orl	%eax, %ecx
+
+	/* We can't reuse either of the old comparisons as since we mask
+	   of zeros after first zero (instead of using the full
+	   comparison) we can't gurantee no interference between match
+	   after end of string and valid match.  */
+	pmovmskb %xmm4, %eax
+	pmovmskb %xmm7, %edx
+	sall	$16, %edx
+	orl	%edx, %eax
+
+	leal	-1(%ecx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(first_loop_old_match)
+	bsrl	%eax, %eax
+	addq	%rdi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	/* Save minimum state for getting most recent match. We can
+	   throw out all previous work.  */
+	.p2align 4
+L(second_loop_match):
+	movq	%rdi, %rsi
+	movaps	%xmm4, %xmm2
+	movaps	%xmm7, %xmm3
+
+	.p2align 4
+L(second_loop):
+	movaps	(VEC_SIZE * 2)(%rdi), %xmm4
+	movaps	(VEC_SIZE * 3)(%rdi), %xmm5
+	/* Since SSE2 no pminud so wcsrchr needs seperate logic for
+	   detecting zero. Note if this is found to be a bottleneck it
+	   may be worth adding an SSE4.1 wcsrchr implementation.  */
+#ifdef USE_AS_WCSRCHR
+	movaps	%xmm5, %xmm6
+	pxor	%xmm8, %xmm8
+
+	PCMPEQ	%xmm8, %xmm5
+	PCMPEQ	%xmm4, %xmm8
+	por	%xmm5, %xmm8
+#else
+	movaps	%xmm5, %xmm6
+	PMINU	%xmm4, %xmm5
+#endif
+
+	movaps	%xmm4, %xmm9
+	PCMPEQ	%xmm0, %xmm4
+	PCMPEQ	%xmm0, %xmm6
+	movaps	%xmm6, %xmm7
+	por	%xmm4, %xmm6
+#ifndef USE_AS_WCSRCHR
+	pxor	%xmm8, %xmm8
+	PCMPEQ	%xmm5, %xmm8
 #endif
 
-#include "../strrchr.S"
+	pmovmskb %xmm8, %ecx
+	pmovmskb %xmm6, %eax
+
+	addq	$(VEC_SIZE * 2), %rdi
+	/* Either null term or new occurence of CHAR.  */
+	addl	%ecx, %eax
+	jz	L(second_loop)
+
+	/* No null term so much be new occurence of CHAR.  */
+	testl	%ecx, %ecx
+	jz	L(second_loop_match)
+
+
+	subl	%ecx, %eax
+	jnz	L(second_loop_new_match)
+
+L(second_loop_old_match):
+	pmovmskb %xmm2, %ecx
+	pmovmskb %xmm3, %eax
+	sall	$16, %eax
+	orl	%ecx, %eax
+	bsrl	%eax, %eax
+	addq	%rsi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(second_loop_new_match):
+	pxor	%xmm6, %xmm6
+	PCMPEQ	%xmm9, %xmm6
+	pmovmskb %xmm6, %eax
+	sall	$16, %ecx
+	orl	%eax, %ecx
+
+	/* We can't reuse either of the old comparisons as since we mask
+	   of zeros after first zero (instead of using the full
+	   comparison) we can't gurantee no interference between match
+	   after end of string and valid match.  */
+	pmovmskb %xmm4, %eax
+	pmovmskb %xmm7, %edx
+	sall	$16, %edx
+	orl	%edx, %eax
+
+	leal	-1(%ecx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(second_loop_old_match)
+	bsrl	%eax, %eax
+	addq	%rdi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4,, 4
+L(cross_page):
+	movq	%rdi, %rsi
+	andq	$-VEC_SIZE, %rsi
+	movaps	(%rsi), %xmm1
+	pxor	%xmm2, %xmm2
+	PCMPEQ	%xmm1, %xmm2
+	pmovmskb %xmm2, %edx
+	movl	%edi, %ecx
+	andl	$(VEC_SIZE - 1), %ecx
+	sarl	%cl, %edx
+	jz	L(cross_page_continue)
+	PCMPEQ	%xmm0, %xmm1
+	pmovmskb %xmm1, %eax
+	sarl	%cl, %eax
+	leal	-1(%rdx), %ecx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(ret1)
+	bsrl	%eax, %eax
+	addq	%rdi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+L(ret1):
+	ret
+END(STRRCHR)
diff --git a/sysdeps/x86_64/multiarch/wcsrchr-sse2.S b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
index 69d2f3cdb1..d9259720f8 100644
--- a/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
+++ b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
@@ -17,6 +17,12 @@
    <https://www.gnu.org/licenses/>.  */
 
 #if IS_IN (libc)
-# define STRRCHR	__wcsrchr_sse2
+# ifndef STRRCHR
+#  define STRRCHR	__wcsrchr_sse2
+# endif
 #endif
-#include "../wcsrchr.S"
+
+#define USE_AS_WCSRCHR	1
+#define NO_PMINU	1
+
+#include "strrchr-sse2.S"
diff --git a/sysdeps/x86_64/strrchr.S b/sysdeps/x86_64/strrchr.S
index 4d7ba4ceb2..f39da60454 100644
--- a/sysdeps/x86_64/strrchr.S
+++ b/sysdeps/x86_64/strrchr.S
@@ -16,363 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
-
-#include <sysdep.h>
-
-#ifndef STRRCHR
-# define STRRCHR	strrchr
-#endif
-
-#ifdef USE_AS_WCSRCHR
-# define PCMPEQ	pcmpeqd
-# define CHAR_SIZE	4
-# define PMINU	pminud
-#else
-# define PCMPEQ	pcmpeqb
-# define CHAR_SIZE	1
-# define PMINU	pminub
-#endif
-
-#define PAGE_SIZE	4096
-#define VEC_SIZE	16
-
-	.text
-ENTRY(STRRCHR)
-	movd	%esi, %xmm0
-	movq	%rdi, %rax
-	andl	$(PAGE_SIZE - 1), %eax
-#ifndef USE_AS_WCSRCHR
-	punpcklbw %xmm0, %xmm0
-	punpcklwd %xmm0, %xmm0
-#endif
-	pshufd	$0, %xmm0, %xmm0
-	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
-	ja	L(cross_page)
-
-L(cross_page_continue):
-	movups	(%rdi), %xmm1
-	pxor	%xmm2, %xmm2
-	PCMPEQ	%xmm1, %xmm2
-	pmovmskb %xmm2, %ecx
-	testl	%ecx, %ecx
-	jz	L(aligned_more)
-
-	PCMPEQ	%xmm0, %xmm1
-	pmovmskb %xmm1, %eax
-	leal	-1(%rcx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(ret0)
-	bsrl	%eax, %eax
-	addq	%rdi, %rax
-	/* We are off by 3 for wcsrchr if search CHAR is non-zero. If
-	   search CHAR is zero we are correct. Either way `andq
-	   -CHAR_SIZE, %rax` gets the correct result.  */
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-L(ret0):
-	ret
-
-	/* Returns for first vec x1/x2 have hard coded backward search
-	   path for earlier matches.  */
-	.p2align 4
-L(first_vec_x0_test):
-	PCMPEQ	%xmm0, %xmm1
-	pmovmskb %xmm1, %eax
-	testl	%eax, %eax
-	jz	L(ret0)
-	bsrl	%eax, %eax
-	addq	%r8, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(first_vec_x1):
-	PCMPEQ	%xmm0, %xmm2
-	pmovmskb %xmm2, %eax
-	leal	-1(%rcx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(first_vec_x0_test)
-	bsrl	%eax, %eax
-	leaq	(VEC_SIZE)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(first_vec_x1_test):
-	PCMPEQ	%xmm0, %xmm2
-	pmovmskb %xmm2, %eax
-	testl	%eax, %eax
-	jz	L(first_vec_x0_test)
-	bsrl	%eax, %eax
-	leaq	(VEC_SIZE)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(first_vec_x2):
-	PCMPEQ	%xmm0, %xmm3
-	pmovmskb %xmm3, %eax
-	leal	-1(%rcx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(first_vec_x1_test)
-	bsrl	%eax, %eax
-	leaq	(VEC_SIZE * 2)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(aligned_more):
-	/* Save original pointer if match was in VEC 0.  */
-	movq	%rdi, %r8
-	andq	$-VEC_SIZE, %rdi
-
-	movaps	VEC_SIZE(%rdi), %xmm2
-	pxor	%xmm3, %xmm3
-	PCMPEQ	%xmm2, %xmm3
-	pmovmskb %xmm3, %ecx
-	testl	%ecx, %ecx
-	jnz	L(first_vec_x1)
-
-	movaps	(VEC_SIZE * 2)(%rdi), %xmm3
-	pxor	%xmm4, %xmm4
-	PCMPEQ	%xmm3, %xmm4
-	pmovmskb %xmm4, %ecx
-	testl	%ecx, %ecx
-	jnz	L(first_vec_x2)
-
-	addq	$VEC_SIZE, %rdi
-	/* Save pointer again before realigning.  */
-	movq	%rdi, %rsi
-	andq	$-(VEC_SIZE * 2), %rdi
-	.p2align 4
-L(first_loop):
-	/* Do 2x VEC at a time.  */
-	movaps	(VEC_SIZE * 2)(%rdi), %xmm4
-	movaps	(VEC_SIZE * 3)(%rdi), %xmm5
-	/* Since SSE2 no pminud so wcsrchr needs seperate logic for
-	   detecting zero. Note if this is found to be a bottleneck it
-	   may be worth adding an SSE4.1 wcsrchr implementation.  */
-#ifdef USE_AS_WCSRCHR
-	movaps	%xmm5, %xmm6
-	pxor	%xmm8, %xmm8
-
-	PCMPEQ	%xmm8, %xmm5
-	PCMPEQ	%xmm4, %xmm8
-	por	%xmm5, %xmm8
-#else
-	movaps	%xmm5, %xmm6
-	PMINU	%xmm4, %xmm5
-#endif
-
-	movaps	%xmm4, %xmm9
-	PCMPEQ	%xmm0, %xmm4
-	PCMPEQ	%xmm0, %xmm6
-	movaps	%xmm6, %xmm7
-	por	%xmm4, %xmm6
-#ifndef USE_AS_WCSRCHR
-	pxor	%xmm8, %xmm8
-	PCMPEQ	%xmm5, %xmm8
-#endif
-	pmovmskb %xmm8, %ecx
-	pmovmskb %xmm6, %eax
-
-	addq	$(VEC_SIZE * 2), %rdi
-	/* Use `addl` 1) so we can undo it with `subl` and 2) it can
-	   macro-fuse with `jz`.  */
-	addl	%ecx, %eax
-	jz	L(first_loop)
-
-	/* Check if there is zero match.  */
-	testl	%ecx, %ecx
-	jz	L(second_loop_match)
-
-	/* Check if there was a match in last iteration.  */
-	subl	%ecx, %eax
-	jnz	L(new_match)
-
-L(first_loop_old_match):
-	PCMPEQ	%xmm0, %xmm2
-	PCMPEQ	%xmm0, %xmm3
-	pmovmskb %xmm2, %ecx
-	pmovmskb %xmm3, %eax
-	addl	%eax, %ecx
-	jz	L(first_vec_x0_test)
-	/* NB: We could move this shift to before the branch and save a
-	   bit of code size / performance on the fall through. The
-	   branch leads to the null case which generally seems hotter
-	   than char in first 3x VEC.  */
-	sall	$16, %eax
-	orl	%ecx, %eax
-
-	bsrl	%eax, %eax
-	addq	%rsi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(new_match):
-	pxor	%xmm6, %xmm6
-	PCMPEQ	%xmm9, %xmm6
-	pmovmskb %xmm6, %eax
-	sall	$16, %ecx
-	orl	%eax, %ecx
-
-	/* We can't reuse either of the old comparisons as since we mask
-	   of zeros after first zero (instead of using the full
-	   comparison) we can't gurantee no interference between match
-	   after end of string and valid match.  */
-	pmovmskb %xmm4, %eax
-	pmovmskb %xmm7, %edx
-	sall	$16, %edx
-	orl	%edx, %eax
-
-	leal	-1(%ecx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(first_loop_old_match)
-	bsrl	%eax, %eax
-	addq	%rdi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	/* Save minimum state for getting most recent match. We can
-	   throw out all previous work.  */
-	.p2align 4
-L(second_loop_match):
-	movq	%rdi, %rsi
-	movaps	%xmm4, %xmm2
-	movaps	%xmm7, %xmm3
-
-	.p2align 4
-L(second_loop):
-	movaps	(VEC_SIZE * 2)(%rdi), %xmm4
-	movaps	(VEC_SIZE * 3)(%rdi), %xmm5
-	/* Since SSE2 no pminud so wcsrchr needs seperate logic for
-	   detecting zero. Note if this is found to be a bottleneck it
-	   may be worth adding an SSE4.1 wcsrchr implementation.  */
-#ifdef USE_AS_WCSRCHR
-	movaps	%xmm5, %xmm6
-	pxor	%xmm8, %xmm8
-
-	PCMPEQ	%xmm8, %xmm5
-	PCMPEQ	%xmm4, %xmm8
-	por	%xmm5, %xmm8
-#else
-	movaps	%xmm5, %xmm6
-	PMINU	%xmm4, %xmm5
-#endif
-
-	movaps	%xmm4, %xmm9
-	PCMPEQ	%xmm0, %xmm4
-	PCMPEQ	%xmm0, %xmm6
-	movaps	%xmm6, %xmm7
-	por	%xmm4, %xmm6
-#ifndef USE_AS_WCSRCHR
-	pxor	%xmm8, %xmm8
-	PCMPEQ	%xmm5, %xmm8
-#endif
-
-	pmovmskb %xmm8, %ecx
-	pmovmskb %xmm6, %eax
-
-	addq	$(VEC_SIZE * 2), %rdi
-	/* Either null term or new occurence of CHAR.  */
-	addl	%ecx, %eax
-	jz	L(second_loop)
-
-	/* No null term so much be new occurence of CHAR.  */
-	testl	%ecx, %ecx
-	jz	L(second_loop_match)
-
-
-	subl	%ecx, %eax
-	jnz	L(second_loop_new_match)
-
-L(second_loop_old_match):
-	pmovmskb %xmm2, %ecx
-	pmovmskb %xmm3, %eax
-	sall	$16, %eax
-	orl	%ecx, %eax
-	bsrl	%eax, %eax
-	addq	%rsi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(second_loop_new_match):
-	pxor	%xmm6, %xmm6
-	PCMPEQ	%xmm9, %xmm6
-	pmovmskb %xmm6, %eax
-	sall	$16, %ecx
-	orl	%eax, %ecx
-
-	/* We can't reuse either of the old comparisons as since we mask
-	   of zeros after first zero (instead of using the full
-	   comparison) we can't gurantee no interference between match
-	   after end of string and valid match.  */
-	pmovmskb %xmm4, %eax
-	pmovmskb %xmm7, %edx
-	sall	$16, %edx
-	orl	%edx, %eax
-
-	leal	-1(%ecx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(second_loop_old_match)
-	bsrl	%eax, %eax
-	addq	%rdi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4,, 4
-L(cross_page):
-	movq	%rdi, %rsi
-	andq	$-VEC_SIZE, %rsi
-	movaps	(%rsi), %xmm1
-	pxor	%xmm2, %xmm2
-	PCMPEQ	%xmm1, %xmm2
-	pmovmskb %xmm2, %edx
-	movl	%edi, %ecx
-	andl	$(VEC_SIZE - 1), %ecx
-	sarl	%cl, %edx
-	jz	L(cross_page_continue)
-	PCMPEQ	%xmm0, %xmm1
-	pmovmskb %xmm1, %eax
-	sarl	%cl, %eax
-	leal	-1(%rdx), %ecx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(ret1)
-	bsrl	%eax, %eax
-	addq	%rdi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-L(ret1):
-	ret
-END(STRRCHR)
-
-#ifndef USE_AS_WCSRCHR
-	weak_alias (STRRCHR, rindex)
-	libc_hidden_builtin_def (STRRCHR)
-#endif
+#define STRRCHR	strrchr
+#include "multiarch/strrchr-sse2.S"
+weak_alias (strrchr, rindex)
+libc_hidden_builtin_def (strrchr)
diff --git a/sysdeps/x86_64/wcsrchr.S b/sysdeps/x86_64/wcsrchr.S
index 2b80efc5ef..1d4b1eb21c 100644
--- a/sysdeps/x86_64/wcsrchr.S
+++ b/sysdeps/x86_64/wcsrchr.S
@@ -16,12 +16,5 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
-
-#define USE_AS_WCSRCHR	1
-#define NO_PMINU	1
-
-#ifndef STRRCHR
-# define STRRCHR	wcsrchr
-#endif
-
-#include "../strrchr.S"
+#define STRRCHR	wcsrchr
+#include "multiarch/wcsrchr-sse2.S"