about summary refs log tree commit diff
path: root/sysdeps
diff options
context:
space:
mode:
authorWilco Dijkstra <wilco.dijkstra@arm.com>2023-01-11 13:51:17 +0000
committerWilco Dijkstra <wilco.dijkstra@arm.com>2024-04-08 16:42:14 +0100
commit1296aa9b0b145b8a4c82ece6c6ea2f4afa87d21b (patch)
treecd725f8319f60cddee486eb2f071f69aa0f3a966 /sysdeps
parent34424b6770b25b09ace2849f34499845aa65b155 (diff)
downloadglibc-1296aa9b0b145b8a4c82ece6c6ea2f4afa87d21b.tar.gz
glibc-1296aa9b0b145b8a4c82ece6c6ea2f4afa87d21b.tar.xz
glibc-1296aa9b0b145b8a4c82ece6c6ea2f4afa87d21b.zip
AArch64: Optimize memrchr
Optimize the main loop - large strings are 43% faster on modern CPUs.

Reviewed-by: Szabolcs Nagy <szabolcs.nagy@arm.com>
(cherry picked from commit 00776241776e67fc666b896c1e85770f4f3ec1e1)
Diffstat (limited to 'sysdeps')
-rw-r--r--sysdeps/aarch64/memrchr.S20
1 files changed, 11 insertions, 9 deletions
diff --git a/sysdeps/aarch64/memrchr.S b/sysdeps/aarch64/memrchr.S
index 5179320720..428af51f70 100644
--- a/sysdeps/aarch64/memrchr.S
+++ b/sysdeps/aarch64/memrchr.S
@@ -26,7 +26,6 @@
  * MTE compatible.
  */
 
-/* Arguments and results.  */
 #define srcin		x0
 #define chrin		w1
 #define cntin		x2
@@ -77,31 +76,34 @@ ENTRY (__memrchr)
 	csel	result, result, xzr, hi
 	ret
 
+	nop
 L(start_loop):
-	sub	tmp, end, src
-	subs	cntrem, cntin, tmp
+	subs	cntrem, src, srcin
 	b.ls	L(nomatch)
 
 	/* Make sure that it won't overread by a 16-byte chunk */
-	add	tmp, cntrem, 15
-	tbnz	tmp, 4, L(loop32_2)
+	sub	cntrem, cntrem, 1
+	tbz	cntrem, 4, L(loop32_2)
+	add	src, src, 16
 
-	.p2align 4
+	.p2align 5
 L(loop32):
-	ldr	qdata, [src, -16]!
+	ldr	qdata, [src, -32]!
 	cmeq	vhas_chr.16b, vdata.16b, vrepchr.16b
 	umaxp	vend.16b, vhas_chr.16b, vhas_chr.16b		/* 128->64 */
 	fmov	synd, dend
 	cbnz	synd, L(end)
 
 L(loop32_2):
-	ldr	qdata, [src, -16]!
+	ldr	qdata, [src, -16]
 	subs	cntrem, cntrem, 32
 	cmeq	vhas_chr.16b, vdata.16b, vrepchr.16b
-	b.ls	L(end)
+	b.lo	L(end_2)
 	umaxp	vend.16b, vhas_chr.16b, vhas_chr.16b		/* 128->64 */
 	fmov	synd, dend
 	cbz	synd, L(loop32)
+L(end_2):
+	sub	src, src, 16
 L(end):
 	shrn	vend.8b, vhas_chr.8h, 4		/* 128->64 */
 	fmov	synd, dend