about summary refs log tree commit diff
path: root/sysdeps/aarch64/strrchr.S
blob: dc306432fa7aa55e4cc99dc611d248d6edca1f58 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
/* strrchr: find the last instance of a character in a string.

   Copyright (C) 2014-2021 Free Software Foundation, Inc.

   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library.  If not, see
   <https://www.gnu.org/licenses/>.  */

#include <sysdep.h>

/* Assumptions:
 *
 * ARMv8-a, AArch64
 * Neon Available.
 * MTE compatible.
 */

/* Arguments and results.  */
#define srcin		x0
#define chrin		w1
#define result		x0

#define src		x2
#define tmp		x3
#define wtmp		w3
#define synd		x3
#define shift		x4
#define src_match	x4
#define nul_match	x5
#define chr_match	x6

#define vrepchr		v0
#define vdata		v1
#define vhas_nul	v2
#define vhas_chr	v3
#define vrepmask	v4
#define vrepmask2	v5
#define vend		v5
#define dend		d5

/* Core algorithm.

   For each 16-byte chunk we calculate a 64-bit syndrome value, with
   four bits per byte (LSB is always in bits 0 and 1, for both big
   and little-endian systems).  For each tuple, bits 0-1 are set if
   the relevant byte matched the requested character; bits 2-3 are set
   if the relevant byte matched the NUL end of string.  */

ENTRY(strrchr)
	PTR_ARG (0)
	bic	src, srcin, 15
	dup	vrepchr.16b, chrin
	mov	wtmp, 0x3003
	dup	vrepmask.8h, wtmp
	tst	srcin, 15
	beq	L(loop1)

	ld1	{vdata.16b}, [src], 16
	cmeq	vhas_nul.16b, vdata.16b, 0
	cmeq	vhas_chr.16b, vdata.16b, vrepchr.16b
	mov	wtmp, 0xf00f
	dup	vrepmask2.8h, wtmp
	bit	vhas_nul.16b, vhas_chr.16b, vrepmask.16b
	and	vhas_nul.16b, vhas_nul.16b, vrepmask2.16b
	addp	vend.16b, vhas_nul.16b, vhas_nul.16b
	lsl	shift, srcin, 2
	fmov	synd, dend
	lsr	synd, synd, shift
	lsl	synd, synd, shift
	ands	nul_match, synd, 0xcccccccccccccccc
	bne	L(tail)
	cbnz	synd, L(loop2)

	.p2align 5
L(loop1):
	ld1	{vdata.16b}, [src], 16
	cmeq	vhas_chr.16b, vdata.16b, vrepchr.16b
	cmhs	vhas_nul.16b, vhas_chr.16b, vdata.16b
	umaxp	vend.16b, vhas_nul.16b, vhas_nul.16b
	fmov	synd, dend
	cbz	synd, L(loop1)

	cmeq	vhas_nul.16b, vdata.16b, 0
	bit	vhas_nul.16b, vhas_chr.16b, vrepmask.16b
	bic	vhas_nul.8h, 0x0f, lsl 8
	addp	vend.16b, vhas_nul.16b, vhas_nul.16b
	fmov	synd, dend
	ands	nul_match, synd, 0xcccccccccccccccc
	beq	L(loop2)

L(tail):
	sub	nul_match, nul_match, 1
	and	chr_match, synd, 0x3333333333333333
	ands	chr_match, chr_match, nul_match
	sub	result, src, 1
	clz	tmp, chr_match
	sub	result, result, tmp, lsr 2
	csel	result, result, xzr, ne
	ret

	.p2align 4
L(loop2):
	cmp	synd, 0
	csel	src_match, src, src_match, ne
	csel	chr_match, synd, chr_match, ne
	ld1	{vdata.16b}, [src], 16
	cmeq	vhas_nul.16b, vdata.16b, 0
	cmeq	vhas_chr.16b, vdata.16b, vrepchr.16b
	bit	vhas_nul.16b, vhas_chr.16b, vrepmask.16b
	umaxp	vend.16b, vhas_nul.16b, vhas_nul.16b
	fmov	synd, dend
	tst	synd, 0xcccccccccccccccc
	beq	L(loop2)

	bic	vhas_nul.8h, 0x0f, lsl 8
	addp	vend.16b, vhas_nul.16b, vhas_nul.16b
	fmov	synd, dend
	and	nul_match, synd, 0xcccccccccccccccc
	sub	nul_match, nul_match, 1
	and	tmp, synd, 0x3333333333333333
	ands	tmp, tmp, nul_match
	csel	chr_match, tmp, chr_match, ne
	csel	src_match, src, src_match, ne
	sub	src_match, src_match, 1
	clz	tmp, chr_match
	sub	result, src_match, tmp, lsr 2
	ret

END(strrchr)
weak_alias (strrchr, rindex)
libc_hidden_builtin_def (strrchr)