about summary refs log tree commit diff
path: root/sysdeps/aarch64/multiarch/memchr_nosimd.S
blob: 0080f53931af53f6c2b4f44ad9d97af14b2a68fe (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
/* memchr - find a character in a memory zone using base integer registers

   Copyright (C) 2018-2021 Free Software Foundation, Inc.

   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library.  If not, see
   <https://www.gnu.org/licenses/>.  */

#include <sysdep.h>

/* Assumptions:
 *
 * ARMv8-a, AArch64
 * Use base integer registers.
 */

#ifndef MEMCHR
# define MEMCHR __memchr_nosimd
#endif

/* Arguments and results.  */
#define srcin		x0
#define chrin		x1
#define cntin		x2

#define result		x0

#define repchr		x1

#define tmp1		x2
#define tmp2		x3
#define tmp3		x4
#define tmp4		x5

#define src		x6
#define srcend		x7
#define srcend16	x8

#define anymore		x9

#define zeroones	x10

#define data1		x11
#define data2		x12

#define has_chr1	x13
#define has_chr2	x14

#define REP8_01		0x0101010101010101
#define REP8_7f		0x7f7f7f7f7f7f7f7f


ENTRY_ALIGN (MEMCHR, 6)

	PTR_ARG (0)
	SIZE_ARG (2)

	/* Do not dereference srcin if no bytes to compare. */
	cbz	cntin, L(none_chr)

	/* Start address is 16-byte aligned or not? */
	tst	srcin, 15
	bic	src, srcin, 15

	mov	zeroones, REP8_01
	and	repchr, chrin, 255
	/* Generate a qword integer as |c|c|c|c|c|c|c|c|. */
	mul	repchr, repchr, zeroones

	add	srcend, srcin, cntin
	/*
	 * srcend16 is address of the block following the last block.
	 *
	 * [A block is 16-byte aligned and sized.]
	 */
	add	srcend16, srcend, 15
	bic	srcend16, srcend16, 15

	b.eq	L(loop)

	/* Load the first block containing start address. */
	ldp	data1, data2, [src], 16

	lsl	tmp1, srcin, 3
	mov	tmp2, ~0
#ifdef __AARCH64EB__
	lsr	tmp3, tmp2, tmp1
#else
	lsl	tmp3, tmp2, tmp1
#endif
	/* Start address is in the first or the second qword? */
	tst	srcin, 8

	/*
	 * Transform any byte in the block to zero using XOR operation,
	 * if that byte equals the char to search. In this way, searching
	 * the char becomes detecting zero in the resulting two qwords.
	 */
	eor	data1, data1, repchr
	eor	data2, data2, repchr

	/*
	 * Set those unused bytes(before start address) to 0xff, so
	 * that they will not hit any zero detection.
	 */
	orn	tmp1, data1, tmp3
	orn	tmp2, data2, tmp3

	csinv	data1, tmp1, xzr, eq
	csel	data2, data2, tmp2, eq

	/*
	 * When the first and last block are the same, there are two cases:
	 *  o. Memory range to search is just in one block.
	 *      ( start address - end address) < 0
	 *
	 *  o. Memory range is so large that end address wrap-around.
	 *      ( start address - end address) > 0
	 */
	cmp	srcin, srcend
	ccmp	src, srcend16, 0, mi
	csetm	anymore, ne
	b	L(find_chr)

	.p2align 4
L(loop):
	ldp	data1, data2, [src], 16

	subs	anymore, src, srcend16

	/*
	 * Transform any byte in the block to zero using XOR operation,
	 * if that byte equals the char to search.
	 */
	eor	data1, data1, repchr
	eor	data2, data2, repchr

L(find_chr):
	/*
	 * Use the following integer test to find out if any byte in a
	 * qword is zero. If do not contain zero-valued byte, test result
	 * is zero.
	 *
	 *  (qword - 0x0101010101010101) & ~(qword) & 0x8080808080808080
	 * =
	 *  (qword - 0x0101010101010101) & ~(qword  | 0x7f7f7f7f7f7f7f7f)
	 *
	 */
	sub	tmp1, data1, zeroones
	sub	tmp2, data2, zeroones

	orr	tmp3, data1, REP8_7f
	orr	tmp4, data2, REP8_7f

	bic	has_chr1, tmp1, tmp3
	bic	has_chr2, tmp2, tmp4

	orr	tmp1, has_chr1, has_chr2
	ccmp	tmp1, 0, 0, ne

	b.eq	L(loop)

	cbz	has_chr1, 1f
	sub	result, src, 16
#ifdef __AARCH64EB__
	rev	data1, data1
#else
	rev	has_chr1, has_chr1
#endif
	b	L(done)

1:	cbz	has_chr2, L(none_chr)
	sub	result, src, 8
#ifdef __AARCH64EB__
	rev	data1, data2
#else
	rev	has_chr1, has_chr2
#endif

L(done):
#ifdef __AARCH64EB__
	/*
	 * For big-endian, can not directly use has_chr1/has_chr2 because
	 * two qwords has been reversed after loading from memory.
	 * Thus, have to perform char detection on two qwords again, which
	 * should be byte-swapped this time.
	 */
	sub	tmp1, data1, zeroones
	orr	tmp3, data1, REP8_7f
	bic	has_chr1, tmp1, tmp3
	rev	has_chr1, has_chr1
#endif

	/*
	 * If the specified char is found in a qword, the corresponding
	 * byte of in has_chr has value of 1, while this is only true for
	 * the first occurrence, not other occurrences.
	 */
	cmp	anymore, 0
	clz	tmp1, has_chr1
	add	result, result, tmp1, lsr 3
	ccmp	result, srcend, 8, eq	/* NZCV = 8000 */
	csel	result, result, xzr, mi
	ret

L(none_chr):
	mov	result, 0
	ret

END (MEMCHR)
libc_hidden_builtin_def (MEMCHR)