about summary refs log tree commit diff
path: root/sysdeps/x86_64/multiarch/memchr-evex.S
blob: 04c336988a6936b58bf9bd481a40bd1c49042447 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
/* memchr/wmemchr optimized with 256-bit EVEX instructions.
   Copyright (C) 2021-2024 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

#include <isa-level.h>
#include <sysdep.h>

#if ISA_SHOULD_BUILD (4)

# ifndef VEC_SIZE
#  include "x86-evex256-vecs.h"
# endif

# ifndef MEMCHR
#  define MEMCHR	__memchr_evex
# endif

# ifdef USE_AS_WMEMCHR
#  define PC_SHIFT_GPR	rcx
#  define VPTESTN	vptestnmd
#  define VPBROADCAST	vpbroadcastd
#  define VPMINU	vpminud
#  define VPCMP	vpcmpd
#  define VPCMPEQ	vpcmpeqd
#  define CHAR_SIZE	4

#  define USE_WIDE_CHAR
# else
#  define PC_SHIFT_GPR	rdi
#  define VPTESTN	vptestnmb
#  define VPBROADCAST	vpbroadcastb
#  define VPMINU	vpminub
#  define VPCMP	vpcmpb
#  define VPCMPEQ	vpcmpeqb
#  define CHAR_SIZE	1
# endif

# include "reg-macros.h"


/* If not in an RTM and VEC_SIZE != 64 (the VEC_SIZE = 64
   doesn't have VEX encoding), use VEX encoding in loop so we
   can use vpcmpeqb + vptern which is more efficient than the
   EVEX alternative.  */
# if defined USE_IN_RTM || VEC_SIZE == 64
#  undef COND_VZEROUPPER
#  undef VZEROUPPER_RETURN
#  undef VZEROUPPER

#  define COND_VZEROUPPER
#  define VZEROUPPER_RETURN	ret
#  define VZEROUPPER

#  define USE_TERN_IN_LOOP	0
# else
#  define USE_TERN_IN_LOOP	1
#  undef VZEROUPPER
#  define VZEROUPPER	vzeroupper
# endif

# if USE_TERN_IN_LOOP
	/* Resulting bitmask for vpmovmskb has 4-bits set for each wchar
	   so we don't want to multiply resulting index.  */
#  define TERN_CHAR_MULT	1

#  ifdef USE_AS_WMEMCHR
#   define TEST_END()	inc %VRCX
#  else
#   define TEST_END()	add %rdx, %rcx
#  endif
# else
#  define TERN_CHAR_MULT	CHAR_SIZE
#  define TEST_END()	KORTEST %k2, %k3
# endif

# if defined USE_AS_WMEMCHR || !USE_TERN_IN_LOOP
#  ifndef USE_AS_WMEMCHR
#   define GPR_X0_IS_RET	1
#  else
#   define GPR_X0_IS_RET	0
#  endif
#  define GPR_X0	rax
# else
#  define GPR_X0_IS_RET	0
#  define GPR_X0	rdx
# endif

# define CHAR_PER_VEC	(VEC_SIZE / CHAR_SIZE)

# if CHAR_PER_VEC == 64
#  define LAST_VEC_OFFSET	(VEC_SIZE * 3)
# else
#  define LAST_VEC_OFFSET	(VEC_SIZE * 2)
# endif
# if CHAR_PER_VEC >= 32
#  define MASK_GPR(...)	VGPR(__VA_ARGS__)
# elif CHAR_PER_VEC == 16
#  define MASK_GPR(reg)	VGPR_SZ(reg, 16)
# else
#  define MASK_GPR(reg)	VGPR_SZ(reg, 8)
# endif

# define VMATCH	VMM(0)
# define VMATCH_LO	VMM_lo(0)

# define PAGE_SIZE	4096


	.section SECTION(.text), "ax", @progbits
ENTRY_P2ALIGN (MEMCHR, 6)
	/* Check for zero length.  */
	test	%RDX_LP, %RDX_LP
	jz	L(zero_0)

# ifdef __ILP32__
	/* Clear the upper 32 bits.  */
	movl	%edx, %edx
# endif
	VPBROADCAST %esi, %VMATCH
	/* Check if we may cross page boundary with one vector load.  */
	movl	%edi, %eax
	andl	$(PAGE_SIZE - 1), %eax
	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
	ja	L(page_cross)

	VPCMPEQ	(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRAX
# ifndef USE_AS_WMEMCHR
	/* If rcx is zero then tzcnt -> CHAR_PER_VEC.  NB: there is a
	   already a dependency between rcx and rsi so no worries about
	   false-dep here.  */
	tzcnt	%VRAX, %VRSI
	/* If rdx <= rsi then either 1) rcx was non-zero (there was a
	   match) but it was out of bounds or 2) rcx was zero and rdx
	   was <= VEC_SIZE so we are done scanning.  */
	cmpq	%rsi, %rdx
	/* NB: Use branch to return zero/non-zero.  Common usage will
	   branch on result of function (if return is null/non-null).
	   This branch can be used to predict the ensuing one so there
	   is no reason to extend the data-dependency with cmovcc.  */
	jbe	L(zero_0)

	/* If rcx is zero then len must be > RDX, otherwise since we
	   already tested len vs lzcnt(rcx) (in rsi) we are good to
	   return this match.  */
	test	%VRAX, %VRAX
	jz	L(more_1x_vec)
	leaq	(%rdi, %rsi), %rax
# else

	/* We can't use the `tzcnt` trick for wmemchr because CHAR_SIZE
	   > 1 so if rcx is tzcnt != CHAR_PER_VEC.  */
	cmpq	$CHAR_PER_VEC, %rdx
	ja	L(more_1x_vec)
	tzcnt	%VRAX, %VRAX
	cmpl	%eax, %edx
	jbe	L(zero_0)
L(first_vec_x0_ret):
	leaq	(%rdi, %rax, CHAR_SIZE), %rax
# endif
	ret

	/* Only fits in first cache line for VEC_SIZE == 32.  */
# if VEC_SIZE == 32
	.p2align 4,, 2
L(zero_0):
	xorl	%eax, %eax
	ret
# endif

	.p2align 4,, 9
L(more_1x_vec):
# ifdef USE_AS_WMEMCHR
	/* If wmemchr still need to test if there was a match in first
	   VEC.  Use bsf to test here so we can reuse
	   L(first_vec_x0_ret).  */
	bsf	%VRAX, %VRAX
	jnz	L(first_vec_x0_ret)
# endif

L(page_cross_continue):
# ifdef USE_AS_WMEMCHR
	/* We can't use end of the buffer to re-calculate length for
	   wmemchr as len * CHAR_SIZE may overflow.  */
	leaq	-(VEC_SIZE + CHAR_SIZE)(%rdi), %rax
	andq	$(VEC_SIZE * -1), %rdi
	subq	%rdi, %rax
	sarq	$2, %rax
	addq	%rdx, %rax
# else
	leaq	-(VEC_SIZE + 1)(%rdx, %rdi), %rax
	andq	$(VEC_SIZE * -1), %rdi
	subq	%rdi, %rax
# endif

	/* rax contains remaining length - 1.  -1 so we can get imm8
	   encoding in a few additional places saving code size.  */

	/* Needed regardless of remaining length.  */
	VPCMPEQ	VEC_SIZE(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRDX

	/* We cannot fold the above `sub %rdi, %rax` with the `cmp
	   $(CHAR_PER_VEC * 2), %rax` because its possible for a very
	   large length to overflow and cause the subtract to carry
	   despite length being above CHAR_PER_VEC * 2.  */
	cmpq	$(CHAR_PER_VEC * 2 - 1), %rax
	ja	L(more_2x_vec)
L(last_2x_vec):

	test	%VRDX, %VRDX
	jnz	L(first_vec_x1_check)

	/* Check the end of data.  NB: use 8-bit operations to save code
	   size.  We no longer need the full-width of eax and will
	   perform a write-only operation over eax so there will be no
	   partial-register stalls.  */
	subb	$(CHAR_PER_VEC * 1 - 1), %al
	jle	L(zero_0)

	VPCMPEQ	(VEC_SIZE * 2)(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRCX
# ifdef USE_AS_WMEMCHR
	/* For wmemchr against we can't take advantage of tzcnt(0) ==
	   VEC_SIZE as CHAR_PER_VEC != VEC_SIZE.  */
	test	%VRCX, %VRCX
	jz	L(zero_0)
# endif
	tzcnt	%VRCX, %VRCX
	cmp	%cl, %al

	/* Same CFG for VEC_SIZE == 64 and VEC_SIZE == 32.  We give
	   fallthrough to L(zero_0) for VEC_SIZE == 64 here as there is
	   not enough space before the next cache line to fit the `lea`
	   for return.  */
# if VEC_SIZE == 64
	ja	L(first_vec_x2_ret)
L(zero_0):
	xorl	%eax, %eax
	ret
# else
	jbe	L(zero_0)
	leaq	(VEC_SIZE * 2)(%rdi, %rcx, CHAR_SIZE), %rax
	ret
# endif

	.p2align 4,, 5
L(first_vec_x1_check):
	bsf	%VRDX, %VRDX
	cmpb	%dl, %al
	jb	L(zero_4)
	leaq	(VEC_SIZE * 1)(%rdi, %rdx, CHAR_SIZE), %rax
	ret

	/* Fits at the end of the cache line here for VEC_SIZE == 32.
	 */
# if VEC_SIZE == 32
L(zero_4):
	xorl	%eax, %eax
	ret
# endif


	.p2align 4,, 4
L(first_vec_x2):
	bsf	%VRCX, %VRCX
L(first_vec_x2_ret):
	leaq	(VEC_SIZE * 2)(%rdi, %rcx, CHAR_SIZE), %rax
	ret

	/* Fits at the end of the cache line here for VEC_SIZE == 64.
	 */
# if VEC_SIZE == 64
L(zero_4):
	xorl	%eax, %eax
	ret
# endif

	.p2align 4,, 4
L(first_vec_x1):
	bsf	%VRDX, %VRDX
	leaq	(VEC_SIZE * 1)(%rdi, %rdx, CHAR_SIZE), %rax
	ret


	.p2align 4,, 5
L(more_2x_vec):
	/* Length > VEC_SIZE * 2 so check first 2x VEC before rechecking
	   length.  */


	/* Already computed matches for first VEC in rdx.  */
	test	%VRDX, %VRDX
	jnz	L(first_vec_x1)


	VPCMPEQ	(VEC_SIZE * 2)(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRCX
	test	%VRCX, %VRCX
	jnz	L(first_vec_x2)

	/* Needed regardless of next length check.  */
	VPCMPEQ	(VEC_SIZE * 3)(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRCX

	/* Check if we are near the end.  */
	cmpq	$(CHAR_PER_VEC * 4 - 1), %rax
	ja	L(more_4x_vec)

	test	%VRCX, %VRCX
	jnz	L(first_vec_x3_check)

	/* Use 8-bit instructions to save code size.  We won't use full-
	   width eax again and will perform a write-only operation to
	   eax so no worries about partial-register stalls.  */
	subb	$(CHAR_PER_VEC * 3), %al
	jb	L(zero_2)
L(last_vec_check):
	VPCMPEQ	(VEC_SIZE * 4)(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRCX
# ifdef USE_AS_WMEMCHR
	/* For wmemchr against we can't take advantage of tzcnt(0) ==
	   VEC_SIZE as CHAR_PER_VEC != VEC_SIZE.  */
	test	%VRCX, %VRCX
	jz	L(zero_2)
# endif
	tzcnt	%VRCX, %VRCX
	cmp	%cl, %al
	jae	L(first_vec_x4_ret)
L(zero_2):
	xorl	%eax, %eax
	ret

	/* Fits at the end of the cache line here for VEC_SIZE == 64.
	   For VEC_SIZE == 32 we put the return label at the end of
	   L(first_vec_x4).  */
# if VEC_SIZE == 64
L(first_vec_x4_ret):
	leaq	(VEC_SIZE * 4)(%rdi, %rcx, CHAR_SIZE), %rax
	ret
# endif

	.p2align 4,, 6
L(first_vec_x4):
	bsf	%VRCX, %VRCX
# if VEC_SIZE == 32
	/* Place L(first_vec_x4_ret) here as we can't fit it in the same
	   cache line as where it is called from so we might as well
	   save code size by reusing return of L(first_vec_x4).  */
L(first_vec_x4_ret):
# endif
	leaq	(VEC_SIZE * 4)(%rdi, %rcx, CHAR_SIZE), %rax
	ret

	.p2align 4,, 6
L(first_vec_x3_check):
	/* Need to adjust remaining length before checking.  */
	addb	$-(CHAR_PER_VEC * 2), %al
	bsf	%VRCX, %VRCX
	cmpb	%cl, %al
	jb	L(zero_2)
	leaq	(VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %rax
	ret

	.p2align 4,, 6
L(first_vec_x3):
	bsf	%VRCX, %VRCX
	leaq	(VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %rax
	ret

	.p2align 4,, 3
# if !USE_TERN_IN_LOOP
	.p2align 4,, 10
# endif
L(more_4x_vec):
	test	%VRCX, %VRCX
	jnz	L(first_vec_x3)

	VPCMPEQ	(VEC_SIZE * 4)(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRCX
	test	%VRCX, %VRCX
	jnz	L(first_vec_x4)

	subq	$-(VEC_SIZE * 5), %rdi
	subq	$(CHAR_PER_VEC * 8), %rax
	jb	L(last_4x_vec)

# ifdef USE_AS_WMEMCHR
	movl	%edi, %ecx
# else
	addq	%rdi, %rax
# endif


# if VEC_SIZE == 64
	/* use xorb to do `andq $-(VEC_SIZE * 4), %rdi`. No evex
	   processor has partial register stalls (all have merging
	   uop). If that changes this can be removed.  */
	xorb	%dil, %dil
# else
	andq	$-(VEC_SIZE * 4), %rdi
# endif

# ifdef USE_AS_WMEMCHR
	subl	%edi, %ecx
	sarl	$2, %ecx
	addq	%rcx, %rax
# else
	subq	%rdi, %rax
# endif



# if USE_TERN_IN_LOOP
	/* copy VMATCH to low ymm so we can use vpcmpeq which is not
	   encodable with EVEX registers.  NB: this is VEC_SIZE == 32
	   only as there is no way to encode vpcmpeq with zmm0-15.  */
	vmovdqa64 %VMATCH, %VMATCH_LO
# endif

	.p2align 4,, 11
L(loop_4x_vec):
	/* Two versions of the loop.  One that does not require
	   vzeroupper by not using ymmm0-15 and another does that
	   require vzeroupper because it uses ymmm0-15.  The reason why
	   ymm0-15 is used at all is because there is no EVEX encoding
	   vpcmpeq and with vpcmpeq this loop can be performed more
	   efficiently.  The non-vzeroupper version is safe for RTM
	   while the vzeroupper version should be preferred if RTM are
	   not supported.   Which loop version we use is determined by
	   USE_TERN_IN_LOOP.  */

# if USE_TERN_IN_LOOP
	/* Since vptern can only take 3x vectors fastest to do 1 vec
	   separately with EVEX vpcmp.  */
#  ifdef USE_AS_WMEMCHR
	/* vptern can only accept masks for epi32/epi64 so can only save
	   instruction using not equals mask on vptern with wmemchr.
	 */
	VPCMP	$4, (VEC_SIZE * 0)(%rdi), %VMATCH, %k1
#  else
	VPCMPEQ	(VEC_SIZE * 0)(%rdi), %VMATCH, %k1
#  endif
	/* Compare 3x with vpcmpeq and or them all together with vptern.
	 */
	VPCMPEQ	(VEC_SIZE * 1)(%rdi), %VMATCH_LO, %VMM_lo(2)
	VPCMPEQ	(VEC_SIZE * 2)(%rdi), %VMATCH_LO, %VMM_lo(3)
	VPCMPEQ	(VEC_SIZE * 3)(%rdi), %VMATCH_LO, %VMM_lo(4)
#  ifdef USE_AS_WMEMCHR
	/* This takes the not of or between VEC_lo(2), VEC_lo(3),
	   VEC_lo(4) as well as combines result from VEC(0) with zero
	   mask.  */
	vpternlogd $1, %VMM_lo(2), %VMM_lo(3), %VMM_lo(4){%k1}{z}
	vpmovmskb %VMM_lo(4), %VRCX
#  else
	/* 254 is mask for oring VEC_lo(2), VEC_lo(3), VEC_lo(4) into
	   VEC_lo(4).  */
	vpternlogd $254, %VMM_lo(2), %VMM_lo(3), %VMM_lo(4)
	vpmovmskb %VMM_lo(4), %VRCX
	KMOV	%k1, %edx
#  endif

# else
	/* Loop version that uses EVEX encoding.  */
	VPCMP	$4, (VEC_SIZE * 0)(%rdi), %VMATCH, %k1
	vpxorq	(VEC_SIZE * 1)(%rdi), %VMATCH, %VMM(2)
	vpxorq	(VEC_SIZE * 2)(%rdi), %VMATCH, %VMM(3)
	VPCMPEQ	(VEC_SIZE * 3)(%rdi), %VMATCH, %k3
	VPMINU	%VMM(2), %VMM(3), %VMM(3){%k1}{z}
	VPTESTN	%VMM(3), %VMM(3), %k2
# endif


	TEST_END ()
	jnz	L(loop_vec_ret)

	subq	$-(VEC_SIZE * 4), %rdi

	subq	$(CHAR_PER_VEC * 4), %rax
	jae	L(loop_4x_vec)

	/* COND_VZEROUPPER is vzeroupper if we use the VEX encoded loop.
	 */
	COND_VZEROUPPER

	.p2align 4,, 10
L(last_4x_vec):
	/* For CHAR_PER_VEC == 64 we don't need to mask as we use 8-bit
	   instructions on eax from here on out.  */
# if CHAR_PER_VEC != 64
	andl	$(CHAR_PER_VEC * 4 - 1), %eax
# endif
	VPCMPEQ	(VEC_SIZE * 0)(%rdi), %VMATCH, %k0
	subq	$(VEC_SIZE * 1), %rdi
	KMOV	%k0, %VRDX
	cmpb	$(CHAR_PER_VEC * 2 - 1), %al
	jbe	L(last_2x_vec)
	test	%VRDX, %VRDX
	jnz	L(last_vec_x1_novzero)

	VPCMPEQ	(VEC_SIZE * 2)(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRDX
	test	%VRDX, %VRDX
	jnz	L(last_vec_x2_novzero)

	VPCMPEQ	(VEC_SIZE * 3)(%rdi), %VMATCH, %k0
	KMOV	%k0, %VRCX
	test	%VRCX, %VRCX
	jnz	L(first_vec_x3_check)

	subb	$(CHAR_PER_VEC * 3), %al
	jae	L(last_vec_check)

	xorl	%eax, %eax
	ret

# if defined USE_AS_WMEMCHR && USE_TERN_IN_LOOP
L(last_vec_x2_novzero):
	addq	$VEC_SIZE, %rdi
L(last_vec_x1_novzero):
	bsf	%VRDX, %VRDX
	leaq	(VEC_SIZE * 1)(%rdi, %rdx, CHAR_SIZE), %rax
	ret
# endif

# if CHAR_PER_VEC == 64
	/* Since we can't combine the last 2x VEC when CHAR_PER_VEC ==
	   64 it needs a separate return label.  */
	.p2align 4,, 4
L(last_vec_x2):
L(last_vec_x2_novzero):
	bsf	%VRDX, %VRDX
	leaq	(VEC_SIZE * 2)(%rdi, %rdx, TERN_CHAR_MULT), %rax
	ret
# endif

	.p2align 4,, 4
L(loop_vec_ret):
# if defined USE_AS_WMEMCHR || !USE_TERN_IN_LOOP
	KMOV	%k1, %VRAX
	inc	%MASK_GPR(rax)
# else
	test	%VRDX, %VRDX
# endif
	jnz	L(last_vec_x0)


# if USE_TERN_IN_LOOP
	vpmovmskb %VMM_lo(2), %VRDX
# else
	VPTESTN	%VMM(2), %VMM(2), %k1
	KMOV	%k1, %VRDX
# endif
	test	%VRDX, %VRDX
	jnz	L(last_vec_x1)


# if USE_TERN_IN_LOOP
	vpmovmskb %VMM_lo(3), %VRDX
# else
	KMOV	%k2, %VRDX
# endif

	/* No longer need any of the lo vecs (ymm0-15) so vzeroupper
	   (only if used VEX encoded loop).  */
	COND_VZEROUPPER

	/* Separate logic for CHAR_PER_VEC == 64 vs the rest.  For
	   CHAR_PER_VEC we test the last 2x VEC separately, for
	   CHAR_PER_VEC <= 32 we can combine the results from the 2x
	   VEC in a single GPR.  */
# if CHAR_PER_VEC == 64
#  if USE_TERN_IN_LOOP
#   error "Unsupported"
#  endif


	/* If CHAR_PER_VEC == 64 we can't combine the last two VEC.  */
	test	%VRDX, %VRDX
	jnz	L(last_vec_x2)
	KMOV	%k3, %VRDX
# else
	/* CHAR_PER_VEC <= 32 so we can combine the results from the
	   last 2x VEC.  */

#  if !USE_TERN_IN_LOOP
	KMOV	%k3, %VRCX
#  endif
	salq	$(VEC_SIZE / TERN_CHAR_MULT), %rcx
	addq	%rcx, %rdx
#  if !defined USE_AS_WMEMCHR || !USE_TERN_IN_LOOP
L(last_vec_x2_novzero):
#  endif
# endif
	bsf	%rdx, %rdx
	leaq	(LAST_VEC_OFFSET)(%rdi, %rdx, TERN_CHAR_MULT), %rax
	ret

	.p2align 4,, 8
L(last_vec_x1):
	COND_VZEROUPPER
# if !defined USE_AS_WMEMCHR || !USE_TERN_IN_LOOP
L(last_vec_x1_novzero):
# endif
	bsf	%VRDX, %VRDX
	leaq	(VEC_SIZE * 1)(%rdi, %rdx, TERN_CHAR_MULT), %rax
	ret


	.p2align 4,, 4
L(last_vec_x0):
	COND_VZEROUPPER
	bsf	%VGPR(GPR_X0), %VGPR(GPR_X0)
# if GPR_X0_IS_RET
	addq	%rdi, %rax
# else
	leaq	(%rdi, %GPR_X0, CHAR_SIZE), %rax
# endif
	ret

	.p2align 4,, 6
L(page_cross):
	/* Need to preserve eax to compute inbound bytes we are
	   checking.  */
# ifdef USE_AS_WMEMCHR
	movl	%eax, %ecx
# else
	xorl	%ecx, %ecx
	subl	%eax, %ecx
# endif

	xorq	%rdi, %rax
	VPCMPEQ	(PAGE_SIZE - VEC_SIZE)(%rax), %VMATCH, %k0
	KMOV	%k0, %VRAX

# ifdef USE_AS_WMEMCHR
	/* NB: Divide by CHAR_SIZE to shift out out of bounds bytes.  */
	shrl	$2, %ecx
	andl	$(CHAR_PER_VEC - 1), %ecx
# endif


	shrx	%VGPR(PC_SHIFT_GPR), %VRAX, %VRAX

# ifdef USE_AS_WMEMCHR
	negl	%ecx
# endif

	/* mask lower bits from ecx (negative eax) to get bytes till
	   next VEC.  */
	andl	$(CHAR_PER_VEC - 1), %ecx

	/* Check if VEC is entirely contained in the remainder of the
	   page.  */
	cmpq	%rcx, %rdx
	jbe	L(page_cross_ret)

	/* Length crosses the page so if rax is zero (no matches)
	   continue.  */
	test	%VRAX, %VRAX
	jz	L(page_cross_continue)

	/* if rdx > rcx then any match here must be in [buf:buf + len].
	 */
	tzcnt	%VRAX, %VRAX
# ifdef USE_AS_WMEMCHR
	leaq	(%rdi, %rax, CHAR_SIZE), %rax
# else
	addq	%rdi, %rax
# endif
	ret

	.p2align 4,, 2
L(page_cross_zero):
	xorl	%eax, %eax
	ret

	.p2align 4,, 4
L(page_cross_ret):
	/* Search is entirely contained in page cross case.  */
# ifdef USE_AS_WMEMCHR
	test	%VRAX, %VRAX
	jz	L(page_cross_zero)
# endif
	tzcnt	%VRAX, %VRAX
	cmpl	%eax, %edx
	jbe	L(page_cross_zero)
# ifdef USE_AS_WMEMCHR
	leaq	(%rdi, %rax, CHAR_SIZE), %rax
# else
	addq	%rdi, %rax
# endif
	ret
END (MEMCHR)
#endif