about summary refs log tree commit diff
path: root/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
blob: 03de0ab907db3255f48edcce41783d3048da5014 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
/* memset with unaligned store and rep stosb
   Copyright (C) 2016-2022 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

/* memset is implemented as:
   1. Use overlapping store to avoid branch.
   2. If size is less than VEC, use integer register stores.
   3. If size is from VEC_SIZE to 2 * VEC_SIZE, use 2 VEC stores.
   4. If size is from 2 * VEC_SIZE to 4 * VEC_SIZE, use 4 VEC stores.
   5. If size is more to 4 * VEC_SIZE, align to 4 * VEC_SIZE with
      4 VEC stores and store 4 * VEC at a time until done.  */

#include <sysdep.h>

#ifndef MEMSET_CHK_SYMBOL
# define MEMSET_CHK_SYMBOL(p,s)		MEMSET_SYMBOL(p, s)
#endif

#ifndef WMEMSET_CHK_SYMBOL
# define WMEMSET_CHK_SYMBOL(p,s)	WMEMSET_SYMBOL(p, s)
#endif

#ifndef VZEROUPPER
# if VEC_SIZE > 16
#  define VZEROUPPER			vzeroupper
#  define VZEROUPPER_SHORT_RETURN	vzeroupper; ret
# else
#  define VZEROUPPER
# endif
#endif

#ifndef VZEROUPPER_SHORT_RETURN
# define VZEROUPPER_SHORT_RETURN	rep; ret
#endif

#ifndef MOVQ
# if VEC_SIZE > 16
#  define MOVQ				vmovq
#  define MOVD				vmovd
# else
#  define MOVQ				movq
#  define MOVD				movd
# endif
#endif

#if VEC_SIZE == 64
# define LOOP_4X_OFFSET	(VEC_SIZE * 4)
#else
# define LOOP_4X_OFFSET	(0)
#endif

#if defined USE_WITH_EVEX || defined USE_WITH_AVX512
# define END_REG	rcx
# define LOOP_REG	rdi
# define LESS_VEC_REG	rax
#else
# define END_REG	rdi
# define LOOP_REG	rdx
# define LESS_VEC_REG	rdi
#endif

#ifdef USE_XMM_LESS_VEC
# define XMM_SMALL	1
#else
# define XMM_SMALL	0
#endif

#ifdef USE_LESS_VEC_MASK_STORE
# define SET_REG64	rcx
# define SET_REG32	ecx
# define SET_REG16	cx
# define SET_REG8	cl
#else
# define SET_REG64	rsi
# define SET_REG32	esi
# define SET_REG16	si
# define SET_REG8	sil
#endif

#define PAGE_SIZE 4096

/* Macro to calculate size of small memset block for aligning
   purposes.  */
#define SMALL_MEMSET_ALIGN(mov_sz,	ret_sz)	(2 * (mov_sz) + (ret_sz) + 1)


#ifndef SECTION
# error SECTION is not defined!
#endif

	.section SECTION(.text), "ax", @progbits
#if IS_IN (libc)
# if defined SHARED
ENTRY_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned))
	cmp	%RDX_LP, %RCX_LP
	jb	HIDDEN_JUMPTARGET (__chk_fail)
END_CHK (WMEMSET_CHK_SYMBOL (__wmemset_chk, unaligned))
# endif

ENTRY (WMEMSET_SYMBOL (__wmemset, unaligned))
	shl	$2, %RDX_LP
	WMEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi)
	WMEMSET_VDUP_TO_VEC0_LOW()
	cmpq	$VEC_SIZE, %rdx
	jb	L(less_vec_from_wmemset)
	WMEMSET_VDUP_TO_VEC0_HIGH()
	jmp	L(entry_from_wmemset)
END (WMEMSET_SYMBOL (__wmemset, unaligned))
#endif

#if defined SHARED && IS_IN (libc)
ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned))
	cmp	%RDX_LP, %RCX_LP
	jb	HIDDEN_JUMPTARGET (__chk_fail)
END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned))
#endif

ENTRY (MEMSET_SYMBOL (__memset, unaligned))
	MEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi)
# ifdef __ILP32__
	/* Clear the upper 32 bits.  */
	mov	%edx, %edx
# endif
	cmpq	$VEC_SIZE, %rdx
	jb	L(less_vec)
	MEMSET_VDUP_TO_VEC0_HIGH()
L(entry_from_wmemset):
	cmpq	$(VEC_SIZE * 2), %rdx
	ja	L(more_2x_vec)
	/* From VEC and to 2 * VEC.  No branch when size == VEC_SIZE.  */
	VMOVU	%VMM(0), -VEC_SIZE(%rdi,%rdx)
	VMOVU	%VMM(0), (%rdi)
	VZEROUPPER_RETURN
#if defined USE_MULTIARCH && IS_IN (libc)
END (MEMSET_SYMBOL (__memset, unaligned))

# if defined SHARED && IS_IN (libc)
ENTRY_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms))
	cmp	%RDX_LP, %RCX_LP
	jb	HIDDEN_JUMPTARGET (__chk_fail)
END_CHK (MEMSET_CHK_SYMBOL (__memset_chk, unaligned_erms))
# endif

ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6)
	MEMSET_SET_VEC0_AND_SET_RETURN (%esi, %rdi)
# ifdef __ILP32__
	/* Clear the upper 32 bits.  */
	mov	%edx, %edx
# endif
	cmp	$VEC_SIZE, %RDX_LP
	jb	L(less_vec)
	MEMSET_VDUP_TO_VEC0_HIGH ()
	cmp	$(VEC_SIZE * 2), %RDX_LP
	ja	L(stosb_more_2x_vec)
	/* From VEC and to 2 * VEC.  No branch when size == VEC_SIZE.  */
	VMOVU	%VMM(0), (%rdi)
	VMOVU	%VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
	VZEROUPPER_RETURN
#endif

	.p2align 4,, 4
L(last_2x_vec):
#ifdef USE_LESS_VEC_MASK_STORE
	VMOVU	%VMM(0), (VEC_SIZE * -2)(%rdi, %rdx)
	VMOVU	%VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
#else
	VMOVU	%VMM(0), (VEC_SIZE * -2)(%rdi)
	VMOVU	%VMM(0), (VEC_SIZE * -1)(%rdi)
#endif
	VZEROUPPER_RETURN

	/* If have AVX512 mask instructions put L(less_vec) close to
	   entry as it doesn't take much space and is likely a hot target.
	 */
#ifdef USE_LESS_VEC_MASK_STORE
	.p2align 4,, 10
L(less_vec):
L(less_vec_from_wmemset):
	/* Less than 1 VEC.  */
# if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64
#  error Unsupported VEC_SIZE!
# endif
	/* Clear high bits from edi. Only keeping bits relevant to page
	   cross check. Note that we are using rax which is set in
	   MEMSET_VDUP_TO_VEC0_AND_SET_RETURN as ptr from here on out.  */
	andl	$(PAGE_SIZE - 1), %edi
	/* Check if VEC_SIZE store cross page. Mask stores suffer
	   serious performance degradation when it has to fault supress.
	 */
	cmpl	$(PAGE_SIZE - VEC_SIZE), %edi
	/* This is generally considered a cold target.  */
	ja	L(cross_page)
# if VEC_SIZE > 32
	movq	$-1, %rcx
	bzhiq	%rdx, %rcx, %rcx
	kmovq	%rcx, %k1
# else
	movl	$-1, %ecx
	bzhil	%edx, %ecx, %ecx
	kmovd	%ecx, %k1
# endif
	vmovdqu8 %VMM(0), (%rax){%k1}
	VZEROUPPER_RETURN

# if defined USE_MULTIARCH && IS_IN (libc)
	/* Include L(stosb_local) here if including L(less_vec) between
	   L(stosb_more_2x_vec) and ENTRY. This is to cache align the
	   L(stosb_more_2x_vec) target.  */
	.p2align 4,, 10
L(stosb_local):
	movzbl	%sil, %eax
	mov	%RDX_LP, %RCX_LP
	mov	%RDI_LP, %RDX_LP
	rep	stosb
	mov	%RDX_LP, %RAX_LP
	VZEROUPPER_RETURN
# endif
#endif

#if defined USE_MULTIARCH && IS_IN (libc)
	.p2align 4
L(stosb_more_2x_vec):
	cmp	__x86_rep_stosb_threshold(%rip), %RDX_LP
	ja	L(stosb_local)
#endif
	/* Fallthrough goes to L(loop_4x_vec). Tests for memset (2x, 4x]
	   and (4x, 8x] jump to target.  */
L(more_2x_vec):
	/* Store next 2x vec regardless.  */
	VMOVU	%VMM(0), (%rdi)
	VMOVU	%VMM(0), (VEC_SIZE * 1)(%rdi)


	/* Two different methods of setting up pointers / compare. The two
	   methods are based on the fact that EVEX/AVX512 mov instructions take
	   more bytes then AVX2/SSE2 mov instructions. As well that EVEX/AVX512
	   machines also have fast LEA_BID. Both setup and END_REG to avoid complex
	   address mode. For EVEX/AVX512 this saves code size and keeps a few
	   targets in one fetch block. For AVX2/SSE2 this helps prevent AGU
	   bottlenecks.  */
#if !(defined USE_WITH_EVEX || defined USE_WITH_AVX512)
	/* If AVX2/SSE2 compute END_REG (rdi) with ALU.  */
	addq	%rdx, %END_REG
#endif

	cmpq	$(VEC_SIZE * 4), %rdx
	jbe	L(last_2x_vec)


#if defined USE_WITH_EVEX || defined USE_WITH_AVX512
	/* If EVEX/AVX512 compute END_REG - (VEC_SIZE * 4 + LOOP_4X_OFFSET) with
	   LEA_BID.  */

	/* END_REG is rcx for EVEX/AVX512.  */
	leaq	-(VEC_SIZE * 4 + LOOP_4X_OFFSET)(%rdi, %rdx), %END_REG
#endif

	/* Store next 2x vec regardless.  */
	VMOVU	%VMM(0), (VEC_SIZE * 2)(%rax)
	VMOVU	%VMM(0), (VEC_SIZE * 3)(%rax)


#if defined USE_WITH_EVEX || defined USE_WITH_AVX512
	/* If LOOP_4X_OFFSET don't readjust LOOP_REG (rdi), just add
	   extra offset to addresses in loop. Used for AVX512 to save space
	   as no way to get (VEC_SIZE * 4) in imm8.  */
# if LOOP_4X_OFFSET == 0
	subq	$-(VEC_SIZE * 4), %LOOP_REG
# endif
	/* Avoid imm32 compare here to save code size.  */
	cmpq	%rdi, %rcx
#else
	addq	$-(VEC_SIZE * 4), %END_REG
	cmpq	$(VEC_SIZE * 8), %rdx
#endif
	jbe	L(last_4x_vec)
#if !(defined USE_WITH_EVEX || defined USE_WITH_AVX512)
	/* Set LOOP_REG (rdx).  */
	leaq	(VEC_SIZE * 4)(%rax), %LOOP_REG
#endif
	/* Align dst for loop.  */
	andq	$(VEC_SIZE * -2), %LOOP_REG
	.p2align 4
L(loop):
	VMOVA	%VMM(0), LOOP_4X_OFFSET(%LOOP_REG)
	VMOVA	%VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG)
	VMOVA	%VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG)
	VMOVA	%VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG)
	subq	$-(VEC_SIZE * 4), %LOOP_REG
	cmpq	%END_REG, %LOOP_REG
	jb	L(loop)
	.p2align 4,, MOV_SIZE
L(last_4x_vec):
	VMOVU	%VMM(0), LOOP_4X_OFFSET(%END_REG)
	VMOVU	%VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG)
	VMOVU	%VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG)
	VMOVU	%VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG)
L(return_vzeroupper):
#if VEC_SIZE > 16
	ZERO_UPPER_VEC_REGISTERS_RETURN
#else
	ret
#endif

	.p2align 4,, 10
#ifndef USE_LESS_VEC_MASK_STORE
# if defined USE_MULTIARCH && IS_IN (libc)
	/* If no USE_LESS_VEC_MASK put L(stosb_local) here. Will be in
	   range for 2-byte jump encoding.  */
L(stosb_local):
	movzbl	%sil, %eax
	mov	%RDX_LP, %RCX_LP
	mov	%RDI_LP, %RDX_LP
	rep	stosb
	mov	%RDX_LP, %RAX_LP
	VZEROUPPER_RETURN
# endif
	/* Define L(less_vec) only if not otherwise defined.  */
	.p2align 4
L(less_vec):
	/* Broadcast esi to partial register (i.e VEC_SIZE == 32 broadcast to
	   xmm). This is only does anything for AVX2.  */
	MEMSET_VDUP_TO_VEC0_LOW ()
L(less_vec_from_wmemset):
#endif
L(cross_page):
#if VEC_SIZE > 32
	cmpl	$32, %edx
	jge	L(between_32_63)
#endif
#if VEC_SIZE > 16
	cmpl	$16, %edx
	jge	L(between_16_31)
#endif
#ifndef USE_XMM_LESS_VEC
	MOVQ	%VMM_128(0), %SET_REG64
#endif
	cmpl	$8, %edx
	jge	L(between_8_15)
	cmpl	$4, %edx
	jge	L(between_4_7)
	cmpl	$1, %edx
	jg	L(between_2_3)
	jl	L(between_0_0)
	movb	%SET_REG8, (%LESS_VEC_REG)
L(between_0_0):
	ret

	/* Align small targets only if not doing so would cross a fetch line.
	 */
#if VEC_SIZE > 32
	.p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, RET_SIZE)
	/* From 32 to 63.  No branch when size == 32.  */
L(between_32_63):
	VMOVU	%VMM_256(0), (%LESS_VEC_REG)
	VMOVU	%VMM_256(0), -32(%LESS_VEC_REG, %rdx)
	VZEROUPPER_RETURN
#endif

#if VEC_SIZE >= 32
	.p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, 1)
L(between_16_31):
	/* From 16 to 31.  No branch when size == 16.  */
	VMOVU	%VMM_128(0), (%LESS_VEC_REG)
	VMOVU	%VMM_128(0), -16(%LESS_VEC_REG, %rdx)
	ret
#endif

	/* Move size is 3 for SSE2, EVEX, and AVX512. Move size is 4 for AVX2.
	 */
	.p2align 4,, SMALL_MEMSET_ALIGN(3 + XMM_SMALL, 1)
L(between_8_15):
	/* From 8 to 15.  No branch when size == 8.  */
#ifdef USE_XMM_LESS_VEC
	MOVQ	%VMM_128(0), (%rdi)
	MOVQ	%VMM_128(0), -8(%rdi, %rdx)
#else
	movq	%SET_REG64, (%LESS_VEC_REG)
	movq	%SET_REG64, -8(%LESS_VEC_REG, %rdx)
#endif
	ret

	/* Move size is 2 for SSE2, EVEX, and AVX512. Move size is 4 for AVX2.
	 */
	.p2align 4,, SMALL_MEMSET_ALIGN(2 << XMM_SMALL, 1)
L(between_4_7):
	/* From 4 to 7.  No branch when size == 4.  */
#ifdef USE_XMM_LESS_VEC
	MOVD	%VMM_128(0), (%rdi)
	MOVD	%VMM_128(0), -4(%rdi, %rdx)
#else
	movl	%SET_REG32, (%LESS_VEC_REG)
	movl	%SET_REG32, -4(%LESS_VEC_REG, %rdx)
#endif
	ret

	/* 4 * XMM_SMALL for the third mov for AVX2.  */
	.p2align 4,, 4 * XMM_SMALL + SMALL_MEMSET_ALIGN(3, 1)
L(between_2_3):
	/* From 2 to 3.  No branch when size == 2.  */
#ifdef USE_XMM_LESS_VEC
	movb	%SET_REG8, (%rdi)
	movb	%SET_REG8, 1(%rdi)
	movb	%SET_REG8, -1(%rdi, %rdx)
#else
	movw	%SET_REG16, (%LESS_VEC_REG)
	movb	%SET_REG8, -1(%LESS_VEC_REG, %rdx)
#endif
	ret
END (MEMSET_SYMBOL (__memset, unaligned_erms))