summary refs log tree commit diff
path: root/sysdeps/x86_64/memset.S
blob: 6b718b71268c82603cee2c9d22f7c3ad68271a22 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
/* memset/bzero -- set memory area to CH/0
   Optimized version for x86-64.
   Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   Contributed by Andreas Jaeger <aj@suse.de>.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, write to the Free
   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
   02111-1307 USA.  */

#include <sysdep.h>
#include "asm-syntax.h"
#include "bp-sym.h"
#include "bp-asm.h"

/* BEWARE: `#ifdef memset' means that memset is redefined as `bzero' */
#define BZERO_P (defined memset)

/* This is somehow experimental and could made dependend on the cache
   size.  */
#define LARGE $120000

        .text
#if !BZERO_P && defined PIC && !defined NOT_IN_libc
ENTRY (__memset_chk)
	cmpq	%rdx, %rcx
	jb	HIDDEN_JUMPTARGET (__chk_fail)
END (__memset_chk)
#endif
ENTRY (memset)
#if BZERO_P
	mov	%rsi,%rdx	/* Adjust parameter.  */
	xorq	%rsi,%rsi	/* Fill with 0s.  */
#endif
	cmp	$0x7,%rdx	/* Check for small length.  */
	mov	%rdi,%rcx	/* Save ptr as return value.  */
	jbe	7f

#if BZERO_P
	mov	%rsi,%r8	/* Just copy 0.  */
#else
	/* Populate 8 bit data to full 64-bit.  */
	movabs	$0x0101010101010101,%r8
	movzbl	%sil,%eax
	imul	%rax,%r8
#endif
	test	$0x7,%edi	/* Check for alignment.  */
	je	2f

	.p2align 4
1:	/* Align ptr to 8 byte.  */
	mov	%sil,(%rcx)
	dec	%rdx
	inc	%rcx
	test	$0x7,%ecx
	jne	1b

2:	/* Check for really large regions.  */
	mov	%rdx,%rax
	shr	$0x6,%rax
	je	4f
	cmp	LARGE, %rdx
	jae	11f

	.p2align 4
3:	/* Copy 64 bytes.  */
	mov	%r8,(%rcx)
	mov	%r8,0x8(%rcx)
	mov	%r8,0x10(%rcx)
	mov	%r8,0x18(%rcx)
	mov	%r8,0x20(%rcx)
	mov	%r8,0x28(%rcx)
	mov	%r8,0x30(%rcx)
	mov	%r8,0x38(%rcx)
	add	$0x40,%rcx
	dec	%rax
	jne	3b

4:	/* Copy final bytes.  */
	and	$0x3f,%edx
	mov	%rdx,%rax
	shr	$0x3,%rax
	je	6f

5:	/* First in chunks of 8 bytes.  */
	mov	%r8,(%rcx)
	add	$0x8,%rcx
	dec	%rax
	jne	5b
6:
	and	$0x7,%edx
7:
	test	%rdx,%rdx
	je	9f
8:	/* And finally as bytes (up to 7).  */
	mov	%sil,(%rcx)
	inc	%rcx
	dec	%rdx
	jne	8b
9:
#if BZERO_P
	nop
#else
	/* Load result (only if used as memset).  */
	mov	%rdi,%rax	/* start address of destination is result */
#endif
	retq

	.p2align 4
11:	/* Copy 64 bytes without polluting the cache.  */
	/* We could use	movntdq    %xmm0,(%rcx) here to further
	   speed up for large cases but let's not use XMM registers.  */
	movnti	%r8,(%rcx)
	movnti  %r8,0x8(%rcx)
	movnti  %r8,0x10(%rcx)
	movnti  %r8,0x18(%rcx)
	movnti  %r8,0x20(%rcx)
	movnti  %r8,0x28(%rcx)
	movnti  %r8,0x30(%rcx)
	movnti  %r8,0x38(%rcx)
	add	$0x40,%rcx
	dec	%rax
	jne	11b
	jmp	4b

END (memset)
#if !BZERO_P
libc_hidden_builtin_def (memset)
#endif

#if !BZERO_P && defined PIC && !defined NOT_IN_libc
strong_alias (__memset_chk, __memset_zero_constant_len_parameter)
	.section .gnu.warning.__memset_zero_constant_len_parameter
	.string "memset used with constant zero length parameter; this could be due to transposed parameters"
#endif