1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
|
#ifndef X86_64_MATH_PRIVATE_H
#define X86_64_MATH_PRIVATE_H 1
#include <fenv.h>
#define math_opt_barrier(x) \
({ __typeof(x) __x; \
if (sizeof (x) <= sizeof (double)) \
__asm ("" : "=x" (__x) : "0" (x)); \
else \
__asm ("" : "=t" (__x) : "0" (x)); \
__x; })
#define math_force_eval(x) \
do { \
if (sizeof (x) <= sizeof (double)) \
__asm __volatile ("" : : "x" (x)); \
else \
__asm __volatile ("" : : "f" (x)); \
} while (0)
/* We can do a few things better on x86-64. */
#if defined __AVX__ || defined SSE2AVX
# define MOVD "vmovd"
# define STMXCSR "vstmxcsr"
# define LDMXCSR "vldmxcsr"
#else
# define MOVD "movd"
# define STMXCSR "stmxcsr"
# define LDMXCSR "ldmxcsr"
#endif
/* Direct movement of float into integer register. */
#define EXTRACT_WORDS64(i, d) \
do { \
long int i_; \
asm (MOVD " %1, %0" : "=rm" (i_) : "x" ((double) (d))); \
(i) = i_; \
} while (0)
/* And the reverse. */
#define INSERT_WORDS64(d, i) \
do { \
long int i_ = i; \
double d__; \
asm (MOVD " %1, %0" : "=x" (d__) : "rm" (i_)); \
d = d__; \
} while (0)
/* Direct movement of float into integer register. */
#define GET_FLOAT_WORD(i, d) \
do { \
int i_; \
asm (MOVD " %1, %0" : "=rm" (i_) : "x" ((float) (d))); \
(i) = i_; \
} while (0)
/* And the reverse. */
#define SET_FLOAT_WORD(f, i) \
do { \
int i_ = i; \
float f__; \
asm (MOVD " %1, %0" : "=x" (f__) : "rm" (i_)); \
f = f__; \
} while (0)
/* Specialized variants of the <fenv.h> interfaces which only handle
either the FPU or the SSE unit. */
static __always_inline void
libc_feholdexcept (fenv_t *e)
{
unsigned int mxcsr;
asm (STMXCSR " %0" : "=m" (*&mxcsr));
e->__mxcsr = mxcsr;
mxcsr = (mxcsr | 0x1f80) & ~0x3f;
asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr));
}
#define libc_feholdexcept libc_feholdexcept
#define libc_feholdexceptf libc_feholdexcept
static __always_inline void
libc_feholdexcept_setround (fenv_t *e, int r)
{
unsigned int mxcsr;
asm (STMXCSR " %0" : "=m" (*&mxcsr));
e->__mxcsr = mxcsr;
mxcsr = ((mxcsr | 0x1f80) & ~0x603f) | (r << 3);
asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr));
}
#define libc_feholdexcept_setround libc_feholdexcept_setround
#define libc_feholdexcept_setroundf libc_feholdexcept_setround
static __always_inline int
libc_fetestexcept (int e)
{
unsigned int mxcsr;
asm volatile (STMXCSR " %0" : "=m" (*&mxcsr));
return mxcsr & e & FE_ALL_EXCEPT;
}
#define libc_fetestexcept libc_fetestexcept
#define libc_fetestexceptf libc_fetestexcept
static __always_inline void
libc_fesetenv (fenv_t *e)
{
asm volatile (LDMXCSR " %0" : : "m" (e->__mxcsr));
}
#define libc_fesetenv libc_fesetenv
#define libc_fesetenvf libc_fesetenv
static __always_inline void
libc_feupdateenv (fenv_t *e)
{
unsigned int mxcsr;
asm volatile (STMXCSR " %0" : "=m" (*&mxcsr));
asm volatile (LDMXCSR " %0" : : "m" ((e)->__mxcsr));
__feraiseexcept (mxcsr & FE_ALL_EXCEPT);
}
#define libc_feupdateenv libc_feupdateenv
#define libc_feupdateenvf libc_feupdateenv
static __always_inline void
libc_feholdsetround (fenv_t *e, int r)
{
unsigned int mxcsr;
asm (STMXCSR " %0" : "=m" (*&mxcsr));
e->__mxcsr = mxcsr;
mxcsr = (mxcsr & ~0x6000) | (r << 3);
asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr));
}
#define libc_feholdsetround libc_feholdsetround
#define libc_feholdsetroundf libc_feholdsetround
static __always_inline void
libc_feresetround (fenv_t *e)
{
unsigned int mxcsr;
asm (STMXCSR " %0" : "=m" (*&mxcsr));
mxcsr = (mxcsr & ~0x6000) | (e->__mxcsr & 0x6000);
asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr));
}
#define libc_feresetround libc_feresetround
#define libc_feresetroundf libc_feresetround
#include_next <math_private.h>
extern __always_inline double
__ieee754_sqrt (double d)
{
double res;
#if defined __AVX__ || defined SSE2AVX
asm ("vsqrtsd %1, %0, %0" : "=x" (res) : "xm" (d));
#else
asm ("sqrtsd %1, %0" : "=x" (res) : "xm" (d));
#endif
return res;
}
extern __always_inline float
__ieee754_sqrtf (float d)
{
float res;
#if defined __AVX__ || defined SSE2AVX
asm ("vsqrtss %1, %0, %0" : "=x" (res) : "xm" (d));
#else
asm ("sqrtss %1, %0" : "=x" (res) : "xm" (d));
#endif
return res;
}
extern __always_inline long double
__ieee754_sqrtl (long double d)
{
long double res;
asm ("fsqrt" : "=t" (res) : "0" (d));
return res;
}
#ifdef __SSE4_1__
extern __always_inline double
__rint (double d)
{
double res;
# if defined __AVX__ || defined SSE2AVX
asm ("vroundsd $4, %1, %0, %0" : "=x" (res) : "xm" (d));
# else
asm ("roundsd $4, %1, %0" : "=x" (res) : "xm" (d));
# endif
return res;
}
extern __always_inline float
__rintf (float d)
{
float res;
# if defined __AVX__ || defined SSE2AVX
asm ("vroundss $4, %1, %0, %0" : "=x" (res) : "xm" (d));
# else
asm ("roundss $4, %1, %0" : "=x" (res) : "xm" (d));
# endif
return res;
}
extern __always_inline double
__floor (double d)
{
double res;
# if defined __AVX__ || defined SSE2AVX
asm ("vroundsd $1, %1, %0, %0" : "=x" (res) : "xm" (d));
# else
asm ("roundsd $1, %1, %0" : "=x" (res) : "xm" (d));
# endif
return res;
}
extern __always_inline float
__floorf (float d)
{
float res;
# if defined __AVX__ || defined SSE2AVX
asm ("vroundss $1, %1, %0, %0" : "=x" (res) : "xm" (d));
# else
asm ("roundss $1, %1, %0" : "=x" (res) : "xm" (d));
# endif
return res;
}
#endif /* __SSE4_1__ */
#endif /* X86_64_MATH_PRIVATE_H */
|
