1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
|
/* Function tanhf vectorized with AVX-512.
Copyright (C) 2021-2023 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
https://www.gnu.org/licenses/. */
/*
* ALGORITHM DESCRIPTION:
*
* NOTE: Since the hyperbolic tangent function is odd
* (tanh(x) = -tanh(-x)), below algorithm deals with the absolute
* value of the argument |x|: tanh(x) = sign(x) * tanh(|x|)
*
* We use a table lookup method to compute tanh(|x|).
* The basic idea is to split the input range into a number of subintervals
* and to approximate tanh(.) with a polynomial on each of them.
*
* IEEE SPECIAL CONDITIONS:
* x = [+, -]0, r = [+, -]0
* x = +Inf, r = +1
* x = -Inf, r = -1
* x = QNaN, r = QNaN
* x = SNaN, r = QNaN
*
*
* ALGORITHM DETAILS
* We handle special values in a callout function, aside from main path
* computations. "Special" for this algorithm are:
* INF, NAN, |x| > HUGE_THRESHOLD
*
*
* Main path computations are organized as follows:
* Actually we split the interval [0, SATURATION_THRESHOLD)
* into a number of subintervals. On each subinterval we approximate tanh(.)
* with a minimax polynomial of pre-defined degree. Polynomial coefficients
* are computed beforehand and stored in table. We also use
*
* y := |x| + B,
*
* here B depends on subinterval and is used to make argument
* closer to zero.
* We also add large fake interval [SATURATION_THRESHOLD, HUGE_THRESHOLD],
* where 1.0 + 0.0*y + 0.0*y^2 ... coefficients are stored - just to
* preserve main path computation logic but return 1.0 for all arguments.
*
* Hence reconstruction looks as follows:
* we extract proper polynomial and range reduction coefficients
* (Pj and B), corresponding to subinterval, to which |x| belongs,
* and return
*
* r := sign(x) * (P0 + P1 * y + ... + Pn * y^n)
*
* NOTE: we use multiprecision technique to multiply and sum the first
* K terms of the polynomial. So Pj, j = 0..K are stored in
* table each as a pair of target precision numbers (Pj and PLj) to
* achieve wider than target precision.
*
*
*/
/* Offsets for data table __svml_stanh_data_internal_avx512. Ordered
by use in the function. On cold-starts this might help the
prefetcher. Possibly a better idea is to interleave start/end so
that the prefetcher is less likely to detect a stream and pull
irrelivant lines into cache. */
/* Offsets for data table __svml_stanh_data_internal. 4 bytes each.
*/
#define _iExpMantMask_UISA 0
#define _iMinIdxOfsMask_UISA 4
#define _iMaxIdxMask_UISA 8
#define _iExpMask 12
/* Offsets for data table __svml_stanh_data_internal_al64. 64 bytes
each. */
#define _sC_lo 0
#define _sC_hi 64
#define _sP7_lo 128
#define _sP7_hi 192
#define _sSignMask 256
#define _sP6_lo 320
#define _sP6_hi 384
#define _sP5_lo 448
#define _sP5_hi 512
#define _sP4_lo 576
#define _sP4_hi 640
#define _sP3_lo 704
#define _sP3_hi 768
#define _sP2_lo 832
#define _sP2_hi 896
#define _sP0_lo 960
#define _sP0_hi 1024
#include <sysdep.h>
#define TANHF_DATA(x) ((x)+__svml_stanh_data_internal_al64)
#define TANHF_DATA_UNALIGNED(x) ((x)+__svml_stanh_data_internal)
.section .text.evex512, "ax", @progbits
ENTRY(_ZGVeN16v_tanhf_skx)
/* Here huge arguments, INF and NaNs are filtered out to callout. */
vpandd TANHF_DATA_UNALIGNED(_iExpMantMask_UISA)(%rip){1to16}, %zmm0, %zmm1
vpsubd TANHF_DATA_UNALIGNED(_iMinIdxOfsMask_UISA)(%rip){1to16}, %zmm1, %zmm2
/* Selection arguments between [0, 0x03e00000] into zmm3. */
vpxord %zmm3, %zmm3, %zmm3
vpmaxsd %zmm3, %zmm2, %zmm3
vpminsd TANHF_DATA_UNALIGNED(_iMaxIdxMask_UISA)(%rip){1to16}, %zmm3, %zmm3
/* Setup permute indices in zmm3. */
vpsrld $21, %zmm3, %zmm3
/* Store if there are any special cases in k1. */
vpcmpd $6, TANHF_DATA_UNALIGNED(_iExpMask)(%rip){1to16}, %zmm1, %k1
vmovaps TANHF_DATA(_sC_lo)(%rip), %zmm5
vpermt2ps TANHF_DATA(_sC_hi)(%rip), %zmm3, %zmm5
vmovaps TANHF_DATA(_sP7_lo)(%rip), %zmm2
vpermt2ps TANHF_DATA(_sP7_hi)(%rip), %zmm3, %zmm2
/* Store absolute values of inputs in zmm1. */
vmovaps TANHF_DATA(_sSignMask)(%rip), %zmm4
vandnps %zmm0, %zmm4, %zmm1
vsubps {rn-sae}, %zmm5, %zmm1, %zmm1
vmovaps TANHF_DATA(_sP6_lo)(%rip), %zmm5
vpermt2ps TANHF_DATA(_sP6_hi)(%rip), %zmm3, %zmm5
vmovaps TANHF_DATA(_sP5_lo)(%rip), %zmm6
vpermt2ps TANHF_DATA(_sP5_hi)(%rip), %zmm3, %zmm6
vfmadd213ps {rn-sae}, %zmm5, %zmm1, %zmm2
vfmadd213ps {rn-sae}, %zmm6, %zmm1, %zmm2
vmovaps TANHF_DATA(_sP4_lo)(%rip), %zmm7
vpermt2ps TANHF_DATA(_sP4_hi)(%rip), %zmm3, %zmm7
vmovaps TANHF_DATA(_sP3_lo)(%rip), %zmm8
vpermt2ps TANHF_DATA(_sP3_hi)(%rip), %zmm3, %zmm8
vfmadd213ps {rn-sae}, %zmm7, %zmm1, %zmm2
vfmadd213ps {rn-sae}, %zmm8, %zmm1, %zmm2
vmovaps TANHF_DATA(_sP2_lo)(%rip), %zmm9
vpermt2ps TANHF_DATA(_sP2_hi)(%rip), %zmm3, %zmm9
vmovaps TANHF_DATA(_sP0_lo)(%rip), %zmm10
vpermt2ps TANHF_DATA(_sP0_hi)(%rip), %zmm3, %zmm10
vfmadd213ps {rn-sae}, %zmm9, %zmm1, %zmm2
vfmadd213ps {rn-sae}, %zmm10, %zmm1, %zmm2
kmovw %k1, %edx
testl %edx, %edx
/* Go to special inputs processing branch. */
jne L(SPECIAL_VALUES_BRANCH)
# LOE rbx r12 r13 r14 r15 zmm0 zmm2 zmm4
/* Wait until after branch of write over zmm0. */
vpternlogd $0xec, %zmm4, %zmm2, %zmm0
/* No stack restoration on the fastpath. */
ret
/* Cold case. edx has 1s where there was a special value that
needs to be handled by a tanhf call. Optimize for code size
more so than speed here. */
L(SPECIAL_VALUES_BRANCH):
# LOE rbx rdx r12 r13 r14 r15 zmm0 zmm2 zmm4
/* Use r13 to save/restore the stack. This allows us to use rbp as
callee save register saving code size. */
pushq %r13
cfi_adjust_cfa_offset(8)
cfi_offset(r13, -16)
/* Need to callee save registers to preserve state across tanhf calls.
*/
pushq %rbx
cfi_adjust_cfa_offset(8)
cfi_offset(rbx, -24)
pushq %rbp
cfi_adjust_cfa_offset(8)
cfi_offset(rbp, -32)
movq %rsp, %r13
cfi_def_cfa_register(r13)
/* Align stack and make room for 2x zmm vectors. */
andq $-64, %rsp
addq $-128, %rsp
/* Save original input (zmm0 unchanged up to this point). */
vmovaps %zmm0, 64(%rsp)
/* Save all already computed inputs. */
vpternlogd $0xec, %zmm4, %zmm2, %zmm0
vmovaps %zmm0, (%rsp)
vzeroupper
/* edx has 1s where there was a special value that needs to be handled
by a tanhf call. */
movl %edx, %ebx
L(SPECIAL_VALUES_LOOP):
# LOE rbx rbp r12 r13 r14 r15
/* use rbp as index for special value that is saved across calls to
tanhf. We technically don't need a callee save register here as offset
to rsp is always [0, 56] so we can restore rsp by realigning to 64.
Essentially the tradeoff is 1 extra save/restore vs 2 extra instructions
in the loop. Realigning also costs more code size. */
xorl %ebp, %ebp
tzcntl %ebx, %ebp
/* Scalar math function call to process special input. */
vmovss 64(%rsp, %rbp, 4), %xmm0
call tanhf@PLT
/* No good way to avoid the store-forwarding fault this will cause on
return. `lfence` avoids the SF fault but at greater cost as it
serialized stack/callee save restoration. */
vmovss %xmm0, (%rsp, %rbp, 4)
blsrl %ebx, %ebx
jnz L(SPECIAL_VALUES_LOOP)
# LOE r12 r13 r14 r15
/* All results have been written to (%rsp). */
vmovaps (%rsp), %zmm0
/* Restore rsp. */
movq %r13, %rsp
cfi_def_cfa_register(rsp)
/* Restore callee save registers. */
popq %rbp
cfi_adjust_cfa_offset(-8)
cfi_restore(rbp)
popq %rbx
cfi_adjust_cfa_offset(-8)
cfi_restore(rbp)
popq %r13
cfi_adjust_cfa_offset(-8)
cfi_restore(r13)
ret
END(_ZGVeN16v_tanhf_skx)
.section .rodata, "a"
.align 16
#ifdef __svml_stanh_data_internal_typedef
typedef unsigned int VUINT32;
typedef struct
{
__declspec(align(4)) VUINT32 _iExpMantMask_UISA[1][1];
__declspec(align(4)) VUINT32 _iMinIdxOfsMask_UISA[1][1];
__declspec(align(4)) VUINT32 _iMaxIdxMask_UISA[1][1];
__declspec(align(4)) VUINT32 _iExpMask[1][1];
__declspec(align(64)) VUINT32 _sC_lo[16][1];
__declspec(align(64)) VUINT32 _sC_hi[16][1];
__declspec(align(64)) VUINT32 _sP7_lo[16][1];
__declspec(align(64)) VUINT32 _sP7_hi[16][1];
__declspec(align(64)) VUINT32 _sSignMask[16][1];
__declspec(align(64)) VUINT32 _sP6_lo[16][1];
__declspec(align(64)) VUINT32 _sP6_hi[16][1];
__declspec(align(64)) VUINT32 _sP5_lo[16][1];
__declspec(align(64)) VUINT32 _sP5_hi[16][1];
__declspec(align(64)) VUINT32 _sP4_lo[16][1];
__declspec(align(64)) VUINT32 _sP4_hi[16][1];
__declspec(align(64)) VUINT32 _sP3_lo[16][1];
__declspec(align(64)) VUINT32 _sP3_hi[16][1];
__declspec(align(64)) VUINT32 _sP2_lo[16][1];
__declspec(align(64)) VUINT32 _sP2_hi[16][1];
__declspec(align(64)) VUINT32 _sP0_lo[16][1];
__declspec(align(64)) VUINT32 _sP0_hi[16][1];
} __svml_stanh_data_internal;
#endif
__svml_stanh_data_internal:
.align 4
/* _iExpMantMask_UISA */
.long 0x7fe00000
.align 4
/* _iMinIdxOfsMask_UISA */
.long 0x3d400000
.align 4
/* _iMaxIdxMask_UISA */
.long 0x03e00000
.align 4
/* _iExpMask */
.long 0x7f000000
.align 64
__svml_stanh_data_internal_al64:
.align 64
/* _sC_lo */
.long 0x00000000, 0x3d700000, 0x3d900000, 0x3db00000
.long 0x3dd00000, 0x3df00000, 0x3e100000, 0x3e300000
.long 0x3e500000, 0x3e700000, 0x3e900000, 0x3eb00000
.long 0x3ed00000, 0x3ef00000, 0x3f100000, 0x3f300000
.align 64
/* _sC_hi */
.long 0x3f500000, 0x3f700000, 0x3f900000, 0x3fb00000
.long 0x3fd00000, 0x3ff00000, 0x40100000, 0x40300000
.long 0x40500000, 0x40700000, 0x40900000, 0x40b00000
.long 0x40d00000, 0x40f00000, 0x41100000, 0x00000000
.align 64
/* _sP7_lo */
.long 0xbc0e2f66, 0x460bda12, 0x43d638ef, 0xc3e11c3e
.long 0xc2baa4e9, 0xc249da2d, 0xc1859b82, 0x40dd5b57
.long 0x40494640, 0x40c730a8, 0xbf0f160e, 0x3e30e76f
.long 0xbea81387, 0xbdb26a1c, 0xbd351e57, 0xbb4c01a0
.align 64
/* _sP7_hi */
.long 0x3c1d7bfb, 0x3c722cd1, 0x3c973f1c, 0x3c33a31b
.long 0x3b862ef4, 0x3a27b3d0, 0xba3b5907, 0xba0efc22
.long 0xb97f9f0f, 0xb8c8af50, 0xb7bdddfb, 0xb64f2950
.long 0xb4e085b1, 0xb3731dfa, 0xb15a1f04, 0x00000000
.align 64
/* _sSignMask */
.long 0x80000000, 0x80000000, 0x80000000, 0x80000000
.long 0x80000000, 0x80000000, 0x80000000, 0x80000000
.long 0x80000000, 0x80000000, 0x80000000, 0x80000000
.long 0x80000000, 0x80000000, 0x80000000, 0x80000000
.align 64
/* _sP6_lo */
.long 0x3e0910e9, 0x43761143, 0x4165ecdc, 0xc190f756
.long 0xc08c097d, 0xc02ba813, 0xbf7f6bda, 0x3f2b1dc0
.long 0x3ece105d, 0x3f426a94, 0xbadb0dc4, 0x3da43b17
.long 0xbd51ab88, 0xbcaea23d, 0xbd3b6d8d, 0xbd6caaad
.align 64
/* _sP6_hi */
.long 0xbd795bed, 0xbd5fddda, 0xbd038f3b, 0xbc1cad63
.long 0x3abb4766, 0x3b95f10b, 0x3b825873, 0x3afaea66
.long 0x3a49f878, 0x39996bf3, 0x388f3e6c, 0x371bb0e3
.long 0x35a8a5e6, 0x34369b17, 0x322487b0, 0x00000000
.align 64
/* _sP5_lo */
.long 0xb76dd6b9, 0xbe1c276d, 0x3c1dcf2f, 0x3dc1a78d
.long 0x3d96f985, 0x3da2b61b, 0x3dc13397, 0x3dd2f670
.long 0x3df48a0a, 0x3e06c5a8, 0x3e1a3aba, 0x3e27c405
.long 0x3e2e78d0, 0x3e2c3e44, 0x3e1d3097, 0x3df4a8f4
.align 64
/* _sP5_hi */
.long 0x3da38508, 0x3d31416a, 0x3b562657, 0xbcaeeac9
.long 0xbcce9419, 0xbcaaeac4, 0xbc49e7d0, 0xbba71ddd
.long 0xbb003b0e, 0xba3f9a05, 0xb92c08a7, 0xb7ba9232
.long 0xb64a0b0f, 0xb4dac169, 0xb2ab78ac, 0x00000000
.align 64
/* _sP4_lo */
.long 0xbeaaaaa5, 0xbeab0612, 0xbea7f01f, 0xbea4e120
.long 0xbea387b7, 0xbea15962, 0xbe9d57f7, 0xbe976b5a
.long 0xbe90230d, 0xbe880dff, 0xbe7479b3, 0xbe4c3d88
.long 0xbe212482, 0xbdeb8cba, 0xbd5e78ad, 0x3c6b5e6e
.align 64
/* _sP4_hi */
.long 0x3d839143, 0x3dc21ee1, 0x3de347af, 0x3dcbec96
.long 0x3d99ef2d, 0x3d542ea1, 0x3cdde701, 0x3c2cca67
.long 0x3b81cb27, 0x3ac073a1, 0x39ac3032, 0x383a94d9
.long 0x36ca081d, 0x355abd4c, 0x332b3cb6, 0x00000000
.align 64
/* _sP3_lo */
.long 0xb0343c7b, 0xbd6ee69d, 0xbd8f0da7, 0xbdae477d
.long 0xbdcd2a1f, 0xbdeba80d, 0xbe0c443b, 0xbe293cf3
.long 0xbe44f282, 0xbe5f3651, 0xbe81c7c0, 0xbe96d7ca
.long 0xbea7fb8e, 0xbeb50e9e, 0xbec12efe, 0xbec4be92
.align 64
/* _sP3_hi */
.long 0xbebce070, 0xbead510e, 0xbe8ef7d6, 0xbe4b8704
.long 0xbe083237, 0xbdaf7449, 0xbd2e1ec4, 0xbc83bf06
.long 0xbbc3e0b5, 0xbb10aadc, 0xba0157db, 0xb88c18f2
.long 0xb717b096, 0xb5a43bae, 0xb383012c, 0x00000000
.align 64
/* _sP2_lo */
.long 0x3f800000, 0x3f7f1f84, 0x3f7ebd11, 0x3f7e1e5f
.long 0x3f7d609f, 0x3f7c842d, 0x3f7b00e5, 0x3f789580
.long 0x3f75b8ad, 0x3f726fd9, 0x3f6cc59b, 0x3f63fb92
.long 0x3f59ff97, 0x3f4f11d7, 0x3f3d7573, 0x3f24f360
.align 64
/* _sP2_hi */
.long 0x3f0cbfe7, 0x3eec1a69, 0x3eb0a801, 0x3e6753a2
.long 0x3e132f1a, 0x3db7e7d3, 0x3d320845, 0x3c84d3d4
.long 0x3bc477b7, 0x3b10d3da, 0x3a01601e, 0x388c1a3b
.long 0x3717b0da, 0x35a43bce, 0x338306c6, 0x00000000
.align 64
/* _sP0_lo */
.long 0x00000000, 0x3d6fb9c9, 0x3d8fc35f, 0x3daf9169
.long 0x3dcf49ab, 0x3deee849, 0x3e0f0ee8, 0x3e2e4984
.long 0x3e4d2f8e, 0x3e6bb32e, 0x3e8c51cd, 0x3ea96163
.long 0x3ec543f1, 0x3edfd735, 0x3f028438, 0x3f18abf0
.align 64
/* _sP0_hi */
.long 0x3f2bc480, 0x3f3bec1c, 0x3f4f2e5b, 0x3f613c53
.long 0x3f6ce37d, 0x3f743c4f, 0x3f7a5feb, 0x3f7dea85
.long 0x3f7f3b3d, 0x3f7fb78c, 0x3f7fefd4, 0x3f7ffdd0
.long 0x3f7fffb4, 0x3f7ffff6, 0x3f7fffff, 0x3f800000
.align 64
.type __svml_stanh_data_internal_al64, @object
.size __svml_stanh_data_internal_al64, .-__svml_stanh_data_internal_al64
.type __svml_stanh_data_internal, @object
.size __svml_stanh_data_internal, .-__svml_stanh_data_internal
|
