From 8700a7851bccce63335f937f930382de58b8a249 Mon Sep 17 00:00:00 2001 From: "H.J. Lu" Date: Wed, 26 Dec 2018 06:56:04 -0800 Subject: x86-64: Vectorize sincosf_poly and update s_sincosf-fma.c Add and include it in s_sincosf.h to allow vectorized sincosf_poly. Add x86 sincosf_poly.h to vectorize sincosf_poly. On Broadwell, bench-sincosf shows: Before After Improvement max 160.273 114.198 40% min 6.25 5.625 11% mean 13.0325 10.6462 22% Vectorized sincosf_poly shows Before After Improvement max 138.653 114.198 21% min 5.004 5.625 -11% mean 11.5934 10.6462 9% Tested on x86-64 and i686 as well as with build-many-glibcs.py. * sysdeps/ieee754/flt-32/s_sincosf.h: Include . (sincos_t, sincosf_poly, sinf_poly): Moved to ... * sysdeps/ieee754/flt-32/sincosf_poly.h: Here. New file. * sysdeps/x86/fpu/s_sincosf_data.c: New file. * sysdeps/x86/fpu/sincosf_poly.h: Likewise. * sysdeps/x86_64/fpu/multiarch/s_sincosf-fma.c: Just include . --- sysdeps/x86/fpu/s_sincosf_data.c | 68 ++++++++++++++++++++++++ sysdeps/x86/fpu/sincosf_poly.h | 111 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 sysdeps/x86/fpu/s_sincosf_data.c create mode 100644 sysdeps/x86/fpu/sincosf_poly.h (limited to 'sysdeps/x86') diff --git a/sysdeps/x86/fpu/s_sincosf_data.c b/sysdeps/x86/fpu/s_sincosf_data.c new file mode 100644 index 0000000000..b219644884 --- /dev/null +++ b/sysdeps/x86/fpu/s_sincosf_data.c @@ -0,0 +1,68 @@ +/* Compute sine and cosine of argument. + Copyright (C) 2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include +#include +#include +#include + +/* The constants and polynomials for sine and cosine. The 2nd entry + computes -cos (x) rather than cos (x) to get negation for free. */ +const sincos_t __sincosf_table[2] = +{ + { + { 1.0, -1.0, -1.0, 1.0 }, +#if TOINT_INTRINSICS + 0x1.45F306DC9C883p-1, +#else + 0x1.45F306DC9C883p+23, +#endif + 0x1.921FB54442D18p0, + 0x1p0, + -0x1.ffffffd0c621cp-2, + { -0x1.555545995a603p-3, 0x1.55553e1068f19p-5 }, + { 0x1.1107605230bc4p-7, -0x1.6c087e89a359dp-10 }, + { -0x1.994eb3774cf24p-13, 0x1.99343027bf8c3p-16 } + }, + { + { 1.0, -1.0, -1.0, 1.0 }, +#if TOINT_INTRINSICS + 0x1.45F306DC9C883p-1, +#else + 0x1.45F306DC9C883p+23, +#endif + 0x1.921FB54442D18p0, + -0x1p0, + 0x1.ffffffd0c621cp-2, + { -0x1.555545995a603p-3, -0x1.55553e1068f19p-5 }, + { 0x1.1107605230bc4p-7, 0x1.6c087e89a359dp-10 }, + { -0x1.994eb3774cf24p-13, -0x1.99343027bf8c3p-16 } + } +}; + +/* Table with 4/PI to 192 bit precision. To avoid unaligned accesses + only 8 new bits are added per entry, making the table 4 times larger. */ +const uint32_t __inv_pio4[24] = +{ + 0xa2, 0xa2f9, 0xa2f983, 0xa2f9836e, + 0xf9836e4e, 0x836e4e44, 0x6e4e4415, 0x4e441529, + 0x441529fc, 0x1529fc27, 0x29fc2757, 0xfc2757d1, + 0x2757d1f5, 0x57d1f534, 0xd1f534dd, 0xf534ddc0, + 0x34ddc0db, 0xddc0db62, 0xc0db6295, 0xdb629599, + 0x6295993c, 0x95993c43, 0x993c4390, 0x3c439041 +}; diff --git a/sysdeps/x86/fpu/sincosf_poly.h b/sysdeps/x86/fpu/sincosf_poly.h new file mode 100644 index 0000000000..396ba8993d --- /dev/null +++ b/sysdeps/x86/fpu/sincosf_poly.h @@ -0,0 +1,111 @@ +/* Used by sinf, cosf and sincosf functions. X86-64 version. + Copyright (C) 2018 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +typedef double v2df_t __attribute__ ((vector_size (2 * sizeof (double)))); + +#ifdef __SSE2_MATH__ +typedef float v4sf_t __attribute__ ((vector_size (4 * sizeof (float)))); + +static inline void +v2df_to_sf (v2df_t v2df, float *f0p, float *f1p) +{ + v4sf_t v4sf = __builtin_ia32_cvtpd2ps (v2df); + *f0p = v4sf[0]; + *f1p = v4sf[1]; +} +#else +static inline void +v2df_to_sf (v2df_t v2df, float *f0p, float *f1p) +{ + *f0p = (float) v2df[0]; + *f1p = (float) v2df[1]; +} +#endif + +/* The constants and polynomials for sine and cosine. */ +typedef struct +{ + double sign[4]; /* Sign of sine in quadrants 0..3. */ + double hpi_inv; /* 2 / PI ( * 2^24 if !TOINT_INTRINSICS). */ + double hpi; /* PI / 2. */ + /* Cosine polynomial: c0, c1, c2, c3, c4. + Sine polynomial: s1, s2, s3. */ + double c0, c1; + v2df_t s1c2, s2c3, s3c4; +} sincos_t; + +/* Compute the sine and cosine of inputs X and X2 (X squared), using the + polynomial P and store the results in SINP and COSP. N is the quadrant, + if odd the cosine and sine polynomials are swapped. */ +static inline void +sincosf_poly (double x, double x2, const sincos_t *p, int n, float *sinp, + float *cosp) +{ + v2df_t vx2x2 = { x2, x2 }; + v2df_t vxx2 = { x, x2 }; + v2df_t vx3x4, vs1c2; + + vx3x4 = vx2x2 * vxx2; + vs1c2 = p->s2c3 + vx2x2 * p->s3c4; + + /* Swap sin/cos result based on quadrant. */ + if (n & 1) + { + float *tmp = cosp; + cosp = sinp; + sinp = tmp; + } + + double c1 = p->c0 + x2 * p->c1; + v2df_t vxc1 = { x, c1 }; + v2df_t vx5x6 = vx3x4 * vx2x2; + + v2df_t vsincos = vxc1 + vx3x4 * p->s1c2; + vsincos = vsincos + vx5x6 * vs1c2; + v2df_to_sf (vsincos, sinp, cosp); +} + +/* Return the sine of inputs X and X2 (X squared) using the polynomial P. + N is the quadrant, and if odd the cosine polynomial is used. */ +static inline float +sinf_poly (double x, double x2, const sincos_t *p, int n) +{ + double x3, x4, x6, x7, s, c, c1, c2, s1; + + if ((n & 1) == 0) + { + x3 = x * x2; + s1 = p->s2c3[0] + x2 * p->s3c4[0]; + + x7 = x3 * x2; + s = x + x3 * p->s1c2[0]; + + return s + x7 * s1; + } + else + { + x4 = x2 * x2; + c2 = p->s2c3[1] + x2 * p->s3c4[1]; + c1 = p->c0 + x2 * p->c1; + + x6 = x4 * x2; + c = c1 + x4 * p->s1c2[1]; + + return c + x6 * c2; + } +} -- cgit 1.4.1