diff options
| -rw-r--r-- | src/internal/libm.h | 6 | ||||
| -rw-r--r-- | src/math/powf.c | 406 | ||||
| -rw-r--r-- | src/math/powf_data.c | 34 | ||||
| -rw-r--r-- | src/math/powf_data.h | 26 |
4 files changed, 232 insertions, 240 deletions
diff --git a/src/internal/libm.h b/src/internal/libm.h index 98bf5c68..9cd105fc 100644 --- a/src/internal/libm.h +++ b/src/internal/libm.h @@ -64,6 +64,12 @@ union ldshape { /* Support signaling NaNs. */ #define WANT_SNAN 0 +#if WANT_SNAN +#error SNaN is unsupported +#else +#define issignalingf_inline(x) 0 +#endif + #ifndef TOINT_INTRINSICS #define TOINT_INTRINSICS 0 #endif diff --git a/src/math/powf.c b/src/math/powf.c index 427c8965..de8fab54 100644 --- a/src/math/powf.c +++ b/src/math/powf.c @@ -1,259 +1,185 @@ -/* origin: FreeBSD /usr/src/lib/msun/src/e_powf.c */ /* - * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. - */ -/* - * ==================================================== - * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. - * - * Developed at SunPro, a Sun Microsystems, Inc. business. - * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice - * is preserved. - * ==================================================== + * Copyright (c) 2017-2018, Arm Limited. + * SPDX-License-Identifier: MIT */ +#include <math.h> +#include <stdint.h> #include "libm.h" +#include "exp2f_data.h" +#include "powf_data.h" -static const float -bp[] = {1.0, 1.5,}, -dp_h[] = { 0.0, 5.84960938e-01,}, /* 0x3f15c000 */ -dp_l[] = { 0.0, 1.56322085e-06,}, /* 0x35d1cfdc */ -two24 = 16777216.0, /* 0x4b800000 */ -huge = 1.0e30, -tiny = 1.0e-30, -/* poly coefs for (3/2)*(log(x)-2s-2/3*s**3 */ -L1 = 6.0000002384e-01, /* 0x3f19999a */ -L2 = 4.2857143283e-01, /* 0x3edb6db7 */ -L3 = 3.3333334327e-01, /* 0x3eaaaaab */ -L4 = 2.7272811532e-01, /* 0x3e8ba305 */ -L5 = 2.3066075146e-01, /* 0x3e6c3255 */ -L6 = 2.0697501302e-01, /* 0x3e53f142 */ -P1 = 1.6666667163e-01, /* 0x3e2aaaab */ -P2 = -2.7777778450e-03, /* 0xbb360b61 */ -P3 = 6.6137559770e-05, /* 0x388ab355 */ -P4 = -1.6533901999e-06, /* 0xb5ddea0e */ -P5 = 4.1381369442e-08, /* 0x3331bb4c */ -lg2 = 6.9314718246e-01, /* 0x3f317218 */ -lg2_h = 6.93145752e-01, /* 0x3f317200 */ -lg2_l = 1.42860654e-06, /* 0x35bfbe8c */ -ovt = 4.2995665694e-08, /* -(128-log2(ovfl+.5ulp)) */ -cp = 9.6179670095e-01, /* 0x3f76384f =2/(3ln2) */ -cp_h = 9.6191406250e-01, /* 0x3f764000 =12b cp */ -cp_l = -1.1736857402e-04, /* 0xb8f623c6 =tail of cp_h */ -ivln2 = 1.4426950216e+00, /* 0x3fb8aa3b =1/ln2 */ -ivln2_h = 1.4426879883e+00, /* 0x3fb8aa00 =16b 1/ln2*/ -ivln2_l = 7.0526075433e-06; /* 0x36eca570 =1/ln2 tail*/ +/* +POWF_LOG2_POLY_ORDER = 5 +EXP2F_TABLE_BITS = 5 -float powf(float x, float y) +ULP error: 0.82 (~ 0.5 + relerr*2^24) +relerr: 1.27 * 2^-26 (Relative error ~= 128*Ln2*relerr_log2 + relerr_exp2) +relerr_log2: 1.83 * 2^-33 (Relative error of logx.) +relerr_exp2: 1.69 * 2^-34 (Relative error of exp2(ylogx).) +*/ + +#define N (1 << POWF_LOG2_TABLE_BITS) +#define T __powf_log2_data.tab +#define A __powf_log2_data.poly +#define OFF 0x3f330000 + +/* Subnormal input is normalized so ix has negative biased exponent. + Output is multiplied by N (POWF_SCALE) if TOINT_INTRINICS is set. */ +static inline double_t log2_inline(uint32_t ix) { - float z,ax,z_h,z_l,p_h,p_l; - float y1,t1,t2,r,s,sn,t,u,v,w; - int32_t i,j,k,yisint,n; - int32_t hx,hy,ix,iy,is; + double_t z, r, r2, r4, p, q, y, y0, invc, logc; + uint32_t iz, top, tmp; + int k, i; - GET_FLOAT_WORD(hx, x); - GET_FLOAT_WORD(hy, y); - ix = hx & 0x7fffffff; - iy = hy & 0x7fffffff; + /* x = 2^k z; where z is in range [OFF,2*OFF] and exact. + The range is split into N subintervals. + The ith subinterval contains z and c is near its center. */ + tmp = ix - OFF; + i = (tmp >> (23 - POWF_LOG2_TABLE_BITS)) % N; + top = tmp & 0xff800000; + iz = ix - top; + k = (int32_t)top >> (23 - POWF_SCALE_BITS); /* arithmetic shift */ + invc = T[i].invc; + logc = T[i].logc; + z = (double_t)asfloat(iz); - /* x**0 = 1, even if x is NaN */ - if (iy == 0) - return 1.0f; - /* 1**y = 1, even if y is NaN */ - if (hx == 0x3f800000) - return 1.0f; - /* NaN if either arg is NaN */ - if (ix > 0x7f800000 || iy > 0x7f800000) - return x + y; + /* log2(x) = log1p(z/c-1)/ln2 + log2(c) + k */ + r = z * invc - 1; + y0 = logc + (double_t)k; - /* determine if y is an odd int when x < 0 - * yisint = 0 ... y is not an integer - * yisint = 1 ... y is an odd int - * yisint = 2 ... y is an even int - */ - yisint = 0; - if (hx < 0) { - if (iy >= 0x4b800000) - yisint = 2; /* even integer y */ - else if (iy >= 0x3f800000) { - k = (iy>>23) - 0x7f; /* exponent */ - j = iy>>(23-k); - if ((j<<(23-k)) == iy) - yisint = 2 - (j & 1); - } - } + /* Pipelined polynomial evaluation to approximate log1p(r)/ln2. */ + r2 = r * r; + y = A[0] * r + A[1]; + p = A[2] * r + A[3]; + r4 = r2 * r2; + q = A[4] * r + y0; + q = p * r2 + q; + y = y * r4 + q; + return y; +} - /* special value of y */ - if (iy == 0x7f800000) { /* y is +-inf */ - if (ix == 0x3f800000) /* (-1)**+-inf is 1 */ - return 1.0f; - else if (ix > 0x3f800000) /* (|x|>1)**+-inf = inf,0 */ - return hy >= 0 ? y : 0.0f; - else /* (|x|<1)**+-inf = 0,inf */ - return hy >= 0 ? 0.0f: -y; - } - if (iy == 0x3f800000) /* y is +-1 */ - return hy >= 0 ? x : 1.0f/x; - if (hy == 0x40000000) /* y is 2 */ - return x*x; - if (hy == 0x3f000000) { /* y is 0.5 */ - if (hx >= 0) /* x >= +0 */ - return sqrtf(x); - } +#undef N +#undef T +#define N (1 << EXP2F_TABLE_BITS) +#define T __exp2f_data.tab +#define SIGN_BIAS (1 << (EXP2F_TABLE_BITS + 11)) - ax = fabsf(x); - /* special value of x */ - if (ix == 0x7f800000 || ix == 0 || ix == 0x3f800000) { /* x is +-0,+-inf,+-1 */ - z = ax; - if (hy < 0) /* z = (1/|x|) */ - z = 1.0f/z; - if (hx < 0) { - if (((ix-0x3f800000)|yisint) == 0) { - z = (z-z)/(z-z); /* (-1)**non-int is NaN */ - } else if (yisint == 1) - z = -z; /* (x<0)**odd = -(|x|**odd) */ - } - return z; - } +/* The output of log2 and thus the input of exp2 is either scaled by N + (in case of fast toint intrinsics) or not. The unscaled xd must be + in [-1021,1023], sign_bias sets the sign of the result. */ +static inline float exp2_inline(double_t xd, uint32_t sign_bias) +{ + uint64_t ki, ski, t; + double_t kd, z, r, r2, y, s; - sn = 1.0f; /* sign of result */ - if (hx < 0) { - if (yisint == 0) /* (x<0)**(non-int) is NaN */ - return (x-x)/(x-x); - if (yisint == 1) /* (x<0)**(odd int) */ - sn = -1.0f; - } +#if TOINT_INTRINSICS +#define C __exp2f_data.poly_scaled + /* N*x = k + r with r in [-1/2, 1/2] */ + kd = roundtoint(xd); /* k */ + ki = converttoint(xd); +#else +#define C __exp2f_data.poly +#define SHIFT __exp2f_data.shift_scaled + /* x = k/N + r with r in [-1/(2N), 1/(2N)] */ + kd = eval_as_double(xd + SHIFT); + ki = asuint64(kd); + kd -= SHIFT; /* k/N */ +#endif + r = xd - kd; - /* |y| is huge */ - if (iy > 0x4d000000) { /* if |y| > 2**27 */ - /* over/underflow if x is not close to one */ - if (ix < 0x3f7ffff8) - return hy < 0 ? sn*huge*huge : sn*tiny*tiny; - if (ix > 0x3f800007) - return hy > 0 ? sn*huge*huge : sn*tiny*tiny; - /* now |1-x| is tiny <= 2**-20, suffice to compute - log(x) by x-x^2/2+x^3/3-x^4/4 */ - t = ax - 1; /* t has 20 trailing zeros */ - w = (t*t)*(0.5f - t*(0.333333333333f - t*0.25f)); - u = ivln2_h*t; /* ivln2_h has 16 sig. bits */ - v = t*ivln2_l - w*ivln2; - t1 = u + v; - GET_FLOAT_WORD(is, t1); - SET_FLOAT_WORD(t1, is & 0xfffff000); - t2 = v - (t1-u); - } else { - float s2,s_h,s_l,t_h,t_l; - n = 0; - /* take care subnormal number */ - if (ix < 0x00800000) { - ax *= two24; - n -= 24; - GET_FLOAT_WORD(ix, ax); - } - n += ((ix)>>23) - 0x7f; - j = ix & 0x007fffff; - /* determine interval */ - ix = j | 0x3f800000; /* normalize ix */ - if (j <= 0x1cc471) /* |x|<sqrt(3/2) */ - k = 0; - else if (j < 0x5db3d7) /* |x|<sqrt(3) */ - k = 1; - else { - k = 0; - n += 1; - ix -= 0x00800000; - } - SET_FLOAT_WORD(ax, ix); + /* exp2(x) = 2^(k/N) * 2^r ~= s * (C0*r^3 + C1*r^2 + C2*r + 1) */ + t = T[ki % N]; + ski = ki + sign_bias; + t += ski << (52 - EXP2F_TABLE_BITS); + s = asdouble(t); + z = C[0] * r + C[1]; + r2 = r * r; + y = C[2] * r + 1; + y = z * r2 + y; + y = y * s; + return eval_as_float(y); +} - /* compute s = s_h+s_l = (x-1)/(x+1) or (x-1.5)/(x+1.5) */ - u = ax - bp[k]; /* bp[0]=1.0, bp[1]=1.5 */ - v = 1.0f/(ax+bp[k]); - s = u*v; - s_h = s; - GET_FLOAT_WORD(is, s_h); - SET_FLOAT_WORD(s_h, is & 0xfffff000); - /* t_h=ax+bp[k] High */ - is = ((ix>>1) & 0xfffff000) | 0x20000000; - SET_FLOAT_WORD(t_h, is + 0x00400000 + (k<<21)); - t_l = ax - (t_h - bp[k]); - s_l = v*((u - s_h*t_h) - s_h*t_l); - /* compute log(ax) */ - s2 = s*s; - r = s2*s2*(L1+s2*(L2+s2*(L3+s2*(L4+s2*(L5+s2*L6))))); - r += s_l*(s_h+s); - s2 = s_h*s_h; - t_h = 3.0f + s2 + r; - GET_FLOAT_WORD(is, t_h); - SET_FLOAT_WORD(t_h, is & 0xfffff000); - t_l = r - ((t_h - 3.0f) - s2); - /* u+v = s*(1+...) */ - u = s_h*t_h; - v = s_l*t_h + t_l*s; - /* 2/(3log2)*(s+...) */ - p_h = u + v; - GET_FLOAT_WORD(is, p_h); - SET_FLOAT_WORD(p_h, is & 0xfffff000); - p_l = v - (p_h - u); - z_h = cp_h*p_h; /* cp_h+cp_l = 2/(3*log2) */ - z_l = cp_l*p_h + p_l*cp+dp_l[k]; - /* log2(ax) = (s+..)*2/(3*log2) = n + dp_h + z_h + z_l */ - t = (float)n; - t1 = (((z_h + z_l) + dp_h[k]) + t); - GET_FLOAT_WORD(is, t1); - SET_FLOAT_WORD(t1, is & 0xfffff000); - t2 = z_l - (((t1 - t) - dp_h[k]) - z_h); - } +/* Returns 0 if not int, 1 if odd int, 2 if even int. The argument is + the bit representation of a non-zero finite floating-point value. */ +static inline int checkint(uint32_t iy) +{ + int e = iy >> 23 & 0xff; + if (e < 0x7f) + return 0; + if (e > 0x7f + 23) + return 2; + if (iy & ((1 << (0x7f + 23 - e)) - 1)) + return 0; + if (iy & (1 << (0x7f + 23 - e))) + return 1; + return 2; +} + +static inline int zeroinfnan(uint32_t ix) +{ + return 2 * ix - 1 >= 2u * 0x7f800000 - 1; +} - /* split up y into y1+y2 and compute (y1+y2)*(t1+t2) */ - GET_FLOAT_WORD(is, y); - SET_FLOAT_WORD(y1, is & 0xfffff000); - p_l = (y-y1)*t1 + y*t2; - p_h = y1*t1; - z = p_l + p_h; - GET_FLOAT_WORD(j, z); - if (j > 0x43000000) /* if z > 128 */ - return sn*huge*huge; /* overflow */ - else if (j == 0x43000000) { /* if z == 128 */ - if (p_l + ovt > z - p_h) - return sn*huge*huge; /* overflow */ - } else if ((j&0x7fffffff) > 0x43160000) /* z < -150 */ // FIXME: check should be (uint32_t)j > 0xc3160000 - return sn*tiny*tiny; /* underflow */ - else if (j == 0xc3160000) { /* z == -150 */ - if (p_l <= z-p_h) - return sn*tiny*tiny; /* underflow */ +float powf(float x, float y) +{ + uint32_t sign_bias = 0; + uint32_t ix, iy; + + ix = asuint(x); + iy = asuint(y); + if (predict_false(ix - 0x00800000 >= 0x7f800000 - 0x00800000 || + zeroinfnan(iy))) { + /* Either (x < 0x1p-126 or inf or nan) or (y is 0 or inf or nan). */ + if (predict_false(zeroinfnan(iy))) { + if (2 * iy == 0) + return issignalingf_inline(x) ? x + y : 1.0f; + if (ix == 0x3f800000) + return issignalingf_inline(y) ? x + y : 1.0f; + if (2 * ix > 2u * 0x7f800000 || + 2 * iy > 2u * 0x7f800000) + return x + y; + if (2 * ix == 2 * 0x3f800000) + return 1.0f; + if ((2 * ix < 2 * 0x3f800000) == !(iy & 0x80000000)) + return 0.0f; /* |x|<1 && y==inf or |x|>1 && y==-inf. */ + return y * y; + } + if (predict_false(zeroinfnan(ix))) { + float_t x2 = x * x; + if (ix & 0x80000000 && checkint(iy) == 1) + x2 = -x2; + /* Without the barrier some versions of clang hoist the 1/x2 and + thus division by zero exception can be signaled spuriously. */ + return iy & 0x80000000 ? fp_barrierf(1 / x2) : x2; + } + /* x and y are non-zero finite. */ + if (ix & 0x80000000) { + /* Finite x < 0. */ + int yint = checkint(iy); + if (yint == 0) + return __math_invalidf(x); + if (yint == 1) + sign_bias = SIGN_BIAS; + ix &= 0x7fffffff; + } + if (ix < 0x00800000) { + /* Normalize subnormal x so exponent becomes negative. */ + ix = asuint(x * 0x1p23f); + ix &= 0x7fffffff; + ix -= 23 << 23; + } } - /* - * compute 2**(p_h+p_l) - */ - i = j & 0x7fffffff; - k = (i>>23) - 0x7f; - n = 0; - if (i > 0x3f000000) { /* if |z| > 0.5, set n = [z+0.5] */ - n = j + (0x00800000>>(k+1)); - k = ((n&0x7fffffff)>>23) - 0x7f; /* new k for n */ - SET_FLOAT_WORD(t, n & ~(0x007fffff>>k)); - n = ((n&0x007fffff)|0x00800000)>>(23-k); - if (j < 0) - n = -n; - p_h -= t; + double_t logx = log2_inline(ix); + double_t ylogx = y * logx; /* cannot overflow, y is single prec. */ + if (predict_false((asuint64(ylogx) >> 47 & 0xffff) >= + asuint64(126.0 * POWF_SCALE) >> 47)) { + /* |y*log(x)| >= 126. */ + if (ylogx > 0x1.fffffffd1d571p+6 * POWF_SCALE) + return __math_oflowf(sign_bias); + if (ylogx <= -150.0 * POWF_SCALE) + return __math_uflowf(sign_bias); } - t = p_l + p_h; - GET_FLOAT_WORD(is, t); - SET_FLOAT_WORD(t, is & 0xffff8000); - u = t*lg2_h; - v = (p_l-(t-p_h))*lg2 + t*lg2_l; - z = u + v; - w = v - (z - u); - t = z*z; - t1 = z - t*(P1+t*(P2+t*(P3+t*(P4+t*P5)))); - r = (z*t1)/(t1-2.0f) - (w+z*w); - z = 1.0f - (r - z); - GET_FLOAT_WORD(j, z); - j += n<<23; - if ((j>>23) <= 0) /* subnormal output */ - z = scalbnf(z, n); - else - SET_FLOAT_WORD(z, j); - return sn*z; + return exp2_inline(ylogx, sign_bias); } diff --git a/src/math/powf_data.c b/src/math/powf_data.c new file mode 100644 index 00000000..13e1d9a0 --- /dev/null +++ b/src/math/powf_data.c @@ -0,0 +1,34 @@ +/* + * Data definition for powf. + * + * Copyright (c) 2017-2018, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +#include "powf_data.h" + +const struct powf_log2_data __powf_log2_data = { + .tab = { + { 0x1.661ec79f8f3bep+0, -0x1.efec65b963019p-2 * POWF_SCALE }, + { 0x1.571ed4aaf883dp+0, -0x1.b0b6832d4fca4p-2 * POWF_SCALE }, + { 0x1.49539f0f010bp+0, -0x1.7418b0a1fb77bp-2 * POWF_SCALE }, + { 0x1.3c995b0b80385p+0, -0x1.39de91a6dcf7bp-2 * POWF_SCALE }, + { 0x1.30d190c8864a5p+0, -0x1.01d9bf3f2b631p-2 * POWF_SCALE }, + { 0x1.25e227b0b8eap+0, -0x1.97c1d1b3b7afp-3 * POWF_SCALE }, + { 0x1.1bb4a4a1a343fp+0, -0x1.2f9e393af3c9fp-3 * POWF_SCALE }, + { 0x1.12358f08ae5bap+0, -0x1.960cbbf788d5cp-4 * POWF_SCALE }, + { 0x1.0953f419900a7p+0, -0x1.a6f9db6475fcep-5 * POWF_SCALE }, + { 0x1p+0, 0x0p+0 * POWF_SCALE }, + { 0x1.e608cfd9a47acp-1, 0x1.338ca9f24f53dp-4 * POWF_SCALE }, + { 0x1.ca4b31f026aap-1, 0x1.476a9543891bap-3 * POWF_SCALE }, + { 0x1.b2036576afce6p-1, 0x1.e840b4ac4e4d2p-3 * POWF_SCALE }, + { 0x1.9c2d163a1aa2dp-1, 0x1.40645f0c6651cp-2 * POWF_SCALE }, + { 0x1.886e6037841edp-1, 0x1.88e9c2c1b9ff8p-2 * POWF_SCALE }, + { 0x1.767dcf5534862p-1, 0x1.ce0a44eb17bccp-2 * POWF_SCALE }, + }, + .poly = { + 0x1.27616c9496e0bp-2 * POWF_SCALE, -0x1.71969a075c67ap-2 * POWF_SCALE, + 0x1.ec70a6ca7baddp-2 * POWF_SCALE, -0x1.7154748bef6c8p-1 * POWF_SCALE, + 0x1.71547652ab82bp0 * POWF_SCALE, + } +}; diff --git a/src/math/powf_data.h b/src/math/powf_data.h new file mode 100644 index 00000000..5b136e28 --- /dev/null +++ b/src/math/powf_data.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017-2018, Arm Limited. + * SPDX-License-Identifier: MIT + */ +#ifndef _POWF_DATA_H +#define _POWF_DATA_H + +#include "libm.h" +#include "exp2f_data.h" + +#define POWF_LOG2_TABLE_BITS 4 +#define POWF_LOG2_POLY_ORDER 5 +#if TOINT_INTRINSICS +#define POWF_SCALE_BITS EXP2F_TABLE_BITS +#else +#define POWF_SCALE_BITS 0 +#endif +#define POWF_SCALE ((double)(1 << POWF_SCALE_BITS)) +extern hidden const struct powf_log2_data { + struct { + double invc, logc; + } tab[1 << POWF_LOG2_TABLE_BITS]; + double poly[POWF_LOG2_POLY_ORDER]; +} __powf_log2_data; + +#endif |