math: new powf

from https://github.com/ARM-software/optimized-routines,
commit 04884bd04eac4b251da4026900010ea7d8850edc

POWF_SCALE != 1.0 case only matters if TOINT_INTRINSICS is set, which
is currently not supported for any target.

SNaN is not supported, it would require an issignalingf
implementation.

code size change: -816 bytes.
benchmark on x86_64 before, after, speedup:

-Os:
  powf rthruput:  95.14 ns/call 20.04 ns/call 4.75x
   powf latency: 137.00 ns/call 34.98 ns/call 3.92x
-O3:
  powf rthruput:  92.48 ns/call 13.67 ns/call 6.77x
   powf latency: 131.11 ns/call 35.15 ns/call 3.73x

3 files changed, 226 insertions, 240 deletions

diff --git a/src/math/powf.c b/src/math/powf.c
index 427c8965..de8fab54 100644
--- a/src/math/powf.c
+++ b/src/math/powf.c
@@ -1,259 +1,185 @@
-/* origin: FreeBSD /usr/src/lib/msun/src/e_powf.c */
 /*
- * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com.
- */
-/*
- * ====================================================
- * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
- *
- * Developed at SunPro, a Sun Microsystems, Inc. business.
- * Permission to use, copy, modify, and distribute this
- * software is freely granted, provided that this notice
- * is preserved.
- * ====================================================
+ * Copyright (c) 2017-2018, Arm Limited.
+ * SPDX-License-Identifier: MIT
  */
 
+#include <math.h>
+#include <stdint.h>
 #include "libm.h"
+#include "exp2f_data.h"
+#include "powf_data.h"
 
-static const float
-bp[]   = {1.0, 1.5,},
-dp_h[] = { 0.0, 5.84960938e-01,}, /* 0x3f15c000 */
-dp_l[] = { 0.0, 1.56322085e-06,}, /* 0x35d1cfdc */
-two24  =  16777216.0,  /* 0x4b800000 */
-huge   =  1.0e30,
-tiny   =  1.0e-30,
-/* poly coefs for (3/2)*(log(x)-2s-2/3*s**3 */
-L1 =  6.0000002384e-01, /* 0x3f19999a */
-L2 =  4.2857143283e-01, /* 0x3edb6db7 */
-L3 =  3.3333334327e-01, /* 0x3eaaaaab */
-L4 =  2.7272811532e-01, /* 0x3e8ba305 */
-L5 =  2.3066075146e-01, /* 0x3e6c3255 */
-L6 =  2.0697501302e-01, /* 0x3e53f142 */
-P1 =  1.6666667163e-01, /* 0x3e2aaaab */
-P2 = -2.7777778450e-03, /* 0xbb360b61 */
-P3 =  6.6137559770e-05, /* 0x388ab355 */
-P4 = -1.6533901999e-06, /* 0xb5ddea0e */
-P5 =  4.1381369442e-08, /* 0x3331bb4c */
-lg2     =  6.9314718246e-01, /* 0x3f317218 */
-lg2_h   =  6.93145752e-01,   /* 0x3f317200 */
-lg2_l   =  1.42860654e-06,   /* 0x35bfbe8c */
-ovt     =  4.2995665694e-08, /* -(128-log2(ovfl+.5ulp)) */
-cp      =  9.6179670095e-01, /* 0x3f76384f =2/(3ln2) */
-cp_h    =  9.6191406250e-01, /* 0x3f764000 =12b cp */
-cp_l    = -1.1736857402e-04, /* 0xb8f623c6 =tail of cp_h */
-ivln2   =  1.4426950216e+00, /* 0x3fb8aa3b =1/ln2 */
-ivln2_h =  1.4426879883e+00, /* 0x3fb8aa00 =16b 1/ln2*/
-ivln2_l =  7.0526075433e-06; /* 0x36eca570 =1/ln2 tail*/
+/*
+POWF_LOG2_POLY_ORDER = 5
+EXP2F_TABLE_BITS = 5
 
-float powf(float x, float y)
+ULP error: 0.82 (~ 0.5 + relerr*2^24)
+relerr: 1.27 * 2^-26 (Relative error ~= 128*Ln2*relerr_log2 + relerr_exp2)
+relerr_log2: 1.83 * 2^-33 (Relative error of logx.)
+relerr_exp2: 1.69 * 2^-34 (Relative error of exp2(ylogx).)
+*/
+
+#define N (1 << POWF_LOG2_TABLE_BITS)
+#define T __powf_log2_data.tab
+#define A __powf_log2_data.poly
+#define OFF 0x3f330000
+
+/* Subnormal input is normalized so ix has negative biased exponent.
+   Output is multiplied by N (POWF_SCALE) if TOINT_INTRINICS is set.  */
+static inline double_t log2_inline(uint32_t ix)
 {
-	float z,ax,z_h,z_l,p_h,p_l;
-	float y1,t1,t2,r,s,sn,t,u,v,w;
-	int32_t i,j,k,yisint,n;
-	int32_t hx,hy,ix,iy,is;
+	double_t z, r, r2, r4, p, q, y, y0, invc, logc;
+	uint32_t iz, top, tmp;
+	int k, i;
 
-	GET_FLOAT_WORD(hx, x);
-	GET_FLOAT_WORD(hy, y);
-	ix = hx & 0x7fffffff;
-	iy = hy & 0x7fffffff;
+	/* x = 2^k z; where z is in range [OFF,2*OFF] and exact.
+	   The range is split into N subintervals.
+	   The ith subinterval contains z and c is near its center.  */
+	tmp = ix - OFF;
+	i = (tmp >> (23 - POWF_LOG2_TABLE_BITS)) % N;
+	top = tmp & 0xff800000;
+	iz = ix - top;
+	k = (int32_t)top >> (23 - POWF_SCALE_BITS); /* arithmetic shift */
+	invc = T[i].invc;
+	logc = T[i].logc;
+	z = (double_t)asfloat(iz);
 
-	/* x**0 = 1, even if x is NaN */
-	if (iy == 0)
-		return 1.0f;
-	/* 1**y = 1, even if y is NaN */
-	if (hx == 0x3f800000)
-		return 1.0f;
-	/* NaN if either arg is NaN */
-	if (ix > 0x7f800000 || iy > 0x7f800000)
-		return x + y;
+	/* log2(x) = log1p(z/c-1)/ln2 + log2(c) + k */
+	r = z * invc - 1;
+	y0 = logc + (double_t)k;
 
-	/* determine if y is an odd int when x < 0
-	 * yisint = 0       ... y is not an integer
-	 * yisint = 1       ... y is an odd int
-	 * yisint = 2       ... y is an even int
-	 */
-	yisint  = 0;
-	if (hx < 0) {
-		if (iy >= 0x4b800000)
-			yisint = 2; /* even integer y */
-		else if (iy >= 0x3f800000) {
-			k = (iy>>23) - 0x7f;         /* exponent */
-			j = iy>>(23-k);
-			if ((j<<(23-k)) == iy)
-				yisint = 2 - (j & 1);
-		}
-	}
+	/* Pipelined polynomial evaluation to approximate log1p(r)/ln2.  */
+	r2 = r * r;
+	y = A[0] * r + A[1];
+	p = A[2] * r + A[3];
+	r4 = r2 * r2;
+	q = A[4] * r + y0;
+	q = p * r2 + q;
+	y = y * r4 + q;
+	return y;
+}
 
-	/* special value of y */
-	if (iy == 0x7f800000) {  /* y is +-inf */
-		if (ix == 0x3f800000)      /* (-1)**+-inf is 1 */
-			return 1.0f;
-		else if (ix > 0x3f800000)  /* (|x|>1)**+-inf = inf,0 */
-			return hy >= 0 ? y : 0.0f;
-		else                       /* (|x|<1)**+-inf = 0,inf */
-			return hy >= 0 ? 0.0f: -y;
-	}
-	if (iy == 0x3f800000)    /* y is +-1 */
-		return hy >= 0 ? x : 1.0f/x;
-	if (hy == 0x40000000)    /* y is 2 */
-		return x*x;
-	if (hy == 0x3f000000) {  /* y is  0.5 */
-		if (hx >= 0)     /* x >= +0 */
-			return sqrtf(x);
-	}
+#undef N
+#undef T
+#define N (1 << EXP2F_TABLE_BITS)
+#define T __exp2f_data.tab
+#define SIGN_BIAS (1 << (EXP2F_TABLE_BITS + 11))
 
-	ax = fabsf(x);
-	/* special value of x */
-	if (ix == 0x7f800000 || ix == 0 || ix == 0x3f800000) { /* x is +-0,+-inf,+-1 */
-		z = ax;
-		if (hy < 0)  /* z = (1/|x|) */
-			z = 1.0f/z;
-		if (hx < 0) {
-			if (((ix-0x3f800000)|yisint) == 0) {
-				z = (z-z)/(z-z); /* (-1)**non-int is NaN */
-			} else if (yisint == 1)
-				z = -z;          /* (x<0)**odd = -(|x|**odd) */
-		}
-		return z;
-	}
+/* The output of log2 and thus the input of exp2 is either scaled by N
+   (in case of fast toint intrinsics) or not.  The unscaled xd must be
+   in [-1021,1023], sign_bias sets the sign of the result.  */
+static inline float exp2_inline(double_t xd, uint32_t sign_bias)
+{
+	uint64_t ki, ski, t;
+	double_t kd, z, r, r2, y, s;
 
-	sn = 1.0f; /* sign of result */
-	if (hx < 0) {
-		if (yisint == 0) /* (x<0)**(non-int) is NaN */
-			return (x-x)/(x-x);
-		if (yisint == 1) /* (x<0)**(odd int) */
-			sn = -1.0f;
-	}
+#if TOINT_INTRINSICS
+#define C __exp2f_data.poly_scaled
+	/* N*x = k + r with r in [-1/2, 1/2] */
+	kd = roundtoint(xd); /* k */
+	ki = converttoint(xd);
+#else
+#define C __exp2f_data.poly
+#define SHIFT __exp2f_data.shift_scaled
+	/* x = k/N + r with r in [-1/(2N), 1/(2N)] */
+	kd = eval_as_double(xd + SHIFT);
+	ki = asuint64(kd);
+	kd -= SHIFT; /* k/N */
+#endif
+	r = xd - kd;
 
-	/* |y| is huge */
-	if (iy > 0x4d000000) { /* if |y| > 2**27 */
-		/* over/underflow if x is not close to one */
-		if (ix < 0x3f7ffff8)
-			return hy < 0 ? sn*huge*huge : sn*tiny*tiny;
-		if (ix > 0x3f800007)
-			return hy > 0 ? sn*huge*huge : sn*tiny*tiny;
-		/* now |1-x| is tiny <= 2**-20, suffice to compute
-		   log(x) by x-x^2/2+x^3/3-x^4/4 */
-		t = ax - 1;     /* t has 20 trailing zeros */
-		w = (t*t)*(0.5f - t*(0.333333333333f - t*0.25f));
-		u = ivln2_h*t;  /* ivln2_h has 16 sig. bits */
-		v = t*ivln2_l - w*ivln2;
-		t1 = u + v;
-		GET_FLOAT_WORD(is, t1);
-		SET_FLOAT_WORD(t1, is & 0xfffff000);
-		t2 = v - (t1-u);
-	} else {
-		float s2,s_h,s_l,t_h,t_l;
-		n = 0;
-		/* take care subnormal number */
-		if (ix < 0x00800000) {
-			ax *= two24;
-			n -= 24;
-			GET_FLOAT_WORD(ix, ax);
-		}
-		n += ((ix)>>23) - 0x7f;
-		j = ix & 0x007fffff;
-		/* determine interval */
-		ix = j | 0x3f800000;     /* normalize ix */
-		if (j <= 0x1cc471)       /* |x|<sqrt(3/2) */
-			k = 0;
-		else if (j < 0x5db3d7)   /* |x|<sqrt(3)   */
-			k = 1;
-		else {
-			k = 0;
-			n += 1;
-			ix -= 0x00800000;
-		}
-		SET_FLOAT_WORD(ax, ix);
+	/* exp2(x) = 2^(k/N) * 2^r ~= s * (C0*r^3 + C1*r^2 + C2*r + 1) */
+	t = T[ki % N];
+	ski = ki + sign_bias;
+	t += ski << (52 - EXP2F_TABLE_BITS);
+	s = asdouble(t);
+	z = C[0] * r + C[1];
+	r2 = r * r;
+	y = C[2] * r + 1;
+	y = z * r2 + y;
+	y = y * s;
+	return eval_as_float(y);
+}
 
-		/* compute s = s_h+s_l = (x-1)/(x+1) or (x-1.5)/(x+1.5) */
-		u = ax - bp[k];   /* bp[0]=1.0, bp[1]=1.5 */
-		v = 1.0f/(ax+bp[k]);
-		s = u*v;
-		s_h = s;
-		GET_FLOAT_WORD(is, s_h);
-		SET_FLOAT_WORD(s_h, is & 0xfffff000);
-		/* t_h=ax+bp[k] High */
-		is = ((ix>>1) & 0xfffff000) | 0x20000000;
-		SET_FLOAT_WORD(t_h, is + 0x00400000 + (k<<21));
-		t_l = ax - (t_h - bp[k]);
-		s_l = v*((u - s_h*t_h) - s_h*t_l);
-		/* compute log(ax) */
-		s2 = s*s;
-		r = s2*s2*(L1+s2*(L2+s2*(L3+s2*(L4+s2*(L5+s2*L6)))));
-		r += s_l*(s_h+s);
-		s2 = s_h*s_h;
-		t_h = 3.0f + s2 + r;
-		GET_FLOAT_WORD(is, t_h);
-		SET_FLOAT_WORD(t_h, is & 0xfffff000);
-		t_l = r - ((t_h - 3.0f) - s2);
-		/* u+v = s*(1+...) */
-		u = s_h*t_h;
-		v = s_l*t_h + t_l*s;
-		/* 2/(3log2)*(s+...) */
-		p_h = u + v;
-		GET_FLOAT_WORD(is, p_h);
-		SET_FLOAT_WORD(p_h, is & 0xfffff000);
-		p_l = v - (p_h - u);
-		z_h = cp_h*p_h;  /* cp_h+cp_l = 2/(3*log2) */
-		z_l = cp_l*p_h + p_l*cp+dp_l[k];
-		/* log2(ax) = (s+..)*2/(3*log2) = n + dp_h + z_h + z_l */
-		t = (float)n;
-		t1 = (((z_h + z_l) + dp_h[k]) + t);
-		GET_FLOAT_WORD(is, t1);
-		SET_FLOAT_WORD(t1, is & 0xfffff000);
-		t2 = z_l - (((t1 - t) - dp_h[k]) - z_h);
-	}
+/* Returns 0 if not int, 1 if odd int, 2 if even int.  The argument is
+   the bit representation of a non-zero finite floating-point value.  */
+static inline int checkint(uint32_t iy)
+{
+	int e = iy >> 23 & 0xff;
+	if (e < 0x7f)
+		return 0;
+	if (e > 0x7f + 23)
+		return 2;
+	if (iy & ((1 << (0x7f + 23 - e)) - 1))
+		return 0;
+	if (iy & (1 << (0x7f + 23 - e)))
+		return 1;
+	return 2;
+}
+
+static inline int zeroinfnan(uint32_t ix)
+{
+	return 2 * ix - 1 >= 2u * 0x7f800000 - 1;
+}
 
-	/* split up y into y1+y2 and compute (y1+y2)*(t1+t2) */
-	GET_FLOAT_WORD(is, y);
-	SET_FLOAT_WORD(y1, is & 0xfffff000);
-	p_l = (y-y1)*t1 + y*t2;
-	p_h = y1*t1;
-	z = p_l + p_h;
-	GET_FLOAT_WORD(j, z);
-	if (j > 0x43000000)          /* if z > 128 */
-		return sn*huge*huge;  /* overflow */
-	else if (j == 0x43000000) {  /* if z == 128 */
-		if (p_l + ovt > z - p_h)
-			return sn*huge*huge;  /* overflow */
-	} else if ((j&0x7fffffff) > 0x43160000)  /* z < -150 */ // FIXME: check should be  (uint32_t)j > 0xc3160000
-		return sn*tiny*tiny;  /* underflow */
-	else if (j == 0xc3160000) {  /* z == -150 */
-		if (p_l <= z-p_h)
-			return sn*tiny*tiny;  /* underflow */
+float powf(float x, float y)
+{
+	uint32_t sign_bias = 0;
+	uint32_t ix, iy;
+
+	ix = asuint(x);
+	iy = asuint(y);
+	if (predict_false(ix - 0x00800000 >= 0x7f800000 - 0x00800000 ||
+			  zeroinfnan(iy))) {
+		/* Either (x < 0x1p-126 or inf or nan) or (y is 0 or inf or nan).  */
+		if (predict_false(zeroinfnan(iy))) {
+			if (2 * iy == 0)
+				return issignalingf_inline(x) ? x + y : 1.0f;
+			if (ix == 0x3f800000)
+				return issignalingf_inline(y) ? x + y : 1.0f;
+			if (2 * ix > 2u * 0x7f800000 ||
+			    2 * iy > 2u * 0x7f800000)
+				return x + y;
+			if (2 * ix == 2 * 0x3f800000)
+				return 1.0f;
+			if ((2 * ix < 2 * 0x3f800000) == !(iy & 0x80000000))
+				return 0.0f; /* |x|<1 && y==inf or |x|>1 && y==-inf.  */
+			return y * y;
+		}
+		if (predict_false(zeroinfnan(ix))) {
+			float_t x2 = x * x;
+			if (ix & 0x80000000 && checkint(iy) == 1)
+				x2 = -x2;
+			/* Without the barrier some versions of clang hoist the 1/x2 and
+			   thus division by zero exception can be signaled spuriously.  */
+			return iy & 0x80000000 ? fp_barrierf(1 / x2) : x2;
+		}
+		/* x and y are non-zero finite.  */
+		if (ix & 0x80000000) {
+			/* Finite x < 0.  */
+			int yint = checkint(iy);
+			if (yint == 0)
+				return __math_invalidf(x);
+			if (yint == 1)
+				sign_bias = SIGN_BIAS;
+			ix &= 0x7fffffff;
+		}
+		if (ix < 0x00800000) {
+			/* Normalize subnormal x so exponent becomes negative.  */
+			ix = asuint(x * 0x1p23f);
+			ix &= 0x7fffffff;
+			ix -= 23 << 23;
+		}
 	}
-	/*
-	 * compute 2**(p_h+p_l)
-	 */
-	i = j & 0x7fffffff;
-	k = (i>>23) - 0x7f;
-	n = 0;
-	if (i > 0x3f000000) {   /* if |z| > 0.5, set n = [z+0.5] */
-		n = j + (0x00800000>>(k+1));
-		k = ((n&0x7fffffff)>>23) - 0x7f;  /* new k for n */
-		SET_FLOAT_WORD(t, n & ~(0x007fffff>>k));
-		n = ((n&0x007fffff)|0x00800000)>>(23-k);
-		if (j < 0)
-			n = -n;
-		p_h -= t;
+	double_t logx = log2_inline(ix);
+	double_t ylogx = y * logx; /* cannot overflow, y is single prec.  */
+	if (predict_false((asuint64(ylogx) >> 47 & 0xffff) >=
+			  asuint64(126.0 * POWF_SCALE) >> 47)) {
+		/* |y*log(x)| >= 126.  */
+		if (ylogx > 0x1.fffffffd1d571p+6 * POWF_SCALE)
+			return __math_oflowf(sign_bias);
+		if (ylogx <= -150.0 * POWF_SCALE)
+			return __math_uflowf(sign_bias);
 	}
-	t = p_l + p_h;
-	GET_FLOAT_WORD(is, t);
-	SET_FLOAT_WORD(t, is & 0xffff8000);
-	u = t*lg2_h;
-	v = (p_l-(t-p_h))*lg2 + t*lg2_l;
-	z = u + v;
-	w = v - (z - u);
-	t = z*z;
-	t1 = z - t*(P1+t*(P2+t*(P3+t*(P4+t*P5))));
-	r = (z*t1)/(t1-2.0f) - (w+z*w);
-	z = 1.0f - (r - z);
-	GET_FLOAT_WORD(j, z);
-	j += n<<23;
-	if ((j>>23) <= 0)  /* subnormal output */
-		z = scalbnf(z, n);
-	else
-		SET_FLOAT_WORD(z, j);
-	return sn*z;
+	return exp2_inline(ylogx, sign_bias);
 }
diff --git a/src/math/powf_data.c b/src/math/powf_data.c
new file mode 100644
index 00000000..13e1d9a0
--- /dev/null
+++ b/src/math/powf_data.c
@@ -0,0 +1,34 @@
+/*
+ * Data definition for powf.
+ *
+ * Copyright (c) 2017-2018, Arm Limited.
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "powf_data.h"
+
+const struct powf_log2_data __powf_log2_data = {
+  .tab = {
+  { 0x1.661ec79f8f3bep+0, -0x1.efec65b963019p-2 * POWF_SCALE },
+  { 0x1.571ed4aaf883dp+0, -0x1.b0b6832d4fca4p-2 * POWF_SCALE },
+  { 0x1.49539f0f010bp+0, -0x1.7418b0a1fb77bp-2 * POWF_SCALE },
+  { 0x1.3c995b0b80385p+0, -0x1.39de91a6dcf7bp-2 * POWF_SCALE },
+  { 0x1.30d190c8864a5p+0, -0x1.01d9bf3f2b631p-2 * POWF_SCALE },
+  { 0x1.25e227b0b8eap+0, -0x1.97c1d1b3b7afp-3 * POWF_SCALE },
+  { 0x1.1bb4a4a1a343fp+0, -0x1.2f9e393af3c9fp-3 * POWF_SCALE },
+  { 0x1.12358f08ae5bap+0, -0x1.960cbbf788d5cp-4 * POWF_SCALE },
+  { 0x1.0953f419900a7p+0, -0x1.a6f9db6475fcep-5 * POWF_SCALE },
+  { 0x1p+0, 0x0p+0 * POWF_SCALE },
+  { 0x1.e608cfd9a47acp-1, 0x1.338ca9f24f53dp-4 * POWF_SCALE },
+  { 0x1.ca4b31f026aap-1, 0x1.476a9543891bap-3 * POWF_SCALE },
+  { 0x1.b2036576afce6p-1, 0x1.e840b4ac4e4d2p-3 * POWF_SCALE },
+  { 0x1.9c2d163a1aa2dp-1, 0x1.40645f0c6651cp-2 * POWF_SCALE },
+  { 0x1.886e6037841edp-1, 0x1.88e9c2c1b9ff8p-2 * POWF_SCALE },
+  { 0x1.767dcf5534862p-1, 0x1.ce0a44eb17bccp-2 * POWF_SCALE },
+  },
+  .poly = {
+  0x1.27616c9496e0bp-2 * POWF_SCALE, -0x1.71969a075c67ap-2 * POWF_SCALE,
+  0x1.ec70a6ca7baddp-2 * POWF_SCALE, -0x1.7154748bef6c8p-1 * POWF_SCALE,
+  0x1.71547652ab82bp0 * POWF_SCALE,
+  }
+};
diff --git a/src/math/powf_data.h b/src/math/powf_data.h
new file mode 100644
index 00000000..5b136e28
--- /dev/null
+++ b/src/math/powf_data.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2017-2018, Arm Limited.
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef _POWF_DATA_H
+#define _POWF_DATA_H
+
+#include "libm.h"
+#include "exp2f_data.h"
+
+#define POWF_LOG2_TABLE_BITS 4
+#define POWF_LOG2_POLY_ORDER 5
+#if TOINT_INTRINSICS
+#define POWF_SCALE_BITS EXP2F_TABLE_BITS
+#else
+#define POWF_SCALE_BITS 0
+#endif
+#define POWF_SCALE ((double)(1 << POWF_SCALE_BITS))
+extern hidden const struct powf_log2_data {
+	struct {
+		double invc, logc;
+	} tab[1 << POWF_LOG2_TABLE_BITS];
+	double poly[POWF_LOG2_POLY_ORDER];
+} __powf_log2_data;
+
+#endif