Fix scalbn, scalbln integer overflow.

15 files changed, 121 insertions, 40 deletions

diff --git a/ChangeLog b/ChangeLog
index cb3e30b534..b856c4ba5a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,24 @@
 2012-03-02  Joseph Myers  <joseph@codesourcery.com>
 
+	[BZ #10135]
+	* sysdeps/ieee754/dbl-64/s_scalbln.c (__scalbln): First test for
+	small n, then large n, before computing and testing k+n.
+	* sysdeps/ieee754/dbl-64/s_scalbn.c (__scalbn): Likewise.
+	* sysdeps/ieee754/dbl-64/wordsize-64/s_scalbln.c (__scalbln):
+	Likewise.
+	* sysdeps/ieee754/dbl-64/wordsize-64/s_scalbn.c (__scalbn):
+	Likewise.
+	* sysdeps/ieee754/flt-32/s_scalblnf.c (__scalblnf): Likewise.
+	* sysdeps/ieee754/flt-32/s_scalbnf.c (__scalbnf): Likewise.
+	* sysdeps/ieee754/ldbl-128/s_scalblnl.c (__scalblnl): Likewise.
+	* sysdeps/ieee754/ldbl-128/s_scalbnl.c (__scalbnl): Likewise.
+	* sysdeps/ieee754/ldbl-128ibm/s_scalblnl.c (__scalblnl): Likewise.
+	* sysdeps/ieee754/ldbl-128ibm/s_scalbnl.c (__scalbnl): Likewise.
+	* sysdeps/ieee754/ldbl-96/s_scalblnl.c (__scalblnl): Likewise.
+	* sysdeps/ieee754/ldbl-96/s_scalbnl.c (__scalbnl): Likewise.
+	* math/libm-test.inc (scalbn_test): Add more tests.
+	(scalbln_test): Likewise.
+
 	* manual/filesys.texi (mode_t): Describe constraints on size and
 	signedness, not exact equivalence to a particular type.
 	(ino_t): Likewise.
diff --git a/NEWS b/NEWS
index 6c0bc4acab..5d7d093598 100644
--- a/NEWS
+++ b/NEWS
@@ -10,10 +10,10 @@ Version 2.16
 * The following bugs are resolved with this release:
 
   174, 350, 411, 2541, 2547, 2548, 3335, 3992, 4026, 4108, 4596, 4822, 5077,
-  5461, 5805, 5993, 6884, 6907, 9739, 9902, 10110, 10140, 10210, 11174,
-  11322, 11365, 11494, 12047, 13058, 13525, 13526, 13527, 13528, 13529,
-  13530, 13531, 13532, 13533, 13547, 13551, 13552, 13553, 13555, 13559,
-  13583, 13618, 13637, 13695, 13704, 13706, 13738, 13786
+  5461, 5805, 5993, 6884, 6907, 9739, 9902, 10110, 10135, 10140, 10210,
+  11174, 11322, 11365, 11494, 12047, 13058, 13525, 13526, 13527, 13528,
+  13529, 13530, 13531, 13532, 13533, 13547, 13551, 13552, 13553, 13555,
+  13559, 13583, 13618, 13637, 13695, 13704, 13706, 13738, 13786
 
 * ISO C11 support:
 
diff --git a/math/libm-test.inc b/math/libm-test.inc
index 5bc0d40872..ef7a253662 100644
--- a/math/libm-test.inc
+++ b/math/libm-test.inc
@@ -5790,6 +5790,15 @@ scalbn_test (void)
 
   TEST_fi_f (scalbn, 1, 0L, 1);
 
+  TEST_fi_f (scalbn, 1, INT_MAX, plus_infty);
+  TEST_fi_f (scalbn, 1, INT_MIN, plus_zero);
+  TEST_fi_f (scalbn, max_value, INT_MAX, plus_infty);
+  TEST_fi_f (scalbn, max_value, INT_MIN, plus_zero);
+  TEST_fi_f (scalbn, min_value, INT_MAX, plus_infty);
+  TEST_fi_f (scalbn, min_value, INT_MIN, plus_zero);
+  TEST_fi_f (scalbn, min_value / 4, INT_MAX, plus_infty);
+  TEST_fi_f (scalbn, min_value / 4, INT_MIN, plus_zero);
+
   END (scalbn);
 }
 
@@ -5812,6 +5821,35 @@ scalbln_test (void)
 
   TEST_fl_f (scalbln, 1, 0L, 1);
 
+  TEST_fi_f (scalbln, 1, INT_MAX, plus_infty);
+  TEST_fi_f (scalbln, 1, INT_MIN, plus_zero);
+  TEST_fi_f (scalbln, max_value, INT_MAX, plus_infty);
+  TEST_fi_f (scalbln, max_value, INT_MIN, plus_zero);
+  TEST_fi_f (scalbln, min_value, INT_MAX, plus_infty);
+  TEST_fi_f (scalbln, min_value, INT_MIN, plus_zero);
+  TEST_fi_f (scalbln, min_value / 4, INT_MAX, plus_infty);
+  TEST_fi_f (scalbln, min_value / 4, INT_MIN, plus_zero);
+
+  TEST_fi_f (scalbln, 1, LONG_MAX, plus_infty);
+  TEST_fi_f (scalbln, 1, LONG_MIN, plus_zero);
+  TEST_fi_f (scalbln, max_value, LONG_MAX, plus_infty);
+  TEST_fi_f (scalbln, max_value, LONG_MIN, plus_zero);
+  TEST_fi_f (scalbln, min_value, LONG_MAX, plus_infty);
+  TEST_fi_f (scalbln, min_value, LONG_MIN, plus_zero);
+  TEST_fi_f (scalbln, min_value / 4, LONG_MAX, plus_infty);
+  TEST_fi_f (scalbln, min_value / 4, LONG_MIN, plus_zero);
+
+#if LONG_MAX >= 0x100000000
+  TEST_fi_f (scalbln, 1, 0x88000000L, plus_infty);
+  TEST_fi_f (scalbln, 1, -0x88000000L, plus_zero);
+  TEST_fi_f (scalbln, max_value, 0x88000000L, plus_infty);
+  TEST_fi_f (scalbln, max_value, -0x88000000L, plus_zero);
+  TEST_fi_f (scalbln, min_value, 0x88000000L, plus_infty);
+  TEST_fi_f (scalbln, min_value, -0x88000000L, plus_zero);
+  TEST_fi_f (scalbln, min_value / 4, 0x88000000L, plus_infty);
+  TEST_fi_f (scalbln, min_value / 4, -0x88000000L, plus_zero);
+#endif
+
   END (scalbn);
 }
 
diff --git a/sysdeps/ieee754/dbl-64/s_scalbln.c b/sysdeps/ieee754/dbl-64/s_scalbln.c
index 89174b47fb..b5903c97d0 100644
--- a/sysdeps/ieee754/dbl-64/s_scalbln.c
+++ b/sysdeps/ieee754/dbl-64/s_scalbln.c
@@ -38,11 +38,13 @@ __scalbln (double x, long int n)
 	    k = ((hx&0x7ff00000)>>20) - 54;
 	    }
 	if (__builtin_expect(k==0x7ff, 0)) return x+x;	/* NaN or Inf */
-	k = k+n;
-	if (__builtin_expect(n> 50000 || k >  0x7fe, 0))
-	  return huge*__copysign(huge,x); /* overflow  */
 	if (__builtin_expect(n< -50000, 0))
 	  return tiny*__copysign(tiny,x); /*underflow*/
+	if (__builtin_expect(n> 50000 || k+n > 0x7fe, 0))
+	  return huge*__copysign(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (__builtin_expect(k > 0, 1))		/* normal result */
 	    {SET_HIGH_WORD(x,(hx&0x800fffff)|(k<<20)); return x;}
 	if (k <= -54)
diff --git a/sysdeps/ieee754/dbl-64/s_scalbn.c b/sysdeps/ieee754/dbl-64/s_scalbn.c
index 1e67dbe5eb..c2488fbbee 100644
--- a/sysdeps/ieee754/dbl-64/s_scalbn.c
+++ b/sysdeps/ieee754/dbl-64/s_scalbn.c
@@ -38,11 +38,13 @@ __scalbn (double x, int n)
 	    k = ((hx&0x7ff00000)>>20) - 54;
 	    }
 	if (__builtin_expect(k==0x7ff, 0)) return x+x;	/* NaN or Inf */
-	k = k+n;
-	if (__builtin_expect(n> 50000 || k >  0x7fe, 0))
-	  return huge*__copysign(huge,x); /* overflow  */
 	if (__builtin_expect(n< -50000, 0))
 	  return tiny*__copysign(tiny,x); /*underflow*/
+	if (__builtin_expect(n> 50000 || k+n > 0x7fe, 0))
+	  return huge*__copysign(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (__builtin_expect(k > 0, 1))		/* normal result */
 	    {SET_HIGH_WORD(x,(hx&0x800fffff)|(k<<20)); return x;}
 	if (k <= -54)
diff --git a/sysdeps/ieee754/dbl-64/wordsize-64/s_scalbln.c b/sysdeps/ieee754/dbl-64/wordsize-64/s_scalbln.c
index d6b97b5483..1d0da687c1 100644
--- a/sysdeps/ieee754/dbl-64/wordsize-64/s_scalbln.c
+++ b/sysdeps/ieee754/dbl-64/wordsize-64/s_scalbln.c
@@ -39,11 +39,13 @@ __scalbln (double x, long int n)
 	    k = ((ix >> 52) & 0x7ff) - 54;
 	    }
 	if (__builtin_expect(k==0x7ff, 0)) return x+x;	/* NaN or Inf */
-	k = k+n;
-	if (__builtin_expect(n> 50000 || k >  0x7fe, 0))
-	  return huge*__copysign(huge,x); /* overflow  */
 	if (__builtin_expect(n< -50000, 0))
 	  return tiny*__copysign(tiny,x); /*underflow*/
+	if (__builtin_expect(n> 50000 || k+n > 0x7fe, 0))
+	  return huge*__copysign(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (__builtin_expect(k > 0, 1))		/* normal result */
 	    {INSERT_WORDS64(x,(ix&UINT64_C(0x800fffffffffffff))|(k<<52));
 	      return x;}
diff --git a/sysdeps/ieee754/dbl-64/wordsize-64/s_scalbn.c b/sysdeps/ieee754/dbl-64/wordsize-64/s_scalbn.c
index 7f0e21f646..e183c3875f 100644
--- a/sysdeps/ieee754/dbl-64/wordsize-64/s_scalbn.c
+++ b/sysdeps/ieee754/dbl-64/wordsize-64/s_scalbn.c
@@ -39,11 +39,13 @@ __scalbn (double x, int n)
 	    k = ((ix >> 52) & 0x7ff) - 54;
 	    }
 	if (__builtin_expect(k==0x7ff, 0)) return x+x;	/* NaN or Inf */
-	k = k+n;
-	if (__builtin_expect(n> 50000 || k >  0x7fe, 0))
-	  return huge*__copysign(huge,x); /* overflow  */
 	if (__builtin_expect(n< -50000, 0))
 	  return tiny*__copysign(tiny,x); /*underflow*/
+	if (__builtin_expect(n> 50000 || k+n > 0x7fe, 0))
+	  return huge*__copysign(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (__builtin_expect(k > 0, 1))		/* normal result */
 	    {INSERT_WORDS64(x,(ix&UINT64_C(0x800fffffffffffff))|(k<<52));
 	      return x;}
diff --git a/sysdeps/ieee754/flt-32/s_scalblnf.c b/sysdeps/ieee754/flt-32/s_scalblnf.c
index 5256c32592..2a2d7ab651 100644
--- a/sysdeps/ieee754/flt-32/s_scalblnf.c
+++ b/sysdeps/ieee754/flt-32/s_scalblnf.c
@@ -35,11 +35,13 @@ __scalblnf (float x, long int n)
 	    k = ((ix&0x7f800000)>>23) - 25;
 	    }
 	if (__builtin_expect(k==0xff, 0)) return x+x;	/* NaN or Inf */
-	k = k+n;
-	if (__builtin_expect(n> 50000 || k >  0xfe, 0))
-	  return huge*copysignf(huge,x); /* overflow  */
 	if (__builtin_expect(n< -50000, 0))
 	  return tiny*copysignf(tiny,x);	/*underflow*/
+	if (__builtin_expect(n> 50000 || k+n > 0xfe, 0))
+	  return huge*copysignf(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (__builtin_expect(k > 0, 1))		/* normal result */
 	    {SET_FLOAT_WORD(x,(ix&0x807fffff)|(k<<23)); return x;}
 	if (k <= -25)
diff --git a/sysdeps/ieee754/flt-32/s_scalbnf.c b/sysdeps/ieee754/flt-32/s_scalbnf.c
index 3be2925a03..a7cb1a1a49 100644
--- a/sysdeps/ieee754/flt-32/s_scalbnf.c
+++ b/sysdeps/ieee754/flt-32/s_scalbnf.c
@@ -35,11 +35,13 @@ __scalbnf (float x, int n)
 	    k = ((ix&0x7f800000)>>23) - 25;
 	    }
 	if (__builtin_expect(k==0xff, 0)) return x+x;	/* NaN or Inf */
-	k = k+n;
-	if (__builtin_expect(n> 50000 || k >  0xfe, 0))
-	  return huge*__copysignf(huge,x); /* overflow  */
 	if (__builtin_expect(n< -50000, 0))
 	  return tiny*__copysignf(tiny,x);	/*underflow*/
+	if (__builtin_expect(n> 50000 || k+n > 0xfe, 0))
+	  return huge*__copysignf(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (__builtin_expect(k > 0, 1))		/* normal result */
 	    {SET_FLOAT_WORD(x,(ix&0x807fffff)|(k<<23)); return x;}
 	if (k <= -25)
diff --git a/sysdeps/ieee754/ldbl-128/s_scalblnl.c b/sysdeps/ieee754/ldbl-128/s_scalblnl.c
index f5624e2a73..9ad976bb11 100644
--- a/sysdeps/ieee754/ldbl-128/s_scalblnl.c
+++ b/sysdeps/ieee754/ldbl-128/s_scalblnl.c
@@ -46,10 +46,12 @@ long double __scalblnl (long double x, long int n)
 	    k = ((hx>>48)&0x7fff) - 114;
 	}
         if (k==0x7fff) return x+x;		/* NaN or Inf */
-        k = k+n;
-        if (n> 50000 || k > 0x7ffe)
-	  return huge*__copysignl(huge,x); /* overflow  */
 	if (n< -50000) return tiny*__copysignl(tiny,x); /*underflow*/
+        if (n> 50000 || k+n > 0x7ffe)
+	  return huge*__copysignl(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+        k = k+n;
         if (k > 0) 				/* normal result */
 	    {SET_LDOUBLE_MSW64(x,(hx&0x8000ffffffffffffULL)|(k<<48)); return x;}
         if (k <= -114)
diff --git a/sysdeps/ieee754/ldbl-128/s_scalbnl.c b/sysdeps/ieee754/ldbl-128/s_scalbnl.c
index b9de0f74fc..a928ecccec 100644
--- a/sysdeps/ieee754/ldbl-128/s_scalbnl.c
+++ b/sysdeps/ieee754/ldbl-128/s_scalbnl.c
@@ -46,10 +46,12 @@ long double __scalbnl (long double x, int n)
 	    k = ((hx>>48)&0x7fff) - 114;
 	}
         if (k==0x7fff) return x+x;		/* NaN or Inf */
-        k = k+n;
-        if (n> 50000 || k > 0x7ffe)
-	  return huge*__copysignl(huge,x); /* overflow  */
 	if (n< -50000) return tiny*__copysignl(tiny,x); /*underflow*/
+        if (n> 50000 || k+n > 0x7ffe)
+	  return huge*__copysignl(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+        k = k+n;
         if (k > 0) 				/* normal result */
 	    {SET_LDOUBLE_MSW64(x,(hx&0x8000ffffffffffffULL)|(k<<48)); return x;}
         if (k <= -114)
diff --git a/sysdeps/ieee754/ldbl-128ibm/s_scalblnl.c b/sysdeps/ieee754/ldbl-128ibm/s_scalblnl.c
index a1632e77fc..2c30d1c78a 100644
--- a/sysdeps/ieee754/ldbl-128ibm/s_scalblnl.c
+++ b/sysdeps/ieee754/ldbl-128ibm/s_scalblnl.c
@@ -52,10 +52,12 @@ long double __scalblnl (long double x, long int n)
 	    k = ((hx>>52)&0x7ff) - 54;
 	}
 	else if (k==0x7ff) return x+x;		/* NaN or Inf */
-	k = k+n;
-	if (n> 50000 || k > 0x7fe)
-	  return huge*__copysignl(huge,x); /* overflow */
 	if (n< -50000) return tiny*__copysignl(tiny,x); /*underflow */
+	if (n> 50000 || k+n > 0x7fe)
+	  return huge*__copysignl(huge,x); /* overflow */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (k > 0) {				/* normal result */
 	    hx = (hx&0x800fffffffffffffULL)|(k<<52);
 	    if ((lx & 0x7fffffffffffffffULL) == 0) { /* low part +-0 */
diff --git a/sysdeps/ieee754/ldbl-128ibm/s_scalbnl.c b/sysdeps/ieee754/ldbl-128ibm/s_scalbnl.c
index a52cd42f0b..48102efcf5 100644
--- a/sysdeps/ieee754/ldbl-128ibm/s_scalbnl.c
+++ b/sysdeps/ieee754/ldbl-128ibm/s_scalbnl.c
@@ -52,10 +52,12 @@ long double __scalbnl (long double x, int n)
 	    k = ((hx>>52)&0x7ff) - 54;
 	}
 	else if (k==0x7ff) return x+x;		/* NaN or Inf */
-	k = k+n;
-	if (n> 50000 || k > 0x7fe)
-	  return huge*__copysignl(huge,x); /* overflow */
 	if (n< -50000) return tiny*__copysignl(tiny,x); /*underflow */
+	if (n> 50000 || k+n > 0x7fe)
+	  return huge*__copysignl(huge,x); /* overflow */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (k > 0) {				/* normal result */
 	    hx = (hx&0x800fffffffffffffULL)|(k<<52);
 	    if ((lx & 0x7fffffffffffffffULL) == 0) { /* low part +-0 */
diff --git a/sysdeps/ieee754/ldbl-96/s_scalblnl.c b/sysdeps/ieee754/ldbl-96/s_scalblnl.c
index ada587b9d5..755a212555 100644
--- a/sysdeps/ieee754/ldbl-96/s_scalblnl.c
+++ b/sysdeps/ieee754/ldbl-96/s_scalblnl.c
@@ -43,11 +43,13 @@ __scalblnl (long double x, long int n)
 	    k = (hx&0x7fff) - 63;
 	    }
 	if (__builtin_expect(k==0x7fff, 0)) return x+x;	/* NaN or Inf */
-	k = k+n;
-	if (__builtin_expect(n> 50000 || k > 0x7ffe, 0))
-	  return huge*__copysignl(huge,x); /* overflow  */
 	if (__builtin_expect(n< -50000, 0))
 	  return tiny*__copysignl(tiny,x);
+	if (__builtin_expect(n> 50000 || k+n > 0x7ffe, 0))
+	  return huge*__copysignl(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (__builtin_expect(k > 0, 1))		/* normal result */
 	    {SET_LDOUBLE_EXP(x,(es&0x8000)|k); return x;}
 	if (k <= -63)
diff --git a/sysdeps/ieee754/ldbl-96/s_scalbnl.c b/sysdeps/ieee754/ldbl-96/s_scalbnl.c
index 6a41920aca..6946cf232f 100644
--- a/sysdeps/ieee754/ldbl-96/s_scalbnl.c
+++ b/sysdeps/ieee754/ldbl-96/s_scalbnl.c
@@ -43,11 +43,13 @@ __scalbnl (long double x, int n)
 	    k = (hx&0x7fff) - 64;
 	    }
 	if (__builtin_expect(k==0x7fff, 0)) return x+x;	/* NaN or Inf */
-	k = k+n;
-	if (__builtin_expect(n> 50000 || k > 0x7ffe, 0))
-	  return huge*__copysignl(huge,x); /* overflow  */
 	if (__builtin_expect(n< -50000, 0))
 	  return tiny*__copysignl(tiny,x);
+	if (__builtin_expect(n> 50000 || k+n > 0x7ffe, 0))
+	  return huge*__copysignl(huge,x); /* overflow  */
+	/* Now k and n are bounded we know that k = k+n does not
+	   overflow.  */
+	k = k+n;
 	if (__builtin_expect(k > 0, 1))		/* normal result */
 	    {SET_LDOUBLE_EXP(x,(es&0x8000)|k); return x;}
 	if (k <= -64)