about summary refs log tree commit diff
path: root/misc/hsearch_r.c
diff options
context:
space:
mode:
authorFlorian Weimer <fweimer@redhat.com>2016-01-28 13:59:11 +0100
committerFlorian Weimer <fweimer@redhat.com>2016-01-28 13:59:11 +0100
commitbae7c7c764413b23e61cb099ce33be4c4ee259bb (patch)
tree8c643811b62f09a257324e9cc64dad1c17d94d2e /misc/hsearch_r.c
parent8a71d2e27fd067a85059aefb93c9ce83142b03e9 (diff)
downloadglibc-bae7c7c764413b23e61cb099ce33be4c4ee259bb.tar.gz
glibc-bae7c7c764413b23e61cb099ce33be4c4ee259bb.tar.xz
glibc-bae7c7c764413b23e61cb099ce33be4c4ee259bb.zip
Improve check against integer wraparound in hcreate_r [BZ #18240]
Diffstat (limited to 'misc/hsearch_r.c')
-rw-r--r--misc/hsearch_r.c35
1 files changed, 17 insertions, 18 deletions
diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c
index f6f16edd10..1fca6b3222 100644
--- a/misc/hsearch_r.c
+++ b/misc/hsearch_r.c
@@ -46,15 +46,12 @@ static int
 isprime (unsigned int number)
 {
   /* no even number will be passed */
-  unsigned int div = 3;
-
-  while (div * div < number && number % div != 0)
-    div += 2;
-
-  return number % div != 0;
+  for (unsigned int div = 3; div <= number / div; div += 2)
+    if (number % div == 0)
+      return 0;
+  return 1;
 }
 
-
 /* Before using the hash table we must allocate memory for it.
    Test for an existing table are done. We allocate one element
    more as the found prime number says. This is done for more effective
@@ -71,13 +68,6 @@ __hcreate_r (size_t nel, struct hsearch_data *htab)
       return 0;
     }
 
-  if (nel >= SIZE_MAX / sizeof (_ENTRY))
-    {
-      __set_errno (ENOMEM);
-      return 0;
-    }
-
-
   /* There is still another table active. Return with error. */
   if (htab->table != NULL)
     return 0;
@@ -86,10 +76,19 @@ __hcreate_r (size_t nel, struct hsearch_data *htab)
      use will not work.  */
   if (nel < 3)
     nel = 3;
-  /* Change nel to the first prime number not smaller as nel. */
-  nel |= 1;      /* make odd */
-  while (!isprime (nel))
-    nel += 2;
+
+  /* Change nel to the first prime number in the range [nel, UINT_MAX - 2],
+     The '- 2' means 'nel += 2' cannot overflow.  */
+  for (nel |= 1; ; nel += 2)
+    {
+      if (UINT_MAX - 2 < nel)
+	{
+	  __set_errno (ENOMEM);
+	  return 0;
+	}
+      if (isprime (nel))
+	break;
+    }
 
   htab->size = nel;
   htab->filled = 0;