dynarray: Set errno on overflow-induced allocation failure

This allows the caller to return directly on such an error, with an
appropriate errno value.

(cherry picked from commit 5898f4548efdcd7c0fd437a74eeb80facc51a117)

4 files changed, 49 insertions, 3 deletions

diff --git a/ChangeLog b/ChangeLog
index 1cbf15267d..66b74d4728 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2017-08-30  Florian Weimer  <fweimer@redhat.com>
+
+	* malloc/dynarray_emplace_enlarge.c
+	(__libc_dynarray_emplace_enlarge): Set errno on overflow.
+	* malloc/dynarray_resize.c (__libc_dynarray_resize): Likewise.
+	* malloc/tst-dynarray.c (test_long_overflow): New function.
+	(do_test): Call it.
+
 2017-06-13  Florian Weimer  <fweimer@redhat.com>
 
 	* malloc/dynarray-skeleton.c: List begin/end as defined functions.
diff --git a/malloc/dynarray_emplace_enlarge.c b/malloc/dynarray_emplace_enlarge.c
index dfc70017ce..0fb032765c 100644
--- a/malloc/dynarray_emplace_enlarge.c
+++ b/malloc/dynarray_emplace_enlarge.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include <dynarray.h>
+#include <errno.h>
 #include <malloc-internal.h>
 #include <stdlib.h>
 #include <string.h>
@@ -43,8 +44,11 @@ __libc_dynarray_emplace_enlarge (struct dynarray_header *list,
     {
       new_allocated = list->allocated + list->allocated / 2 + 1;
       if (new_allocated <= list->allocated)
-        /* Overflow.  */
-        return false;
+        {
+          /* Overflow.  */
+          __set_errno (ENOMEM);
+          return false;
+        }
     }
 
   size_t new_size;
diff --git a/malloc/dynarray_resize.c b/malloc/dynarray_resize.c
index e6dc9fbc68..63c981bf61 100644
--- a/malloc/dynarray_resize.c
+++ b/malloc/dynarray_resize.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include <dynarray.h>
+#include <errno.h>
 #include <malloc-internal.h>
 #include <stdlib.h>
 #include <string.h>
@@ -38,7 +39,11 @@ __libc_dynarray_resize (struct dynarray_header *list, size_t size,
 
   size_t new_size_bytes;
   if (check_mul_overflow_size_t (size, element_size, &new_size_bytes))
-    return false;
+    {
+      /* Overflow.  */
+      __set_errno (ENOMEM);
+      return false;
+    }
   void *new_array;
   if (list->array == scratch)
     {
diff --git a/malloc/tst-dynarray.c b/malloc/tst-dynarray.c
index 2206d75e31..d11f7bb8a3 100644
--- a/malloc/tst-dynarray.c
+++ b/malloc/tst-dynarray.c
@@ -18,6 +18,9 @@
 
 #include "tst-dynarray-shared.h"
 
+#include <errno.h>
+#include <stdint.h>
+
 #define DYNARRAY_STRUCT dynarray_long
 #define DYNARRAY_ELEMENT long
 #define DYNARRAY_PREFIX dynarray_long_
@@ -463,6 +466,31 @@ test_long_init (void)
   }
 }
 
+/* Test overflow in resize.  */
+static void
+test_long_overflow (void)
+{
+  {
+    struct dynarray_long dyn;
+    dynarray_long_init (&dyn);
+    errno = EINVAL;
+    TEST_VERIFY (!dynarray_long_resize
+                 (&dyn, (SIZE_MAX / sizeof (long)) + 1));
+    TEST_VERIFY (errno == ENOMEM);
+    TEST_VERIFY (dynarray_long_has_failed (&dyn));
+  }
+
+  {
+    struct dynarray_long_noscratch dyn;
+    dynarray_long_noscratch_init (&dyn);
+    errno = EINVAL;
+    TEST_VERIFY (!dynarray_long_noscratch_resize
+                 (&dyn, (SIZE_MAX / sizeof (long)) + 1));
+    TEST_VERIFY (errno == ENOMEM);
+    TEST_VERIFY (dynarray_long_noscratch_has_failed (&dyn));
+  }
+}
+
 /* Test NUL-terminated string construction with the add function and
    the simple finalize function.  */
 static void
@@ -538,6 +566,7 @@ do_test (void)
   test_int ();
   test_str ();
   test_long_init ();
+  test_long_overflow ();
   test_zstr ();
   return 0;
 }