Add reallocarray function

The reallocarray function is an extension from OpenBSD.  It is an
integer-overflow-safe replacement for realloc(p, X*Y) and
malloc(X*Y) (realloc(NULL, X*Y)).  It can therefore help in preventing
certain security issues in code.

This is an updated version of a patch originally submitted by Rüdiger
Sonderfeld in May 2014 [1].

Checked on i686-linux-gnu and x86_64-linux-gnu.

[1] <https://sourceware.org/ml/libc-alpha/2014-05/msg00481.html>.

2017-05-30  Dennis Wölfing  <denniswoelfing@gmx.de>
            Rüdiger Sonderfeld  <ruediger@c-plusplus.de>

	* include/stdlib.h (__libc_reallocarray): New declaration.
	* malloc/Makefile (routines): Add reallocarray.
	(tests): Add tst-reallocarray.c.
	* malloc/Versions: Add reallocarray and __libc_reallocarray.
	* malloc/malloc-internal.h (check_mul_overflow_size_t): New inline
	function.
	* malloc/malloc.h (reallocarray): New declaration.
	* stdlib/stdlib.h (reallocarray): Likewise.
	* malloc/reallocarray.c: New file.
	* malloc/tst-reallocarray.c: New test file.
	* manual/memory.texi: Document reallocarray.
	* sysdeps/unix/sysv/linux/aarch64/libc.abilist: Add reallocarray.
	* sysdeps/unix/sysv/linux/alpha/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/arm/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/hppa/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/i386/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/ia64/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/microblaze/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/nios2/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist:
	Likewise.
	* sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist:
	Likewise.
	* sysdeps/unix/sysv/linux/powerpc/powerpc64/libc-le.abilist: Likewise.
	* sysdeps/unix/sysv/linux/powerpc/powerpc64/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/sh/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/tile/tilegx/tilegx32/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/tile/tilegx/tilegx64/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/tilepro/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/64/libc.abilist: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist: Likewise.

6 files changed, 188 insertions, 2 deletions

diff --git a/malloc/Makefile b/malloc/Makefile
index d0f23f7bf3..b7d4c63920 100644
--- a/malloc/Makefile
+++ b/malloc/Makefile
@@ -26,7 +26,7 @@ dist-headers := malloc.h
 headers := $(dist-headers) obstack.h mcheck.h
 tests := mallocbug tst-malloc tst-valloc tst-calloc tst-obstack \
 	 tst-mcheck tst-mallocfork tst-trim1 \
-	 tst-malloc-usable tst-realloc tst-posix_memalign \
+	 tst-malloc-usable tst-realloc tst-reallocarray tst-posix_memalign \
 	 tst-pvalloc tst-memalign tst-mallopt \
 	 tst-malloc-backtrace tst-malloc-thread-exit \
 	 tst-malloc-thread-fail tst-malloc-fork-deadlock \
@@ -49,7 +49,7 @@ endif
 tests += $(tests-static)
 test-srcs = tst-mtrace
 
-routines = malloc morecore mcheck mtrace obstack \
+routines = malloc morecore mcheck mtrace obstack reallocarray \
   scratch_buffer_grow scratch_buffer_grow_preserve \
   scratch_buffer_set_array_size
 
diff --git a/malloc/Versions b/malloc/Versions
index e34ab177be..23aafb5ccc 100644
--- a/malloc/Versions
+++ b/malloc/Versions
@@ -62,6 +62,7 @@ libc {
     aligned_alloc;
   }
   GLIBC_2.26 {
+    reallocarray;
   }
   GLIBC_PRIVATE {
     # Internal startup hook for libpthread.
@@ -74,5 +75,8 @@ libc {
     __libc_scratch_buffer_grow;
     __libc_scratch_buffer_grow_preserve;
     __libc_scratch_buffer_set_array_size;
+
+    # Internal name for reallocarray
+    __libc_reallocarray;
   }
 }
diff --git a/malloc/malloc-internal.h b/malloc/malloc-internal.h
index de6103d7e1..dbd801a58e 100644
--- a/malloc/malloc-internal.h
+++ b/malloc/malloc-internal.h
@@ -81,5 +81,24 @@ void __malloc_fork_unlock_parent (void) internal_function attribute_hidden;
 /* Called in the child process after a fork.  */
 void __malloc_fork_unlock_child (void) internal_function attribute_hidden;
 
+/* Set *RESULT to LEFT * RIGHT.  Return true if the multiplication
+   overflowed.  */
+static inline bool
+check_mul_overflow_size_t (size_t left, size_t right, size_t *result)
+{
+#if __GNUC__ >= 5
+  return __builtin_mul_overflow (left, right, result);
+#else
+  /* size_t is unsigned so the behavior on overflow is defined.  */
+  *result = left * right;
+  size_t half_size_t = ((size_t) 1) << (8 * sizeof (size_t) / 2);
+  if (__glibc_unlikely ((left | right) >= half_size_t))
+    {
+      if (__glibc_unlikely (right != 0 && *result / right != left))
+        return true;
+    }
+  return false;
+#endif
+}
 
 #endif /* _MALLOC_INTERNAL_H */
diff --git a/malloc/malloc.h b/malloc/malloc.h
index 274c0958e4..339ab64c7d 100644
--- a/malloc/malloc.h
+++ b/malloc/malloc.h
@@ -49,6 +49,14 @@ __THROW __attribute_malloc__ __wur;
 extern void *realloc (void *__ptr, size_t __size)
 __THROW __attribute_warn_unused_result__;
 
+/* Re-allocate the previously allocated block in PTR, making the new
+   block large enough for NMEMB elements of SIZE bytes each.  */
+/* __attribute_malloc__ is not used, because if reallocarray returns
+   the same pointer that was passed to it, aliasing needs to be allowed
+   between objects pointed by the old and new pointers.  */
+extern void *reallocarray (void *__ptr, size_t __nmemb, size_t __size)
+__THROW __attribute_warn_unused_result__;
+
 /* Free a block allocated by `malloc', `realloc' or `calloc'.  */
 extern void free (void *__ptr) __THROW;
 
diff --git a/malloc/reallocarray.c b/malloc/reallocarray.c
new file mode 100644
index 0000000000..07562c30c9
--- /dev/null
+++ b/malloc/reallocarray.c
@@ -0,0 +1,37 @@
+/* Change the size of an allocated block.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public License as
+   published by the Free Software Foundation; either version 2.1 of the
+   License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; see the file COPYING.LIB.  If
+   not, see <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <malloc.h>
+#include <malloc/malloc-internal.h>
+
+void *
+__libc_reallocarray (void *optr, size_t nmemb, size_t elem_size)
+{
+  size_t bytes;
+  if (check_mul_overflow_size_t (nmemb, elem_size, &bytes))
+    {
+      __set_errno (ENOMEM);
+      return 0;
+    }
+  else
+    return realloc (optr, bytes);
+}
+libc_hidden_def (__libc_reallocarray)
+
+weak_alias (__libc_reallocarray, reallocarray)
diff --git a/malloc/tst-reallocarray.c b/malloc/tst-reallocarray.c
new file mode 100644
index 0000000000..f1cbf7fe0a
--- /dev/null
+++ b/malloc/tst-reallocarray.c
@@ -0,0 +1,118 @@
+/* Test for reallocarray.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <malloc.h>
+#include <string.h>
+#include <support/check.h>
+
+static int
+do_test (void)
+{
+  void *ptr = NULL;
+  void *ptr2 = NULL;
+  unsigned char *c;
+  size_t i;
+  int ok;
+  const size_t max = ~(size_t)0;
+  size_t a, b;
+
+  /* Test overflow detection.  */
+  errno = 0;
+  ptr = reallocarray (NULL, max, 2);
+  TEST_VERIFY (!ptr);
+  TEST_VERIFY (errno == ENOMEM);
+
+  errno = 0;
+  ptr = reallocarray (NULL, 2, max);
+  TEST_VERIFY (!ptr);
+  TEST_VERIFY (errno == ENOMEM);
+
+  a = 65537;
+  b = max/65537 + 1;
+  errno = 0;
+  ptr = reallocarray (NULL, a, b);
+  TEST_VERIFY (!ptr);
+  TEST_VERIFY (errno == ENOMEM);
+
+  errno = 0;
+  ptr = reallocarray (NULL, b, a);
+  TEST_VERIFY (!ptr);
+  TEST_VERIFY (errno == ENOMEM);
+
+  /* Test realloc-like behavior.  */
+  /* Allocate memory like malloc.  */
+  ptr = reallocarray (NULL, 10, 2);
+  TEST_VERIFY_EXIT (ptr);
+  TEST_VERIFY_EXIT (malloc_usable_size (ptr) >= 10*2);
+
+  memset (ptr, 0xAF, 10*2);
+
+  /* Enlarge buffer.   */
+  ptr2 = reallocarray (ptr, 20, 2);
+  TEST_VERIFY (ptr2);
+  if (ptr2)
+    ptr = ptr2;
+  TEST_VERIFY (malloc_usable_size (ptr) >= 20*2);
+
+  c = ptr;
+  ok = 1;
+  for (i = 0; i < 10*2; ++i)
+    {
+      if (c[i] != 0xAF)
+        ok = 0;
+    }
+  TEST_VERIFY (ok);
+
+  /* Decrease buffer size.  */
+  ptr2 = reallocarray (ptr, 5, 3);
+  TEST_VERIFY (ptr2);
+  if (ptr2)
+    ptr = ptr2;
+  TEST_VERIFY_EXIT (malloc_usable_size (ptr) >= 5*3);
+
+  c = ptr;
+  ok = 1;
+  for (i = 0; i < 5*3; ++i)
+    {
+      if (c[i] != 0xAF)
+        ok = 0;
+    }
+  TEST_VERIFY (ok);
+
+  /* Overflow should leave buffer untouched.  */
+  errno = 0;
+  ptr2 = reallocarray (ptr, 2, ~(size_t)0);
+  TEST_VERIFY (!ptr2);
+  TEST_VERIFY (errno == ENOMEM);
+
+  c = ptr;
+  ok = 1;
+  for (i = 0; i < 5*3; ++i)
+    {
+      if (c[i] != 0xAF)
+        ok = 0;
+    }
+  TEST_VERIFY (ok);
+
+  free (ptr);
+
+  return 0;
+}
+
+#include <support/test-driver.c>