malloc: make malloc fail with requests larger than PTRDIFF_MAX (BZ#23741)

As discussed previously on libc-alpha [1], this patch follows up the idea
and add both the __attribute_alloc_size__ on malloc functions (malloc,
calloc, realloc, reallocarray, valloc, pvalloc, and memalign) and limit
maximum requested allocation size to up PTRDIFF_MAX (taking into
consideration internal padding and alignment).

This aligns glibc with gcc expected size defined by default warning
-Walloc-size-larger-than value which warns for allocation larger than
PTRDIFF_MAX.  It also aligns with gcc expectation regarding libc and
expected size, such as described in PR#67999 [2] and previously discussed
ISO C11 issues [3] on libc-alpha.

From the RFC thread [4] and previous discussion, it seems that consensus
is only to limit such requested size for malloc functions, not the system
allocation one (mmap, sbrk, etc.).

The implementation changes checked_request2size to check for both overflow
and maximum object size up to PTRDIFF_MAX. No additional checks are done
on sysmalloc, so it can still issue mmap with values larger than
PTRDIFF_T depending on the requested size.

The __attribute_alloc_size__ is for functions that return a pointer only,
which means it cannot be applied to posix_memalign (see remarks in GCC
PR#87683 [5]). The runtimes checks to limit maximum requested allocation
size does applies to posix_memalign.

Checked on x86_64-linux-gnu and i686-linux-gnu.

[1] https://sourceware.org/ml/libc-alpha/2018-11/msg00223.html
[2] https://gcc.gnu.org/bugzilla//show_bug.cgi?id=67999
[3] https://sourceware.org/ml/libc-alpha/2011-12/msg00066.html
[4] https://sourceware.org/ml/libc-alpha/2018-11/msg00224.html
[5] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87683

	[BZ #23741]
	* malloc/hooks.c (malloc_check, realloc_check): Use
	__builtin_add_overflow on overflow check and adapt to
	checked_request2size change.
	* malloc/malloc.c (__libc_malloc, __libc_realloc, _mid_memalign,
	__libc_pvalloc, __libc_calloc, _int_memalign): Limit maximum
	allocation size to PTRDIFF_MAX.
	(REQUEST_OUT_OF_RANGE): Remove macro.
	(checked_request2size): Change to inline function and limit maximum
	requested size to PTRDIFF_MAX.
	(__libc_malloc, __libc_realloc, _int_malloc, _int_memalign): Limit
	maximum allocation size to PTRDIFF_MAX.
	(_mid_memalign): Use _int_memalign call for overflow check.
	(__libc_pvalloc): Use __builtin_add_overflow on overflow check.
	(__libc_calloc): Use __builtin_mul_overflow for overflow check and
	limit maximum requested size to PTRDIFF_MAX.
	* malloc/malloc.h (malloc, calloc, realloc, reallocarray, memalign,
	valloc, pvalloc): Add __attribute_alloc_size__.
	* stdlib/stdlib.h (malloc, realloc, reallocarray, valloc): Likewise.
	* malloc/tst-malloc-too-large.c (do_test): Add check for allocation
	larger than PTRDIFF_MAX.
	* malloc/tst-memalign.c (do_test): Disable -Walloc-size-larger-than=
	around tests of malloc with negative sizes.
	* malloc/tst-posix_memalign.c (do_test): Likewise.
	* malloc/tst-pvalloc.c (do_test): Likewise.
	* malloc/tst-valloc.c (do_test): Likewise.
	* malloc/tst-reallocarray.c (do_test): Replace call to reallocarray
	with resulting size allocation larger than PTRDIFF_MAX with
	reallocarray_nowarn.
	(reallocarray_nowarn): New function.
	* NEWS: Mention the malloc function semantic change.

1 files changed, 49 insertions, 0 deletions

diff --git a/malloc/tst-malloc-too-large.c b/malloc/tst-malloc-too-large.c
index 15e25f558e..c1c8cb88cc 100644
--- a/malloc/tst-malloc-too-large.c
+++ b/malloc/tst-malloc-too-large.c
@@ -72,13 +72,28 @@ test_large_allocations (size_t size)
   void * ptr_to_realloc;
 
   test_setup ();
+  DIAG_PUSH_NEEDS_COMMENT;
+#if __GNUC_PREREQ (7, 0)
+  /* GCC 7 warns about too-large allocations; here we want to test
+     that they fail.  */
+  DIAG_IGNORE_NEEDS_COMMENT (7, "-Walloc-size-larger-than=");
+#endif
   TEST_VERIFY (malloc (size) == NULL);
+#if __GNUC_PREREQ (7, 0)
+  DIAG_POP_NEEDS_COMMENT;
+#endif
   TEST_VERIFY (errno == ENOMEM);
 
   ptr_to_realloc = malloc (16);
   TEST_VERIFY_EXIT (ptr_to_realloc != NULL);
   test_setup ();
+#if __GNUC_PREREQ (7, 0)
+  DIAG_IGNORE_NEEDS_COMMENT (7, "-Walloc-size-larger-than=");
+#endif
   TEST_VERIFY (realloc (ptr_to_realloc, size) == NULL);
+#if __GNUC_PREREQ (7, 0)
+  DIAG_POP_NEEDS_COMMENT;
+#endif
   TEST_VERIFY (errno == ENOMEM);
   free (ptr_to_realloc);
 
@@ -135,7 +150,13 @@ test_large_aligned_allocations (size_t size)
   for (align = 1; align <= pagesize; align *= 2)
     {
       test_setup ();
+#if __GNUC_PREREQ (7, 0)
+      DIAG_IGNORE_NEEDS_COMMENT (7, "-Walloc-size-larger-than=");
+#endif
       TEST_VERIFY (memalign (align, size) == NULL);
+#if __GNUC_PREREQ (7, 0)
+      DIAG_POP_NEEDS_COMMENT;
+#endif
       TEST_VERIFY (errno == ENOMEM);
 
       /* posix_memalign expects an alignment that is a power of 2 *and* a
@@ -151,7 +172,13 @@ test_large_aligned_allocations (size_t size)
       if ((size % align) == 0)
         {
           test_setup ();
+#if __GNUC_PREREQ (7, 0)
+	  DIAG_IGNORE_NEEDS_COMMENT (7, "-Walloc-size-larger-than=");
+#endif
           TEST_VERIFY (aligned_alloc (align, size) == NULL);
+#if __GNUC_PREREQ (7, 0)
+	  DIAG_POP_NEEDS_COMMENT;
+#endif
           TEST_VERIFY (errno == ENOMEM);
         }
     }
@@ -159,11 +186,23 @@ test_large_aligned_allocations (size_t size)
   /* Both valloc and pvalloc return page-aligned memory.  */
 
   test_setup ();
+#if __GNUC_PREREQ (7, 0)
+  DIAG_IGNORE_NEEDS_COMMENT (7, "-Walloc-size-larger-than=");
+#endif
   TEST_VERIFY (valloc (size) == NULL);
+#if __GNUC_PREREQ (7, 0)
+  DIAG_POP_NEEDS_COMMENT;
+#endif
   TEST_VERIFY (errno == ENOMEM);
 
   test_setup ();
+#if __GNUC_PREREQ (7, 0)
+  DIAG_IGNORE_NEEDS_COMMENT (7, "-Walloc-size-larger-than=");
+#endif
   TEST_VERIFY (pvalloc (size) == NULL);
+#if __GNUC_PREREQ (7, 0)
+  DIAG_POP_NEEDS_COMMENT;
+#endif
   TEST_VERIFY (errno == ENOMEM);
 }
 
@@ -226,6 +265,16 @@ do_test (void)
       test_large_aligned_allocations (SIZE_MAX - i);
     }
 
+  /* Allocation larger than PTRDIFF_MAX does play well with C standard,
+     since pointer subtraction within the object might overflow ptrdiff_t
+     resulting in undefined behavior.  To prevent it malloc function fail
+     for such allocations.  */
+  for (size_t i = 1; i <= FOURTEEN_ON_BITS; i++)
+    {
+      test_large_allocations (PTRDIFF_MAX + i);
+      test_large_aligned_allocations (PTRDIFF_MAX + i);
+    }
+
 #if __WORDSIZE >= 64
   /* On 64-bit targets, we need to test a much wider range of too-large
      sizes, so we test at intervals of (1 << 50) that allocation sizes