Handle large requests.

(cherry picked from commit 3b11189345d0080527a76e3bf867da395a1b0261)

2 files changed, 26 insertions, 1 deletions

diff --git a/ChangeLog b/ChangeLog
index a929f59f15..8037d6361d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+2010-10-03  Ulrich Drepper  <drepper@gmail.com>
+
+	[BZ #12005]
+	* malloc/mcheck.c: Handle large requests.
+
 2010-09-15  Joseph Myers  <joseph@codesourcery.com>
 
 	* sysdeps/unix/sysv/linux/getdents.c (__GETDENTS): When
diff --git a/malloc/mcheck.c b/malloc/mcheck.c
index 524acc755c..e2eb83f41d 100644
--- a/malloc/mcheck.c
+++ b/malloc/mcheck.c
@@ -1,5 +1,6 @@
 /* Standard debugging hooks for `malloc'.
-   Copyright (C) 1990-1997,1999,2000-2002,2007 Free Software Foundation, Inc.
+   Copyright (C) 1990-1997,1999,2000-2002,2007,2010
+   Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Written May 1989 by Mike Haertel.
 
@@ -25,6 +26,7 @@
 # include <stdint.h>
 # include <stdio.h>
 # include <libintl.h>
+# include <errno.h>
 #endif
 
 /* Old hook values.  */
@@ -209,6 +211,12 @@ mallochook (__malloc_size_t size, const __ptr_t caller)
   if (pedantic)
     mcheck_check_all ();
 
+  if (size > ~((size_t) 0) - (sizeof (struct hdr) + 1))
+    {
+      __set_errno (ENOMEM);
+      return NULL;
+    }
+
   __malloc_hook = old_malloc_hook;
   if (old_malloc_hook != NULL)
     hdr = (struct hdr *) (*old_malloc_hook) (sizeof (struct hdr) + size + 1,
@@ -241,6 +249,12 @@ memalignhook (__malloc_size_t alignment, __malloc_size_t size,
 
   slop = (sizeof *hdr + alignment - 1) & -alignment;
 
+  if (size > ~((size_t) 0) - (slop + 1))
+    {
+      __set_errno (ENOMEM);
+      return NULL;
+    }
+
   __memalign_hook = old_memalign_hook;
   if (old_memalign_hook != NULL)
     block = (*old_memalign_hook) (alignment, slop + size + 1, caller);
@@ -276,6 +290,12 @@ reallochook (__ptr_t ptr, __malloc_size_t size, const __ptr_t caller)
   if (pedantic)
     mcheck_check_all ();
 
+  if (size > ~((size_t) 0) - (sizeof (struct hdr) + 1))
+    {
+      __set_errno (ENOMEM);
+      return NULL;
+    }
+
   if (ptr)
     {
       hdr = ((struct hdr *) ptr) - 1;