* elf/dl-sysdep.c (_dl_important_hwcaps): Add integer overflow check.

	* elf/dl-minimal.c (__libc_memalign): Likewise.  Handle malloc (0).
	Return NULL if mmap failed instead of asserting it does not.
	(calloc): Check for integer overflow.

	* elf/dl-minimal.c (__strtoul_internal): Fix parsing of numbers bigger
	than LONG_MAX / 10.

3 files changed, 44 insertions, 8 deletions

diff --git a/ChangeLog b/ChangeLog
index 0a48667197..c1cdbdb2e7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2007-07-01  Jakub Jelinek  <jakub@redhat.com>
+
+	* elf/dl-sysdep.c (_dl_important_hwcaps): Add integer overflow check.
+	* elf/dl-minimal.c (__libc_memalign): Likewise.  Handle malloc (0).
+	Return NULL if mmap failed instead of asserting it does not.
+	(calloc): Check for integer overflow.
+
+	* elf/dl-minimal.c (__strtoul_internal): Fix parsing of numbers bigger
+	than LONG_MAX / 10.
+
 2007-07-03  Jakub Jelinek  <jakub@redhat.com>
 
 	[BZ #4702]
diff --git a/elf/dl-minimal.c b/elf/dl-minimal.c
index 45524088e9..5079c449f6 100644
--- a/elf/dl-minimal.c
+++ b/elf/dl-minimal.c
@@ -75,14 +75,21 @@ __libc_memalign (size_t align, size_t n)
   alloc_ptr = (void *) 0 + (((alloc_ptr - (void *) 0) + align - 1)
 			    & ~(align - 1));
 
-  if (alloc_ptr + n >= alloc_end)
+  if (alloc_ptr + n >= alloc_end || n >= -(uintptr_t) alloc_ptr)
     {
       /* Insufficient space left; allocate another page.  */
       caddr_t page;
       size_t nup = (n + GLRO(dl_pagesize) - 1) & ~(GLRO(dl_pagesize) - 1);
+      if (__builtin_expect (nup == 0, 0))
+	{
+	  if (n)
+	    return NULL;
+	  nup = GLRO(dl_pagesize);
+	}
       page = __mmap (0, nup, PROT_READ|PROT_WRITE,
 		     MAP_ANON|MAP_PRIVATE, _dl_zerofd, 0);
-      assert (page != MAP_FAILED);
+      if (page == MAP_FAILED)
+	return NULL;
       if (page != alloc_end)
 	alloc_ptr = page;
       alloc_end = page + nup;
@@ -108,7 +115,14 @@ calloc (size_t nmemb, size_t size)
   /* New memory from the trivial malloc above is always already cleared.
      (We make sure that's true in the rare occasion it might not be,
      by clearing memory in free, below.)  */
-  return malloc (nmemb * size);
+  size_t bytes = nmemb * size;
+
+#define HALF_SIZE_T (((size_t) 1) << (8 * sizeof (size_t) / 2))
+  if (__builtin_expect ((nmemb | size) >= HALF_SIZE_T, 0)
+      && size != 0 && bytes / size != nmemb)
+    return NULL;
+
+  return malloc (bytes);
 }
 
 /* This will rarely be called.  */
@@ -264,7 +278,7 @@ __strtoul_internal (const char *nptr, char **endptr, int base, int group)
   while (*nptr >= '0' && *nptr <= '9')
     {
       unsigned long int digval = *nptr - '0';
-      if (result > LONG_MAX / 10
+      if (result > ULONG_MAX / 10
 	  || (result == ULONG_MAX / 10 && digval > ULONG_MAX % 10))
 	{
 	  errno = ERANGE;
diff --git a/elf/dl-sysdep.c b/elf/dl-sysdep.c
index d06ce1754e..ec3f0e3474 100644
--- a/elf/dl-sysdep.c
+++ b/elf/dl-sysdep.c
@@ -1,5 +1,5 @@
 /* Operating system support for run-time dynamic linker.  Generic Unix version.
-   Copyright (C) 1995-1998, 2000-2005, 2006 Free Software Foundation, Inc.
+   Copyright (C) 1995-1998, 2000-2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -460,9 +460,21 @@ _dl_important_hwcaps (const char *platform, size_t platform_len, size_t *sz,
     total = temp[0].len + 1;
   else
     {
-      total = (1UL << (cnt - 2)) * (temp[0].len + temp[cnt - 1].len + 2);
-      for (n = 1; n + 1 < cnt; ++n)
-	total += (1UL << (cnt - 3)) * (temp[n].len + 1);
+      total = temp[0].len + temp[cnt - 1].len + 2;
+      if (cnt > 2)
+	{
+	  total <<= 1;
+	  for (n = 1; n + 1 < cnt; ++n)
+	    total += temp[n].len + 1;
+	  if (cnt > 3
+	      && (cnt >= sizeof (size_t) * 8
+		  || total + (sizeof (*result) << 3)
+		     >= (1UL << (sizeof (size_t) * 8 - cnt + 3))))
+	    _dl_signal_error (ENOMEM, NULL, NULL,
+			      N_("Xcannot create capability list"));
+
+	  total <<= cnt - 3;
+	}
     }
 
   /* The result structure: we use a very compressed way to store the