* elf/dl-close.c: Include stddef.h.

	(_dl_close): If called recursively, just remember GC needs to be rerun
	and decrease l_direct_opencount.  Avoid GC if l_direct_opencount
	decreased to 1.  Rerun GC at the end if any destructor unloaded some
	additional libraries.
	* elf/Makefile: Add rules to build and run unload6 test.
	* elf/unload6.c: New test.
	* elf/unload6mod1.c: New file.
	* elf/unload6mod2.c: New file.
	* elf/unload6mod3.c: New file.

	* malloc/hooks.c (mem2chunk_check): Add magic_p argument, set *magic_p
	if magic_p is not NULL.
	(top_check): Invoke MALLOC_FAILURE_ACTION if MORECORE failed.
	(malloc_check): Fail if sz == -1.
	(free_check): Adjust mem2chunk_check caller.
	(realloc_check): Likewise.  Fail if bytes == -1.  If bytes == 0 and
	oldmem != NULL, call free_check and return NULL.  If reallocating
	and returning NULL, invert magic byte again to make oldmem valid
	region for further checking.
	(memalign_check): Fail if bytes == -1.
	* malloc/Makefile: Add rules to build and run tst-mcheck.
	* malloc/tst-mcheck.c: New test.

3 files changed, 133 insertions, 10 deletions

diff --git a/malloc/Makefile b/malloc/Makefile
index 45e8f59ab4..398dd2b031 100644
--- a/malloc/Makefile
+++ b/malloc/Makefile
@@ -1,4 +1,5 @@
-# Copyright (C) 1991-1999,2000,2001,2002,2003 Free Software Foundation, Inc.
+# Copyright (C) 1991-1999, 2000, 2001, 2002, 2003, 2005
+# Free Software Foundation, Inc.
 # This file is part of the GNU C Library.
 
 # The GNU C Library is free software; you can redistribute it and/or
@@ -26,7 +27,7 @@ all:
 dist-headers := malloc.h
 headers := $(dist-headers) obstack.h mcheck.h
 tests := mallocbug tst-malloc tst-valloc tst-calloc tst-obstack \
-	 tst-mallocstate
+	 tst-mallocstate tst-mcheck
 test-srcs = tst-mtrace
 
 distribute = thread-m.h mtrace.pl mcheck-init.c stackinfo.h memusage.h \
@@ -120,6 +121,8 @@ endif
 endif
 endif
 
+tst-mcheck-ENV = MALLOC_CHECK_=3
+
 # Uncomment this for test releases.  For public releases it is too expensive.
 #CPPFLAGS-malloc.o += -DMALLOC_DEBUG=1
 
diff --git a/malloc/hooks.c b/malloc/hooks.c
index a5c97f3133..0f8f274e38 100644
--- a/malloc/hooks.c
+++ b/malloc/hooks.c
@@ -1,5 +1,5 @@
 /* Malloc implementation for multiple threads without lock contention.
-   Copyright (C) 2001, 2002, 2003, 2004 Free Software Foundation, Inc.
+   Copyright (C) 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Wolfram Gloger <wg@malloc.de>, 2001.
 
@@ -146,9 +146,9 @@ mem2mem_check(ptr, sz) Void_t *ptr; size_t sz;
 static mchunkptr
 internal_function
 #if __STD_C
-mem2chunk_check(Void_t* mem)
+mem2chunk_check(Void_t* mem, unsigned char **magic_p)
 #else
-mem2chunk_check(mem) Void_t* mem;
+mem2chunk_check(mem, magic_p) Void_t* mem; unsigned char **magic_p;
 #endif
 {
   mchunkptr p;
@@ -173,7 +173,6 @@ mem2chunk_check(mem) Void_t* mem;
     for(sz += SIZE_SZ-1; (c = ((unsigned char*)p)[sz]) != magic; sz -= c) {
       if(c<=0 || sz<(c+2*SIZE_SZ)) return NULL;
     }
-    ((unsigned char*)p)[sz] ^= 0xFF;
   } else {
     unsigned long offset, page_mask = malloc_getpagesize-1;
 
@@ -193,8 +192,10 @@ mem2chunk_check(mem) Void_t* mem;
     for(sz -= 1; (c = ((unsigned char*)p)[sz]) != magic; sz -= c) {
       if(c<=0 || sz<(c+2*SIZE_SZ)) return NULL;
     }
-    ((unsigned char*)p)[sz] ^= 0xFF;
   }
+  ((unsigned char*)p)[sz] ^= 0xFF;
+  if (magic_p)
+    *magic_p = (unsigned char *)p + sz;
   return p;
 }
 
@@ -232,7 +233,11 @@ top_check()
   sbrk_size = front_misalign + mp_.top_pad + MINSIZE;
   sbrk_size += pagesz - ((unsigned long)(brk + sbrk_size) & (pagesz - 1));
   new_brk = (char*)(MORECORE (sbrk_size));
-  if (new_brk == (char*)(MORECORE_FAILURE)) return -1;
+  if (new_brk == (char*)(MORECORE_FAILURE))
+    {
+      MALLOC_FAILURE_ACTION;
+      return -1;
+    }
   /* Call the `morecore' hook if necessary.  */
   if (__after_morecore_hook)
     (*__after_morecore_hook) ();
@@ -253,6 +258,11 @@ malloc_check(sz, caller) size_t sz; const Void_t *caller;
 {
   Void_t *victim;
 
+  if (sz+1 == 0) {
+    MALLOC_FAILURE_ACTION;
+    return NULL;
+  }
+
   (void)mutex_lock(&main_arena.mutex);
   victim = (top_check() >= 0) ? _int_malloc(&main_arena, sz+1) : NULL;
   (void)mutex_unlock(&main_arena.mutex);
@@ -270,7 +280,7 @@ free_check(mem, caller) Void_t* mem; const Void_t *caller;
 
   if(!mem) return;
   (void)mutex_lock(&main_arena.mutex);
-  p = mem2chunk_check(mem);
+  p = mem2chunk_check(mem, NULL);
   if(!p) {
     (void)mutex_unlock(&main_arena.mutex);
 
@@ -302,10 +312,19 @@ realloc_check(oldmem, bytes, caller)
   mchunkptr oldp;
   INTERNAL_SIZE_T nb, oldsize;
   Void_t* newmem = 0;
+  unsigned char *magic_p;
 
+  if (bytes+1 == 0) {
+    MALLOC_FAILURE_ACTION;
+    return NULL;
+  }
   if (oldmem == 0) return malloc_check(bytes, NULL);
+  if (bytes == 0) {
+    free_check (oldmem, NULL);
+    return NULL;
+  }
   (void)mutex_lock(&main_arena.mutex);
-  oldp = mem2chunk_check(oldmem);
+  oldp = mem2chunk_check(oldmem, &magic_p);
   (void)mutex_unlock(&main_arena.mutex);
   if(!oldp) {
     malloc_printerr(check_action, "realloc(): invalid pointer", oldmem);
@@ -357,6 +376,12 @@ realloc_check(oldmem, bytes, caller)
 #if HAVE_MMAP
   }
 #endif
+
+  /* mem2chunk_check changed the magic byte in the old chunk.
+     If newmem is NULL, then the old chunk will still be used though,
+     so we need to invert that change here.  */
+  if (newmem == NULL) *magic_p ^= 0xFF;
+
   (void)mutex_unlock(&main_arena.mutex);
 
   return mem2mem_check(newmem, bytes);
@@ -376,6 +401,10 @@ memalign_check(alignment, bytes, caller)
   if (alignment <= MALLOC_ALIGNMENT) return malloc_check(bytes, NULL);
   if (alignment <  MINSIZE) alignment = MINSIZE;
 
+  if (bytes+1 == 0) {
+    MALLOC_FAILURE_ACTION;
+    return NULL;
+  }
   checked_request2size(bytes+1, nb);
   (void)mutex_lock(&main_arena.mutex);
   mem = (top_check() >= 0) ? _int_memalign(&main_arena, alignment, bytes+1) :
diff --git a/malloc/tst-mcheck.c b/malloc/tst-mcheck.c
new file mode 100644
index 0000000000..16784912a9
--- /dev/null
+++ b/malloc/tst-mcheck.c
@@ -0,0 +1,91 @@
+/* Copyright (C) 2005 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Jakub Jelinek <jakub@redhat.com>, 2005.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static int errors = 0;
+
+static void
+merror (const char *msg)
+{
+  ++errors;
+  printf ("Error: %s\n", msg);
+}
+
+int
+main (void)
+{
+  void *p, *q;
+
+  errno = 0;
+
+  p = malloc (-1);
+
+  if (p != NULL)
+    merror ("malloc (-1) succeeded.");
+  else if (errno != ENOMEM)
+    merror ("errno is not set correctly.");
+
+  p = malloc (10);
+  if (p == NULL)
+    merror ("malloc (10) failed.");
+
+  p = realloc (p, 0);
+  if (p != NULL)
+    merror ("realloc (p, 0) failed.");
+
+  p = malloc (0);
+  if (p == NULL)
+    merror ("malloc (0) failed.");
+
+  p = realloc (p, 0);
+  if (p != NULL)
+    merror ("realloc (p, 0) failed.");
+
+  q = malloc (256);
+  if (q == NULL)
+    merror ("malloc (256) failed.");
+
+  p = malloc (512);
+  if (p == NULL)
+    merror ("malloc (512) failed.");
+
+  if (realloc (p, -256) != NULL)
+    merror ("realloc (p, -256) succeeded.");
+  else if (errno != ENOMEM)
+    merror ("errno is not set correctly.");
+
+  free (p);
+
+  p = malloc (512);
+  if (p == NULL)
+    merror ("malloc (512) failed.");
+
+  if (realloc (p, -1) != NULL)
+    merror ("realloc (p, -1) succeeded.");
+  else if (errno != ENOMEM)
+    merror ("errno is not set correctly.");
+
+  free (p);
+  free (q);
+
+  return errors != 0;
+}