mcheck: Wean away from malloc hooks [BZ #23489]

Split the mcheck implementation into the debugging hooks and API so
that the API can be replicated in libc and libc_malloc_debug.so.  The
libc APIs always result in failure.

The mcheck implementation has also been moved entirely into
libc_malloc_debug.so and with it, all of the hook initialization code
can now be moved into the debug library.  Now the initialization can
be done independently of libc internals.

With this patch, libc_malloc_debug.so can no longer be used with older
libcs, which is not its goal anyway.  tst-vfork3 breaks due to this
since it spawns shell scripts, which in turn execute using the system
glibc.  Move the test to tests-container so that only the built glibc
is used.

This move also fixes bugs in the mcheck version of memalign and
realloc, thus allowing removal of the tests from tests-mcheck
exclusion list.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Tested-by: Carlos O'Donell <carlos@redhat.com>

1 files changed, 21 insertions, 377 deletions

diff --git a/malloc/mcheck.c b/malloc/mcheck.c
index 1e68cedbf5..74c20ffe25 100644
--- a/malloc/mcheck.c
+++ b/malloc/mcheck.c
@@ -1,4 +1,4 @@
-/* Standard debugging hooks for `malloc'.
+/* The mcheck() interface.
    Copyright (C) 1990-2021 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Written May 1989 by Mike Haertel.
@@ -17,402 +17,46 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
-#ifndef _MALLOC_INTERNAL
-# define _MALLOC_INTERNAL
-# include <malloc.h>
-# include <malloc-size.h>
-# include <mcheck.h>
-# include <stdint.h>
-# include <stdio.h>
-# include <libintl.h>
-# include <errno.h>
-# include <malloc-internal.h>
-#endif
-
-/* Old hook values.  */
-static void (*old_free_hook)(void *ptr, const void *);
-static void *(*old_malloc_hook) (size_t size, const void *);
-static void *(*old_memalign_hook) (size_t alignment, size_t size,
-				   const void *);
-static void *(*old_realloc_hook) (void *ptr, size_t size,
-				  const void *);
-
-/* Function to call when something awful happens.  */
-static void (*abortfunc) (enum mcheck_status);
-
-/* Arbitrary magical numbers.  */
-#define MAGICWORD       0xfedabeeb
-#define MAGICFREE       0xd8675309
-#define MAGICBYTE       ((char) 0xd7)
-#define MALLOCFLOOD     ((char) 0x93)
-#define FREEFLOOD       ((char) 0x95)
-
-struct hdr
-{
-  size_t size;                  /* Exact size requested by user.  */
-  unsigned long int magic;      /* Magic number to check header integrity.  */
-  struct hdr *prev;
-  struct hdr *next;
-  void *block;                  /* Real block allocated, for memalign.  */
-  unsigned long int magic2;     /* Extra, keeps us doubleword aligned.  */
-} __attribute__ ((aligned (MALLOC_ALIGNMENT)));
-
-/* This is the beginning of the list of all memory blocks allocated.
-   It is only constructed if the pedantic testing is requested.  */
-static struct hdr *root;
-
-static int mcheck_used;
-
-/* Nonzero if pedentic checking of all blocks is requested.  */
-static int pedantic;
-
-#if defined _LIBC || defined STDC_HEADERS || defined USG
-# include <string.h>
-# define flood memset
+#if !IS_IN (libc)
+# include "mcheck-impl.c"
 #else
-static void flood (void *, int, size_t);
-static void
-flood (void *ptr, int val, size_t size)
-{
-  char *cp = ptr;
-  while (size--)
-    *cp++ = val;
-}
+# include <mcheck.h>
 #endif
 
-static enum mcheck_status
-checkhdr (const struct hdr *hdr)
-{
-  enum mcheck_status status;
-
-  if (!mcheck_used)
-    /* Maybe the mcheck used is disabled?  This happens when we find
-       an error and report it.  */
-    return MCHECK_OK;
-
-  switch (hdr->magic ^ ((uintptr_t) hdr->prev + (uintptr_t) hdr->next))
-    {
-    default:
-      status = MCHECK_HEAD;
-      break;
-    case MAGICFREE:
-      status = MCHECK_FREE;
-      break;
-    case MAGICWORD:
-      if (((char *) &hdr[1])[hdr->size] != MAGICBYTE)
-        status = MCHECK_TAIL;
-      else if ((hdr->magic2 ^ (uintptr_t) hdr->block) != MAGICWORD)
-        status = MCHECK_HEAD;
-      else
-        status = MCHECK_OK;
-      break;
-    }
-  if (status != MCHECK_OK)
-    {
-      mcheck_used = 0;
-      (*abortfunc) (status);
-      mcheck_used = 1;
-    }
-  return status;
-}
-
 void
 mcheck_check_all (void)
 {
-  /* Walk through all the active blocks and test whether they were tampered
-     with.  */
-  struct hdr *runp = root;
-
-  /* Temporarily turn off the checks.  */
-  pedantic = 0;
-
-  while (runp != NULL)
-    {
-      (void) checkhdr (runp);
-
-      runp = runp->next;
-    }
-
-  /* Turn checks on again.  */
-  pedantic = 1;
-}
-#ifdef _LIBC
-libc_hidden_def (mcheck_check_all)
+#if !IS_IN (libc)
+  __mcheck_checkptr (NULL);
 #endif
-
-static void
-unlink_blk (struct hdr *ptr)
-{
-  if (ptr->next != NULL)
-    {
-      ptr->next->prev = ptr->prev;
-      ptr->next->magic = MAGICWORD ^ ((uintptr_t) ptr->next->prev
-                                      + (uintptr_t) ptr->next->next);
-    }
-  if (ptr->prev != NULL)
-    {
-      ptr->prev->next = ptr->next;
-      ptr->prev->magic = MAGICWORD ^ ((uintptr_t) ptr->prev->prev
-                                      + (uintptr_t) ptr->prev->next);
-    }
-  else
-    root = ptr->next;
-}
-
-static void
-link_blk (struct hdr *hdr)
-{
-  hdr->prev = NULL;
-  hdr->next = root;
-  root = hdr;
-  hdr->magic = MAGICWORD ^ (uintptr_t) hdr->next;
-
-  /* And the next block.  */
-  if (hdr->next != NULL)
-    {
-      hdr->next->prev = hdr;
-      hdr->next->magic = MAGICWORD ^ ((uintptr_t) hdr
-                                      + (uintptr_t) hdr->next->next);
-    }
 }
-static void
-freehook (void *ptr, const void *caller)
-{
-  if (pedantic)
-    mcheck_check_all ();
-  if (ptr)
-    {
-      struct hdr *hdr = ((struct hdr *) ptr) - 1;
-      checkhdr (hdr);
-      hdr->magic = MAGICFREE;
-      hdr->magic2 = MAGICFREE;
-      unlink_blk (hdr);
-      hdr->prev = hdr->next = NULL;
-      flood (ptr, FREEFLOOD, hdr->size);
-      ptr = hdr->block;
-    }
-  __free_hook = old_free_hook;
-  if (old_free_hook != NULL)
-    (*old_free_hook)(ptr, caller);
-  else
-    free (ptr);
-  __free_hook = freehook;
-}
-
-static void *
-mallochook (size_t size, const void *caller)
-{
-  struct hdr *hdr;
-
-  if (pedantic)
-    mcheck_check_all ();
-
-  if (size > ~((size_t) 0) - (sizeof (struct hdr) + 1))
-    {
-      __set_errno (ENOMEM);
-      return NULL;
-    }
-
-  __malloc_hook = old_malloc_hook;
-  if (old_malloc_hook != NULL)
-    hdr = (struct hdr *) (*old_malloc_hook)(sizeof (struct hdr) + size + 1,
-                                            caller);
-  else
-    hdr = (struct hdr *) malloc (sizeof (struct hdr) + size + 1);
-  __malloc_hook = mallochook;
-  if (hdr == NULL)
-    return NULL;
-
-  hdr->size = size;
-  link_blk (hdr);
-  hdr->block = hdr;
-  hdr->magic2 = (uintptr_t) hdr ^ MAGICWORD;
-  ((char *) &hdr[1])[size] = MAGICBYTE;
-  flood ((void *) (hdr + 1), MALLOCFLOOD, size);
-  return (void *) (hdr + 1);
-}
-
-static void *
-memalignhook (size_t alignment, size_t size,
-              const void *caller)
-{
-  struct hdr *hdr;
-  size_t slop;
-  char *block;
-
-  if (pedantic)
-    mcheck_check_all ();
-
-  slop = (sizeof *hdr + alignment - 1) & - alignment;
-
-  if (size > ~((size_t) 0) - (slop + 1))
-    {
-      __set_errno (ENOMEM);
-      return NULL;
-    }
-
-  __memalign_hook = old_memalign_hook;
-  if (old_memalign_hook != NULL)
-    block = (*old_memalign_hook)(alignment, slop + size + 1, caller);
-  else
-    block = memalign (alignment, slop + size + 1);
-  __memalign_hook = memalignhook;
-  if (block == NULL)
-    return NULL;
-
-  hdr = ((struct hdr *) (block + slop)) - 1;
-
-  hdr->size = size;
-  link_blk (hdr);
-  hdr->block = (void *) block;
-  hdr->magic2 = (uintptr_t) block ^ MAGICWORD;
-  ((char *) &hdr[1])[size] = MAGICBYTE;
-  flood ((void *) (hdr + 1), MALLOCFLOOD, size);
-  return (void *) (hdr + 1);
-}
-
-static void *
-reallochook (void *ptr, size_t size, const void *caller)
-{
-  if (size == 0)
-    {
-      freehook (ptr, caller);
-      return NULL;
-    }
-
-  struct hdr *hdr;
-  size_t osize;
-
-  if (pedantic)
-    mcheck_check_all ();
-
-  if (size > ~((size_t) 0) - (sizeof (struct hdr) + 1))
-    {
-      __set_errno (ENOMEM);
-      return NULL;
-    }
-
-  if (ptr)
-    {
-      hdr = ((struct hdr *) ptr) - 1;
-      osize = hdr->size;
-
-      checkhdr (hdr);
-      unlink_blk (hdr);
-      if (size < osize)
-        flood ((char *) ptr + size, FREEFLOOD, osize - size);
-    }
-  else
-    {
-      osize = 0;
-      hdr = NULL;
-    }
-  __free_hook = old_free_hook;
-  __malloc_hook = old_malloc_hook;
-  __memalign_hook = old_memalign_hook;
-  __realloc_hook = old_realloc_hook;
-  if (old_realloc_hook != NULL)
-    hdr = (struct hdr *) (*old_realloc_hook)((void *) hdr,
-                                             sizeof (struct hdr) + size + 1,
-                                             caller);
-  else
-    hdr = (struct hdr *) realloc ((void *) hdr,
-                                  sizeof (struct hdr) + size + 1);
-  __free_hook = freehook;
-  __malloc_hook = mallochook;
-  __memalign_hook = memalignhook;
-  __realloc_hook = reallochook;
-  if (hdr == NULL)
-    return NULL;
-
-  hdr->size = size;
-  link_blk (hdr);
-  hdr->block = hdr;
-  hdr->magic2 = (uintptr_t) hdr ^ MAGICWORD;
-  ((char *) &hdr[1])[size] = MAGICBYTE;
-  if (size > osize)
-    flood ((char *) (hdr + 1) + osize, MALLOCFLOOD, size - osize);
-  return (void *) (hdr + 1);
-}
-
-__attribute__ ((noreturn))
-static void
-mabort (enum mcheck_status status)
-{
-  const char *msg;
-  switch (status)
-    {
-    case MCHECK_OK:
-      msg = _ ("memory is consistent, library is buggy\n");
-      break;
-    case MCHECK_HEAD:
-      msg = _ ("memory clobbered before allocated block\n");
-      break;
-    case MCHECK_TAIL:
-      msg = _ ("memory clobbered past end of allocated block\n");
-      break;
-    case MCHECK_FREE:
-      msg = _ ("block freed twice\n");
-      break;
-    default:
-      msg = _ ("bogus mcheck_status, library is buggy\n");
-      break;
-    }
-#ifdef _LIBC
-  __libc_fatal (msg);
-#else
-  fprintf (stderr, "mcheck: %s", msg);
-  fflush (stderr);
-  abort ();
-#endif
-}
-
-/* Memory barrier so that GCC does not optimize out the argument.  */
-#define malloc_opt_barrier(x) \
-  ({ __typeof (x) __x = x; __asm ("" : "+m" (__x)); __x; })
 
 int
 mcheck (void (*func) (enum mcheck_status))
 {
-  abortfunc = (func != NULL) ? func : &mabort;
-
-  /* These hooks may not be safely inserted if malloc is already in use.  */
-  if (__malloc_initialized <= 0 && !mcheck_used)
-    {
-      /* We call malloc() once here to ensure it is initialized.  */
-      void *p = malloc (0);
-      /* GCC might optimize out the malloc/free pair without a barrier.  */
-      p = malloc_opt_barrier (p);
-      free (p);
-
-      old_free_hook = __free_hook;
-      __free_hook = freehook;
-      old_malloc_hook = __malloc_hook;
-      __malloc_hook = mallochook;
-      old_memalign_hook = __memalign_hook;
-      __memalign_hook = memalignhook;
-      old_realloc_hook = __realloc_hook;
-      __realloc_hook = reallochook;
-      mcheck_used = 1;
-    }
-
-  return mcheck_used ? 0 : -1;
-}
-#ifdef _LIBC
-libc_hidden_def (mcheck)
+#if IS_IN (libc)
+  return -1;
+#else
+  return __mcheck_initialize (func, false);
 #endif
+}
 
 int
 mcheck_pedantic (void (*func) (enum mcheck_status))
 {
-  int res = mcheck (func);
-  if (res == 0)
-    pedantic = 1;
-  return res;
+#if IS_IN (libc)
+  return -1;
+#else
+  return __mcheck_initialize (func, true);
+#endif
 }
 
 enum mcheck_status
 mprobe (void *ptr)
 {
-  return mcheck_used ? checkhdr (((struct hdr *) ptr) - 1) : MCHECK_DISABLED;
+#if IS_IN (libc)
+  return MCHECK_DISABLED;
+#else
+  return __mcheck_checkptr (ptr);
+#endif
 }