Move malloc hooks into a compat DSO

Remove all malloc hook uses from core malloc functions and move it
into a new library libc_malloc_debug.so.  With this, the hooks now no
longer have any effect on the core library.

libc_malloc_debug.so is a malloc interposer that needs to be preloaded
to get hooks functionality back so that the debugging features that
depend on the hooks, i.e. malloc-check, mcheck and mtrace work again.
Without the preloaded DSO these debugging features will be nops.
These features will be ported away from hooks in subsequent patches.

Similarly, legacy applications that need hooks functionality need to
preload libc_malloc_debug.so.

The symbols exported by libc_malloc_debug.so are maintained at exactly
the same version as libc.so.

Finally, static binaries will no longer be able to use malloc
debugging features since they cannot preload the debugging DSO.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Tested-by: Carlos O'Donell <carlos@redhat.com>

1 files changed, 189 insertions, 0 deletions

diff --git a/malloc/malloc-debug.c b/malloc/malloc-debug.c
new file mode 100644
index 0000000000..41dfcd3369
--- /dev/null
+++ b/malloc/malloc-debug.c
@@ -0,0 +1,189 @@
+/* Malloc debug DSO.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public License as
+   published by the Free Software Foundation; either version 2.1 of the
+   License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; see the file COPYING.LIB.  If
+   not, see <https://www.gnu.org/licenses/>.  */
+
+#include <atomic.h>
+#include <libc-symbols.h>
+#include <shlib-compat.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/param.h>
+
+/* Support only the glibc allocators.  */
+extern void *__libc_malloc (size_t);
+extern void __libc_free (void *);
+extern void *__libc_realloc (void *, size_t);
+extern void *__libc_memalign (size_t, size_t);
+extern void *__libc_valloc (size_t);
+extern void *__libc_pvalloc (size_t);
+extern void *__libc_calloc (size_t, size_t);
+
+#define DEBUG_FN(fn) \
+  static __typeof (__libc_ ## fn) __debug_ ## fn
+
+DEBUG_FN(malloc);
+DEBUG_FN(free);
+DEBUG_FN(realloc);
+DEBUG_FN(memalign);
+DEBUG_FN(valloc);
+DEBUG_FN(pvalloc);
+DEBUG_FN(calloc);
+
+extern void (*__free_hook) (void *, const void *);
+compat_symbol_reference (libc, __free_hook, __free_hook, GLIBC_2_0);
+extern void * (*__malloc_hook) (size_t, const void *);
+compat_symbol_reference (libc, __malloc_hook, __malloc_hook, GLIBC_2_0);
+extern void * (*__realloc_hook) (void *, size_t, const void *);
+compat_symbol_reference (libc, __realloc_hook, __realloc_hook, GLIBC_2_0);
+extern void * (*__memalign_hook) (size_t, size_t, const void *);
+compat_symbol_reference (libc, __memalign_hook, __memalign_hook, GLIBC_2_0);
+
+static size_t pagesize;
+
+/* The allocator functions.  */
+
+static void *
+__debug_malloc (size_t bytes)
+{
+  void *(*hook) (size_t, const void *) = atomic_forced_read (__malloc_hook);
+  if (__builtin_expect (hook != NULL, 0))
+    return (*hook)(bytes, RETURN_ADDRESS (0));
+
+  return __libc_malloc (bytes);
+}
+strong_alias (__debug_malloc, malloc)
+
+static void
+__debug_free (void *mem)
+{
+  void (*hook) (void *, const void *) = atomic_forced_read (__free_hook);
+  if (__builtin_expect (hook != NULL, 0))
+    {
+      (*hook)(mem, RETURN_ADDRESS (0));
+      return;
+    }
+  __libc_free (mem);
+}
+strong_alias (__debug_free, free)
+
+static void *
+__debug_realloc (void *oldmem, size_t bytes)
+{
+  void *(*hook) (void *, size_t, const void *) =
+    atomic_forced_read (__realloc_hook);
+  if (__builtin_expect (hook != NULL, 0))
+    return (*hook)(oldmem, bytes, RETURN_ADDRESS (0));
+
+  return __libc_realloc (oldmem, bytes);
+}
+strong_alias (__debug_realloc, realloc)
+
+static void *
+_debug_mid_memalign (size_t alignment, size_t bytes, const void *address)
+{
+  void *(*hook) (size_t, size_t, const void *) =
+    atomic_forced_read (__memalign_hook);
+  if (__builtin_expect (hook != NULL, 0))
+    return (*hook)(alignment, bytes, address);
+
+  return __libc_memalign (alignment, bytes);
+}
+
+static void *
+__debug_memalign (size_t alignment, size_t bytes)
+{
+  return _debug_mid_memalign (alignment, bytes, RETURN_ADDRESS (0));
+}
+strong_alias (__debug_memalign, memalign)
+strong_alias (__debug_memalign, aligned_alloc)
+
+static void *
+__debug_pvalloc (size_t bytes)
+{
+  size_t rounded_bytes;
+
+  if (!pagesize)
+    pagesize = sysconf (_SC_PAGESIZE);
+
+  /* ALIGN_UP with overflow check.  */
+  if (__glibc_unlikely (__builtin_add_overflow (bytes,
+						pagesize - 1,
+						&rounded_bytes)))
+    {
+      errno = ENOMEM;
+      return NULL;
+    }
+  rounded_bytes = rounded_bytes & -(pagesize - 1);
+
+  return _debug_mid_memalign (pagesize, rounded_bytes, RETURN_ADDRESS (0));
+}
+strong_alias (__debug_pvalloc, pvalloc)
+
+static void *
+__debug_valloc (size_t bytes)
+{
+  if (!pagesize)
+    pagesize = sysconf (_SC_PAGESIZE);
+
+  return _debug_mid_memalign (pagesize, bytes, RETURN_ADDRESS (0));
+}
+strong_alias (__debug_valloc, valloc)
+
+static int
+__debug_posix_memalign (void **memptr, size_t alignment, size_t bytes)
+{
+  /* Test whether the SIZE argument is valid.  It must be a power of
+     two multiple of sizeof (void *).  */
+  if (alignment % sizeof (void *) != 0
+      || !powerof2 (alignment / sizeof (void *))
+      || alignment == 0)
+    return EINVAL;
+
+  *memptr = _debug_mid_memalign (alignment, bytes, RETURN_ADDRESS (0));
+
+  if (*memptr == NULL)
+    return ENOMEM;
+
+  return 0;
+}
+strong_alias (__debug_posix_memalign, posix_memalign)
+
+static void *
+__debug_calloc (size_t nmemb, size_t size)
+{
+  void *(*hook) (size_t, const void *) = atomic_forced_read (__malloc_hook);
+  if (__builtin_expect (hook != NULL, 0))
+    {
+      size_t bytes;
+
+      if (__glibc_unlikely (__builtin_mul_overflow (nmemb, size, &bytes)))
+	{
+	  errno = ENOMEM;
+	  return NULL;
+	}
+
+      void *mem = (*hook)(bytes, RETURN_ADDRESS (0));
+
+      if (mem != NULL)
+	memset (mem, 0, bytes);
+
+      return mem;
+    }
+
+  return __libc_calloc (nmemb, size);
+}
+strong_alias (__debug_calloc, calloc)