libc: Extend __libc_freeres framework (Bug 23329).

The __libc_freeres framework does not extend to non-libc.so objects.
This causes problems in general for valgrind and mtrace detecting
unfreed objects in both libdl.so and libpthread.so.  This change is
a pre-requisite to properly moving the malloc hooks out of malloc
since such a move now requires precise accounting of all allocated
data before destructors are run.

This commit adds a proper hook in libc.so.6 for both libdl.so and
for libpthread.so, this ensures that shm-directory.c which uses
freeit () to free memory is called properly.  We also remove the
nptl_freeres hook and fall back to using weak-ref-and-check idiom
for a loaded libpthread.so, thus making this process similar for
all DSOs.

Lastly we follow best practice and use explicit free calls for
both libdl.so and libpthread.so instead of the generic hook process
which has undefined order.

Tested on x86_64 with no regressions.

Signed-off-by: DJ Delorie <dj@redhat.com>
Signed-off-by: Carlos O'Donell <carlos@redhat.com>

3 files changed, 78 insertions, 12 deletions

diff --git a/include/dlfcn.h b/include/dlfcn.h
index c231309083..0dc57dbe22 100644
--- a/include/dlfcn.h
+++ b/include/dlfcn.h
@@ -155,6 +155,8 @@ extern void __libc_register_dl_open_hook (struct link_map *map)
 extern void __libc_register_dlfcn_hook (struct link_map *map)
      attribute_hidden;
 #endif
-#endif
 
+extern void __dlerror_main_freeres (void) attribute_hidden;
+
+#endif
 #endif
diff --git a/include/libc-symbols.h b/include/libc-symbols.h
index 6137304b0b..8b9273c13a 100644
--- a/include/libc-symbols.h
+++ b/include/libc-symbols.h
@@ -217,16 +217,6 @@
   static const char __evoke_link_warning_##symbol[]	\
     __attribute__ ((used, section (".gnu.warning." #symbol __sec_comment))) \
     = msg;
-#define libc_freeres_ptr(decl) \
-  __make_section_unallocated ("__libc_freeres_ptrs, \"aw\", %nobits") \
-  decl __attribute__ ((section ("__libc_freeres_ptrs" __sec_comment)))
-#define __libc_freeres_fn_section \
-  __attribute__ ((section ("__libc_freeres_fn")))
-
-#define libc_freeres_fn(name)	\
-  static void name (void) __attribute_used__ __libc_freeres_fn_section;	\
-  text_set_element (__libc_subfreeres, name);				\
-  static void name (void)
 
 /* A canned warning for sysdeps/stub functions.  */
 #define	stub_warning(name) \
@@ -244,6 +234,79 @@ requires at runtime the shared libraries from the glibc version used \
 for linking")
 #endif
 
+/* Resource Freeing Hooks:
+
+   Normally a process exits and the OS cleans up any allocated
+   memory.  However, when tooling like mtrace or valgrind is monitoring
+   the process we need to free all resources that are part of the
+   process in order to provide the consistency required to track
+   memory leaks.
+
+   A single public API exists and is __libc_freeres(), and this is used
+   by applications like valgrind to freee resouces.
+
+   There are 3 cases:
+
+   (a) __libc_freeres
+
+	In this case all you need to do is define the freeing routine:
+
+	foo.c:
+	libfoo_freeres_fn (foo_freeres)
+	{
+	  complex_free (mem);
+	}
+
+	This ensures the function is called at the right point to free
+	resources.
+
+   (b) __libc_freeres_ptr
+
+	The framework for (a) iterates over the list of pointers-to-free
+	in (b) and frees them.
+
+	foo.c:
+	libc_freeres_ptr (static char *foo_buffer);
+
+	Freeing these resources alaways happens last and is equivalent
+	to registering a function that does 'free (foo_buffer)'.
+
+   (c) Explicit lists of free routines to call or objects to free.
+
+	It is the intended goal to remove (a) and (b) which have some
+	non-determinism based on link order, and instead use explicit
+	lists of functions and frees to resolve cleanup ordering issues
+	and make it easy to debug and maintain.
+
+	As of today the following subsystems use (c):
+
+	Per-thread cleanup:
+	* malloc/thread-freeres.c
+
+	libdl cleanup:
+	* dlfcn/dlfreeres.c
+
+	libpthread cleanup:
+	* nptl/nptlfreeres.c
+
+	So if you need any shutdown routines to run you should add them
+	directly to the appropriate subsystem's shutdown list.  */
+
+/* Resource pointers to free in libc.so.  */
+#define libc_freeres_ptr(decl) \
+  __make_section_unallocated ("__libc_freeres_ptrs, \"aw\", %nobits") \
+  decl __attribute__ ((section ("__libc_freeres_ptrs" __sec_comment)))
+
+/* Resource freeing functions from libc.so go in this section.  */
+#define __libc_freeres_fn_section \
+  __attribute__ ((section ("__libc_freeres_fn")))
+
+/* Resource freeing functions for libc.so.  */
+#define libc_freeres_fn(name) \
+  static void name (void) __attribute_used__ __libc_freeres_fn_section;	\
+  text_set_element (__libc_subfreeres, name);				\
+  static void name (void)
+
 /* Declare SYMBOL to be TYPE (`function' or `object') of SIZE bytes
    alias to ORIGINAL, when the assembler supports such declarations
    (such as in ELF).
diff --git a/include/set-hooks.h b/include/set-hooks.h
index 0207656bb9..b3bd8b4092 100644
--- a/include/set-hooks.h
+++ b/include/set-hooks.h
@@ -22,11 +22,12 @@
 #define __need_size_t
 #include <stddef.h>
 #include <sys/cdefs.h>
+#include <libc-symbols.h>
 
 #ifdef symbol_set_define
 /* Define a hook variable called NAME.  Functions put on this hook take
    arguments described by PROTO.  Use `text_set_element (NAME, FUNCTION)'
-   from gnu-stabs.h to add a function to the hook.  */
+   from include/libc-symbols.h to add a function to the hook.  */
 
 # define DEFINE_HOOK(NAME, PROTO)		\
   typedef void __##NAME##_hook_function_t PROTO; \