muntrace: reset file and hooks before finalizing the stream

fclose will call free, invoking its hook, then fprintf which would indirectly
try to allocate a buffer, and this can cause malloc to be used (thus its hook
to be invoked) if libio uses malloc instead of mmap; given any malloc/free hook
locks the internal lock, this leads to a deadlock.

To prevent this hook roundtrip at muntrace, first unset MALLSTREAM and the
hooks, and only after that close the trace file.

2 files changed, 10 insertions, 2 deletions

diff --git a/ChangeLog b/ChangeLog
index fbac1c5d5d..1031d4d5c1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,8 @@
 2012-11-19  Pino Toscano  <toscano.pino@tiscali.it>
 
+	* malloc/mtrace.c (muntrace): Reset MALLSTREAM and the hooks before
+	finalizing MALLSTREAM.
+
 	* sysdeps/mach/hurd/syncfs.c: New file.
 
 2012-11-19  Siddhesh Poyarekar  <siddhesh@redhat.com>
diff --git a/malloc/mtrace.c b/malloc/mtrace.c
index d7a032a86a..3f02c71822 100644
--- a/malloc/mtrace.c
+++ b/malloc/mtrace.c
@@ -364,11 +364,16 @@ muntrace ()
   if (mallstream == NULL)
     return;
 
-  fprintf (mallstream, "= End\n");
-  fclose (mallstream);
+  /* Do the reverse of what done in mtrace: first reset the hooks and
+     MALLSTREAM, and only after that write the trailer and close the
+     file.  */
+  FILE *f = mallstream;
   mallstream = NULL;
   __free_hook = tr_old_free_hook;
   __malloc_hook = tr_old_malloc_hook;
   __realloc_hook = tr_old_realloc_hook;
   __memalign_hook = tr_old_memalign_hook;
+
+  fprintf (f, "= End\n");
+  fclose (f);
 }