nptl: Avoid fork handler lock for async-signal-safe fork [BZ #24161]

Commit 27761a1042daf01987e7d79636d0c41511c6df3c ("Refactor atfork
handlers") introduced a lock, atfork_lock, around fork handler list
accesses.  It turns out that this lock occasionally results in
self-deadlocks in malloc/tst-mallocfork2:

(gdb) bt
#0  __lll_lock_wait_private ()
    at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:63
#1  0x00007f160c6f927a in __run_fork_handlers (who=(unknown: 209394016),
    who@entry=atfork_run_prepare) at register-atfork.c:116
#2  0x00007f160c6b7897 in __libc_fork () at ../sysdeps/nptl/fork.c:58
#3  0x00000000004027d6 in sigusr1_handler (signo=<optimized out>)
    at tst-mallocfork2.c:80
#4  sigusr1_handler (signo=<optimized out>) at tst-mallocfork2.c:64
#5  <signal handler called>
#6  0x00007f160c6f92e4 in __run_fork_handlers (who=who@entry=atfork_run_parent)
    at register-atfork.c:136
#7  0x00007f160c6b79a2 in __libc_fork () at ../sysdeps/nptl/fork.c:152
#8  0x0000000000402567 in do_test () at tst-mallocfork2.c:156
#9  0x0000000000402dd2 in support_test_main (argc=1, argv=0x7ffc81ef1ab0,
    config=config@entry=0x7ffc81ef1970) at support_test_main.c:350
#10 0x0000000000402362 in main (argc=<optimized out>, argv=<optimized out>)
    at ../support/test-driver.c:168

If no locking happens in the single-threaded case (where fork is
expected to be async-signal-safe), this deadlock is avoided.
(pthread_atfork is not required to be async-signal-safe, so a fork
call from a signal handler interrupting pthread_atfork is not
a problem.)

1 files changed, 5 insertions, 3 deletions

diff --git a/sysdeps/nptl/fork.h b/sysdeps/nptl/fork.h
index a1c3b26b68..99ed76034b 100644
--- a/sysdeps/nptl/fork.h
+++ b/sysdeps/nptl/fork.h
@@ -52,9 +52,11 @@ enum __run_fork_handler_type
    - atfork_run_child: run all the CHILD_HANDLER and unlocks the internal
 		       lock.
    - atfork_run_parent: run all the PARENT_HANDLER and unlocks the internal
-			lock.  */
-extern void __run_fork_handlers (enum __run_fork_handler_type who)
-  attribute_hidden;
+			lock.
+
+   Perform locking only if DO_LOCKING.  */
+extern void __run_fork_handlers (enum __run_fork_handler_type who,
+				 _Bool do_locking) attribute_hidden;
 
 /* C library side function to register new fork handlers.  */
 extern int __register_atfork (void (*__prepare) (void),