enable private futex for process-local robust mutexes

the kernel always uses non-private wake when walking the robust list
when a thread or process exits, so it's not able to wake waiters
listening with the private futex flag. this problem is solved by doing
the equivalent in userspace as the last step of pthread_exit.

care is taken to remove mutexes from the robust list before unlocking
them so that the kernel will not attempt to access them again,
possibly after another thread locks them. this removal code can treat
the list as singly-linked, since no further code which would add or
remove items is able to run at this point. moreover, the pending
pointer is not needed since the mutexes being unlocked are all
process-local; in the case of asynchronous process termination, they
all cease to exist.

since a process-local robust mutex cannot come into existence without
a call to pthread_mutexattr_setrobust in the same process, the code
for userspace robust list processing is put in that source file, and
a weak alias to a dummy function is used to avoid pulling in this
bloat as part of pthread_exit in static-linked programs.

3 files changed, 25 insertions, 1 deletions

diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c
index e77e54a5..c8c117b9 100644
--- a/src/thread/pthread_create.c
+++ b/src/thread/pthread_create.c
@@ -10,6 +10,7 @@ static void dummy_0()
 weak_alias(dummy_0, __acquire_ptc);
 weak_alias(dummy_0, __release_ptc);
 weak_alias(dummy_0, __pthread_tsd_run_dtors);
+weak_alias(dummy_0, __do_private_robust_list);
 
 _Noreturn void pthread_exit(void *result)
 {
@@ -63,6 +64,8 @@ _Noreturn void pthread_exit(void *result)
 			a_dec(&libc.bytelocale_cnt_minus_1);
 	}
 
+	__do_private_robust_list();
+
 	if (self->detached && self->map_base) {
 		/* Detached threads must avoid the kernel clear_child_tid
 		 * feature, since the virtual address will have been
diff --git a/src/thread/pthread_mutex_init.c b/src/thread/pthread_mutex_init.c
index b83edd0f..acf45a74 100644
--- a/src/thread/pthread_mutex_init.c
+++ b/src/thread/pthread_mutex_init.c
@@ -4,6 +4,5 @@ int pthread_mutex_init(pthread_mutex_t *restrict m, const pthread_mutexattr_t *r
 {
 	*m = (pthread_mutex_t){0};
 	if (a) m->_m_type = a->__attr;
-	if (m->_m_type & 4) m->_m_type |= 128U;
 	return 0;
 }
diff --git a/src/thread/pthread_mutexattr_setrobust.c b/src/thread/pthread_mutexattr_setrobust.c
index dcfa4cf1..8948cbaf 100644
--- a/src/thread/pthread_mutexattr_setrobust.c
+++ b/src/thread/pthread_mutexattr_setrobust.c
@@ -1,4 +1,26 @@
 #include "pthread_impl.h"
+#include <stddef.h>
+
+void __do_private_robust_list()
+{
+	pthread_t self = __pthread_self();
+	void **p, **prev, **next;
+	pthread_mutex_t *m;
+
+	for (prev=0, p=self->robust_list.head; p; p=next) {
+		next = *p;
+		m = (void *)((char *)p - offsetof(pthread_mutex_t, _m_next));
+		if (!(m->_m_type & 128)) {
+			int waiters = m->_m_waiters;
+			if (prev) *prev = next;
+			else self->robust_list.head = next;
+			int cont = a_swap(&m->_m_lock, self->tid|0x40000000);
+			if (cont < 0 || waiters) __wake(&m->_m_lock, 1, 1);
+		} else {
+			prev = p;
+		}
+	}
+}
 
 int pthread_mutexattr_setrobust(pthread_mutexattr_t *a, int robust)
 {