overhaul cancellation to fix resource leaks and dangerous behavior with signals

this commit addresses two issues:

1. a race condition, whereby a cancellation request occurring after a
syscall returned from kernelspace but before the subsequent
CANCELPT_END would cause cancellable resource-allocating syscalls
(like open) to leak resources.

2. signal handlers invoked while the thread was blocked at a
cancellation point behaved as if asynchronous cancellation mode wer in
effect, resulting in potentially dangerous state corruption if a
cancellation request occurs.

the glibc/nptl implementation of threads shares both of these issues.

with this commit, both are fixed. however, cancellation points
encountered in a signal handler will not be acted upon if the signal
was received while the thread was already at a cancellation point.
they will of course be acted upon after the signal handler returns, so
in real-world usage where signal handlers quickly return, it should
not be a problem. it's possible to solve this problem too by having
sigaction() wrap all signal handlers with a function that uses a
pthread_cleanup handler to catch cancellation, patch up the saved
context, and return into the cancellable function that will catch and
act upon the cancellation. however that would be a lot of complexity
for minimal if any benefit...

1 files changed, 11 insertions, 5 deletions

diff --git a/src/thread/pthread_create.c b/src/thread/pthread_create.c
index 17a47f6a..785a82b8 100644
--- a/src/thread/pthread_create.c
+++ b/src/thread/pthread_create.c
@@ -42,6 +42,12 @@ void __pthread_unwind_next(struct __ptcb *cb)
 static void docancel(struct pthread *self)
 {
 	struct __ptcb cb = { .__next = self->cancelbuf };
+	sigset_t set;
+	self->canceldisable = 1;
+	self->cancelasync = 0;
+	sigemptyset(&set);
+	sigaddset(&set, SIGCANCEL);
+	__libc_sigprocmask(SIG_UNBLOCK, &set, 0);
 	__pthread_unwind_next(&cb);
 }
 
@@ -50,17 +56,17 @@ static void cancel_handler(int sig, siginfo_t *si, void *ctx)
 	struct pthread *self = __pthread_self();
 	if (si->si_code > 0 || si->si_pid != self->pid) return;
 	self->cancel = 1;
-	if (self->canceldisable || (!self->cancelasync && !self->cancelpoint))
-		return;
-	docancel(self);
+	if (self->canceldisable) return;
+	if (self->cancelasync || (self->cancelpoint==1 && PC_AT_SYS(ctx)))
+		docancel(self);
 }
 
 static void cancelpt(int x)
 {
 	struct pthread *self = __pthread_self();
 	if (self->canceldisable) return;
-	self->cancelpoint = x;
-	if (self->cancel) docancel(self);
+	if ((self->cancelpoint+=x)==1 && x>=0 && self->cancel)
+		docancel(self);
 }
 
 /* "rsyscall" is a mechanism by which a thread can synchronously force all