fix some bugs in setxid and update setrlimit to use __synccall

setrlimit is supposed to be per-process, not per-thread, but again
linux gets it wrong. work around this in userspace. not only is it
needed for correctness; setxid also depends on the resource limits for
all threads being the same to avoid situations where temporarily
unlimiting the limit succeeds in some threads but fails in others.

2 files changed, 33 insertions, 10 deletions

diff --git a/src/misc/setrlimit.c b/src/misc/setrlimit.c
index 68bd9d74..bf03fe6f 100644
--- a/src/misc/setrlimit.c
+++ b/src/misc/setrlimit.c
@@ -1,11 +1,36 @@
 #include <sys/resource.h>
+#include <errno.h>
 #include "syscall.h"
 #include "libc.h"
 
-int setrlimit(int resource, const struct rlimit *rlim)
+int __setrlimit(int resource, const struct rlimit *rlim)
 {
 	long k_rlim[2] = { rlim->rlim_cur, rlim->rlim_max };
-	return syscall(SYS_setrlimit, resource, k_rlim);
+	return __syscall(SYS_setrlimit, resource, k_rlim);
+}
+
+struct ctx {
+	const struct rlimit *rlim;
+	int res;
+	int err;
+};
+
+static void do_setrlimit(void *p)
+{
+	struct ctx *c = p;
+	if (c->err) return;
+	c->err = -__setrlimit(c->res, c->rlim);
+}
+
+int setrlimit(int resource, const struct rlimit *rlim)
+{
+	struct ctx c = { .res = resource, .rlim = rlim };
+	__synccall(do_setrlimit, &c);
+	if (c.err) {
+		errno = c.err;
+		return -1;
+	}
+	return 0;
 }
 
 LFS64(setrlimit);
diff --git a/src/unistd/setxid.c b/src/unistd/setxid.c
index d3bfaf62..2f651a11 100644
--- a/src/unistd/setxid.c
+++ b/src/unistd/setxid.c
@@ -11,6 +11,8 @@ struct ctx {
 
 /* We jump through hoops to eliminate the possibility of partial failures. */
 
+int __setrlimit(int, const struct rlimit *);
+
 static void do_setxid(void *p)
 {
 	struct ctx *c = p;
@@ -18,17 +20,13 @@ static void do_setxid(void *p)
 	if (c->rlim && c->id >= 0 && c->id != getuid()) {
 		struct rlimit inf = { RLIM_INFINITY, RLIM_INFINITY }, old;
 		getrlimit(RLIMIT_NPROC, &old);
-		if (setrlimit(RLIMIT_NPROC, &inf) && libc.threads_minus_1) {
-			c->err = errno;
+		if ((c->err = -__setrlimit(RLIMIT_NPROC, &inf)) && libc.threads_minus_1)
 			return;
-		}
-		if (__syscall(c->nr, c->id, c->eid, c->sid))
-			c->err = errno;
-		setrlimit(RLIMIT_NPROC, &old);
+		c->err = -__syscall(c->nr, c->id, c->eid, c->sid);
+		__setrlimit(RLIMIT_NPROC, &old);
 		return;
 	}
-	if (__syscall(c->nr, c->id, c->eid, c->sid))
-		c->err = errno;
+	c->err = -__syscall(c->nr, c->id, c->eid, c->sid);
 }
 
 int __setxid(int nr, int id, int eid, int sid)