fix for broken kernel side RLIM_INFINITY on mips

On 32 bit mips the kernel uses -1UL/2 to mark RLIM_INFINITY (and
this is the definition in the userspace api), but since it is in
the middle of the valid range of limits and limits are often
compared with relational operators, various kernel side logic is
broken if larger than -1UL/2 limits are used. So we truncate the
limits to -1UL/2 in get/setrlimit and prlimit.

Even if the kernel side logic consistently treated -1UL/2 as greater
than any other limit value, there wouldn't be any clean workaround
that allowed using large limits:
* using -1UL/2 as RLIM_INFINITY in userspace would mean different
infinity value for get/setrlimt and prlimit (where infinity is always
-1ULL) and userspace logic could break easily (just like the kernel
is broken now) and more special case code would be needed for mips.
* translating -1UL/2 kernel side value to -1ULL in userspace would
mean that -1UL/2 limit cannot be set (eg. -1UL/2+1 had to be passed
to the kernel instead).

1 files changed, 16 insertions, 1 deletions

diff --git a/src/linux/prlimit.c b/src/linux/prlimit.c
index d1639cca..0fe28e10 100644
--- a/src/linux/prlimit.c
+++ b/src/linux/prlimit.c
@@ -3,9 +3,24 @@
 #include "syscall.h"
 #include "libc.h"
 
+#define FIX(x) do{ if ((x)>=SYSCALL_RLIM_INFINITY) (x)=RLIM_INFINITY; }while(0)
+
 int prlimit(pid_t pid, int resource, const struct rlimit *new_limit, struct rlimit *old_limit)
 {
-	return syscall(SYS_prlimit64, pid, resource, new_limit, old_limit);
+	struct rlimit tmp;
+	int r;
+	if (new_limit && SYSCALL_RLIM_INFINITY != RLIM_INFINITY) {
+		tmp = *new_limit;
+		FIX(tmp.rlim_cur);
+		FIX(tmp.rlim_max);
+		new_limit = &tmp;
+	}
+	r = syscall(SYS_prlimit64, pid, resource, new_limit, old_limit);
+	if (!r && old_limit && SYSCALL_RLIM_INFINITY != RLIM_INFINITY) {
+		FIX(old_limit->rlim_cur);
+		FIX(old_limit->rlim_max);
+	}
+	return r;
 }
 
 #undef prlimit64