MIPS: Add wrappers to get/setrlimit64 to fix RLIM64_INFINITY constant

RLIM64_INFINITY was supposed to be a glibc convention rather than
anything seen by the kernel, but it ended being passed to the kernel
through the prlimit64 syscall.  On O32 and N32 ABIs, we therefore
end-up with different values on the userland and kernel side:

* On the kernel side, the value is defined for all architectures as
  include/uapi/linux/resource.h:

  #define RLIM64_INFINITY           (~0ULL)

* On the GNU libc side, the value is defined in
  ports/sysdeps/unix/sysv/linux/mips/bits/resource.h:

  For the O32 and N32 ABI:
  #  define RLIM64_INFINITY 0x7fffffffffffffffULL

  and for the N64 ABI:
  #  define RLIM64_INFINITY 0xffffffffffffffffUL

This was not a problem until the prlimit64 syscall was wired in the
2.6.36 kernel. Given the GLIBC uses the prlimit64 syscall to implement
getrlimit64 and setrlimit64, pam_limits.so is setting the limits to
a very big value instead of infinity. As a normal user process can
later only decrease the value and not increase it, it will later get
and EPERM error when trying to set the value to infinity with setrlimit.

The GLIBC has this constant for more than 7 years, and as it is defined
in a header file, it means a lot of binaries are in the wild. This patch
fixes that by adding a wrapper to fix the value passed to or received
from the kernel, before or after calling the prlimit64 syscall.

3 files changed, 127 insertions, 0 deletions

diff --git a/ports/ChangeLog.mips b/ports/ChangeLog.mips
index 753e04b908..d2212a1933 100644
--- a/ports/ChangeLog.mips
+++ b/ports/ChangeLog.mips
@@ -1,3 +1,11 @@
+2013-11-27  Aurelien Jarno <aurelien@aurel32.net>
+
+	* sysdeps/unix/sysv/linux/mips/getrlimit64.c: On O32 and N32 ABIs,
+	include the generic getrlimit64 version as __internal_getrlimit64
+	and add a wrapper around it to convert the RLIM64_INFINITY constant
+	between the libc and the kernel version.
+	* sysdeps/unix/sysv/linux/mips/setrlimit64.c: Ditto with setrlimit64.
+
 2013-11-26  Ondřej Bílka  <neleai@seznam.cz>
 	* sysdeps/unix/sysv/linux/mips/bits/ipc.h: Use __glibc_reserved instead __unused.
 	* sysdeps/unix/sysv/linux/mips/bits/msq.h: Likewise.
diff --git a/ports/sysdeps/unix/sysv/linux/mips/getrlimit64.c b/ports/sysdeps/unix/sysv/linux/mips/getrlimit64.c
new file mode 100644
index 0000000000..10a5495f8d
--- /dev/null
+++ b/ports/sysdeps/unix/sysv/linux/mips/getrlimit64.c
@@ -0,0 +1,64 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <sys/resource.h>
+
+#if _MIPS_SIM == _ABIO32 || _MIPS_SIM == _ABIN32
+
+# define getrlimit64 static __internal_getrlimit64
+# undef libc_hidden_def
+# define libc_hidden_def(name)
+# include <sysdeps/unix/sysv/linux/getrlimit64.c>
+# undef getrlimit64
+# undef libc_hidden_def
+# define libc_hidden_def(name) hidden_def (name)
+
+/* RLIM64_INFINITY was supposed to be a glibc convention rather than
+   anything seen by the kernel, but it ended being passed to the kernel
+   through the prlimit64 syscall.  Given that a lot of binaries with
+   the wrong constant value are in the wild, provide a wrapper function
+   fixing the value after the syscall.  */
+
+# define GLIBC_RLIM64_INFINITY		0x7fffffffffffffffULL
+# define KERNEL_RLIM64_INFINITY		0xffffffffffffffffULL
+
+int
+getrlimit64 (enum __rlimit_resource resource,
+	     struct rlimit64 *rlimits)
+{
+  struct rlimit64 krlimits;
+
+  if (__internal_getrlimit64 (resource, &krlimits) < 0)
+    return -1;
+
+  if (krlimits.rlim_cur == KERNEL_RLIM64_INFINITY)
+    rlimits->rlim_cur = GLIBC_RLIM64_INFINITY;
+  else
+    rlimits->rlim_cur = krlimits.rlim_cur;
+  if (krlimits.rlim_max == KERNEL_RLIM64_INFINITY)
+    rlimits->rlim_max = GLIBC_RLIM64_INFINITY;
+  else
+    rlimits->rlim_max = krlimits.rlim_max;
+
+  return 0;
+}
+
+libc_hidden_def (getrlimit64)
+
+#else /* !_ABI_O32 && !_ABI_N32 */
+# include <sysdeps/unix/sysv/linux/getrlimit64.c>
+#endif
diff --git a/ports/sysdeps/unix/sysv/linux/mips/setrlimit64.c b/ports/sysdeps/unix/sysv/linux/mips/setrlimit64.c
new file mode 100644
index 0000000000..8609f694f8
--- /dev/null
+++ b/ports/sysdeps/unix/sysv/linux/mips/setrlimit64.c
@@ -0,0 +1,55 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <sys/resource.h>
+
+#if _MIPS_SIM == _ABIO32 || _MIPS_SIM == _ABIN32
+
+# define setrlimit64 static __internal_setrlimit64
+# include <sysdeps/unix/sysv/linux/setrlimit64.c>
+# undef setrlimit64
+
+/* RLIM64_INFINITY was supposed to be a glibc convention rather than
+   anything seen by the kernel, but it ended being passed to the kernel
+   through the prlimit64 syscall.  Given that a lot of binaries with
+   the wrong constant value are in the wild, provide a wrapper function
+   fixing the value before the syscall.  */
+
+# define GLIBC_RLIM64_INFINITY		0x7fffffffffffffffULL
+# define KERNEL_RLIM64_INFINITY		0xffffffffffffffffULL
+
+int
+setrlimit64 (enum __rlimit_resource resource,
+	     const struct rlimit64 *rlimits)
+{
+  struct rlimit64 krlimits;
+
+  if (rlimits->rlim_cur == GLIBC_RLIM64_INFINITY)
+    krlimits.rlim_cur = KERNEL_RLIM64_INFINITY;
+  else
+    krlimits.rlim_cur = rlimits->rlim_cur;
+  if (rlimits->rlim_max == GLIBC_RLIM64_INFINITY)
+    krlimits.rlim_max = KERNEL_RLIM64_INFINITY;
+  else
+    krlimits.rlim_max = rlimits->rlim_max;
+
+  return __internal_setrlimit64 (resource, &krlimits);
+}
+
+#else /* !_ABI_O32 && !_ABI_N32 */
+# include <sysdeps/unix/sysv/linux/setrlimit64.c>
+#endif