Alpha: Add wrappers to get/setrlimit64 to fix RLIM64_INFINITY constant [BZ #22648]

RLIM64_INFINITY was supposed to be a glibc convention rather than
anything seen by the kernel, but it ended being passed to the kernel
through the prlimit64 syscall.

* On the kernel side, the value is defined for the prlimit64 syscall for
  all architectures in include/uapi/linux/resource.h:

  #define RLIM64_INFINITY           (~0ULL)

* On the kernel side, the value is defined for getrlimit and setrlimit
  in arch/alpha/include/uapi/asm/resource.h

  #define RLIM_INFINITY            0x7ffffffffffffffful

* On the GNU libc side, the value is defined in
  sysdeps/unix/sysv/linux/alpha/bits/resource.h:

  # define RLIM64_INFINITY 0x7fffffffffffffffLL

This was not an issue until the getrlimit and setrlimit glibc functions
have been changed in commit 045c13d185 ("Consolidate Linux setrlimit and
getrlimit implementation") to use the prlimit64 syscall instead of the
getrlimit and setrlimit ones.

This patch fixes that by adding a wrapper to fix the value passed to or
received from the kernel, before or after calling the prlimit64 syscall.

Changelog:
	[BZ #22648]
	* sysdeps/unix/sysv/linux/alpha/getrlimit64.c: New file.
	* sysdeps/unix/sysv/linux/alpha/setrlimit64.c: Ditto.

3 files changed, 131 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index ddd49200ed..489cd1fd84 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2018-01-05  Aurelien Jarno  <aurelien@aurel32.net>
+
+	[BZ #22648]
+	* sysdeps/unix/sysv/linux/alpha/getrlimit64.c: New file.
+	* sysdeps/unix/sysv/linux/alpha/setrlimit64.c: Ditto.
+
 2018-01-04  Joseph Myers  <joseph@codesourcery.com>
 
 	* malloc/tst-malloc-tcache-leak.c (TIMEOUT): Define to 50.
diff --git a/sysdeps/unix/sysv/linux/alpha/getrlimit64.c b/sysdeps/unix/sysv/linux/alpha/getrlimit64.c
new file mode 100644
index 0000000000..908be53180
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/alpha/getrlimit64.c
@@ -0,0 +1,64 @@
+/* Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <sys/types.h>
+
+/* Add this redirection so the strong_alias linking getrlimit64 to
+   {__}getrlimit does not throw a type error.  */
+#undef getrlimit
+#undef __getrlimit
+#define getrlimit getrlimit_redirect
+#define __getrlimit __getrlimit_redirect
+#include <sys/resource.h>
+#undef getrlimit
+#undef __getrlimit
+
+/* RLIM64_INFINITY was supposed to be a glibc convention rather than
+   anything seen by the kernel, but it ended being passed to the kernel
+   through the prlimit64 syscall.  Given that a lot of binaries with
+   the wrong constant value are in the wild, provide a wrapper function
+   fixing the value after the syscall.  */
+#define KERNEL_RLIM64_INFINITY		0xffffffffffffffffULL
+
+int
+__getrlimit64 (enum __rlimit_resource resource, struct rlimit64 *rlimits)
+{
+  struct rlimit64 krlimits;
+
+  if (INLINE_SYSCALL_CALL (prlimit64, 0, resource, NULL, &krlimits) < 0)
+    return -1;
+
+  if (krlimits.rlim_cur == KERNEL_RLIM64_INFINITY)
+    rlimits->rlim_cur = RLIM64_INFINITY;
+  else
+    rlimits->rlim_cur = krlimits.rlim_cur;
+  if (krlimits.rlim_max == KERNEL_RLIM64_INFINITY)
+    rlimits->rlim_max = RLIM64_INFINITY;
+  else
+    rlimits->rlim_max = krlimits.rlim_max;
+
+  return 0;
+}
+libc_hidden_def (__getrlimit64)
+strong_alias (__getrlimit64, __GI_getrlimit)
+strong_alias (__getrlimit64, __GI___getrlimit)
+strong_alias (__getrlimit64, __getrlimit)
+weak_alias (__getrlimit64, getrlimit)
+
+weak_alias (__getrlimit64, getrlimit64)
+libc_hidden_weak (getrlimit64)
diff --git a/sysdeps/unix/sysv/linux/alpha/setrlimit64.c b/sysdeps/unix/sysv/linux/alpha/setrlimit64.c
new file mode 100644
index 0000000000..1b8a95b9e9
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/alpha/setrlimit64.c
@@ -0,0 +1,61 @@
+/* Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <sys/types.h>
+
+/* Add this redirection so the strong_alias linking setrlimit64 to
+   {__}setrlimit does not throw a type error.  */
+#undef setrlimit
+#undef __setrlimit
+#define setrlimit setrlimit_redirect
+#define __setrlimit __setrlimit_redirect
+#include <sys/resource.h>
+#undef setrlimit
+#undef __setrlimit
+
+/* RLIM64_INFINITY was supposed to be a glibc convention rather than
+   anything seen by the kernel, but it ended being passed to the kernel
+   through the prlimit64 syscall.  Given that a lot of binaries with
+   the wrong constant value are in the wild, provide a wrapper function
+   fixing the value before the syscall.  */
+#define KERNEL_RLIM64_INFINITY		0xffffffffffffffffULL
+
+int
+__setrlimit64 (enum __rlimit_resource resource, const struct rlimit64 *rlimits)
+{
+  struct rlimit64 krlimits;
+
+  if (rlimits->rlim_cur == RLIM64_INFINITY)
+    krlimits.rlim_cur = KERNEL_RLIM64_INFINITY;
+  else
+    krlimits.rlim_cur = rlimits->rlim_cur;
+  if (rlimits->rlim_max == RLIM64_INFINITY)
+    krlimits.rlim_max = KERNEL_RLIM64_INFINITY;
+  else
+    krlimits.rlim_max = rlimits->rlim_max;
+
+  return INLINE_SYSCALL_CALL (prlimit64, 0, resource, &krlimits, NULL);
+}
+
+weak_alias (__setrlimit64, setrlimit64)
+
+strong_alias (__setrlimit64, __setrlimit)
+weak_alias (__setrlimit64, setrlimit)
+#ifdef SHARED
+__hidden_ver1 (__setrlimit64, __GI___setrlimit, __setrlimit64);
+#endif