prlimit: Translate old_rlimit from RLIM64_INFINITY to RLIM_INFINITY [BZ #22678]

prlimit called without a new value fails on 32-bit machines if any of
the soft or hard limits are infinity. This is because prlimit does not
translate old_rlimit from RLIM64_INFINITY to RLIM_INFINITY, but checks
that the value returned by the prlimit64 syscall fits into a 32-bit
value, like it is done for example in getrlimit. Note that on the
other hand new_rlimit is correctly translated from RLIM_INFINITY to
RLIM64_INFINITY before calling the syscall.

This patch fixes that.

Changelog:
	[BZ #22678]
	* sysdeps/unix/sysv/linux/prlimit.c (prlimit): Translate
	old_rlimit from RLIM64_INFINITY to RLIM_INFINITY.

1 files changed, 9 insertions, 6 deletions

diff --git a/sysdeps/unix/sysv/linux/prlimit.c b/sysdeps/unix/sysv/linux/prlimit.c
index 9db8e821b3..2fa0642c76 100644
--- a/sysdeps/unix/sysv/linux/prlimit.c
+++ b/sysdeps/unix/sysv/linux/prlimit.c
@@ -50,21 +50,24 @@ prlimit (__pid_t pid, enum __rlimit_resource resource,
     {
       /* The prlimit64 syscall is ill-designed for 32-bit machines.
 	 We have to provide a 32-bit variant since otherwise the LFS
-	 system would not work.  But what shall we do if the syscall
-	 succeeds but the old values do not fit into a rlimit
-	 structure?  We cannot return an error because the operation
-	 itself worked.  Best is perhaps to return RLIM_INFINITY.  */
+	 system would not work.  The infinity value can be translated,
+	 but otherwise what shall we do if the syscall succeeds but the
+	 old values do not fit into a rlimit structure?  We cannot return
+	 an error because the operation itself worked.  Best is perhaps
+	 to return RLIM_INFINITY.  */
       old_rlimit->rlim_cur = old_rlimit64_mem.rlim_cur;
       if (old_rlimit->rlim_cur != old_rlimit64_mem.rlim_cur)
 	{
-	  if (new_rlimit == NULL)
+	  if ((new_rlimit == NULL)
+	      && (old_rlimit64_mem.rlim_cur != RLIM64_INFINITY))
 	    return INLINE_SYSCALL_ERROR_RETURN_VALUE (EOVERFLOW);
 	  old_rlimit->rlim_cur = RLIM_INFINITY;
 	}
       old_rlimit->rlim_max = old_rlimit64_mem.rlim_max;
       if (old_rlimit->rlim_max != old_rlimit64_mem.rlim_max)
 	{
-	  if (new_rlimit == NULL)
+	  if ((new_rlimit == NULL)
+	      && (old_rlimit64_mem.rlim_max != RLIM64_INFINITY))
 	    return INLINE_SYSCALL_ERROR_RETURN_VALUE (EOVERFLOW);
 	  old_rlimit->rlim_max = RLIM_INFINITY;
 	}