hsearch_r: Apply VM size limit in test case

(cherry picked from commit f34f146e682d8d529dcf64b3c2781bf3f2f05f6c)

2 files changed, 26 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index cd343c7090..c2b1307d4d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2016-05-24  Florian Weimer  <fweimer@redhat.com>
+
+	* misc/bug18240.c (do_test): Set RLIMIT_AS.
+
 2016-05-24  Paul Eggert  <eggert@cs.ucla.edu>
 
 	[BZ #18240]
diff --git a/misc/bug18240.c b/misc/bug18240.c
index 4b26865a31..773586ee10 100644
--- a/misc/bug18240.c
+++ b/misc/bug18240.c
@@ -22,6 +22,7 @@
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/resource.h>
 
 static void
 test_size (size_t size)
@@ -58,6 +59,27 @@ test_size (size_t size)
 static int
 do_test (void)
 {
+  /* Limit the size of the process, so that memory allocation will
+     fail without impacting the entire system.  */
+  {
+    struct rlimit limit;
+    if (getrlimit (RLIMIT_AS, &limit) != 0)
+      {
+        printf ("getrlimit (RLIMIT_AS) failed: %m\n");
+        return 1;
+      }
+    long target = 100 * 1024 * 1024;
+    if (limit.rlim_cur == RLIM_INFINITY || limit.rlim_cur > target)
+      {
+        limit.rlim_cur = target;
+        if (setrlimit (RLIMIT_AS, &limit) != 0)
+          {
+            printf ("setrlimit (RLIMIT_AS) failed: %m\n");
+            return 1;
+          }
+      }
+  }
+
   test_size (500);
   test_size (-1);
   test_size (-3);