Check for integer overflow in cache size computation in strcoll

strcoll is implemented using a cache for indices and weights of
collation sequences in the strings so that subsequent passes do not
have to search through collation data again.  For very large string
inputs, the cache size computation could overflow.  In such a case,
use the fallback function that does not cache indices and weights of
collation sequences.

Fixes CVE-2012-4412.

3 files changed, 72 insertions, 1 deletions

diff --git a/string/Makefile b/string/Makefile
index 736282875e..c2c7d376ec 100644
--- a/string/Makefile
+++ b/string/Makefile
@@ -57,6 +57,8 @@ tests		:= tester inl-tester noinl-tester testcopy test-ffs	\
 tests-ifunc := $(strop-tests:%=test-%-ifunc)
 tests += $(tests-ifunc)
 
+xtests = tst-strcoll-overflow
+
 include ../Rules
 
 tester-ENV = LANGUAGE=C
diff --git a/string/strcoll_l.c b/string/strcoll_l.c
index eb042ff2ec..4ee101a118 100644
--- a/string/strcoll_l.c
+++ b/string/strcoll_l.c
@@ -524,7 +524,15 @@ STRCOLL (const STRING_TYPE *s1, const STRING_TYPE *s2, __locale_t l)
   memset (&seq1, 0, sizeof (seq1));
   seq2 = seq1;
 
-  if (! __libc_use_alloca ((s1len + s2len) * (sizeof (int32_t) + 1)))
+  size_t size_max = SIZE_MAX / (sizeof (int32_t) + 1);
+
+  if (MIN (s1len, s2len) > size_max
+      || MAX (s1len, s2len) > size_max - MIN (s1len, s2len))
+    {
+      /* If the strings are long enough to cause overflow in the size request,
+         then skip the allocation and proceed with the non-cached routines.  */
+    }
+  else if (! __libc_use_alloca ((s1len + s2len) * (sizeof (int32_t) + 1)))
     {
       seq1.idxarr = (int32_t *) malloc ((s1len + s2len) * (sizeof (int32_t) + 1));
 
diff --git a/string/tst-strcoll-overflow.c b/string/tst-strcoll-overflow.c
new file mode 100644
index 0000000000..bb665ac514
--- /dev/null
+++ b/string/tst-strcoll-overflow.c
@@ -0,0 +1,61 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <locale.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Verify that strcoll does not crash for large strings for which it cannot
+   cache weight lookup results.  The size is large enough to cause integer
+   overflows on 32-bit as well as buffer overflows on 64-bit.  The test should
+   work reasonably reliably when overcommit is disabled, but it obviously
+   depends on how much memory the system has.  There's a limitation to this
+   test in that it does not run to completion.  Actually collating such a
+   large string can take days and we can't have xcheck running that long.  For
+   that reason, we run the test for about 5 minutes and then assume that
+   everything is fine if there are no crashes.  */
+#define SIZE 0x40000000ul
+
+int
+do_test (void)
+{
+  if (setlocale (LC_COLLATE, "en_GB.UTF-8") == NULL)
+    {
+      puts ("setlocale failed, cannot test for overflow");
+      return 0;
+    }
+
+  char *p = malloc (SIZE);
+
+  if (p == NULL)
+    {
+      puts ("could not allocate memory");
+      return 1;
+    }
+
+  memset (p, 'x', SIZE - 1);
+  p[SIZE - 1] = 0;
+  printf ("%d\n", strcoll (p, p));
+  return 0;
+}
+
+#define TIMEOUT 300
+#define EXPECTED_SIGNAL SIGALRM
+#define TEST_FUNCTION do_test ()
+#include "../test-skeleton.c"