[BZ #5382]

	* nscd/nscd.h (NSCD_THREAD_STACKSIZE): Define.
	* nscd/connections.c (start_threads): Use NSCD_THREAD_STACKSIZE.
	* nscd/mem.c (gc): Don't allocate arrays on stack if it can
	overflow it.

	* sysdeps/unix/sysv/linux/nscd_setup_thread.c (setup_thread):
	Return zero in case thread library is not NPTL.

3 files changed, 55 insertions, 6 deletions

diff --git a/nscd/connections.c b/nscd/connections.c
index d8244b7cb8..abd84b9b63 100644
--- a/nscd/connections.c
+++ b/nscd/connections.c
@@ -1840,7 +1840,7 @@ start_threads (void)
   pthread_attr_init (&attr);
   pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
   /* Use 1MB stacks, twice as much for 64-bit architectures.  */
-  pthread_attr_setstacksize (&attr, 1024 * 1024 * (sizeof (void *) / 4));
+  pthread_attr_setstacksize (&attr, NSCD_THREAD_STACKSIZE);
 
   /* We allow less than LASTDB threads only for debugging.  */
   if (debug_level == 0)
diff --git a/nscd/mem.c b/nscd/mem.c
index d7c59244aa..5c269219b3 100644
--- a/nscd/mem.c
+++ b/nscd/mem.c
@@ -34,6 +34,11 @@
 #include "nscd.h"
 
 
+/* Wrapper functions with error checking for standard functions.  */
+extern void *xmalloc (size_t n);
+extern void *xcalloc (size_t n, size_t s);
+
+
 static int
 sort_he (const void *p1, const void *p2)
 {
@@ -69,6 +74,10 @@ sort_he_data (const void *p1, const void *p2)
 #define ALLBITS ((((BITMAP_T) 1) << BITS) - 1)
 #define HIGHBIT (((BITMAP_T) 1) << (BITS - 1))
 
+/* Maximum size of stack frames we allow the thread to use.  We use
+   80% of the thread stack size.  */
+#define MAX_STACK_USE ((8 * NSCD_THREAD_STACKSIZE) / 10)
+
 
 static void
 markrange (BITMAP_T *mark, ref_t start, size_t len)
@@ -117,13 +126,43 @@ gc (struct database_dyn *db)
      we have to look at the memory.  We use a mark and sweep algorithm
      where the marks are placed in this array.  */
   assert (db->head->first_free % BLOCK_ALIGN == 0);
-  BITMAP_T mark[(db->head->first_free / BLOCK_ALIGN + BITS - 1) / BITS];
-  memset (mark, '\0', sizeof (mark));
+
+  BITMAP_T *mark;
+  bool mark_use_malloc;
+  size_t stack_used = 0;
+  size_t memory_needed = ((db->head->first_free / BLOCK_ALIGN + BITS - 1)
+			  / BITS) * sizeof (BITMAP_T);
+  if (memory_needed <= MAX_STACK_USE)
+    {
+      mark = (BITMAP_T *) alloca (memory_needed);
+      mark_use_malloc = false;
+      memset (mark, '\0', memory_needed);
+      stack_used = memory_needed;
+    }
+  else
+    {
+      mark = (BITMAP_T *) xcalloc (1, memory_needed);
+      mark_use_malloc = true;
+    }
 
   /* Create an array which can hold pointer to all the entries in hash
      entries.  */
-  struct hashentry *he[db->head->nentries];
-  struct hashentry *he_data[db->head->nentries];
+  memory_needed = 2 * db->head->nentries * sizeof (struct hashentry *);
+  struct hashentry **he;
+  struct hashentry **he_data;
+  bool he_use_malloc;
+  if (stack_used + memory_needed <= MAX_STACK_USE)
+    {
+      he = alloca (db->head->nentries * sizeof (struct hashentry *));
+      he_data = alloca (db->head->nentries * sizeof (struct hashentry *));
+      he_use_malloc = false;
+    }
+  else
+    {
+      he = xmalloc (memory_needed);
+      he_data = &he[db->head->nentries * sizeof (struct hashentry *)];
+      he_use_malloc = true;
+    }
 
   size_t cnt = 0;
   for (size_t idx = 0; idx < db->head->module; ++idx)
@@ -455,6 +494,11 @@ gc (struct database_dyn *db)
  out:
   pthread_mutex_unlock (&db->memlock);
   pthread_rwlock_unlock (&db->lock);
+
+  if (he_use_malloc)
+    free (he);
+  if (mark_use_malloc)
+    free (mark);
 }
 
 
@@ -481,7 +525,8 @@ mempool_alloc (struct database_dyn *db, size_t len)
 	{
 	  /* Try to resize the database.  Grow size of 1/8th.  */
 	  size_t oldtotal = (sizeof (struct database_pers_head)
-			     + roundup (db->head->module * sizeof (ref_t), ALIGN)
+			     + roundup (db->head->module * sizeof (ref_t),
+					ALIGN)
 			     + db->head->data_size);
 	  size_t new_data_size = (db->head->data_size
 				  + MAX (2 * len, db->head->data_size / 8));
diff --git a/nscd/nscd.h b/nscd/nscd.h
index 12b1f85f89..7ff3894f18 100644
--- a/nscd/nscd.h
+++ b/nscd/nscd.h
@@ -55,6 +55,10 @@ typedef enum
 #define RESTART_INTERVAL (60 * 60)
 
 
+/* Stack size for worker threads.  */
+#define NSCD_THREAD_STACKSIZE 1024 * 1024 * (sizeof (void *) / 4)
+
+
 /* Structure describing dynamic part of one database.  */
 struct database_dyn
 {