Fix crash when malloc failed, memory leak with uncomputably large numbers.

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@837 9d0c8265-081b-0410-96cb-a4ca84ce46f8

1 files changed, 51 insertions, 16 deletions

diff --git a/lib/libpammap.c b/lib/libpammap.c
index 1ebc639c..6fea0eb9 100644
--- a/lib/libpammap.c
+++ b/lib/libpammap.c
@@ -17,6 +17,7 @@
 
 #include "pm_c_util.h"
 #include "mallocvar.h"
+#include "nstring.h"
 #include "pam.h"
 #include "pammap.h"
 
@@ -399,14 +400,14 @@ pnm_computetuplefreqhash(struct pam *   const pamP,
 
 
 
-tupletable
-pnm_alloctupletable(const struct pam * const pamP, 
-                    unsigned int       const size) {
+static void
+alloctupletable(const struct pam * const pamP, 
+                unsigned int       const size,
+                tupletable *       const tupletableP,
+                const char **      const errorP) {
     
-    tupletable retval;
-
     if (UINT_MAX / sizeof(struct tupleint) < size)
-        pm_error("size %u is too big for arithmetic", size);
+        asprintfN(errorP, "size %u is too big for arithmetic", size);
     else {
         unsigned int const mainTableSize = size * sizeof(struct tupleint *);
         unsigned int const tupleIntSize = 
@@ -418,20 +419,49 @@ pnm_alloctupletable(const struct pam * const pamP,
            as a single malloc block and suballocate internally.
         */
         if ((UINT_MAX - mainTableSize) / tupleIntSize < size)
-            pm_error("size %u is too big for arithmetic", size);
+            asprintfN(errorP, "size %u is too big for arithmetic", size);
         else {
+            unsigned int const allocSize = mainTableSize + size * tupleIntSize;
             void * pool;
-            unsigned int i;
-    
-            pool = malloc(mainTableSize + size * tupleIntSize);
     
-            retval = (tupletable) pool;
+            pool = malloc(allocSize);
 
-            for (i = 0; i < size; ++i)
-                retval[i] = (struct tupleint *)
-                    ((char*)pool + mainTableSize + i * tupleIntSize);
+            if (!pool)
+                asprintfN(errorP, "Unable to allocate %u bytes for a %u-entry "
+                          "tuple table", allocSize, size);
+            else {
+                tupletable const tbl = (tupletable) pool;
+
+                unsigned int i;
+
+                *errorP = NULL;
+
+                for (i = 0; i < size; ++i)
+                    tbl[i] = (struct tupleint *)
+                        ((char*)pool + mainTableSize + i * tupleIntSize);
+
+                *tupletableP = tbl;
+            }
         }
     }
+}
+
+
+
+tupletable
+pnm_alloctupletable(const struct pam * const pamP, 
+                    unsigned int       const size) {
+
+    tupletable retval;
+    const char * error;
+
+    alloctupletable(pamP, size, &retval, &error);
+
+    if (error) {
+        pm_errormsg("%s", error);
+        strfree(error);
+        pm_longjmp();
+    }
     return retval;
 }
 
@@ -478,10 +508,15 @@ tuplehashtotable(const struct pam * const pamP,
    in the table to tuples or anything else in existing space.
 -----------------------------------------------------------------------------*/
     tupletable tupletable;
+    const char * error;
 
-    tupletable = pnm_alloctupletable(pamP, allocsize);
+    alloctupletable(pamP, allocsize, &tupletable, &error);
 
-    if (tupletable != NULL) {
+    if (error) {
+        pm_errormsg("%s", error);
+        strfree(error);
+        pm_longjmp();
+    } else {
         unsigned int i, j;
         /* Loop through the hash table. */
         j = 0;