Validate samples do not exceed maxval; valid image dimension small enough that you can allocate a row buffer; use packed PBM read functions in promoted reads

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@2140 9d0c8265-081b-0410-96cb-a4ca84ce46f8

1 files changed, 45 insertions, 6 deletions

diff --git a/lib/libpgm1.c b/lib/libpgm1.c
index 987ee04e..8d6887f9 100644
--- a/lib/libpgm1.c
+++ b/lib/libpgm1.c
@@ -100,10 +100,13 @@ validateComputableSize(unsigned int const cols,
    you expect.  That failed expectation can be disastrous if you use
    it to allocate memory.
 
+   It is very normal to allocate space for a pixel row, so we make sure
+   the size of a pixel row, in bytes, can be represented by an 'int'.
+
    A common operation is adding 1 or 2 to the highest row or
    column number in the image, so we make sure that's possible.
 -----------------------------------------------------------------------------*/
-    if (cols > INT_MAX - 2)
+    if (cols > INT_MAX / (sizeof(gray)) || cols > INT_MAX - 2)
         pm_error("image width (%u) too large to be processed", cols);
     if (rows > INT_MAX - 2)
         pm_error("image height (%u) too large to be processed", rows);
@@ -172,6 +175,38 @@ pgm_readpgminit(FILE * const fileP,
 
 
 static void
+validateRpgmRow(gray *         const grayrow, 
+                unsigned int   const cols,
+                gray           const maxval,
+                const char **  const errorP) {
+/*----------------------------------------------------------------------------
+  Check for sample values above maxval in input.  
+
+  Note: a program that wants to deal with invalid sample values itself can
+  simply make sure it uses a sufficiently high maxval on the read function
+  call, so this validation never fails.
+-----------------------------------------------------------------------------*/
+    if (maxval == 255 || maxval == 65535) {
+        /* There's no way a sample can be invalid, so we don't need to look at
+           the samples individually.
+        */
+        *errorP = NULL;
+    } else {
+        unsigned int col;
+        for (col = 0; col < cols; ++col) {
+            if (grayrow[col] > maxval) {
+                pm_asprintf(errorP, "value out of bounds (%u > %u)",
+                            grayrow[col], maxval);
+                return;
+            }
+        }
+        *errorP = NULL;
+    }
+}  
+
+
+
+static void
 readRpgmRow(FILE * const fileP,
                gray * const grayrow, 
                int    const cols,
@@ -198,7 +233,6 @@ readRpgmRow(FILE * const fileP,
             pm_asprintf(&error, "Error reading row.  Short read of %u bytes "
                         "instead of %u", (unsigned)rc, bytesPerRow);
         else {
-            error = NULL;
             if (maxval < 256) {
                 unsigned int col;
                 for (col = 0; col < cols; ++col)
@@ -218,6 +252,7 @@ readRpgmRow(FILE * const fileP,
                     grayrow[col] = g;
                 }
             }
+            validateRpgmRow(grayrow, cols, maxval, &error); 
         }
         free(rowBuffer);
     }
@@ -241,7 +276,7 @@ readPbmRow(FILE * const fileP,
     jmp_buf * origJmpbufP;
     bit * bitrow;
     
-    bitrow = pbm_allocrow(cols);
+    bitrow = pbm_allocrow_packed(cols);
     if (setjmp(jmpbuf) != 0) {
         pbm_freerow(bitrow);
         pm_setjmpbuf(origJmpbufP);
@@ -251,10 +286,14 @@ readPbmRow(FILE * const fileP,
 
         pm_setjmpbufsave(&jmpbuf, &origJmpbufP);
 
-        pbm_readpbmrow(fileP, bitrow, cols, format);
-        for (col = 0; col < cols; ++col)
-            grayrow[col] = (bitrow[col] == PBM_WHITE ) ? maxval : 0;
+        pbm_readpbmrow_packed(fileP, bitrow, cols, format);
 
+        for (col = 0; col < cols; ++col) {
+            grayrow[col] =
+                ((bitrow[col/8] >> (7 - col%8)) & 0x1) == PBM_WHITE ?
+                maxval : 0
+                ;
+        }
         pm_setjmpbuf(origJmpbufP);
     }
     pbm_freerow(bitrow);