Validate samples do not exceed maxval; valid image dimension small enough that you can allocate a row buffer; use packed PBM read functions in promoted reads

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@2140 9d0c8265-081b-0410-96cb-a4ca84ce46f8

4 files changed, 161 insertions, 21 deletions

diff --git a/lib/libpamread.c b/lib/libpamread.c
index 3c8e2e33..cbac18c1 100644
--- a/lib/libpamread.c
+++ b/lib/libpamread.c
@@ -202,6 +202,51 @@ parse4BpsRow(const struct pam *    const pamP,
 
 
 static void
+validatePamRow(const struct pam * const pamP,
+               tuple *            const tuplerow,
+               const char **      const errorP) {
+/*----------------------------------------------------------------------------
+  Check for sample values above maxval in input.  
+
+  Note: a program that wants to deal with invalid sample values itself can
+  simply make sure it sets pamP->maxval sufficiently high, so this validation
+  never fails.
+-----------------------------------------------------------------------------*/
+    /* To save time, skip the test for if the maxval is a saturated value
+       (255, 65535) or format is PBM.
+
+       This is an expensive test, but is skipped in most cases: in practice
+       maxvals other than 255 or 65535 are uncommon.  Thus we do this in a
+       separate pass through the row rather than while reading in the row.
+    */
+
+    if (pamP->maxval == (((sample) 0x1) << pamP->bytes_per_sample*8) - 1 ||
+        PAM_FORMAT_TYPE(pamP->format) == PBM_FORMAT) {
+        /* There's no way a sample can be invalid, so we don't need to
+           look at the samples individually.
+        */
+        *errorP = NULL;
+    } else {
+        unsigned int col;
+        for (col = 0; col < pamP->width; ++col) {
+            unsigned int plane;
+            for (plane = 0; plane < pamP->depth; ++plane) {
+                if (tuplerow[col][plane] > pamP->maxval) {
+                    pm_asprintf(errorP,
+                                "Plane %u sample value %lu exceeds the "
+                                "image maxval of %lu",
+                                plane, tuplerow[col][plane], pamP->maxval);
+                    return;
+                }
+            }
+        }
+        *errorP = NULL;
+    }
+}
+
+
+
+static void
 readRawNonPbmRow(const struct pam * const pamP,
                  tuple *            const tuplerow) {
 
@@ -234,8 +279,10 @@ readRawNonPbmRow(const struct pam * const pamP,
             case 4: parse4BpsRow(pamP, tuplerow, inbuf); break;
             default:
                 pm_asprintf(&error, "invalid bytes per sample passed to "
-                            "pnm_formatpamrow(): %u",  pamP->bytes_per_sample);
+                            "pnm_formatpamrow(): %u", pamP->bytes_per_sample);
             }
+            if (error == NULL)
+                validatePamRow(pamP, tuplerow, &error);
         }
     }
     pnm_freerowimage(inbuf);
@@ -264,7 +311,7 @@ pnm_readpamrow(const struct pam * const pamP,
     /* For speed, we don't check any of the inputs for consistency 
        here (unless it's necessary to avoid crashing).  Any consistency
        checking should have been done by a prior call to 
-       pnm_writepaminit().
+       pnm_readpaminit().
     */  
 
     /* Need a special case for raw PBM because it has multiple tuples (8)
diff --git a/lib/libpgm1.c b/lib/libpgm1.c
index 987ee04e..8d6887f9 100644
--- a/lib/libpgm1.c
+++ b/lib/libpgm1.c
@@ -100,10 +100,13 @@ validateComputableSize(unsigned int const cols,
    you expect.  That failed expectation can be disastrous if you use
    it to allocate memory.
 
+   It is very normal to allocate space for a pixel row, so we make sure
+   the size of a pixel row, in bytes, can be represented by an 'int'.
+
    A common operation is adding 1 or 2 to the highest row or
    column number in the image, so we make sure that's possible.
 -----------------------------------------------------------------------------*/
-    if (cols > INT_MAX - 2)
+    if (cols > INT_MAX / (sizeof(gray)) || cols > INT_MAX - 2)
         pm_error("image width (%u) too large to be processed", cols);
     if (rows > INT_MAX - 2)
         pm_error("image height (%u) too large to be processed", rows);
@@ -172,6 +175,38 @@ pgm_readpgminit(FILE * const fileP,
 
 
 static void
+validateRpgmRow(gray *         const grayrow, 
+                unsigned int   const cols,
+                gray           const maxval,
+                const char **  const errorP) {
+/*----------------------------------------------------------------------------
+  Check for sample values above maxval in input.  
+
+  Note: a program that wants to deal with invalid sample values itself can
+  simply make sure it uses a sufficiently high maxval on the read function
+  call, so this validation never fails.
+-----------------------------------------------------------------------------*/
+    if (maxval == 255 || maxval == 65535) {
+        /* There's no way a sample can be invalid, so we don't need to look at
+           the samples individually.
+        */
+        *errorP = NULL;
+    } else {
+        unsigned int col;
+        for (col = 0; col < cols; ++col) {
+            if (grayrow[col] > maxval) {
+                pm_asprintf(errorP, "value out of bounds (%u > %u)",
+                            grayrow[col], maxval);
+                return;
+            }
+        }
+        *errorP = NULL;
+    }
+}  
+
+
+
+static void
 readRpgmRow(FILE * const fileP,
                gray * const grayrow, 
                int    const cols,
@@ -198,7 +233,6 @@ readRpgmRow(FILE * const fileP,
             pm_asprintf(&error, "Error reading row.  Short read of %u bytes "
                         "instead of %u", (unsigned)rc, bytesPerRow);
         else {
-            error = NULL;
             if (maxval < 256) {
                 unsigned int col;
                 for (col = 0; col < cols; ++col)
@@ -218,6 +252,7 @@ readRpgmRow(FILE * const fileP,
                     grayrow[col] = g;
                 }
             }
+            validateRpgmRow(grayrow, cols, maxval, &error); 
         }
         free(rowBuffer);
     }
@@ -241,7 +276,7 @@ readPbmRow(FILE * const fileP,
     jmp_buf * origJmpbufP;
     bit * bitrow;
     
-    bitrow = pbm_allocrow(cols);
+    bitrow = pbm_allocrow_packed(cols);
     if (setjmp(jmpbuf) != 0) {
         pbm_freerow(bitrow);
         pm_setjmpbuf(origJmpbufP);
@@ -251,10 +286,14 @@ readPbmRow(FILE * const fileP,
 
         pm_setjmpbufsave(&jmpbuf, &origJmpbufP);
 
-        pbm_readpbmrow(fileP, bitrow, cols, format);
-        for (col = 0; col < cols; ++col)
-            grayrow[col] = (bitrow[col] == PBM_WHITE ) ? maxval : 0;
+        pbm_readpbmrow_packed(fileP, bitrow, cols, format);
 
+        for (col = 0; col < cols; ++col) {
+            grayrow[col] =
+                ((bitrow[col/8] >> (7 - col%8)) & 0x1) == PBM_WHITE ?
+                maxval : 0
+                ;
+        }
         pm_setjmpbuf(origJmpbufP);
     }
     pbm_freerow(bitrow);
diff --git a/lib/libpnm1.c b/lib/libpnm1.c
index 536e5dc4..29fee13a 100644
--- a/lib/libpnm1.c
+++ b/lib/libpnm1.c
@@ -72,10 +72,13 @@ validateComputableSize(unsigned int const cols,
    you expect.  That failed expectation can be disastrous if you use
    it to allocate memory.
 
+   It is very normal to allocate space for a pixel row, so we make sure
+   the size of a pixel row, in bytes, can be represented by an 'int'.
+
    A common operation is adding 1 or 2 to the highest row or
    column number in the image, so we make sure that's possible.
 -----------------------------------------------------------------------------*/
-    if (cols > INT_MAX - 2)
+    if (cols > INT_MAX/(sizeof(pixval) * 3) || cols > INT_MAX - 2)
         pm_error("image width (%u) too large to be processed", cols);
     if (rows > INT_MAX - 2)
         pm_error("image height (%u) too large to be processed", rows);
@@ -178,10 +181,10 @@ readpbmrow(FILE * const fileP,
     jmp_buf * origJmpbufP;
     bit * bitrow;
 
-    bitrow = pbm_allocrow(cols);
+    bitrow = pbm_allocrow_packed(cols);
 
     if (setjmp(jmpbuf) != 0) {
-        pbm_freerow(bitrow);
+        pbm_freerow_packed(bitrow);
         pm_setjmpbuf(origJmpbufP);
         pm_longjmp();
     } else {
@@ -189,11 +192,14 @@ readpbmrow(FILE * const fileP,
 
         pm_setjmpbufsave(&jmpbuf, &origJmpbufP);
 
-        pbm_readpbmrow(fileP, bitrow, cols, format);
-
-        for (col = 0; col < cols; ++col)
-            PNM_ASSIGN1(xelrow[col], bitrow[col] == PBM_BLACK ? 0 : maxval);
+        pbm_readpbmrow_packed(fileP, bitrow, cols, format);
 
+        for (col = 0; col < cols; ++col) {
+            pixval const g =
+                ((bitrow[col/8] >> (7 - col%8)) & 0x1) == PBM_WHITE ?
+                maxval : 0;
+            PNM_ASSIGN1(xelrow[col], g);
+        }
         pm_setjmpbuf(origJmpbufP);
     }
     pbm_freerow(bitrow);
diff --git a/lib/libppm1.c b/lib/libppm1.c
index 195aec70..f1eb8b7d 100644
--- a/lib/libppm1.c
+++ b/lib/libppm1.c
@@ -100,10 +100,13 @@ validateComputableSize(unsigned int const cols,
    you expect.  That failed expectation can be disastrous if you use
    it to allocate memory.
 
+   It is very normal to allocate space for a pixel row, so we make sure
+   the size of a pixel row, in bytes, can be represented by an 'int'.
+
    A common operation is adding 1 or 2 to the highest row or
    column number in the image, so we make sure that's possible.
 -----------------------------------------------------------------------------*/
-    if (cols > INT_MAX - 2)
+    if (cols > INT_MAX/(sizeof(pixval) * 3) || cols > INT_MAX - 2)
         pm_error("image width (%u) too large to be processed", cols);
     if (rows > INT_MAX - 2)
         pm_error("image height (%u) too large to be processed", rows);
@@ -224,6 +227,48 @@ interpRasterRowRaw(const unsigned char * const rowBuffer,
 
 
 static void
+validateRppmRow(pixel *       const pixelrow,
+                unsigned int  const cols,
+                pixval        const maxval,
+                const char ** const errorP) {
+/*----------------------------------------------------------------------------
+  Check for sample values above maxval in input.  
+
+  Note: a program that wants to deal with invalid sample values itself can
+  simply make sure it uses a sufficiently high maxval on the read function
+  call, so this validation never fails.
+-----------------------------------------------------------------------------*/
+    if (maxval == 255 || maxval == 65535) {
+        /* There's no way a sample can be invalid, so we don't need to look at
+           the samples individually.
+        */
+        *errorP = NULL;
+    } else {
+        unsigned int col;
+
+        for (col = 0; col < cols; ++col) {
+            pixval const r = PPM_GETR(pixelrow[col]);
+            pixval const g = PPM_GETG(pixelrow[col]);
+            pixval const b = PPM_GETB(pixelrow[col]);
+
+            if (r > maxval)
+                pm_error("Red sample value %u is greater than maxval (%u)",
+                         r, maxval);
+            if (g > maxval)
+                pm_error("Green sample value %u is greater than maxval (%u)",
+                         g, maxval);
+            if (b > maxval)
+                pm_error("Blue sample value %u is greater than maxval (%u)",
+                         b, maxval);
+        }
+    }
+}
+
+
+
+
+
+static void
 readRppmRow(FILE *       const fileP, 
             pixel *      const pixelrow, 
             unsigned int const cols, 
@@ -259,7 +304,8 @@ readRppmRow(FILE *       const fileP,
                             "instead of %u", (unsigned)bytesRead, bytesPerRow);
             else {
                 interpRasterRowRaw(rowBuffer, pixelrow, cols, bytesPerSample);
-                error = NULL;
+
+                validateRppmRow(pixelrow, cols, maxval, &error);
             }
         }
         free(rowBuffer);
@@ -319,10 +365,10 @@ readPbmRow(FILE *       const fileP,
     jmp_buf * origJmpbufP;
     bit * bitrow;
 
-    bitrow = pbm_allocrow(cols);
+    bitrow = pbm_allocrow_packed(cols);
 
     if (setjmp(jmpbuf) != 0) {
-        pbm_freerow(bitrow);
+        pbm_freerow_packed(bitrow);
         pm_setjmpbuf(origJmpbufP);
         pm_longjmp();
     } else {
@@ -330,10 +376,12 @@ readPbmRow(FILE *       const fileP,
 
         pm_setjmpbufsave(&jmpbuf, &origJmpbufP);
 
-        pbm_readpbmrow(fileP, bitrow, cols, format);
+        pbm_readpbmrow_packed(fileP, bitrow, cols, format);
 
         for (col = 0; col < cols; ++col) {
-            pixval const g = (bitrow[col] == PBM_WHITE) ? maxval : 0;
+            pixval const g =
+                ((bitrow[col/8] >> (7 - col%8)) & 0x1) == PBM_WHITE ?
+                maxval : 0;
             PPM_ASSIGN(pixelrow[col], g, g, g);
         }
         pm_setjmpbuf(origJmpbufP);