Validate computable image size and maxval in libnetpbm write init functions

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@4706 9d0c8265-081b-0410-96cb-a4ca84ce46f8

11 files changed, 82 insertions, 70 deletions

diff --git a/lib/libpam.c b/lib/libpam.c
index 07241e9e..206e850f 100644
--- a/lib/libpam.c
+++ b/lib/libpam.c
@@ -132,19 +132,7 @@ validateComputableMaxval(const struct pam * const pamP) {
 /*----------------------------------------------------------------------------
   This is similar to validateComputableSize, but for the maxval.
 -----------------------------------------------------------------------------*/
-    /* Code sometimes allocates an array indexed by sample values and
-       represents the size of that array as an INT.  (UNSIGNED INT would be
-       more proper, but there's no need to be that permissive).
-
-       Code also sometimes iterates through sample values and quits when the
-       value is greater than the maxval.
-    */
-
-    if (pamP->maxval == 0)
-        pm_error("Maxval is zero.  Must be at least one.");
-
-    if (pamP->maxval > INT_MAX-1)
-        pm_error("Maxval (%lu) is too large to be processed", pamP->maxval);
+    pgm_validateComputableMaxval(pamP->maxval);
 }
 
 
@@ -1121,6 +1109,8 @@ pnm_writepaminit(struct pam * const pamP) {
 
     switch (PAM_FORMAT_TYPE(pamP->format)) {
     case PAM_TYPE:
+        validateComputableSize(pamP);
+        validateComputableMaxval(pamP);
         /* See explanation below of why we ignore 'pm_plain_output' here. */
         fprintf(pamP->file, "P7\n");
         writeComments(pamP);
diff --git a/lib/libpbm.h b/lib/libpbm.h
index 827c9d7e..f3e6962a 100644
--- a/lib/libpbm.h
+++ b/lib/libpbm.h
@@ -4,9 +4,15 @@
 #ifndef LIBPBM_H_INCLUDED
 #define LIBPBM_H_INCLUDED
 
+#include <stdio.h>
+
 void
 pbm_readpbminitrest(FILE * file,
                     int  * colsP,
                     int *  rowsP);
 
+void
+pbm_validateComputableSize(unsigned int const cols,
+                           unsigned int const rows);
+
 #endif
diff --git a/lib/libpbm1.c b/lib/libpbm1.c
index d3403311..9e51970c 100644
--- a/lib/libpbm1.c
+++ b/lib/libpbm1.c
@@ -22,6 +22,7 @@
 #include "netpbm/mallocvar.h"
 #include "netpbm/shhopt.h"
 
+#include "libpbm.h"
 #include "pbm.h"
 
 
@@ -85,6 +86,28 @@ pbm_check(FILE *               const fileP,
 
 
 
+void
+pbm_validateComputableSize(unsigned int const cols,
+                           unsigned int const rows) {
+/*----------------------------------------------------------------------------
+   Validate that the dimensions of the image are such that it can be
+   processed in typical ways on this machine without worrying about
+   overflows.  Note that in C, arithmetic is always modulus
+   arithmetic, so if your values are too big, the result is not what
+   you expect.  That failed expectation can be disastrous if you use
+   it to allocate memory.
+
+   See comments at 'validateComputableSize' in libpam.c for details on
+   the purpose of these validations.
+-----------------------------------------------------------------------------*/
+    if (cols > INT_MAX - 10)
+        pm_error("image width (%u) too large to be processed", cols);
+    if (rows > INT_MAX - 10)
+        pm_error("image height (%u) too large to be processed", rows);
+}
+
+
+
 static unsigned int
 bitpop8(unsigned char const x) {
 /*----------------------------------------------------------------------------
diff --git a/lib/libpbm2.c b/lib/libpbm2.c
index 244d5835..c71193c9 100644
--- a/lib/libpbm2.c
+++ b/lib/libpbm2.c
@@ -58,28 +58,6 @@ pbm_readpbminitrest( FILE * const file,
 
 
 
-static void
-validateComputableSize(unsigned int const cols,
-                       unsigned int const rows) {
-/*----------------------------------------------------------------------------
-   Validate that the dimensions of the image are such that it can be
-   processed in typical ways on this machine without worrying about
-   overflows.  Note that in C, arithmetic is always modulus
-   arithmetic, so if your values are too big, the result is not what
-   you expect.  That failed expectation can be disastrous if you use
-   it to allocate memory.
-
-   See comments at 'validateComputableSize' in libpam.c for details on
-   the purpose of these validations.
------------------------------------------------------------------------------*/
-    if (cols > INT_MAX - 10)
-        pm_error("image width (%u) too large to be processed", cols);
-    if (rows > INT_MAX - 10)
-        pm_error("image height (%u) too large to be processed", rows);
-}
-
-
-
 void
 pbm_readpbminit(FILE * const ifP,
                 int *  const colsP,
@@ -115,7 +93,7 @@ pbm_readpbminit(FILE * const ifP,
         pm_error("bad magic number 0x%x - not a PPM, PGM, PBM, or PAM file",
                  realFormat);
     }
-    validateComputableSize(*colsP, *rowsP);
+    pbm_validateComputableSize(*colsP, *rowsP);
 }
 
 
diff --git a/lib/libpbm3.c b/lib/libpbm3.c
index 456d3986..5d4e614b 100644
--- a/lib/libpbm3.c
+++ b/lib/libpbm3.c
@@ -14,6 +14,7 @@
 
 #include "netpbm/pm_c_util.h"
 
+#include "libpbm.h"
 #include "pbm.h"
 
 #ifndef PACKBITS_SSE
@@ -45,6 +46,12 @@ pbm_writepbminit(FILE * const fileP,
                  int    const rows,
                  int    const forceplain) {
 
+    /* For Caller's convenience, we include validating computability of the
+       image dimensions, since Caller may be using them in arithmetic after
+       our return.
+    */
+    pbm_validateComputableSize(cols, rows);
+
     if (!forceplain && !pm_plain_output) {
         fprintf(fileP, "%c%c\n%d %d\n", PBM_MAGIC1, RPBM_MAGIC2, cols, rows);
     } else
diff --git a/lib/libpgm.h b/lib/libpgm.h
index eb292c80..49939a26 100644
--- a/lib/libpgm.h
+++ b/lib/libpgm.h
@@ -12,4 +12,11 @@ pgm_readpgminitrest(FILE * const file,
                     int *  const rowsP,
                     gray * const maxvalP);
 
+void
+pgm_validateComputableSize(unsigned int const cols,
+                           unsigned int const rows);
+
+void
+pgm_validateComputableMaxval(gray const maxval);
+
 #endif
diff --git a/lib/libpgm1.c b/lib/libpgm1.c
index 6feb1c8b..8fa5baf9 100644
--- a/lib/libpgm1.c
+++ b/lib/libpgm1.c
@@ -90,9 +90,9 @@ pgm_readpgminitrest(FILE * const fileP,
 
 
 
-static void
-validateComputableSize(unsigned int const cols,
-                       unsigned int const rows) {
+void
+pgm_validateComputableSize(unsigned int const cols,
+                           unsigned int const rows) {
 /*----------------------------------------------------------------------------
    Validate that the dimensions of the image are such that it can be
    processed in typical ways on this machine without worrying about
@@ -115,8 +115,8 @@ validateComputableSize(unsigned int const cols,
 
 
 
-static void
-validateComputableMaxval(gray const maxval) {
+void
+pgm_validateComputableMaxval(gray const maxval) {
 /*----------------------------------------------------------------------------
   This is similar to validateComputableSize, but for the maxval.
 -----------------------------------------------------------------------------*/
@@ -193,9 +193,9 @@ pgm_readpgminit(FILE * const fileP,
         pm_error("bad magic number 0x%x - not a PPM, PGM, PBM, or PAM file",
                  realFormat);
     }
-    validateComputableSize(*colsP, *rowsP);
+    pgm_validateComputableSize(*colsP, *rowsP);
 
-    validateComputableMaxval(*maxvalP);
+    pgm_validateComputableMaxval(*maxvalP);
 }
 
 
diff --git a/lib/libpgm2.c b/lib/libpgm2.c
index 2e3aba90..ec3539a2 100644
--- a/lib/libpgm2.c
+++ b/lib/libpgm2.c
@@ -15,6 +15,7 @@
 
 #include "netpbm/pm_c_util.h"
 #include "netpbm/mallocvar.h"
+#include "libpgm.h"
 #include "pgm.h"
 
 
@@ -28,6 +29,13 @@ pgm_writepgminit(FILE * const fileP,
 
     bool const plainFormat = forceplain || pm_plain_output;
 
+    /* For Caller's convenience, we include validating computability of the
+       image parameters, since Caller may be using them in arithmetic after
+       our return.
+    */
+    pgm_validateComputableSize(cols, rows);
+    pgm_validateComputableMaxval(maxval);
+
     if (maxval > PGM_OVERALLMAXVAL && !plainFormat)
         pm_error("too-large maxval passed to ppm_writepgminit(): %d.\n"
                  "Maximum allowed by the PGM format is %d.",
diff --git a/lib/libppm.h b/lib/libppm.h
index 0c561022..620768a2 100644
--- a/lib/libppm.h
+++ b/lib/libppm.h
@@ -12,4 +12,8 @@ ppm_readppminitrest(FILE *   const file,
                     int *    const rowsP,
                     pixval * const maxvalP);
 
+void
+ppm_validateComputableSize(unsigned int const cols,
+                           unsigned int const rows);
+
 #endif
diff --git a/lib/libppm1.c b/lib/libppm1.c
index 427adf4d..9df1f213 100644
--- a/lib/libppm1.c
+++ b/lib/libppm1.c
@@ -88,9 +88,9 @@ ppm_readppminitrest(FILE *   const fileP,
 
 
 
-static void
-validateComputableSize(unsigned int const cols,
-                       unsigned int const rows) {
+void
+ppm_validateComputableSize(unsigned int const cols,
+                           unsigned int const rows) {
 /*----------------------------------------------------------------------------
    Validate that the dimensions of the image are such that it can be
    processed in typical ways on this machine without worrying about
@@ -113,28 +113,6 @@ validateComputableSize(unsigned int const cols,
 
 
 
-static void
-validateComputableMaxval(pixval const maxval) {
-/*----------------------------------------------------------------------------
-  This is similar to validateComputableSize, but for the maxval.
------------------------------------------------------------------------------*/
-    /* Code sometimes allocates an array indexed by sample values and
-       represents the size of that array as an INT.  (UNSIGNED INT would be
-       more proper, but there's no need to be that permissive).
-
-       Code also sometimes iterates through sample values and quits when the
-       value is greater than the maxval.
-    */
-
-    if (maxval == 0)
-        pm_error("Maxval is zero.  Must be at least one.");
-
-    if (maxval > INT_MAX-1)
-        pm_error("Maxval (%u) is too large to be processed", maxval);
-}
-
-
-
 void
 ppm_readppminit(FILE *   const fileP,
                 int *    const colsP,
@@ -172,9 +150,9 @@ ppm_readppminit(FILE *   const fileP,
         pm_error("bad magic number 0x%x - not a PPM, PGM, PBM, or PAM file",
                  realFormat);
     }
-    validateComputableSize(*colsP, *rowsP);
+    ppm_validateComputableSize(*colsP, *rowsP);
 
-    validateComputableMaxval(*maxvalP);
+    pgm_validateComputableMaxval(*maxvalP);
 }
 
 
diff --git a/lib/libppm2.c b/lib/libppm2.c
index b4690b71..f54f2cef 100644
--- a/lib/libppm2.c
+++ b/lib/libppm2.c
@@ -16,8 +16,12 @@
 
 #include "netpbm/pm_c_util.h"
 #include "netpbm/mallocvar.h"
+#include "libpgm.h"
+#include "libppm.h"
 #include "ppm.h"
 
+
+
 void
 ppm_writeppminit(FILE*  const fileP,
                  int    const cols,
@@ -27,6 +31,13 @@ ppm_writeppminit(FILE*  const fileP,
 
     bool const plainFormat = forceplain || pm_plain_output;
 
+    /* For Caller's convenience, we include validating computability of the
+       image parameters, since Caller may be using them in arithmetic after
+       our return.
+    */
+    ppm_validateComputableSize(cols, rows);
+    pgm_validateComputableMaxval(maxval);
+
     if (maxval > PPM_OVERALLMAXVAL && !plainFormat)
         pm_error("too-large maxval passed to ppm_writeppminit(): %d."
                  "Maximum allowed by the PPM format is %d.",