fix arithmetic overflows

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@4177 9d0c8265-081b-0410-96cb-a4ca84ce46f8

4 files changed, 28 insertions, 27 deletions

diff --git a/editor/pamcut.c b/editor/pamcut.c
index 1fc9d9b2..7870fd70 100644
--- a/editor/pamcut.c
+++ b/editor/pamcut.c
@@ -712,7 +712,7 @@ extractRowsPBM(const struct pam * const inpamP,
 
     if (leftcol > 0) {
         totalWidth = MAX(rightcol+1, inpamP->width) + 7;
-        if (totalWidth > INT_MAX)
+        if (totalWidth > INT_MAX - 10)
             /* Prevent overflows in pbm_allocrow_packed() */
             pm_error("Specified right edge is too far "
                      "from the right end of input image");
@@ -721,7 +721,7 @@ extractRowsPBM(const struct pam * const inpamP,
         writeOffset = leftcol;
     } else {
         totalWidth = -leftcol + MAX(rightcol+1, inpamP->width);
-        if (totalWidth > INT_MAX)
+        if (totalWidth > INT_MAX - 10)
             pm_error("Specified left/right edge is too far "
                      "from the left/right end of input image");
 
diff --git a/editor/pbmpscale.c b/editor/pbmpscale.c
index 9ab89350..434c7965 100644
--- a/editor/pbmpscale.c
+++ b/editor/pbmpscale.c
@@ -12,7 +12,7 @@
 #define LEFTBITS pm_byteLeftBits
 #define RIGHTBITS pm_byteRightBits
 
-/* Table for translating bit pattern into "corners" flag element */ 
+/* Table for translating bit pattern into "corners" flag element */
 
 unsigned char const
 transTable[512] = {
@@ -141,7 +141,7 @@ validateComputableDimensions(unsigned int const width,
    See validateComputetableSize() in libpam.c
    and pbm_readpbminitrest() in libpbm2.c
 -----------------------------------------------------------------------------*/
-    unsigned int const maxWidthHeight = INT_MAX - 2;
+    unsigned int const maxWidthHeight = INT_MAX - 10;
     unsigned int const maxScaleFactor = maxWidthHeight / MAX(height, width);
 
     if (scaleFactor > maxScaleFactor)
@@ -154,8 +154,8 @@ validateComputableDimensions(unsigned int const width,
 
 static void
 writeBitSpan(unsigned char * const packedBitrow,
-             int             const cols,
-             int             const offset,
+             unsigned int    const cols,
+             unsigned int    const offset,
              int             const color) {
 /*----------------------------------------------------------------------------
    Write white (color="0") or black (="1") bits into packedBitrow[],
@@ -191,8 +191,8 @@ setFlags(const bit *     const prevrow,
          unsigned char * const flags,
          unsigned int    const cols ) {
 /*----------------------------------------------------------------------------
-   Scan one row, examining the row above and row below, and determine 
-   whether there are "corners" for each pixel.  Feed a 9 bit sample into 
+   Scan one row, examining the row above and row below, and determine
+   whether there are "corners" for each pixel.  Feed a 9 bit sample into
    pre-calculated array transTable[512] to calculate all four corner statuses
    at once.
 
@@ -220,7 +220,7 @@ setFlags(const bit *     const prevrow,
        The high byte of the patterns is a mask, which determines which bits are
        not ignored.
     */
-    uint16_t const patterns[] 
+    uint16_t const patterns[]
         = { 0x0000,   0xd555,            /* no corner */
             0x0001,   0xffc1, 0xd514,    /* normal corner */
             0x0002,   0xd554, 0xd515, 0xbea2, 0xdfc0, 0xfd81, 0xfd80, 0xdf80,
@@ -230,7 +230,7 @@ setFlags(const bit *     const prevrow,
     /*
       For example, the NE corner is examined with the following 8 bit sample:
       Current : W : NW : N : NE : E : SE : S
-      (SW is the "square behind") 
+      (SW is the "square behind")
       */
 #endif
 
@@ -257,7 +257,7 @@ setFlags(const bit *     const prevrow,
         sample = ( ( prevrow24 >> ( 8 -offset) ) & 0x01c0 )
             | ( ( thisrow24 >> (11 -offset) ) & 0x0038 )
             | ( ( nextrow24 >> (14 -offset) ) & 0x0007 );
-        
+
         flags[col] =  transTable[sample];
     }
 }
@@ -275,14 +275,14 @@ expandRow(const bit *     const thisrow,
           int             const ucutoff) {
 /*----------------------------------------------------------------------------
   Process one row, using flags array as reference.  If pixel has no corners
-  output a NxN square of the given color, otherwise output with the 
+  output a NxN square of the given color, otherwise output with the
   specified corner area(s) clipped off.
 -----------------------------------------------------------------------------*/
     unsigned int const outcols = cols * scale;
 
     unsigned int i;
     unsigned int col;
-    
+
     for (i = 0; i < scale; ++i) {
         int const zone = (i > ucutoff) - (i < cutoff);
         int const cut1 =
@@ -297,7 +297,7 @@ expandRow(const bit *     const thisrow,
         cut[1] = cut1;
         cut[2] = cut1 ? cut1 - 1 : 0;
         cut[3] = (cut1 && cutoff > 1) ? cut1 - 1 : cut1;
-      
+
         for (col = 0; col < cols; ++col) {
             unsigned int const col8 = col / 8;
             unsigned int const offset = col % 8;
@@ -309,11 +309,11 @@ expandRow(const bit *     const thisrow,
             if (flag == 0x00) {
                 /* There are no corners, no color change */
                 outcol += scale;
-            } else { 
+            } else {
                 switch (zone) {
                 case -1:
                     if (i==0 && flag == 0xff) {
-                        /* No corners, color changed */ 
+                        /* No corners, color changed */
                         cutl = cutr = 0;
                         flags[col] = 0x00;
                             /* Use above skip procedure next cycle */
@@ -330,7 +330,7 @@ expandRow(const bit *     const thisrow,
                     cutr = cut[SE(flag)];
                     break;
                 }
-                
+
                 if (cutl > 0) {
                     writeBitSpan(outrow, cutl, outcol, !pix);
                     outcol += cutl;
@@ -384,10 +384,10 @@ main(int argc, const char ** argv) {
 
     pbm_readpbminit(ifP, &cols, &rows, &format) ;
 
-    validateComputableDimensions(cols, rows, cmdline.scale); 
+    validateComputableDimensions(cols, rows, cmdline.scale);
 
     outcols = cols * cmdline.scale;
-    outrows = rows * cmdline.scale; 
+    outrows = rows * cmdline.scale;
 
     /* Initialize input buffers.
        We add a margin of 8 bits on the right of the three rows.
@@ -402,7 +402,7 @@ main(int argc, const char ** argv) {
     for (i = 0; i < pbm_packed_bytes(cols + 8); ++i)
         edgerow[i] = 0x00;
 
-    /* Add blank bytes at right edges */ 
+    /* Add blank bytes at right edges */
     for (i = 0; i < 3; ++i)
         buffer[i][pbm_packed_bytes(cols + 8) - 1] = 0x00;
 
diff --git a/editor/pnmcat.c b/editor/pnmcat.c
index c660698c..01811cb4 100644
--- a/editor/pnmcat.c
+++ b/editor/pnmcat.c
@@ -115,7 +115,7 @@ parseCommandLine(int argc, const char ** const argv,
         cmdlineP->backcolor = BACK_AUTO;
 
     if (jtop + jbottom + jleft + jright + jcenter > 1)
-        pm_error("You may specify onlyone of -jtop, -jbottom, "
+        pm_error("You may specify only one of -jtop, -jbottom, "
                  "-jleft, and -jright");
     else {
         switch (cmdlineP->orientation) {
@@ -367,7 +367,8 @@ getPbmImageInfo(ImgInfo               const img[],
             switch (backcolor) {
             case BACK_AUTO: {
                 bit bgBit;
-                img2[i].proberow = pbm_allocrow_packed(img[i].cols+7);
+                img2[i].proberow =
+                    pbm_allocrow_packed((unsigned int)img[i].cols + 7);
                 pbm_readpbmrow_bitoffset(
                     img[i].ifP, img2[i].proberow,
                     img[i].cols, img[i].format, img2[i].offset % 8);
diff --git a/editor/pnmpad.c b/editor/pnmpad.c
index 55cdcd6c..1e704724 100644
--- a/editor/pnmpad.c
+++ b/editor/pnmpad.c
@@ -11,7 +11,7 @@
 #include "shhopt.h"
 #include "pnm.h"
 
-#define MAX_WIDTHHEIGHT INT_MAX-10
+#define MAX_WIDTHHEIGHT ((INT_MAX)-10)
     /* The maximum width or height value we can handle without risking
        arithmetic overflow
     */
@@ -268,7 +268,7 @@ validateHorizontalSize(struct cmdlineInfo const cmdline,
         pm_error("The right padding value you specified is too large.");
 
     if ((double) cols +
-        (double) lpad + 
+        (double) lpad +
         (double) rpad +
         (double) mwidthMaxPad > MAX_WIDTHHEIGHT)
         pm_error("Given padding parameters make output width too large "
@@ -378,7 +378,7 @@ computePadSizesOneDim(unsigned int   const unpaddedSize,
         unsigned int const totalPadBeforeMult =
             begPadBeforeMult + endPadBeforeMult;
         double const begFrac =
-            totalPadBeforeMult > 0 ? 
+            totalPadBeforeMult > 0 ?
             (double)begPadBeforeMult / totalPadBeforeMult :
             0.0;
         unsigned int const addlMsizeBegPad = ROUNDU(morePadNeeded * begFrac);
@@ -501,7 +501,7 @@ reportPadSizes(int          const inCols,
 
     unsigned int const outCols = inCols + lpad + rpad;
     unsigned int const outRows = inRows + tpad + bpad;
- 
+
     printf("%u %u %u %u %u %u\n", lpad, rpad, tpad, bpad, outCols, outRows);
 
 }
@@ -547,7 +547,7 @@ padPbm(FILE *       const ifP,
     /* Write top margin */
     for (row = 0; row < tpad; ++row)
         pbm_writepbmrow_packed(stdout, bgrow, newcols, 0);
-    
+
     /* Read rows, shift and write with left and right margins added */
     for (row = 0; row < rows; ++row) {
         pbm_readpbmrow_bitoffset(ifP, newrow, cols, format, lpad);