fix arithmetic overflow, cleanup by afu

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@1143 9d0c8265-081b-0410-96cb-a4ca84ce46f8

1 files changed, 74 insertions, 55 deletions

diff --git a/editor/pbmpscale.c b/editor/pbmpscale.c
index 2e24f3cd..1cf4adf6 100644
--- a/editor/pbmpscale.c
+++ b/editor/pbmpscale.c
@@ -6,10 +6,6 @@
 #include "pbm.h"
 #include "mallocvar.h"
 
-/* prototypes */
-void nextrow_pscale ARGS((FILE *ifd, int row));
-int corner ARGS((int pat));
-
 /* input bitmap size and storage */
 int rows, columns, format ;
 bit *inrow[3] ;
@@ -26,34 +22,52 @@ int yd_pscale[] = {  0, -1, -1, -1, 0, 1, 1,  1 } ;
 #define SW(f) (((f) >> 4) & 3)
 #define NW(f) (((f) >> 6) & 3)
 
-typedef unsigned short sixteenbits ;
+#define MAX(x,y) ((x) > (y) ? (x) : (y))
+
+static void
+validateComputableDimensions(unsigned int const width,
+                             unsigned int const height,
+                             unsigned int const scaleFactor) {
+/*----------------------------------------------------------------------------
+   Make sure that multiplication for output image width and height do not
+   overflow.
+   See validateComputetableSize() in libpam.c
+   and pbm_readpbminitrest() in libpbm2.c
+-----------------------------------------------------------------------------*/
+    unsigned int const maxWidthHeight = INT_MAX - 2;
+    unsigned int const maxScaleFactor = maxWidthHeight / MAX(height, width);
+
+    if (scaleFactor > maxScaleFactor)
+       pm_error("Scale factor '%u' too large.  "
+                "The maximum for this %u x %u input image is %u.",
+                scaleFactor, width, height, maxScaleFactor);
+}
+
 
+static int corner(uint16_t const pat) {
 /* list of corner patterns; bit 7 is current color, bits 0-6 are squares
  * around (excluding square behind), going clockwise.
  * The high byte of the patterns is a mask, which determines which bits are
  * not ignored.
  */
 
-sixteenbits patterns[] = { 0x0000, 0xd555,         /* no corner */
-                           0x0001, 0xffc1, 0xd514, /* normal corner */
-                           0x0002, 0xd554, 0xd515, /* reduced corners */
-                           0xbea2, 0xdfc0, 0xfd81,
-                           0xfd80, 0xdf80,
-                           0x0003, 0xbfa1, 0xfec2 /* reduced if > 1 */
-                           };
+uint16_t const patterns[] 
+      = { 0x0000,   0xd555,           /* no corner */
+  	  0x0001,   0xffc1, 0xd514,   /* normal corner */
+	  0x0002,   0xd554, 0xd515, 0xbea2, 0xdfc0, 0xfd81, 0xfd80, 0xdf80,
+                                      /* reduced corners */
+ 	  0x0003,   0xbfa1, 0xfec2 }; /* reduced if > 1 */
 
 /* search for corner patterns, return type of corner found:
- *  0 = no corner,
+ *  0 = no corner, 
  *  1 = normal corner,
  *  2 = reduced corner,
  *  3 = reduced if cutoff > 1
  */
 
-int corner(pat)
-     int pat;
-{
-   register int i, r=0;
-   for (i = 0; i < sizeof(patterns)/sizeof(sixteenbits); i++)
+   int i, r=0;
+
+   for (i = 0; i < sizeof(patterns)/sizeof(uint16_t); i++)
       if (patterns[i] < 0x100)
          r = patterns[i];
       else if ((pat & (patterns[i] >> 8)) ==
@@ -62,13 +76,8 @@ int corner(pat)
    return 0;
 }
 
-/* get a new row
- */
-
-void nextrow_pscale(ifd, row)
-     FILE *ifd;
-     int row;
-{
+/* get a new row */
+static void nextrow_pscale(FILE * const ifd, int const row) {
    bit *shuffle = inrow[0] ;
    inrow[0] = inrow[1];
    inrow[1] = inrow[2];
@@ -78,18 +87,14 @@ void nextrow_pscale(ifd, row)
          inrow[2] = shuffle = pbm_allocrow(columns);
       pbm_readpbmrow(ifd, inrow[2], columns, format) ;
    } else inrow[2] = NULL; /* discard storage */
-
 }
 
-
-
 int
 main(int argc, char ** argv) {
-
     FILE * ifP;
     bit * outrow;
     unsigned int row;
-    int scale, cutoff, ucutoff ;
+    int scale, outcols, outrows, cutoff, ucutoff ;
     unsigned char *flags;
 
     pbm_init( &argc, argv );
@@ -110,12 +115,15 @@ main(int argc, char ** argv) {
     inrow[0] = inrow[1] = inrow[2] = NULL;
     pbm_readpbminit(ifP, &columns, &rows, &format) ;
 
-    outrow = pbm_allocrow(columns*scale) ;
+    validateComputableDimensions(columns, rows, scale); 
+    outcols= columns * scale;     outrows= rows * scale; 
+
+    outrow = pbm_allocrow(outcols) ;
     MALLOCARRAY(flags, columns);
     if (flags == NULL) 
         pm_error("out of memory") ;
 
-    pbm_writepbminit(stdout, columns*scale, rows*scale, 0) ;
+    pbm_writepbminit(stdout, outcols, outrows, 0) ;
 
     cutoff = scale / 2;
     ucutoff = scale - 1 - cutoff;
@@ -125,32 +133,39 @@ main(int argc, char ** argv) {
         unsigned int i;
         nextrow_pscale(ifP, row+1);
         for (col = 0; col < columns; ++col) {
-            unsigned int i;
-            flags[col] = 0 ;
-            for (i = 0; i != 8; i += 2) {
-                int vec = inrow[thisrow][col] != PBM_WHITE;
-                unsigned int k;
-                for (k = 0; k < 7; ++k) {
-                    int x = col + xd_pscale[(k+i)&7] ;
-                    int y = thisrow + yd_pscale[(k+i)&7] ;
-                    vec <<= 1;
-                    if (x >=0 && x < columns && inrow[y])
-                        vec |= (inrow[y][x] != PBM_WHITE) ;
-                }
-                flags[col] |= corner(vec)<<i ;
-            }
+	  unsigned int i, k;
+	  uint16_t thispoint = (inrow[thisrow][col] != PBM_WHITE) << 7;
+          unsigned char vec = 0;
+
+	  for (k = 0; k < 8; ++k) {
+	    int x = col + xd_pscale[k] ;
+	    int y = thisrow + yd_pscale[k] ;
+	    vec <<= 1;
+	    if (x >=0 && x < columns && inrow[y])
+	      vec |= (inrow[y][x] != PBM_WHITE) ;
+	  }
+
+          vec = (vec >>1 | vec <<7);
+
+	  flags[col] = 0 ;
+	  for (i = 0; i != 8; i += 2) {
+	    flags[col] |= corner(thispoint | (vec & 0x7f) )<<i ;
+	    vec = ( vec >>6 | vec <<2);
+	  }
+
         }
+
         for (i = 0; i < scale; i++) {
-            bit *ptr = outrow ;
-            int zone = (i > ucutoff) - (i < cutoff) ;
-            int cut = (zone < 0) ? (cutoff - i) :
+
+            int const zone = (i > ucutoff) - (i < cutoff) ;
+            int const cut = (zone < 0) ? (cutoff - i) :
                 (zone > 0) ? (i - ucutoff) : 0 ;
+            unsigned int outcol=0;
 
             for (col = 0; col < columns; ++col) {
-                int pix = inrow[thisrow][col] ;
-                int flag = flags[col] ;
+                int const pix = inrow[thisrow][col] ;
+                int const flag = flags[col] ;
                 int cutl, cutr;
-                unsigned int k;
 
                 switch (zone) {
                 case -1:
@@ -190,12 +205,16 @@ main(int argc, char ** argv) {
                     break;
                 default: cutl = 0; cutr = 0;  /* Should never reach here */
                 }
+
+		{
+                unsigned int k;
                 for (k = 0; k < cutl; ++k) /* left part */
-                    *ptr++ = !pix ;
+                    outrow[outcol++] = !pix ;
                 for (k = 0; k < scale-cutl-cutr; ++k)  /* center part */
-                    *ptr++ = pix ;
+                    outrow[outcol++] = pix ;
                 for (k = 0; k < cutr; ++k) /* right part */
-                    *ptr++ = !pix ;
+                    outrow[outcol++] = !pix ;
+		}
             }
             pbm_writepbmrow(stdout, outrow, scale*columns, 0) ;
         }