Fix arithmetic overflow with image dimensions represented by 16 bit integers

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@1136 9d0c8265-081b-0410-96cb-a4ca84ce46f8

2 files changed, 188 insertions, 185 deletions

diff --git a/converter/pbm/pbmtogem.c b/converter/pbm/pbmtogem.c
index cefbdc95..9eab0416 100644
--- a/converter/pbm/pbmtogem.c
+++ b/converter/pbm/pbmtogem.c
@@ -27,17 +27,21 @@
 *  removed rounding of the imagewidth to the next word boundary
 *  removed arbitrary limit to imagewidth
 *  changed pattern length to 1 to simplify locating of compressable parts
-*	in real world images
+*       in real world images
 *  add solid run and pattern run compression
 *
 *  Deficiencies:
 *  Compression of repeated scanlines not added
 *  
-*	Johann Haider (jh@fortec.tuwien.ac.at)
+*       Johann Haider (jh@fortec.tuwien.ac.at)
 *
 * 94/01/31 Andreas Schwab (schwab@ls5.informatik.uni-dortmund.de)
 * Changed to remove architecture dependencies
 * Added compression of repeated scanlines
+*
+* Feb 2010 afu
+* Added dimension check to prevent short int from overflowing
+* Changed code style (ANSI-style function definitions, etc.)
 */
 
 #include <stdio.h>
@@ -47,59 +51,11 @@
 #define SOLID_0 0
 #define SOLID_1 0xff
 #define MINRUN 4
+#define INT16MAX 32767
+
 #define putsolid(v,c) putc((v&0x80)|c, stdout)
 #define putpattern(v,c) putc(0, stdout);putc(c, stdout);putc(v, stdout)
 
-static void putinit ARGS ((int rows, int cols));
-static void putbit ARGS(( bit b ));
-static void putitem ARGS(( void ));
-static void putrow ARGS(( void ));
-static void flushrow ARGS ((void));
-static void putstring ARGS((register unsigned char *p, register int n));
-
-int
-main( argc, argv )
-    int argc;
-    char* argv[];
-    {
-    FILE* ifp;
-    bit* bitrow;
-    register bit* bP;
-    int rows, cols, format, row, col;
-
-    pbm_init( &argc, argv );
-
-    if ( argc > 2 )
-	pm_usage( "[pbmfile]" );
-
-    if ( argc == 2 )
-	ifp = pm_openr( argv[1] );
-    else
-	ifp = stdin;
-
-    pbm_readpbminit( ifp, &cols, &rows, &format );
-
-    bitrow = pbm_allocrow( cols );
-
-    putinit (rows, cols);
-    for ( row = 0; row < rows; ++row )
-	{
-#ifdef DEBUG
-	fprintf (stderr, "row %d\n", row);
-#endif
-	pbm_readpbmrow( ifp, bitrow, cols, format );
-        for ( col = 0, bP = bitrow; col < cols; ++col, ++bP )
-	    putbit( *bP );
-        putrow( );
-        }
-    flushrow ();
-
-    pm_close( ifp );
-
-
-    exit( 0 );
-    }
-
 static short item;
 static int outcol, outmax;
 static short bitsperitem, bitshift;
@@ -107,9 +63,9 @@ static short linerepeat;
 static unsigned char *outrow, *lastrow;
 
 static void
-putinit (rows, cols)
-     int rows, cols;
+putinit (int const rows, int const cols)
 {
+
   if (pm_writebigshort (stdout, (short) 1) == -1 /* Image file version */
       || pm_writebigshort (stdout, (short) 8) == -1 /* Header length */
       || pm_writebigshort (stdout, (short) 1) == -1 /* Number of planes */
@@ -130,17 +86,6 @@ putinit (rows, cols)
 }
 
 static void
-putbit( bit b )
-    {
-    if ( bitsperitem == 8 )
-	putitem( );
-    ++bitsperitem;
-    if ( b == PBM_BLACK )
-	item += 1 << bitshift;
-    --bitshift;
-    }
-
-static void
 putitem( )
     {
     outrow[outcol++] = item;
@@ -149,19 +94,93 @@ putitem( )
     bitshift = 7;
     }
 
+
+static void
+putbit( bit const b )
+    {
+    if ( bitsperitem == 8 )
+        putitem( );
+    ++bitsperitem;
+    if ( b == PBM_BLACK )
+        item += 1 << bitshift;
+    --bitshift;
+    }
+
+
 static void
-putstring (p, n)
-register unsigned char *p;
-register int n;
+putstring ( unsigned char *p, int n)
 {
 #ifdef DEBUG
     fprintf (stderr, "Bitstring, length: %d, pos %d\n", n, outcol);
 #endif
     (void) putc((char) 0x80, stdout);     /* a Bit string */
-    (void) putc(n, stdout);	/* count */
+    (void) putc(n, stdout);     /* count */
     fwrite( p, n, 1, stdout );
 }
 
+
+static void
+flushrow( )
+    {
+    unsigned char *outp, *p, *q;
+    int count;
+    int col = outmax;
+
+    if (linerepeat > 1)
+      {
+        /* Put out line repeat count */
+        fwrite ("\0\0\377", 3, 1, stdout);
+        putchar (linerepeat);
+      }
+    for (outp = p = lastrow; col > 0;)
+    {
+            for (q = p, count=0; (count < col) && (*q == *p); q++,count++);
+            if (count > MINRUN)
+            {
+                if (p > outp)
+                {
+                    putstring (outp, p-outp);
+                    outp = p;
+                }
+                col -= count;
+                switch (*p)
+                {
+                case SOLID_0:
+#ifdef DEBUG
+/*                      if (outcol > 0) */
+                        fprintf (stderr, "Solid run 0, length: %d\n", count);
+#endif
+                        putsolid (SOLID_0, count);
+                        break;
+
+                case SOLID_1:
+#ifdef DEBUG
+                        fprintf (stderr, "Solid run 1, length: %d, pos %d\n", count, outcol);
+#endif
+                        putsolid (SOLID_1, count);
+                        break;
+                default:
+#ifdef DEBUG
+                        fprintf (stderr, "Pattern run, length: %d\n", count);
+#endif
+                        putpattern (*p, count);
+                        break;
+                }
+                outp = p = q;
+            }
+            else
+            {
+                p++;
+                col--;
+            }
+    }           
+    if (p > outp)
+         putstring (outp, p-outp);
+    if (ferror (stdout))
+      pm_error ("write error");
+}
+
+
 static void
 putrow( )
 {
@@ -173,7 +192,7 @@ putrow( )
     {
       unsigned char *temp;
       if (linerepeat != -1) /* Unless first line */
-	flushrow ();
+        flushrow ();
       /* Swap the pointers */
       temp = outrow; outrow = lastrow; lastrow = temp;
       linerepeat = 1;
@@ -183,64 +202,47 @@ putrow( )
     linerepeat++;
 }
 
-static void
-flushrow( )
-    {
-    register unsigned char *outp, *p, *q;
-    register int count;
-    int col = outmax;
 
-    if (linerepeat > 1)
-      {
-	/* Put out line repeat count */
-	fwrite ("\0\0\377", 3, 1, stdout);
-	putchar (linerepeat);
-      }
-    for (outp = p = lastrow; col > 0;)
+int
+main( int argc, char* argv[])
     {
-	    for (q = p, count=0; (count < col) && (*q == *p); q++,count++);
-	    if (count > MINRUN)
-	    {
-		if (p > outp)
-		{
-		    putstring (outp, p-outp);
-		    outp = p;
-		}
-		col -= count;
-		switch (*p)
-		{
-		case SOLID_0:
-#ifdef DEBUG
-/*			if (outcol > 0) */
-			fprintf (stderr, "Solid run 0, length: %d\n", count);
-#endif
-			putsolid (SOLID_0, count);
-			break;
+    FILE* ifp;
+    bit* bitrow;
+    int rows, cols, format, row, col;
 
-		case SOLID_1:
-#ifdef DEBUG
-			fprintf (stderr, "Solid run 1, length: %d, pos %d\n", count, outcol);
-#endif
-			putsolid (SOLID_1, count);
-			break;
-		default:
+    pbm_init( &argc, argv );
+
+    if ( argc > 2 )
+        pm_usage( "[pbmfile]" );
+
+    if ( argc == 2 )
+        ifp = pm_openr( argv[1] );
+    else
+        ifp = stdin;
+
+    pbm_readpbminit( ifp, &cols, &rows, &format );
+
+    if( rows>INT16MAX || cols>INT16MAX )
+      pm_error ("Input image is too large.");
+
+
+    bitrow = pbm_allocrow( cols );
+
+    putinit (rows, cols);
+    for ( row = 0; row < rows; ++row )
+        {
 #ifdef DEBUG
-			fprintf (stderr, "Pattern run, length: %d\n", count);
+        fprintf (stderr, "row %d\n", row);
 #endif
-			putpattern (*p, count);
-			break;
-		}
-		outp = p = q;
-	    }
-	    else
-	    {
-		p++;
-		col--;
-	    }
-    }		
-    if (p > outp)
-         putstring (outp, p-outp);
-    if (ferror (stdout))
-      pm_error ("write error");
-}
+        pbm_readpbmrow( ifp, bitrow, cols, format );
+        for ( col = 0; col < cols; ++col )
+            putbit( bitrow[col] );
+        putrow( );
+        }
+    flushrow ();
+
+    pm_close( ifp );
 
+
+    exit( 0 );
+    }
diff --git a/converter/pbm/pbmtoybm.c b/converter/pbm/pbmtoybm.c
index 508e8e92..88172788 100644
--- a/converter/pbm/pbmtoybm.c
+++ b/converter/pbm/pbmtoybm.c
@@ -9,68 +9,33 @@
 ** copyright notice and this permission notice appear in supporting
 ** documentation.  This software is provided "as is" without express or
 ** implied warranty.
+**
+** Feb 2010 afu
+** Added dimension check to prevent short int from overflowing
+** Changed code style (ANSI-style function definitions, etc.)
 */
 
 #include <stdio.h>
 #include "pbm.h"
 
 #define YBM_MAGIC  ( ( '!' << 8 ) | '!' )
+#define INT16MAX 32767
 
-static void putinit ARGS(( int cols, int rows ));
-static void putbit ARGS(( bit b ));
-static void putrest ARGS(( void ));
-static void putitem ARGS(( void ));
-
-int
-main( argc, argv )
-    int argc;
-    char* argv[];
-    {
-    FILE* ifp;
-    bit* bitrow;
-    register bit* bP;
-    int rows, cols, format, padright, row, col;
-
-
-    pbm_init( &argc, argv );
-
-    if ( argc > 2 )
-	pm_usage( "[pbmfile]" );
-    if ( argc == 2 )
-	ifp = pm_openr( argv[1] );
-    else
-	ifp = stdin;
-
-    pbm_readpbminit( ifp, &cols, &rows, &format );
-    bitrow = pbm_allocrow( cols );
-    
-    /* Compute padding to round cols up to the nearest multiple of 16. */
-    padright = ( ( cols + 15 ) / 16 ) * 16 - cols;
-
-    putinit( cols, rows );
-    for ( row = 0; row < rows; ++row )
-	{
-	pbm_readpbmrow( ifp, bitrow, cols, format );
-        for ( col = 0, bP = bitrow; col < cols; ++col, ++bP )
-	    putbit( *bP );
-	for ( col = 0; col < padright; ++col )
-	    putbit( 0 );
-        }
-
-    if ( ifp != stdin )
-	fclose( ifp );
+static long item;
+static int bitsperitem, bitshift;
 
-    putrest( );
 
-    exit( 0 );
+static void
+putitem( )
+    {
+    pm_writebigshort( stdout, item );
+    item = 0;
+    bitsperitem = 0;
+    bitshift = 0;
     }
 
-static long item;
-static int bitsperitem, bitshift;
-
 static void
-putinit( cols, rows )
-    int cols, rows;
+putinit(int const cols, int const rows )
     {
     pm_writebigshort( stdout, YBM_MAGIC );
     pm_writebigshort( stdout, cols );
@@ -81,13 +46,13 @@ putinit( cols, rows )
     }
 
 static void
-putbit( bit b )
+putbit( bit const b )
     {
     if ( bitsperitem == 16 )
-	putitem( );
+        putitem( );
     ++bitsperitem;
     if ( b == PBM_BLACK )
-	item += 1 << bitshift;
+        item += 1 << bitshift;
     ++bitshift;
     }
 
@@ -95,14 +60,50 @@ static void
 putrest( )
     {
     if ( bitsperitem > 0 )
-	putitem( );
+        putitem( );
     }
 
-static void
-putitem( )
+
+int
+main( int argc, char *argv[] )
     {
-    pm_writebigshort( stdout, item );
-    item = 0;
-    bitsperitem = 0;
-    bitshift = 0;
+    FILE* ifp;
+    bit* bitrow;
+    int rows, cols, format, padright, row, col;
+
+    pbm_init( &argc, argv );
+
+    if ( argc > 2 )
+        pm_usage( "[pbmfile]" );
+    if ( argc == 2 )
+        ifp = pm_openr( argv[1] );
+    else
+        ifp = stdin;
+
+    pbm_readpbminit( ifp, &cols, &rows, &format );
+
+    if( rows>INT16MAX || cols>INT16MAX )
+      pm_error ("Input image is too large.");
+
+    bitrow = pbm_allocrow( cols );
+    
+    /* Compute padding to round cols up to the nearest multiple of 16. */
+    padright = ( ( cols + 15 ) / 16 ) * 16 - cols;
+
+    putinit( cols, rows );
+    for ( row = 0; row < rows; ++row )
+        {
+        pbm_readpbmrow( ifp, bitrow, cols, format );
+        for ( col = 0; col < cols; ++col )
+            putbit(  bitrow[col] );
+        for ( col = 0; col < padright; ++col )
+            putbit( 0 );
+        }
+
+    if ( ifp != stdin )
+        fclose( ifp );
+
+    putrest( );
+
+    exit( 0 );
     }