Fix arithmetic overflow with shear near +/- 90 degrees

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@3765 9d0c8265-081b-0410-96cb-a4ca84ce46f8

1 files changed, 7 insertions, 1 deletions

diff --git a/editor/pnmshear.c b/editor/pnmshear.c
index 1c330bb7..c705c261 100644
--- a/editor/pnmshear.c
+++ b/editor/pnmshear.c
@@ -217,6 +217,7 @@ main(int argc, const char * argv[]) {
     int row;
     xelval maxval, newmaxval;
     double shearfac;
+    double newcolsD;
 
     struct CmdlineInfo cmdline;
 
@@ -242,7 +243,12 @@ main(int argc, const char * argv[]) {
 
     shearfac = fabs(tan(cmdline.angle));
 
-    newcols = rows * shearfac + cols + 0.999999;
+    newcolsD = (double) rows * shearfac + cols + 0.999999;
+    if (newcolsD > INT_MAX-2)
+        pm_error("angle is too close to +/-90 degrees; "
+                 "output image too wide for computation");
+    else
+        newcols = (int) newcolsD;
 
     pnm_writepnminit(stdout, newcols, rows, newmaxval, newformat, 0);
     newxelrow = pnm_allocrow(newcols);