From 770bc8ae01ee5be663cda817f708a3a8c4c03825 Mon Sep 17 00:00:00 2001 From: giraffedata Date: Sun, 22 Mar 2020 01:42:18 +0000 Subject: Fix arithmetic overflow with shear near +/- 90 degrees git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@3765 9d0c8265-081b-0410-96cb-a4ca84ce46f8 --- editor/pnmshear.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'editor') diff --git a/editor/pnmshear.c b/editor/pnmshear.c index 1c330bb7..c705c261 100644 --- a/editor/pnmshear.c +++ b/editor/pnmshear.c @@ -217,6 +217,7 @@ main(int argc, const char * argv[]) { int row; xelval maxval, newmaxval; double shearfac; + double newcolsD; struct CmdlineInfo cmdline; @@ -242,7 +243,12 @@ main(int argc, const char * argv[]) { shearfac = fabs(tan(cmdline.angle)); - newcols = rows * shearfac + cols + 0.999999; + newcolsD = (double) rows * shearfac + cols + 0.999999; + if (newcolsD > INT_MAX-2) + pm_error("angle is too close to +/-90 degrees; " + "output image too wide for computation"); + else + newcols = (int) newcolsD; pnm_writepnminit(stdout, newcols, rows, newmaxval, newformat, 0); newxelrow = pnm_allocrow(newcols); -- cgit 1.4.1