From 425d494b10425d26ad353d32914b1d5e91b773ab Mon Sep 17 00:00:00 2001
From: giraffedata <giraffedata@9d0c8265-081b-0410-96cb-a4ca84ce46f8>
Date: Thu, 12 Jan 2012 16:41:20 +0000
Subject: prevent integer overflow in difference variance

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@1623 9d0c8265-081b-0410-96cb-a4ca84ce46f8
---
 analyzer/pgmtexture.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

(limited to 'analyzer/pgmtexture.c')

diff --git a/analyzer/pgmtexture.c b/analyzer/pgmtexture.c
index 5d618f05..cd66db64 100644
--- a/analyzer/pgmtexture.c
+++ b/analyzer/pgmtexture.c
@@ -707,12 +707,14 @@ f10_dvar (float **     const p,
 /*----------------------------------------------------------------------------
    Difference Variance
 -----------------------------------------------------------------------------*/
-    float pxpy[2 * PGM_MAXMAXVAL];
+    double pxpy[PGM_MAXMAXVAL];
     unsigned int i;
-    unsigned int sqrNg;
-    float sum, sumSqr, var;
+    double sqrNg;  /* Square of 'ng' */
+    double sum;
+    double sumSqr;
+    double var;
 
-    for (i = 0; i <= 2 * ng; ++i)
+    for (i = 0; i < ng; ++i)
         pxpy[i] = 0;
 
     for (i = 0; i < ng; ++i) {
-- 
cgit 1.4.1