diff options
| author | giraffedata <giraffedata@9d0c8265-081b-0410-96cb-a4ca84ce46f8> | 2012-01-12 16:41:20 +0000 |
|---|---|---|
| committer | giraffedata <giraffedata@9d0c8265-081b-0410-96cb-a4ca84ce46f8> | 2012-01-12 16:41:20 +0000 |
| commit | 425d494b10425d26ad353d32914b1d5e91b773ab (patch) | |
| tree | 5452c672c4088299a98d893185060b0221e61d35 /analyzer/pgmtexture.c | |
| parent | 78c0c999ea9da2a7dfa65d3529de72e7844072ec (diff) | |
| download | netpbm-mirror-425d494b10425d26ad353d32914b1d5e91b773ab.tar.gz netpbm-mirror-425d494b10425d26ad353d32914b1d5e91b773ab.tar.xz netpbm-mirror-425d494b10425d26ad353d32914b1d5e91b773ab.zip | |
prevent integer overflow in difference variance
git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@1623 9d0c8265-081b-0410-96cb-a4ca84ce46f8
Diffstat (limited to 'analyzer/pgmtexture.c')
| -rw-r--r-- | analyzer/pgmtexture.c | 10 |
1 files changed, 6 insertions, 4 deletions
diff --git a/analyzer/pgmtexture.c b/analyzer/pgmtexture.c index 5d618f05..cd66db64 100644 --- a/analyzer/pgmtexture.c +++ b/analyzer/pgmtexture.c @@ -707,12 +707,14 @@ f10_dvar (float ** const p, /*---------------------------------------------------------------------------- Difference Variance -----------------------------------------------------------------------------*/ - float pxpy[2 * PGM_MAXMAXVAL]; + double pxpy[PGM_MAXMAXVAL]; unsigned int i; - unsigned int sqrNg; - float sum, sumSqr, var; + double sqrNg; /* Square of 'ng' */ + double sum; + double sumSqr; + double var; - for (i = 0; i <= 2 * ng; ++i) + for (i = 0; i < ng; ++i) pxpy[i] = 0; for (i = 0; i < ng; ++i) { |