From 4b8daa32342e426270aca945853934e8e25f0af1 Mon Sep 17 00:00:00 2001 From: giraffedata Date: Sun, 20 Jan 2019 19:23:11 +0000 Subject: Fix handling of floating point imprecision in unnormalizing samples git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@3514 9d0c8265-081b-0410-96cb-a4ca84ce46f8 --- lib/util/pm_c_util.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib/util') diff --git a/lib/util/pm_c_util.h b/lib/util/pm_c_util.h index 718be369..a093adb6 100644 --- a/lib/util/pm_c_util.h +++ b/lib/util/pm_c_util.h @@ -19,11 +19,13 @@ #undef ROUND #define ROUND(X) (((X) >= 0) ? (int)((X)+0.5) : (int)((X)-0.5)) #undef ROUNDU -#define ROUNDU(X) ((unsigned int)((X)+0.5+1e-15)) - /* We add the 1e-15 epsilon because for repeatability we want exactly half - to round up. But the imprecision of the floating point arithmetic can - cause a number that is supposed to be exactly half to be slightly less - than half in reality. Without this adjustment, it would round down. +#define ROUNDU(X) ((unsigned int)((X)+0.5)) + /* Note that imprecision in floating point arithmetic can make an exact + half fractional part round down instead of up. What should be + 1000.5 might actually be 1000.49999999999. + + Use 'pnm_unnormalized_sample' instead of ROUNDU(samplen*maxval) to get + consistent rounding up when you are unnormalizing sample values. */ /* ROUNDUP rounds up to a specified multiple. E.g. ROUNDUP(22, 8) == 24 */ -- cgit 1.4.1