From e7fe108c55e8896484a614b448d6e8aeba2288d9 Mon Sep 17 00:00:00 2001 From: giraffedata Date: Mon, 7 Sep 2020 21:03:19 +0000 Subject: Fix crash with -compare of PBMs git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@3951 9d0c8265-081b-0410-96cb-a4ca84ce46f8 --- doc/HISTORY | 3 +++ other/pamarith.c | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/doc/HISTORY b/doc/HISTORY index c21ca042..039391db 100644 --- a/doc/HISTORY +++ b/doc/HISTORY @@ -31,6 +31,9 @@ not yet BJH Release 10.92.00 jpeg2ktopam: Fix crash whenever the program fails. Broken in Netpbm 10.42 (March 2008). + pamarith: Fix crash with -compare where inputs are PBM. Broken + in Netpbm 10.14 (February 2003). + pcdovtoppm: Fix some garbage from the conversion from csh to sh in Netpbm 9.12 (March 2001) in conversion from csh arrays that should have stopped it from working at all, despite reports from diff --git a/other/pamarith.c b/other/pamarith.c index e0f049f7..12c102b3 100644 --- a/other/pamarith.c +++ b/other/pamarith.c @@ -213,6 +213,30 @@ functionCategory(enum function const function) { +static int +outFmtForCompare(int const format1, + int const format2) { +/*---------------------------------------------------------------------------- + The format for the output image, given that the function is Compare and + 'format1' and 'format2' are the input image formats. + + Note that Compare is special because its maxval is always 2, and that + won't work for PBM. +-----------------------------------------------------------------------------*/ + int const tentativeFormat = MAX(format1, format2); + + int retval; + + switch (tentativeFormat) { + case PBM_FORMAT: retval = PGM_FORMAT; break; + case RPBM_FORMAT: retval = RPGM_FORMAT; break; + default: retval = tentativeFormat; break; + } + return retval; +} + + + static void computeOutputType(struct pam * const outpamP, struct pam const inpam1, @@ -222,12 +246,17 @@ computeOutputType(struct pam * const outpamP, outpamP->size = sizeof(struct pam); outpamP->len = PAM_STRUCT_SIZE(tuple_type); outpamP->file = stdout; - outpamP->format = MAX(inpam1.format, inpam2.format); outpamP->plainformat = FALSE; outpamP->height = inpam1.height; outpamP->width = inpam1.width; outpamP->depth = MAX(inpam1.depth, inpam2.depth); + + if (function == FN_COMPARE) + outpamP->format = outFmtForCompare(inpam1.format, inpam2.format); + else + outpamP->format = MAX(inpam1.format, inpam2.format); + switch (functionCategory(function)) { case CATEGORY_FRACTIONAL_ARITH: if (function == FN_COMPARE) @@ -835,7 +864,7 @@ main(int argc, const char *argv[]) { switch (functionCategory(cmdline.function)) { case CATEGORY_FRACTIONAL_ARITH: - if (inpam1.maxval == inpam2.maxval) + if (inpam1.maxval == inpam2.maxval && inpam2.maxval == outpam.maxval) doUnNormalizedArith(&inpam1, &inpam2, &outpam, cmdline.function); else doNormalizedArith(&inpam1, &inpam2, &outpam, cmdline.function); -- cgit 1.4.1