about summary refs log tree commit diff
diff options
context:
space:
mode:
authorgiraffedata <giraffedata@9d0c8265-081b-0410-96cb-a4ca84ce46f8>2020-09-07 21:03:19 +0000
committergiraffedata <giraffedata@9d0c8265-081b-0410-96cb-a4ca84ce46f8>2020-09-07 21:03:19 +0000
commite7fe108c55e8896484a614b448d6e8aeba2288d9 (patch)
tree5526947a41252c3e5903f0bd921e9a5ff56bc649
parenta64332d4497828b7ac77395bd8f50d41a004c40e (diff)
downloadnetpbm-mirror-e7fe108c55e8896484a614b448d6e8aeba2288d9.tar.gz
netpbm-mirror-e7fe108c55e8896484a614b448d6e8aeba2288d9.tar.xz
netpbm-mirror-e7fe108c55e8896484a614b448d6e8aeba2288d9.zip
Fix crash with -compare of PBMs
git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@3951 9d0c8265-081b-0410-96cb-a4ca84ce46f8
-rw-r--r--doc/HISTORY3
-rw-r--r--other/pamarith.c33
2 files changed, 34 insertions, 2 deletions
diff --git a/doc/HISTORY b/doc/HISTORY
index c21ca042..039391db 100644
--- a/doc/HISTORY
+++ b/doc/HISTORY
@@ -31,6 +31,9 @@ not yet  BJH  Release 10.92.00
               jpeg2ktopam: Fix crash whenever the program fails.  Broken in
               Netpbm 10.42 (March 2008).
 
+              pamarith: Fix crash with -compare where inputs are PBM.  Broken
+              in Netpbm 10.14 (February 2003).
+
               pcdovtoppm: Fix some garbage from the conversion from csh to sh
               in Netpbm 9.12 (March 2001) in conversion from csh arrays that
               should have stopped it from working at all, despite reports from
diff --git a/other/pamarith.c b/other/pamarith.c
index e0f049f7..12c102b3 100644
--- a/other/pamarith.c
+++ b/other/pamarith.c
@@ -213,6 +213,30 @@ functionCategory(enum function const function) {
 
 
 
+static int
+outFmtForCompare(int const format1,
+                 int const format2) {
+/*----------------------------------------------------------------------------
+   The format for the output image, given that the function is Compare and
+   'format1' and 'format2' are the input image formats.
+
+   Note that Compare is special because its maxval is always 2, and that
+   won't work for PBM.
+-----------------------------------------------------------------------------*/
+    int const tentativeFormat = MAX(format1, format2);
+
+    int retval;
+
+    switch (tentativeFormat) {
+    case PBM_FORMAT:  retval = PGM_FORMAT;      break;
+    case RPBM_FORMAT: retval = RPGM_FORMAT;     break;
+    default:          retval = tentativeFormat; break;
+    }
+    return retval;
+}
+
+
+
 static void
 computeOutputType(struct pam *  const outpamP,
                   struct pam    const inpam1,
@@ -222,12 +246,17 @@ computeOutputType(struct pam *  const outpamP,
     outpamP->size        = sizeof(struct pam);
     outpamP->len         = PAM_STRUCT_SIZE(tuple_type);
     outpamP->file        = stdout;
-    outpamP->format      = MAX(inpam1.format, inpam2.format);
     outpamP->plainformat = FALSE;
     outpamP->height      = inpam1.height;
     outpamP->width       = inpam1.width;
     outpamP->depth       = MAX(inpam1.depth, inpam2.depth);
 
+
+    if (function == FN_COMPARE)
+        outpamP->format = outFmtForCompare(inpam1.format, inpam2.format);
+    else
+        outpamP->format = MAX(inpam1.format, inpam2.format);
+
     switch (functionCategory(function)) {
     case CATEGORY_FRACTIONAL_ARITH:
         if (function == FN_COMPARE)
@@ -835,7 +864,7 @@ main(int argc, const char *argv[]) {
 
     switch (functionCategory(cmdline.function)) {
     case CATEGORY_FRACTIONAL_ARITH:
-        if (inpam1.maxval == inpam2.maxval)
+        if (inpam1.maxval == inpam2.maxval && inpam2.maxval == outpam.maxval)
             doUnNormalizedArith(&inpam1, &inpam2, &outpam, cmdline.function);
         else
             doNormalizedArith(&inpam1, &inpam2, &outpam, cmdline.function);