Fix -changemaxval on PBM input

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@3952 9d0c8265-081b-0410-96cb-a4ca84ce46f8

2 files changed, 34 insertions, 23 deletions

diff --git a/doc/HISTORY b/doc/HISTORY
index 039391db..c4caa57c 100644
--- a/doc/HISTORY
+++ b/doc/HISTORY
@@ -34,6 +34,9 @@ not yet  BJH  Release 10.92.00
               pamarith: Fix crash with -compare where inputs are PBM.  Broken
               in Netpbm 10.14 (February 2003).
 
+              pamfunc: Fix crash with -changemaxval and PBM input.  Always
+              broken.  -changemaxval was new in Netpbm 10.65 (December 2013).
+
               pcdovtoppm: Fix some garbage from the conversion from csh to sh
               in Netpbm 9.12 (March 2001) in conversion from csh arrays that
               should have stopped it from working at all, despite reports from
diff --git a/editor/pamfunc.c b/editor/pamfunc.c
index 03cf2a42..1b59a398 100644
--- a/editor/pamfunc.c
+++ b/editor/pamfunc.c
@@ -266,11 +266,12 @@ validateFunction(struct CmdlineInfo const cmdline,
 static void
 planTransform(struct CmdlineInfo const cmdline,
               sample             const inputMaxval,
+              int                const outputFormat,
               sample *           const outputMaxvalP,
               bool *             const mustChangeRasterP) {
 /*----------------------------------------------------------------------------
    Plan the transform described by 'cmdline', given the maxval of the input
-   image is 'inputMaxval.
+   image is 'inputMaxval and the output format will be 'outputFormat'.
 
    The plan just consists of whether to change the maxval or the raster.
    Some multiplications and divisions can be achieved just by changing the
@@ -278,30 +279,36 @@ planTransform(struct CmdlineInfo const cmdline,
 -----------------------------------------------------------------------------*/
     if (cmdline.changemaxval) {
         /* User allows us to change the maxval, if that makes it easier */
-        if (cmdline.function == FN_MULTIPLY || cmdline.function == FN_DIVIDE) {
-            float const multiplier =
-                cmdline.function == FN_MULTIPLY ? cmdline.u.multiplier :
-                (1/cmdline.u.divisor);
-
-            float const neededMaxval = inputMaxval / multiplier;
-
-            if (neededMaxval + 0.5 < inputMaxval) {
-                /* Lowering the maxval might make some of the sample values
-                   higher than the maxval, so we'd have to modify the raster
-                   to clip them.
-                */
-                *outputMaxvalP     = inputMaxval;
-                *mustChangeRasterP = true;
-            } else if (neededMaxval > PAM_OVERALL_MAXVAL) {
+        if (PNM_FORMAT_TYPE(outputFormat) == PBM_TYPE) {
+            *outputMaxvalP     = inputMaxval;
+            *mustChangeRasterP = true;
+        } else {
+            if (cmdline.function == FN_MULTIPLY ||
+                cmdline.function == FN_DIVIDE) {
+                float const multiplier =
+                    cmdline.function == FN_MULTIPLY ? cmdline.u.multiplier :
+                    (1/cmdline.u.divisor);
+
+                float const neededMaxval = inputMaxval / multiplier;
+
+                if (neededMaxval + 0.5 < inputMaxval) {
+                    /* Lowering the maxval might make some of the sample
+                       values higher than the maxval, so we'd have to modify
+                       the raster to clip them.
+                    */
+                    *outputMaxvalP     = inputMaxval;
+                    *mustChangeRasterP = true;
+                } else if (neededMaxval > PAM_OVERALL_MAXVAL) {
+                    *outputMaxvalP     = inputMaxval;
+                    *mustChangeRasterP = true;
+                } else {
+                    *outputMaxvalP     = ROUNDU(neededMaxval);
+                    *mustChangeRasterP = false;
+                }
+            } else {
                 *outputMaxvalP     = inputMaxval;
                 *mustChangeRasterP = true;
-            } else {
-                *outputMaxvalP     = ROUNDU(neededMaxval);
-                *mustChangeRasterP = false;
             }
-        } else {
-            *outputMaxvalP     = inputMaxval;
-            *mustChangeRasterP = true;
         }
     } else {
         *outputMaxvalP     = inputMaxval;
@@ -412,7 +419,8 @@ main(int argc, const char *argv[]) {
     outpam = inpam;    /* Initial value -- most fields should be same */
     outpam.file = stdout;
 
-    planTransform(cmdline, inpam.maxval, &outpam.maxval, &mustChangeRaster);
+    planTransform(cmdline, inpam.maxval, outpam.format,
+                  &outpam.maxval, &mustChangeRaster);
 
     pnm_writepaminit(&outpam);