Rest of test restructure from yesterday

git-svn-id: http://svn.code.sf.net/p/netpbm/code/trunk@2446 9d0c8265-081b-0410-96cb-a4ca84ce46f8

2 files changed, 253 insertions, 99 deletions

diff --git a/GNUmakefile b/GNUmakefile
index 1e933347..4928c8f2 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -1,12 +1,24 @@
 # Makefile for Netpbm
- 
+
 # Configuration should normally be done in the included file config.mk.
 
 # Targets in this file:
 #
 #   nonmerge:     Build everything, in the source directory.
-#   merge:        Build everything as merged executables, in the source dir
-#   package:      Make a package of Netpbm files ready to install
+#   merge:        Build everything as merged executables, in the source dir.
+#   package:      Make a package of Netpbm files ready to install.
+#
+#   deb:          Make a .deb file in the current dir.
+#
+#   check-tree:     Conduct tests on Netpbm files in the source dir. 
+#   check-package:  Conduct tests on packaged Netpbm files.
+#   check-install:  Conduct tests on installed Netpbm files.
+#   check:          Default check.  Synonym for check-package.
+#
+#   clean:        Delete target executables and intermediate objects.
+#   distclean:    Delete configuration files in addition to the above.
+#
+#   tags:         Generate/update an Emacs tags file, named TAGS.
 #   
 #   The default target is either "merge" or "nonmerge", as determined by
 #   the DEFAULT_TARGET variable set by config.mk.
@@ -60,9 +72,9 @@ VPATH=.:$(SRCDIR)
 
 include $(BUILDDIR)/config.mk
 
-PROG_SUBDIRS = converter analyzer editor generator other test
+PROG_SUBDIRS = converter analyzer editor generator other
 PRODUCT_SUBDIRS = lib $(PROG_SUBDIRS)
-SUPPORT_SUBDIRS = urt buildtools
+SUPPORT_SUBDIRS = urt buildtools test
 
 SUBDIRS = $(PRODUCT_SUBDIRS) $(SUPPORT_SUBDIRS)
 
@@ -72,10 +84,6 @@ NOMERGEBINARIES = netpbm
 
 OBJECTS = netpbm.o
 
-PBM_TESTPREFIX ?= $(PKGDIR)/bin
-PBM_LIBRARY_PATH ?= $(PKGDIR)/lib
-RGBDEF ?= $(SRCDIR)/lib/rgb.txt
-
 default: $(DEFAULT_TARGET)
 	echo "EXISTENCE OF THIS FILE MEANS NETPBM HAS BEEN BUILT." \
 	  >build_complete
@@ -134,6 +142,14 @@ $(TYPEGEN) $(ENDIANGEN): $(BUILDDIR)/buildtools
 inttypes_netpbm.h: $(TYPEGEN)
 	$(TYPEGEN) >$@
 
+
+# testrandom is a utility program used by the make file below.
+TESTRANDOM = $(BUILDDIR)/test/testrandom
+
+$(TESTRANDOM): $(BUILDDIR)/test
+	$(MAKE) -C $(dir $@) -f $(SRCDIR)/test/Makefile \
+	    SRCDIR=$(SRCDIR) BUILDDIR=$(BUILDDIR) $(notdir $@) 
+
 # We run a couple of programs on the build machine in computing the
 # contents of pm_config.h.  We need to give the user a way not to do
 # that or to override the results, because it doesn't work if he's
@@ -354,31 +370,14 @@ netpbm.o: mergetrylist
 install.merge: local.install.merge
 .PHONY: local.install.merge
 local.install.merge:
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm bmptoppm
+	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmnoraw
 	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm gemtopbm
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm icontopbm
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pgmedge
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pgmnorm
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pgmoil
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pgmslice
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pngtopnm
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmarith
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmcomp
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmcut
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmdepth
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmenlarge
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmfile
 	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnminterp
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmnoraw
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmscale
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmsplit
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmtofits
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmnoraw
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmtopnm
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm ppmnorm
+	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pgmoil
 	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm ppmtojpeg
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmtotga
-	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmtouil
+	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm bmptoppm
+	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pgmnorm
+	cd $(PKGDIR)/bin; $(SYMLINKEXE) netpbm pnmfile
 
 ifneq ($(NETPBMLIBTYPE),unixstatic)
 install.lib: lib/install.lib
@@ -442,16 +441,111 @@ install.sharedlibstub:
 deb:
 	buildtools/debian/mkdeb --buildtools=buildtools --pkgdir=$(PKGDIR)
 
+
 .PHONY: check
-check:
-# This works on typical Linux systems
-	if [ ! -d $(RESULTDIR) ]; then mkdir -pv $(RESULTDIR); fi
+.PHONY: check-tree
+.PHONY: check-package
+.PHONY: check-install
+
+# Test files in source tree.
+# This does not work when Netpbm is compiled in a separate build dir.
+
+check-tree : SRCBINDIRS :=./analyzer \
+./converter/other \
+./converter/other/cameratopam \
+./converter/other/fiasco \
+./converter/other/jbig \
+./converter/other/jpeg2000 \
+./converter/other/pamtosvg \
+./converter/other/pnmtopalm \
+./converter/pbm \
+./converter/pbm/pbmtoppa \
+./converter/pgm \
+./converter/ppm \
+./converter/ppm/hpcdtoppm \
+./converter/ppm/ppmtompeg \
+./converter/ppm \
+./editor \
+./editor/pamflip \
+./editor/specialty \
+./generator \
+./other \
+./other/pamx \
+./
+
+# Create colon-separated PATH list from the above.
+# Use realpath function (appears in GNU Make v.3.81) if available.
+
+# Kludge to test whether realpath is available:
+ifeq ($(realpath $(CURDIR)/.),$(CURDIR))
+  check-tree : RBINDIRS :=\
+    $(foreach dir,$(SRCBINDIRS),$(realpath $(BUILDDIR)/$(dir)))
+else
+  check-tree : RBINDIRS :=$(foreach dir,$(SRCBINDIRS),$(BUILDDIR)/$(dir))  
+endif
+
+# Kludge to express characters given special meanings by GNU Make.
+# See GNU Make texinfo manual "Function Call Syntax".
+empty :=
+space := $(empty) $(empty)
+colon :=:
+
+check-tree : PBM_TEST_PATH := $(subst $(space),$(colon),$(RBINDIRS))
+check-tree : PBM_LIBRARY_PATH ?= $(BUILDDIR)/lib
+check-tree : RGBDEF ?= $(SRCDIR)/lib/rgb.txt
+
+
+# Create RESULTDIR.
+# If it already exists, rename and covert to an archive directory.
+# Use numbered backup.
+# TODO: Renaming fails with old versions of mv which do not support -T.  
+
+resultdir-backup: FORCE
+	if [ -d $(RESULTDIR) ]; \
+	   then mv -T --backup=numbered $(RESULTDIR) $(RESULTDIR).bak; \
+	fi; \
+	mkdir -p $(RESULTDIR); \
+
+
+check-tree: $(TESTRANDOM) resultdir-backup
 	cd $(RESULTDIR); \
-	  PBM_TESTPREFIX=$(PBM_TESTPREFIX) \
+	  CHECK_TYPE=tree \
+	  PBM_TEST_PATH=$(PBM_TEST_PATH) BUILDDIR=$(BUILDDIR) \
 	  LD_LIBRARY_PATH=$(PBM_LIBRARY_PATH):${LD_LIBRARY_PATH} \
 	  RGBDEF=$(RGBDEF) \
 	  $(SRCDIR)/test/Execute-Tests 2>&1
 
+# Execute-Tests needs to know BUILDDIR in order to locate testrandom.
+# This applies to all check varieties.
+
+# Check after the packaging stage
+# This works on typical Linux systems.
+# This is the default check.
+
+check-package : PBM_TEST_PATH := $(PKGDIR)/bin
+check-package : PBM_LIBRARY_PATH := $(PKGDIR)/lib
+check-package : RGBDEF ?= $(PKGDIR)/misc/rgb.txt
+check: check-package
+
+check-package: $(TESTRANDOM) resultdir-backup
+	cd $(RESULTDIR); \
+	  CHECK_TYPE=package \
+	  PBM_TEST_PATH=$(PBM_TEST_PATH) BUILDDIR=$(BUILDDIR) \
+	  LD_LIBRARY_PATH=$(PBM_LIBRARY_PATH):${LD_LIBRARY_PATH} \
+	  RGBDEF=$(RGBDEF) \
+	  $(SRCDIR)/test/Execute-Tests 2>&1
+
+
+# Check after install
+check-install: $(TESTRANDOM) resultdir-backup
+	cd $(RESULTDIR); \
+	  CHECK_TYPE=install \
+	  BUILDDIR=$(BUILDDIR) \
+	  RGBDEF=$(RGBDEF) \
+	  $(SRCDIR)/test/Execute-Tests 2>&1
+
+
+
 clean: localclean
 
 .PHONY: localclean
@@ -469,7 +563,7 @@ localdistclean: localclean
 	-rm -f TAGS
 	-rm -f config.mk
 
-# 'tags' generates/updates an Emacs tags file, anmed TAGS, in the current
+# 'tags' generates/updates an Emacs tags file, named TAGS, in the current
 # directory.  Use with Emacs command 'find-tag'.
 
 .PHONY: tags
diff --git a/doc/TESTS b/doc/TESTS
index 7b62a841..65f91d26 100644
--- a/doc/TESTS
+++ b/doc/TESTS
@@ -8,10 +8,11 @@ Contents
   1.4 Repeatability
   1.5 Execution time
   1.6 Testing package in designated directory
-  1.7 Post-install check
-  1.8 Skipping test items
-  1.9 Valgrind
-
+  1.7 Pre-packaging check
+  1.8 Post-install check
+  1.9 Skipping test items
+  1.10 Valgrind
+  
 2. Troubleshooting
   2.1 Missing programs
   2.2 Broken programs
@@ -66,31 +67,47 @@ appear at the end:
 1.3 Prerequisites
 =================
 
-The tests require the Bash command shell.  The script Execute-Tests has some
-expressions unique to bash.  Quite old versions work, at least back to bash
-v. 2.05b.
+The tests require the Bash command shell.  The script Execute-Tests has
+some expressions unique to bash.  Quite old versions work, at least back
+to bash v. 2.05b.
 
 The tests also use the following utilities:
 
-  - awk 
-  - cat
-  - cmp
-  - egrep
-  - head
-  - mktemp
-  - sed
-  - seq
-  - tee
+ - sh
+ - awk
+ - perl
+
+ - cat
+ - cksum
+ - cmp
+ - cp
+ - cut
+ - date
+ - dirname
+ - egrep
+ - fgrep
+ - grep
+ - file
+ - head
+ - mkdir
+ - mktemp
+ - rm
+ - sed
+ - seq
+ - tee
+ - tr
+ - uniq
 
 
 
 1.4 Repeatability
 =================
 
-The current version of the test package produces identical results if you run
-"make check" repeatedly.  The tests contain no random elements; some
-Netpbm programs use randomness internally, but the tests seed their random
-number generators with fixed values so that they too have repeatable results.
+The current version of the test package produces identical results if
+you run "make check" repeatedly.  The tests contain no random elements;
+some Netpbm programs use randomness internally, but the tests seed
+their random number generators with fixed values so that they too have
+repeatable results.
 
 
 
@@ -114,64 +131,99 @@ you must do the same with "make check":
 
 
 
-1.7 Post-install check
-======================
+1.7 Pre-packaging check
+=======================
+
+You can run the tests to check executables after compilation, before
+packaging.  This feature is intended for developers.
+
+   make check-tree
 
-You can run the tests after installation as well as from the package
-directory.  Just set $PBM_TESTPREFIX to the bin directory.  For example, if
-the Netpbm executables are in /usr/local/bin/ do
+This test method is incompatible with merge build.
 
-   PBM_TESTPREFIX=/usr/local/bin make check
+Currently this test method reports several errors when Netpbm is compiled in a
+separate build directory.
 
-If you want to test the executables in the default search path do
 
-   PBM_TESTPREFIX="" make check
 
-If the color dictionary file rgb.txt is in a non-standard location, you must
-set RGBDEF.  If you don't, the tests will report several false positives.
+1.8 Post-install check
+======================
+
+You can run the tests after installation.  Run this way, the tests are of
+programs in the default search path.
+
+  make check-install
 
-   PBM_TESTPREFIX=/usr/local/bin RGBDEF=/etc/colors/rgb.txt make check
+Make sure to set RGBDEF if the color dictionary file rgb.txt is in
+a non-standard location.  This must be an absolute path.  
 
+  RGBDEF=/etc/colors/rgb.txt make check-install
 
 
-1.8 Skipping test items
+
+1.9 Skipping test items
 =======================
 
 The file test/Test-Order is a list of tests which are run.  If you want to
 skip any test, remove the line or comment it out with a "#".
 
+The variable "target", a comma-separated list of Netpbm programs
+provides another way to run only select tests.  For example to run
+only the test scripts which examine giftopnm and pamtogif, do:
+
+  make check target=giftopnm,pamtogif
 
 
-1.9 Valgrind
+
+1.10 Valgrind
 ============
 
 You can run the whole test under Valgrind.  This is an advanced feature
 intended for programmers who work on Netpbm code.
 
-To turn on Valgrind, uncomment this line in test/Execute:
+To turn on Valgrind, set VALGRIND_TESTS to "on":
+
+  make check VALGRIND_TESTS="on"
+
+Valgrind version 3.6.0 or newer is required.  For information on
+valgrind, visit http://www.valgrind.org/ .
+
+Valgrind results are output to files, one per process in the directory
+/tmp/netpbm-test/valgrind .  The file name consists of the test script
+name, process ID and the suffix ".vout", e.g.: "ppmmake.18420.vout" .
+
+Valgrind errors are not reported in the summary report and do not
+influence the success/failure count in any way.  The following awk
+one-liner will report ".vout" files with a positive error count in
+the ERROR SUMMARY line:
 
-# export PBM_TESTPREFIX="valgrind --log-fd=4 "${PBM_TESTPREFIX}
+  awk '/ERROR SUMMARY/ && $4>0 {print FILENAME}' \
+       /tmp/netpbm-test/valgrind/*.vout  
 
-   make check 4> valgrind.log
 
-Note that Valgrind output is directed to file descriptor 4.
+You can add or alter valgrind options by editing this line in
+test/Execute-Tests:
 
-Valgrind version 3.5.0 or newer is recommended.  Older versions do
-not report the command that is being executed.
+  vg_command_base="valgrind --trace-children=yes"
 
-You can add any options for Valgrind should by editing the above mentioned
-line.  To run "valgrind --track-origins=yes", you must make two changes in
+To run "valgrind --track-origins=yes", you must make two changes in
 config.mk:
 
-  - Add -g to CFLAGS.
-  - Turn stripping off.
+  - Add -g to CFLAGS
+  - Turn stripping off: STRIPFLAG =
 
-Valgrind significantly increases execution time.  You should consider paring
-down the items in Test-Order.  You probably don't need to run
-"all-in-place.test", which has restrictions in the way errors are reported
-when PBM_TESTPREFIX is not a simple directory path.  Another test
-"pamslice-roundtrip.test" is time-consuming, you may want to skip this one
-too.
+Valgrind significantly increases execution time.  If ordinary
+"make check" requires 10 seconds, "make check VALGRIND_TESTS=on"
+will require roughly 12 minutes, maybe more.  You should consider
+either setting "target=..." or paring down the items in Test-Order.
+In the latter case, you probably don't need to run "all-in-place.test"
+and "legacy-names.test".
+
+The option "--trace-children-skip" is used to prevent valgrind from
+stepping into child processes that are not relevant.  This option
+appears in valgrind v. 3.6.0.  If you wish to conduct valgrind tests
+with an older version, comment out the line in Execute-Tests with
+"--trace-children-skip".
 
 
 
@@ -181,7 +233,8 @@ too.
 2.1 Missing programs
 ====================
 
-The first test run, "all-in-place.test" detects missing programs.
+The first two tests run, "all-in-place.test" and "legacy-names.test"
+detect missing programs.
 
 If you work around a build glitch with "make --keep-going" you
 will get a few errors here.
@@ -191,10 +244,10 @@ problem, such as a misconfigured dynamic library or a directory-wide
 permission issue.  This kind of failure is known to happen when N is
 set too high with "make -jN" (overambitious parallel make.) 
 
-The current test routines assume a typical build configuration - they are not
-aware of the actual configuration you chose.  If a choice you make during
-configure causes "make" to skip compilation of certain programs, the test
-routines won't know and will report failures.
+The current test routines assume a typical build configuration - they are
+not aware of the actual configuration you chose.  If a choice you make
+during configure causes "make" to skip compilation of certain programs,
+the test routines won't know and will report failures.
 
 For details read 'Netpbm Library Prerequisites':
 http://netpbm.sourceforge.net/prereq.html .
@@ -205,8 +258,8 @@ http://netpbm.sourceforge.net/prereq.html .
 ===================
 
 Broken programs will invariably lead to failures.  Certain programs
-(for instance, image generators 'pbmmake' and 'pgmmake') are used in numerous
-test scripts.  Problems in them will lead to multiple failures.
+(for instance, image generators 'pbmmake' and 'pgmmake') are used in
+numerous test scripts.  Problems in them will lead to multiple failures.
 
 To aid you in this situation each test routine lists the necessary programs
 near the top.
@@ -232,7 +285,7 @@ the color dictionary file rgb.txt.
   pathname of your rgb.txt file or don't use color names.
 
 This is highly unlikely to occur with "make check" right after packaging,
-but may appear after installation.
+but may appear in the post-installation check "make check-install".
 
 To check manually after installation, execute the following and see
 whether the proper output or the error message appears:
@@ -249,6 +302,10 @@ Proper output:
 The simple remedy is properly setting the environment value RGBDEF to
 the location of rgb.txt.
 
+RGBDEF must be an absolute path.  The following will not work:
+
+  RGBDEF=./lib/rgb.txt make check-install
+
 If you want to hardcode the path, modify RGB_DB_PATH in pm_config.h
 and run "make" again.  Note that running "configure" will cause
 pm_config.h to be overwritten; changes by editing will be lost.
@@ -264,17 +321,19 @@ the newly built version is in place and in working condition.
 
 The test routines can test binaries other than the intended
 target, for example pre-compiled binaries distributed in .rpm
-or .deb format.  If PBM_TESTPREFIX is explicitly set to a directory
-that contains programs from such a source, you should expect multiple
-failures due to missing programs, missing features, etc.  If
-PBM_TEXTPREFIX is set to "" (null), the outcome will depend heavily
-on what version has precedence in PATH.
+or .deb format.  If the default binary search path gives priority
+to a directory that contains programs from such a source, you should
+expect multiple failures due to missing features, etc. with
+"make check-install".
 
 Netpbm distributed with Debian or Ubuntu is called "Netpbm-Free" and
 is based on a fork which broke off in 2002.  There are many differences.
 Many tests will fail.  However, the test framework itself is valid for
-these distributions.
+these distributions.  The following procedure will allow you to run
+the tests on installed Netpbm programs, regardless of the version:
 
+   ./configure           # accept the defaults
+   make check-install
 
 
 2.5 System dependent issues
@@ -294,6 +353,7 @@ is the one from glibc and skips certain tests if a different one is
 detected.
 
 
+
 3. Reporting test failures
 ==========================