diff options
| -rw-r--r-- | ChangeLog | 14 | ||||
| -rw-r--r-- | INSTALL | 6 | ||||
| -rw-r--r-- | Makeconfig | 7 | ||||
| -rw-r--r-- | Makefile | 12 | ||||
| -rw-r--r-- | NEWS | 7 | ||||
| -rw-r--r-- | manual/install.texi | 6 | ||||
| -rwxr-xr-x | scripts/evaluate-test.sh | 9 |
7 files changed, 59 insertions, 2 deletions
diff --git a/ChangeLog b/ChangeLog index 99d2caed9b..fb61ad2e91 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +2014-03-14 Joseph Myers <joseph@codesourcery.com> + + * scripts/evaluate-test.sh: Handle fourth argument to determine + whether test run should stop on failure. + * Makeconfig (stop-on-test-failure): New variable. + (evaluate-test): Pass fourth argument to evaluate-test.sh based on + $(stop-on-test-failure). + * Makefile (tests): Give a summary of results from testing and + exit with failure status if they include an ERROR or FAIL. + (xtests): Likewise. + * manual/install.texi (Configuring and compiling): Mention + stop-on-test-failure=y. + * INSTALL: Regenerated. + 2014-03-14 Roland McGrath <roland@hack.frob.com> * scripts/versionlist.awk: New file. diff --git a/INSTALL b/INSTALL index f845940e3e..13fb678abb 100644 --- a/INSTALL +++ b/INSTALL @@ -203,6 +203,12 @@ The tests (and later installation) use some pre-existing files of the system such as `/etc/passwd', `/etc/nsswitch.conf' and others. These files must all contain correct and sensible content. + Normally, `make check' will run all the tests before reporting all +problems found and exiting with error status if any problems occurred. +You can specify `stop-on-test-failure=y' when running `make check' to +make the test run stop and exit with an error status immediately when a +failure occurs. + To format the `GNU C Library Reference Manual' for printing, type `make dvi'. You need a working TeX installation to do this. The distribution builds the on-line formatted version of the manual, as diff --git a/Makeconfig b/Makeconfig index fa9fb63851..3338ab698b 100644 --- a/Makeconfig +++ b/Makeconfig @@ -601,6 +601,12 @@ run-built-tests = yes endif endif +# Whether to stop immediately when a test fails. Nonempty means to +# stop, empty means not to stop. +ifndef stop-on-test-failure +stop-on-test-failure = +endif + # How to run a program we just linked with our library. # The program binary is assumed to be $(word 2,$^). built-program-file = $(dir $(word 2,$^))$(notdir $(word 2,$^)) @@ -1092,6 +1098,7 @@ test-xfail-name = $(strip $(patsubst %.out, %, $(patsubst $(objpfx)%, %, $@))) # XPASS or XFAIL rather than PASS or FAIL. evaluate-test = $(..)scripts/evaluate-test.sh $(test-name) $$? \ $(if $(test-xfail-$(test-xfail-name)),true,false) \ + $(if $(stop-on-test-failure),true,false) \ > $(common-objpfx)$(test-name).test-result endif # Makeconfig not yet included diff --git a/Makefile b/Makefile index 8214dda840..e026fb18c3 100644 --- a/Makefile +++ b/Makefile @@ -324,10 +324,22 @@ tests: $(tests-special) $(..)scripts/merge-test-results.sh -t $(objpfx) subdir-tests.sum \ $(sort $(subdirs) .) \ > $(objpfx)tests.sum + @grep '^ERROR:' $(objpfx)tests.sum || true + @grep '^FAIL:' $(objpfx)tests.sum || true + @echo "Summary of test results:" + @sed 's/:.*//' < $(objpfx)tests.sum | sort | uniq -c + @if grep -q '^ERROR:' $(objpfx)tests.sum; then exit 1; fi + @if grep -q '^FAIL:' $(objpfx)tests.sum; then exit 1; fi xtests: $(..)scripts/merge-test-results.sh -t $(objpfx) subdir-xtests.sum \ $(sort $(subdirs)) \ > $(objpfx)xtests.sum + @grep '^ERROR:' $(objpfx)xtests.sum || true + @grep '^FAIL:' $(objpfx)xtests.sum || true + @echo "Summary of test results for extra tests:" + @sed 's/:.*//' < $(objpfx)xtests.sum | sort | uniq -c + @if grep -q '^ERROR:' $(objpfx)xtests.sum; then exit 1; fi + @if grep -q '^FAIL:' $(objpfx)xtests.sum; then exit 1; fi # The realclean target is just like distclean for the parent, but we want # the subdirs to know the difference in case they care. diff --git a/NEWS b/NEWS index 443b146b0c..c69d40a7ee 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,13 @@ Version 2.20 16611, 16613, 16623, 16632, 16639, 16642, 16670, 16674, 16677, 16683, 16689, 16695, 16701, 16706, 16707. +* Running the testsuite no longer terminates as soon as a test fails. + Instead, a file tests.sum (xtests.sum from "make xcheck") is generated, + with PASS or FAIL lines for individual tests. A summary of the results is + printed, including a list of failing lists, and "make check" exits with + error status if there were any unexpected failures. "make check + stop-on-test-failure=y" may be used to keep the old behavior. + * The am33 port, which had not worked for several years, has been removed from ports. diff --git a/manual/install.texi b/manual/install.texi index 8562bdc04f..d2b8c1edec 100644 --- a/manual/install.texi +++ b/manual/install.texi @@ -235,6 +235,12 @@ The tests (and later installation) use some pre-existing files of the system such as @file{/etc/passwd}, @file{/etc/nsswitch.conf} and others. These files must all contain correct and sensible content. +Normally, @code{make check} will run all the tests before reporting +all problems found and exiting with error status if any problems +occurred. You can specify @samp{stop-on-test-failure=y} when running +@code{make check} to make the test run stop and exit with an error +status immediately when a failure occurs. + To format the @cite{GNU C Library Reference Manual} for printing, type @w{@code{make dvi}}. You need a working @TeX{} installation to do this. The distribution builds the on-line formatted version of the diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh index c8f5012c5a..2a5c15641e 100755 --- a/scripts/evaluate-test.sh +++ b/scripts/evaluate-test.sh @@ -17,12 +17,13 @@ # License along with the GNU C Library; if not, see # <http://www.gnu.org/licenses/>. -# usage: evaluate-test.sh test_name rc xfail +# usage: evaluate-test.sh test_name rc xfail stop_on_failure test_name=$1 rc=$2 orig_rc=$rc xfail=$3 +stop_on_failure=$4 if [ $rc -eq 0 ]; then result="PASS" @@ -37,4 +38,8 @@ fi echo "$result: $test_name" echo "original exit status $orig_rc" -exit $rc +if $stop_on_failure; then + exit $rc +else + exit 0 +fi |