45169/0001: In the test suite, allow test cases to be marked as expected to fail.

See next commit for a use-case.

2 files changed, 31 insertions, 3 deletions

diff --git a/Test/B01cd.ztst b/Test/B01cd.ztst
index f79e18f4b..977cbdfe5 100644
--- a/Test/B01cd.ztst
+++ b/Test/B01cd.ztst
@@ -70,11 +70,13 @@
 # the expected status returned by the code when run, or - if it is
 # irrelevant.  An optional set of single-letter flags follows the status
 # or -.  The following are understood:
-#  . d   Don't diff stdout against the expected stdout.
+#  . d  Don't diff stdout against the expected stdout.
 #   D   Don't diff stderr against the expected stderr.
 #   q   All redirection lines given in the test script (not the lines
 #       actually produced by the test) are subject to ordinary quoted shell
 #       expansion (i.e. not globbing).
+#   f   Test is expected to fail. If the test's exit code, stdout, and stderr
+#       match, report a problem; otherwise, carry on to the next test case.
 # This can be followed by a `:' and a message describing the
 # test, which will be printed if the test fails, along with a
 # description of the failure that occurred.  The `:' and message are
diff --git a/Test/ztst.zsh b/Test/ztst.zsh
index a4c625230..375efd16c 100755
--- a/Test/ztst.zsh
+++ b/Test/ztst.zsh
@@ -366,6 +366,7 @@ ZTST_test() {
   local last match mbegin mend found substlines
   local diff_out diff_err
   local ZTST_skip
+  integer expected_to_fail
 
   while true; do
     rm -f $ZTST_in $ZTST_out $ZTST_err
@@ -460,8 +461,21 @@ $ZTST_curline"
 	fi
       fi
 
+      if [[ $ZTST_flags = *f* ]]; then
+        expected_to_fail=1
+        ZTST_xfail_diff() { ZTST_diff "$@" > /dev/null }
+        ZTST_diff=ZTST_xfail_diff
+      else
+        expected_to_fail=0
+        ZTST_diff=ZTST_diff
+      fi
+
       # First check we got the right status, if specified.
       if [[ $ZTST_xstatus != - && $ZTST_xstatus != $ZTST_status ]]; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "bad status $ZTST_status, expected $ZTST_xstatus from:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
@@ -480,7 +494,11 @@ $(<$ZTST_terr)"
 	rm -rf $ZTST_out
 	print -r -- "${(e)substlines}" >$ZTST_out
       fi
-      if [[ $ZTST_flags != *d* ]] && ! ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then
+      if [[ $ZTST_flags != *d* ]] && ! $ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "output differs from expected as shown above for:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
@@ -492,11 +510,19 @@ $(<$ZTST_terr)}"
 	rm -rf $ZTST_err
 	print -r -- "${(e)substlines}" >$ZTST_err
       fi
-      if [[ $ZTST_flags != *D* ]] && ! ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then
+      if [[ $ZTST_flags != *D* ]] && ! $ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "error output differs from expected as shown above for:
 $ZTST_code"
 	return 1
       fi
+      if (( expected_to_fail )); then
+        ZTST_testfailed "test was expected to fail, but passed."
+        return 1
+      fi
     fi
     ZTST_verbose 1 "Test successful."
     [[ -n $last ]] && break