about summary refs log tree commit diff
diff options
context:
space:
mode:
authorDaniel Shahaf <danielsh@apache.org>2019-12-31 01:52:03 +0000
committerDaniel Shahaf <danielsh@apache.org>2020-01-01 13:55:39 +0000
commit7e2c80fcd5a2aeae299f55e392e00275943f1562 (patch)
treefe0b5efa001da7bacc26b7cf1232a453c5547379
parent6a67d38e969fe0acacf3191291e44a862376321f (diff)
downloadzsh-7e2c80fcd5a2aeae299f55e392e00275943f1562.tar.gz
zsh-7e2c80fcd5a2aeae299f55e392e00275943f1562.tar.xz
zsh-7e2c80fcd5a2aeae299f55e392e00275943f1562.zip
45169/0001: In the test suite, allow test cases to be marked as expected to fail.
See next commit for a use-case.
-rw-r--r--ChangeLog6
-rw-r--r--Test/B01cd.ztst4
-rwxr-xr-xTest/ztst.zsh30
-rw-r--r--Util/ztst-syntax.vim2
4 files changed, 38 insertions, 4 deletions
diff --git a/ChangeLog b/ChangeLog
index 204448465..909bb065a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2020-01-01  Daniel Shahaf  <danielsh@apache.org>
+
+	* 45169/0001: Test/B01cd.ztst, Test/ztst.zsh,
+	Util/ztst-syntax.vim: In the test suite, allow test cases to
+	be marked as expected to fail.
+
 2019-12-31  Daniel Shahaf  <danielsh@apache.org>
 
 	* users/24582 + users/24583: Test/D04parameter.ztst,
diff --git a/Test/B01cd.ztst b/Test/B01cd.ztst
index f79e18f4b..977cbdfe5 100644
--- a/Test/B01cd.ztst
+++ b/Test/B01cd.ztst
@@ -70,11 +70,13 @@
 # the expected status returned by the code when run, or - if it is
 # irrelevant.  An optional set of single-letter flags follows the status
 # or -.  The following are understood:
-#  . d   Don't diff stdout against the expected stdout.
+#  . d  Don't diff stdout against the expected stdout.
 #   D   Don't diff stderr against the expected stderr.
 #   q   All redirection lines given in the test script (not the lines
 #       actually produced by the test) are subject to ordinary quoted shell
 #       expansion (i.e. not globbing).
+#   f   Test is expected to fail. If the test's exit code, stdout, and stderr
+#       match, report a problem; otherwise, carry on to the next test case.
 # This can be followed by a `:' and a message describing the
 # test, which will be printed if the test fails, along with a
 # description of the failure that occurred.  The `:' and message are
diff --git a/Test/ztst.zsh b/Test/ztst.zsh
index a4c625230..375efd16c 100755
--- a/Test/ztst.zsh
+++ b/Test/ztst.zsh
@@ -366,6 +366,7 @@ ZTST_test() {
   local last match mbegin mend found substlines
   local diff_out diff_err
   local ZTST_skip
+  integer expected_to_fail
 
   while true; do
     rm -f $ZTST_in $ZTST_out $ZTST_err
@@ -460,8 +461,21 @@ $ZTST_curline"
 	fi
       fi
 
+      if [[ $ZTST_flags = *f* ]]; then
+        expected_to_fail=1
+        ZTST_xfail_diff() { ZTST_diff "$@" > /dev/null }
+        ZTST_diff=ZTST_xfail_diff
+      else
+        expected_to_fail=0
+        ZTST_diff=ZTST_diff
+      fi
+
       # First check we got the right status, if specified.
       if [[ $ZTST_xstatus != - && $ZTST_xstatus != $ZTST_status ]]; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "bad status $ZTST_status, expected $ZTST_xstatus from:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
@@ -480,7 +494,11 @@ $(<$ZTST_terr)"
 	rm -rf $ZTST_out
 	print -r -- "${(e)substlines}" >$ZTST_out
       fi
-      if [[ $ZTST_flags != *d* ]] && ! ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then
+      if [[ $ZTST_flags != *d* ]] && ! $ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "output differs from expected as shown above for:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
@@ -492,11 +510,19 @@ $(<$ZTST_terr)}"
 	rm -rf $ZTST_err
 	print -r -- "${(e)substlines}" >$ZTST_err
       fi
-      if [[ $ZTST_flags != *D* ]] && ! ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then
+      if [[ $ZTST_flags != *D* ]] && ! $ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "error output differs from expected as shown above for:
 $ZTST_code"
 	return 1
       fi
+      if (( expected_to_fail )); then
+        ZTST_testfailed "test was expected to fail, but passed."
+        return 1
+      fi
     fi
     ZTST_verbose 1 "Test successful."
     [[ -n $last ]] && break
diff --git a/Util/ztst-syntax.vim b/Util/ztst-syntax.vim
index 8d8becfe7..f0c82d74c 100644
--- a/Util/ztst-syntax.vim
+++ b/Util/ztst-syntax.vim
@@ -28,7 +28,7 @@ syn include @zsh                   syntax/zsh.vim
 syn match  ztstPayload             /^\s\+\zs.*/ contains=@zsh
 
 syn match  ztstExitCode            /^\d\+\|^-/                nextgroup=ztstFlags
-syn match  ztstFlags               /[.dDq]*:/       contained nextgroup=ztstTestName contains=ztstColon
+syn match  ztstFlags               /[.dDqf]*:/      contained nextgroup=ztstTestName contains=ztstColon
 syn match  ztstColon               /:/              contained
 syn region ztstTestName            start=// end=/$/ contained