users/23809: ZLE_HIGHLIGHT extensions.

Allow non-termcap use of colours > 7.
Add tests.

3 files changed, 183 insertions, 4 deletions

diff --git a/ChangeLog b/ChangeLog
index ca5137636..0479430c7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 2018-12-30  Peter Stephenson  <p.w.stephenson@ntlworld.com>
 
+	* Sebastian: users/23809: Src/prompt.c,
+	Test/X04zlehighlight.ztst: Add ZLE_HIGHLITHT extensions to
+	termcap for colours > 7, add tests.
+
 	* 43944: Martijn: Doc/Zsh/options.yo, README, Src/math.c,
 	Test/C01arith.ztst: apply NO_UNSET consistently to arithmetic.
 
diff --git a/Src/prompt.c b/Src/prompt.c
index 568bfc2a9..135aca942 100644
--- a/Src/prompt.c
+++ b/Src/prompt.c
@@ -1764,7 +1764,12 @@ output_colour(int colour, int fg_bg, int use_tc, int truecol, char *buf)
 	/* length of hex triplet always 7, don't need sprintf to count */
 	atrlen += buf ? sprintf(ptr, "#%02x%02x%02x", colour >> 16,
 		(colour >> 8) & 0xff, colour & 0xff) : 7;
-    /* colour should only be > 7 if using termcap but let's be safe */
+    /* colour should only be > 7 if using termcap but let's be safe. Update:
+     * currently other places in code don't always imply that colour > 7 =>
+     * using-termcap - if zle_highlight will be non-default, then it will be
+     * used instead of termcap even for colour > 7. Here this just emits the
+     * color number, so it works fine for both zle_highlight and tercap cases
+     */
     } else if (use_tc || colour > 7) {
 	char digbuf[DIGBUFSIZE];
 	sprintf(digbuf, "%d", colour);
@@ -1995,6 +2000,7 @@ set_colour_attribute(zattr atr, int fg_bg, int flags)
     char *ptr;
     int do_free, is_prompt = (flags & TSC_PROMPT) ? 1 : 0;
     int colour, tc, def, use_termcap, use_truecolor;
+    int is_default_zle_highlight = 1;
 
     if (fg_bg == COL_SEQ_FG) {
 	colour = txtchangeget(atr, TXT_ATTR_FG_COL);
@@ -2010,6 +2016,15 @@ set_colour_attribute(zattr atr, int fg_bg, int flags)
 	use_termcap = txtchangeisset(atr, TXT_ATTR_BG_TERMCAP);
     }
 
+    /* Test if current zle_highlight settings are customized, or
+     * the typical "standard" codes */
+    if (0 != strcmp(fg_bg_sequences[fg_bg].start, fg_bg == COL_SEQ_FG ? "\e[3" : "\e[4") ||
+            0 != strcmp(fg_bg_sequences[fg_bg].def, "9") || /* the same in-fix for both FG and BG */
+            0 != strcmp(fg_bg_sequences[fg_bg].end, "m") /* the same suffix for both FG and BG */
+   ) {
+            is_default_zle_highlight = 0;
+    }
+
     /*
      * If we're not restoring the default, and either have a
      * colour value that is too large for ANSI, or have been told
@@ -2020,7 +2035,7 @@ set_colour_attribute(zattr atr, int fg_bg, int flags)
      * highlighting variables, so much of this shouldn't be
      * necessary at this point, but we might as well be safe.
      */
-    if (!def && !use_truecolor && (colour > 7 || use_termcap)) {
+    if (!def && !use_truecolor && (is_default_zle_highlight && (colour > 7 || use_termcap))) {
 	/*
 	 * We can if it's available, and either we couldn't get
 	 * the maximum number of colours, or the colour is in range.
@@ -2046,9 +2061,10 @@ set_colour_attribute(zattr atr, int fg_bg, int flags)
 	}
 	/*
 	 * Nope, that didn't work.
-	 * If 0 to 7, assume standard ANSI works, otherwise it won't.
+	 * If 0 to 7, assume standard ANSI works, if 8 to 255, assume
+         * typical 256-color escapes works, otherwise it won't.
 	 */
-	if (colour > 7)
+	if (colour > 255)
 	    return;
     }
 
@@ -2057,6 +2073,10 @@ set_colour_attribute(zattr atr, int fg_bg, int flags)
 	allocate_colour_buffer();
     }
 
+    /* Build the reset-code: .start + .def + . end
+     * or the typical true-color code: .start + 8;2;%d;%d;%d + .end
+     * or the typical 256-color code: .start + 8;5;%d + .end
+     */
     strcpy(colseq_buf, fg_bg_sequences[fg_bg].start);
 
     ptr = colseq_buf + strlen(colseq_buf);
@@ -2067,6 +2087,8 @@ set_colour_attribute(zattr atr, int fg_bg, int flags)
     } else if (use_truecolor) {
 	ptr += sprintf(ptr, "8;2;%d;%d;%d", colour >> 16,
 		(colour >> 8) & 0xff, colour & 0xff);
+    } else if (colour > 7 && colour <= 255) {
+        ptr += sprintf(ptr, "8;5;%d", colour);
     } else
 	*ptr++ = colour + '0';
     strcpy(ptr, fg_bg_sequences[fg_bg].end);
diff --git a/Test/X04zlehighlight.ztst b/Test/X04zlehighlight.ztst
new file mode 100644
index 000000000..23ad14bd6
--- /dev/null
+++ b/Test/X04zlehighlight.ztst
@@ -0,0 +1,153 @@
+# Tests for region_highlight, true-color support, near-color support
+# Version 0.7 2018-12-06
+%prep
+
+  export TERM=xterm-256color
+  if [[ ${+termcap} != 1 || ${termcap[Co]} != <-> || ${termcap[Co]} -lt 256 ]]; then
+    ZTST_unimplemented="no termcap module OR termcap doesn't support 256 or more colors"
+  elif [[ $OSTYPE == cygwin ]]; then
+    ZTST_unimplemented='the zsh/zpty module does not work on Cygwin'
+  elif zmodload zsh/zpty 2> /dev/null; then
+    zpty_start() {
+      export PS1= PS2=
+      zpty -d
+      zpty zsh "${(q)ZTST_testdir}/../Src/zsh -fiV +Z"
+      zpty -w zsh 'zle_highlight=( fg_start_code:"CDE|3" fg_end_code:"|" bg_start_code:"BCDE|4" bg_end_code:"|" )'
+    }
+    zpty_input() {
+      zpty ${${(M)2:#nonl}:+-n} -w zsh "$1"
+    }
+    zpty_enable_zle() {
+      zpty -w zsh "tcfunc() { REPLY=""; }"
+      # This line will not be echoed back, behaving like ! -o zle
+      zpty -w zsh "setopt zle; zle -T tc tcfunc; unset zle_bracketed_paste"
+    }
+    zpty_line() {
+      setopt localoptions extendedglob noshwordsplit
+      local REPLY cm=$'\r'
+      integer i
+      for (( i = 0; i < ${1:-1}; ++i )); do
+        zpty -r zsh REPLY
+        # P is for "preserve", and induces keeping some
+        # expected color codes to test region_highlight,
+        # etc. - the color codes are made a regular text.
+        [[ "$2" = "p" ]] && {
+            REPLY=${REPLY//(#b)$'\x1b'\[([0-9;]##m)/${match[1]}}
+        } || {
+            REPLY=${REPLY//$'\x1b'\[[0-9;]##m/}   # remove all [0-9]...m codes
+        }
+        # Fix e^Mexit - match ((?)\r(?)), if \2 == \3, then replace with \2
+        # otherwise replace with \1 stripped out of leading/trailing [[:space:]]
+        REPLY=${REPLY//(#b)((?(#c0,1))$cm(?(#c0,1)))/${${${(M)match[2]:#${match[3]}}:+${match[2]}}:-${${match[1]##[[:space:]]##}%%[[:space:]]##}}}
+        [[ -n "$REPLY" ]] && print -r -- ${${REPLY%%[[:space:]]##}##[[:space:]]##}
+      done
+    }
+    zpty_stop() {
+      setopt localoptions extendedglob
+      local REPLY cm=$'\r'
+      # To early Ctrl-D might happen, it was happening when debug
+      # logs were writing to a file slowing down Zle a little
+      LANG=C sleep 0.333
+      # Zle is active, can use Ctrl-D to exit
+      zpty -n -w zsh $'\C-d'
+      # zpty gives no output when piped without these braces (?)
+      # The while loop with // substitution is to convert `e^Mexit'
+      # into `exit' (see zpty_line). The sed commands remove escapes
+      { zpty -r zsh } | sed -e $'/[^\t\r ]/!d' -e $'s/\r$//' -e $'s/\x1b\\[[0-9;]*m//g' | while read REPLY; do REPLY=${REPLY//(#b)((?(#c0,1))$cm(?(#c0,1)))/${${${(M)match[2]:#${match[3]}}:+${match[2]}}:-${${match[1]##[[:space:]]##}%%[[:space:]]##}}}; print -rn -- "$REPLY"; done
+      zpty -d
+      :
+    }
+  else
+    ZTST_unimplemented='the zsh/zpty module is not available'
+  fi
+
+%test
+
+  zpty_start
+  zpty_input 'rh_widget() { BUFFER="true word2 word3"; region_highlight+=( "0 4 fg=196" ); rh2; }'
+  zpty_input 'rh2() { region_highlight=( "2 3 standout" ); };' # note the =, not +=
+  zpty_input 'zle -N rh_widget'
+  zpty_input 'bindkey "\C-a" rh_widget'
+  zpty_enable_zle
+  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
+  zpty_line 1 p       # the line of interest, preserving escapes ("p")
+  zpty_stop
+0:region highlight - standout overlapping on other region_highlight entry
+>0m27m24mtr7mu27me word2 word3
+
+  zpty_start
+  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=green" ); }'
+  zpty_input 'zle -N rh_widget'
+  zpty_input 'bindkey "\C-a" rh_widget'
+  zpty_enable_zle
+  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
+  zpty_line 1 p       # the line of interest, preserving escapes ("p")
+  zpty_stop
+0:basic region_highlight with 8 colors
+>0m27m24mCDE|32|trueCDE|39|
+
+  zpty_start
+  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#040810" ); }'
+  zpty_input 'zle -N rh_widget'
+  zpty_input 'bindkey "\C-a" rh_widget'
+  zpty_enable_zle
+  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
+  zpty_line 1 p       # the line of interest, preserving escapes ("p")
+  zpty_stop
+0:basic region_highlight with true-color (hex-triplets)
+>0m27m24mCDE|38;2;4;8;16|trueCDE|39|
+
+  zpty_start
+  zpty_input 'zmodload zsh/nearcolor'
+  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#040810" ); }'
+  zpty_input 'zle -N rh_widget'
+  zpty_input 'bindkey "\C-a" rh_widget'
+  zpty_enable_zle
+  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
+  zpty_line 1 p       # the line of interest, preserving escapes ("p")
+  zpty_stop
+0:basic region_highlight with near-color (hex-triplets at input)
+>0m27m24mCDE|38;5;232|trueCDE|39|
+
+  zpty_start
+  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=green" ); rh2; }'
+  zpty_input 'rh2() { region_highlight+=( "1 2 fg=red" ); }' # `r' in red; the above line would be too long
+  zpty_input 'zle -N rh_widget'
+  zpty_input 'bindkey "\C-a" rh_widget'
+  zpty_enable_zle
+  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
+  zpty_line 1 p       # the line of interest, preserving escapes ("p")
+  zpty_stop
+0:overlapping region_highlight with 8 colors
+>0m27m24mCDE|32|tCDE|31|rCDE|39|CDE|32|ueCDE|39|
+
+  zpty_start
+  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#00cc00" ); rh2; }'
+  zpty_input 'rh2() { region_highlight+=( "1 2 fg=#cc0000" ); }' # `r' in red; the above line would be too long
+  zpty_input 'zle -N rh_widget'
+  zpty_input 'bindkey "\C-a" rh_widget'
+  zpty_enable_zle
+  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
+  zpty_line 1 p       # the line of interest, preserving escapes ("p")
+  zpty_stop
+0:overlapping region_highlight with true-color
+>0m27m24mCDE|38;2;0;204;0|tCDE|38;2;204;0;0|rCDE|39|CDE|38;2;0;204;0|ueCDE|39|
+
+  zpty_start
+  zpty_input 'zmodload zsh/nearcolor'
+  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#00cc00" ); rh2; }'
+  zpty_input 'rh2() { region_highlight+=( "1 2 fg=#cc0000" ); }' # `r' in red; the above line would be too long
+  zpty_input 'zle -N rh_widget'
+  zpty_input 'bindkey "\C-a" rh_widget'
+  zpty_enable_zle
+  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
+  zpty_line 1 p       # the line of interest, preserving escapes ("p")
+  zpty_stop
+0:overlapping region_highlight with near-color (hex-triplets at input)
+>0m27m24mCDE|38;5;40|tCDE|38;5;160|rCDE|39|CDE|38;5;40|ueCDE|39|
+
+%clean
+
+  zmodload -ui zsh/zpty
+
+# vim:ft=zsh