28528: Rearrange zleparse to lexflags

Add (z+n+)

10 files changed, 112 insertions, 69 deletions

diff --git a/ChangeLog b/ChangeLog
index 05179c468..121da5698 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2010-12-14  Peter Stephenson  <pws@csr.com>
+
+	* 28528: Doc/Zsh/expn.yo, Src/hist.c, Src/lex.c, Src/subst.c,
+	Src/zsh.h, Src/Zle/compcore.c, Src/Zle/compctl.c,
+	Src/Zle/zle_tricky.c, Test/D04parameter.ztst: clear up use of
+	zleparse variable into lexflags; add (z+n+) for splitting with
+	newline treated as ordinary whitespace.
+
 2010-12-13  Peter Stephenson  <pws@csr.com>
 
 	* 28526: Src/lex.c, Test/D04parameter.ztst: zplitting
@@ -13942,5 +13950,5 @@
 
 *****************************************************
 * This is used by the shell to define $ZSH_PATCHLEVEL
-* $Revision: 1.5148 $
+* $Revision: 1.5149 $
 *****************************************************
diff --git a/Doc/Zsh/expn.yo b/Doc/Zsh/expn.yo
index 5768b82df..cf8aee65f 100644
--- a/Doc/Zsh/expn.yo
+++ b/Doc/Zsh/expn.yo
@@ -1009,14 +1009,17 @@ find the words, i.e. taking into account any quoting in the value.
 Comments are not treated specially but as ordinary strings, similar
 to interactive shells with the tt(INTERACTIVE_COMMENTS) option unset.
 
-The flag can take option letters between a following pair of
-`tt(PLUS())' characters.  tt(LPAR()z+PLUS()c+PLUS()RPAR()) causes
-comments to be parsed as a string and retained; any field in the
+The flag can take a combination of option letters between a following
+pair of `tt(PLUS())' characters.  tt(LPAR()z+PLUS()c+PLUS()RPAR())
+causes comments to be parsed as a string and retained; any field in the
 resulting array beginning with an unquoted comment character is a
 comment.  tt(LPAR()z+PLUS()C+PLUS()RPAR()) causes comments to be parsed
 and removed.  The rule for comments is standard: anything between a word
 starting with the third charcter of tt($HISTCHARS), default tt(#), up to
-the next newline is a comment.
+the next newline is a comment.  tt(LPAR()z+PLUS()n+PLUS()RPAR()) causes
+unquoted newlines to be treated as ordinary whitespace, else they are
+treated as if they are shell code delimiters and converted to
+semicolons.
 
 Note that this is done very late, as for the `tt((s))' flag. So to
 access single words in the result, one has to use nested expansions as 
diff --git a/Src/Zle/compcore.c b/Src/Zle/compcore.c
index 415f91f79..2e983f62c 100644
--- a/Src/Zle/compcore.c
+++ b/Src/Zle/compcore.c
@@ -1481,7 +1481,7 @@ set_comp_sep(void)
 
     /* Put the string in the lexer buffer and call the lexer to *
      * get the words we have to expand.                        */
-    zleparse = 1;
+    lexflags = LEXFLAGS_ACTIVE;
     ocs = zlemetacs;
     oll = zlemetall;
     ol = zlemetaline;
@@ -1616,7 +1616,7 @@ set_comp_sep(void)
         }
 	else
 	    p = NULL;
-	if (!got && !zleparse) {
+	if (!got && !lexflags) {
 	    DPUTS(!p, "no current word in substr");
 	    got = 1;
 	    cur = i;
@@ -1634,7 +1634,7 @@ set_comp_sep(void)
     noaliases = ona;
     strinend();
     inpop();
-    errflag = zleparse = 0;
+    errflag = lexflags = 0;
     noerrs = ne;
     lexrestore();
     wb = owb;
diff --git a/Src/Zle/compctl.c b/Src/Zle/compctl.c
index f07f80877..4e28fd67c 100644
--- a/Src/Zle/compctl.c
+++ b/Src/Zle/compctl.c
@@ -2789,7 +2789,7 @@ sep_comp_string(char *ss, char *s, int noffs)
 
     /* Put the string in the lexer buffer and call the lexer to *
      * get the words we have to expand.                        */
-    zleparse = 1;
+    lexflags = LEXFLAGS_ACTIVE;
     addedx = 1;
     noerrs = 1;
     lexsave();
@@ -2828,7 +2828,7 @@ sep_comp_string(char *ss, char *s, int noffs)
 	    addlinknode(foo, (p = ztrdup(tokstr)));
 	else
 	    p = NULL;
-	if (!got && !zleparse) {
+	if (!got && !lexflags) {
 	    DPUTS(!p, "no current word in substr");
 	    got = 1;
 	    cur = i;
@@ -2843,7 +2843,7 @@ sep_comp_string(char *ss, char *s, int noffs)
     noaliases = ona;
     strinend();
     inpop();
-    errflag = zleparse = 0;
+    errflag = lexflags = 0;
     noerrs = ne;
     lexrestore();
     wb = owb;
@@ -3703,7 +3703,7 @@ makecomplistflags(Compctl cc, char *s, int incmd, int compadd)
 
 	/* Put the string in the lexer buffer and call the lexer to *
 	 * get the words we have to expand.                        */
-	zleparse = 1;
+	lexflags = LEXFLAGS_ACTIVE;
 	lexsave();
 	tmpbuf = (char *)zhalloc(strlen(cc->str) + 5);
 	sprintf(tmpbuf, "foo %s", cc->str); /* KLUDGE! */
@@ -3721,7 +3721,7 @@ makecomplistflags(Compctl cc, char *s, int incmd, int compadd)
 	noaliases = ona;
 	strinend();
 	inpop();
-	errflag = zleparse = 0;
+	errflag = lexflags = 0;
 	lexrestore();
 	/* Fine, now do full expansion. */
 	prefork(foo, 0);
diff --git a/Src/Zle/zle_tricky.c b/Src/Zle/zle_tricky.c
index 3e2a35171..9a63b2738 100644
--- a/Src/Zle/zle_tricky.c
+++ b/Src/Zle/zle_tricky.c
@@ -1140,7 +1140,7 @@ get_comp_string(void)
     zsfree(varname);
     varname = NULL;
     insubscr = 0;
-    zleparse = 1;
+    lexflags = LEXFLAGS_ACTIVE;
     clwpos = -1;
     lexsave();
     inpush(dupstrspace(linptr), 0, NULL);
@@ -1244,7 +1244,7 @@ get_comp_string(void)
 	    if (wordpos != redirpos)
 		wordpos = redirpos = 0;
 	}
-	if (!zleparse && !tt0) {
+	if (!lexflags && !tt0) {
 	    /* This is done when the lexer reached the word the cursor is on. */
 	    tt = tokstr ? dupstring(tokstr) : NULL;
 
@@ -1345,7 +1345,7 @@ get_comp_string(void)
 				 (sl - 1) : (zlemetacs_qsub - wb)]);
 	}
     } while (tok != LEXERR && tok != ENDINPUT &&
-	     (tok != SEPER || (zleparse && !tt0)));
+	     (tok != SEPER || (lexflags && !tt0)));
     /* Calculate the number of words stored in the clwords array. */
     clwnum = (tt || !wordpos) ? wordpos : wordpos - 1;
     zsfree(clwords[clwnum]);
@@ -1360,7 +1360,7 @@ get_comp_string(void)
     }
     strinend();
     inpop();
-    errflag = zleparse = 0;
+    errflag = lexflags = 0;
     if (parbegin != -1) {
 	/* We are in command or process substitution if we are not in
 	 * a $((...)). */
@@ -2707,7 +2707,7 @@ doexpandhist(void)
     noaliases = ona;
     strinend();
     inpop();
-    zleparse = 0;
+    lexflags = 0;
     lexrestore();
     expanding = 0;
 
@@ -2807,7 +2807,7 @@ getcurcmd(void)
     int curlincmd;
     char *s = NULL;
 
-    zleparse = 2;
+    lexflags = LEXFLAGS_ACTIVE;
     lexsave();
     metafy_line();
     inpush(dupstrspace(zlemetaline), 0, NULL);
@@ -2825,11 +2825,11 @@ getcurcmd(void)
 	    cmdwe = zlemetall + 1 - inbufct;
 	}
     }
-    while (tok != ENDINPUT && tok != LEXERR && zleparse);
+    while (tok != ENDINPUT && tok != LEXERR && lexflags);
     popheap();
     strinend();
     inpop();
-    errflag = zleparse = 0;
+    errflag = lexflags = 0;
     unmetafy_line();
     lexrestore();
 
diff --git a/Src/hist.c b/Src/hist.c
index 89db826b3..ae7ada755 100644
--- a/Src/hist.c
+++ b/Src/hist.c
@@ -2345,7 +2345,8 @@ readhistfile(char *fn, int err, int readflags)
 		/*
 		 * Attempt to do this using the lexer.
 		 */
-		LinkList wordlist = bufferwords(NULL, pt, NULL, 1);
+		LinkList wordlist = bufferwords(NULL, pt, NULL,
+						LEXFLAGS_COMMENTS_KEEP);
 		LinkNode wordnode;
 		int nwords_max;
 		nwords_max = 2 * countlinknodes(wordlist);
@@ -2905,10 +2906,10 @@ histfileIsLocked(void)
 
 /**/
 mod_export LinkList
-bufferwords(LinkList list, char *buf, int *index, int comments)
+bufferwords(LinkList list, char *buf, int *index, int flags)
 {
     int num = 0, cur = -1, got = 0, ne = noerrs;
-    int owb = wb, owe = we, oadx = addedx, ozp = zleparse, onc = nocomments;
+    int owb = wb, owe = we, oadx = addedx, ozp = lexflags, onc = nocomments;
     int ona = noaliases, ocs = zlemetacs, oll = zlemetall;
     int forloop = 0, rcquotes = opts[RCQUOTES];
     char *p, *addedspaceptr;
@@ -2925,6 +2926,12 @@ bufferwords(LinkList list, char *buf, int *index, int comments)
     addedx = 0;
     noerrs = 1;
     lexsave();
+    lexflags = flags | LEXFLAGS_ACTIVE;
+    /*
+     * Are we handling comments?
+     */
+    nocomments = !(flags & (LEXFLAGS_COMMENTS_KEEP|
+			    LEXFLAGS_COMMENTS_STRIP));
     if (buf) {
 	int l = strlen(buf);
 
@@ -2943,18 +2950,10 @@ bufferwords(LinkList list, char *buf, int *index, int comments)
 	inpush(p, 0, NULL);
 	zlemetall = strlen(p) ;
 	zlemetacs = zlemetall + 1;
-
-	/*
-	 * If comments is non-zero we are handling comments.
-	 * zleparse indicates the mode to the lexer.
-	 */
-	zleparse = 1 + comments;
-	nocomments = !comments;
     } else {
 	int ll, cs;
 	char *linein;
 
-	zleparse = 1;
 	linein = zleentry(ZLE_CMD_GET_LINE, &ll, &cs);
 	zlemetall = ll + 1; /* length of line plus space added below */
 	zlemetacs = cs;
@@ -3096,7 +3095,7 @@ bufferwords(LinkList list, char *buf, int *index, int comments)
 	    }
 	    forloop--;
 	}
-	if (!got && !zleparse) {
+	if (!got && !lexflags) {
 	    got = 1;
 	    cur = num - 1;
 	}
@@ -3121,7 +3120,7 @@ bufferwords(LinkList list, char *buf, int *index, int comments)
     strinend();
     inpop();
     errflag = 0;
-    zleparse = ozp;
+    lexflags = ozp;
     nocomments = onc;
     noerrs = ne;
     lexrestore();
diff --git a/Src/lex.c b/Src/lex.c
index 20511f54e..ca48f3e35 100644
--- a/Src/lex.c
+++ b/Src/lex.c
@@ -117,15 +117,11 @@ mod_export int wb, we;
 mod_export int noaliases;
 
 /*
- * we are parsing a line sent to use by the editor, or some other string
- * that's not part of standard command input (e.g. eval is part of
- * normal command input).
+ * If non-zero, we are parsing a line sent to use by the editor, or some
+ * other string that's not part of standard command input (e.g. eval is
+ * part of normal command input).
  *
- * zleparse = 1 is the normal case.
- * zleparse = 2 is used for word splitting; the difference is we
- *              preserve comments.
- * zleparse = 3 is also for word splitting, here handling comments
- *              but stripping them.
+ * Set of bits from LEXFLAGS_*.
  *
  * Note that although it is passed into the lexer as an input, the
  * lexer can set it to zero after finding the word it's searching for.
@@ -134,7 +130,7 @@ mod_export int noaliases;
  */
 
 /**/
-mod_export int zleparse;
+mod_export int lexflags;
 
 /**/
 mod_export int wordbeg;
@@ -429,7 +425,7 @@ zshlex(void)
 	isnewlin = 0;
     else
 	isnewlin = (inbufct) ? -1 : 1;
-    if (tok == SEMI || tok == NEWLIN)
+    if (tok == SEMI || (tok == NEWLIN && !(lexflags & LEXFLAGS_NEWLINE)))
 	tok = SEPER;
 }
 
@@ -588,9 +584,9 @@ add(int c)
     }
 }
 
-#define SETPARBEGIN {if (zleparse && !(inbufflags & INP_ALIAS) && zlemetacs >= zlemetall+1-inbufct) parbegin = inbufct;}
+#define SETPARBEGIN {if (lexflags && !(inbufflags & INP_ALIAS) && zlemetacs >= zlemetall+1-inbufct) parbegin = inbufct;}
 #define SETPAREND {\
-	    if (zleparse && !(inbufflags & INP_ALIAS) && parbegin != -1 && parend == -1) {\
+	    if (lexflags && !(inbufflags & INP_ALIAS) && parbegin != -1 && parend == -1) {\
 		if (zlemetacs >= zlemetall + 1 - inbufct)\
 		    parbegin = -1;\
 		else\
@@ -760,22 +756,17 @@ gettok(void)
 
     /*
      * Handle comments.  There are some special cases when this
-     * is not normal command input: zleparse implies we are examining
+     * is not normal command input: lexflags implies we are examining
      * a line lexically without it being used for normal command input.
-     * If zleparse is 1 we treat comments as normal for interactive
-     * mode.
-     * If zleparse is 2 (which has actually got nothing to do with zle)
-     * we always handle comments and retain them.
-     * If zleparse is 3 we always handle comments and discard them.
      */
     if (c == hashchar && !nocomments &&
 	(isset(INTERACTIVECOMMENTS) ||
-	 ((zleparse != 1) && !expanding &&
+	 ((!lexflags || (lexflags & LEXFLAGS_COMMENTS)) && !expanding &&
 	  (!interact || unset(SHINSTDIN) || strin)))) {
 	/* History is handled here to prevent extra  *
 	 * newlines being inserted into the history. */
 
-	if (zleparse == 2) {
+	if (lexflags & LEXFLAGS_COMMENTS_KEEP) {
 	    len = 0;
 	    bptr = tokstr = (char *)hcalloc(bsiz = 32);
 	    add(c);
@@ -783,14 +774,14 @@ gettok(void)
 	while ((c = ingetc()) != '\n' && !lexstop) {
 	    hwaddc(c);
 	    addtoline(c);
-	    if (zleparse == 2)
+	    if (lexflags & LEXFLAGS_COMMENTS_KEEP)
 		add(c);
 	}
 
 	if (errflag)
 	    peek = LEXERR;
 	else {
-	    if (zleparse == 2) {
+	    if (lexflags & LEXFLAGS_COMMENTS_KEEP) {
 		*bptr = '\0';
 		if (!lexstop)
 		    hungetc(c);
@@ -805,7 +796,7 @@ gettok(void)
 		 * we don't want a newline token since it's
 		 * treated specially.
 		 */
-		if (zleparse == 3 && lexstop)
+		if ((lexflags & LEXFLAGS_COMMENTS_STRIP) && lexstop)
 		    peek = ENDINPUT;
 		else
 		    peek = NEWLIN;
@@ -1778,7 +1769,7 @@ gotword(void)
     we = zlemetall + 1 - inbufct + (addedx == 2 ? 1 : 0);
     if (zlemetacs <= we) {
 	wb = zlemetall - wordbeg + addedx;
-	zleparse = 0;
+	lexflags = 0;
     }
 }
 
@@ -1813,11 +1804,11 @@ exalias(void)
 	} else
 	    zshlextext = tokstr;
 
-	if (zleparse && !(inbufflags & INP_ALIAS)) {
-	    int zp = zleparse;
+	if (lexflags && !(inbufflags & INP_ALIAS)) {
+	    int zp = lexflags;
 
 	    gotword();
-	    if (zp == 1 && !zleparse) {
+	    if (zp == 1 && !lexflags) {
 		if (zshlextext == copy)
 		    zshlextext = tokstr;
 		return 0;
diff --git a/Src/subst.c b/Src/subst.c
index 799682df2..7ad4aecdf 100644
--- a/Src/subst.c
+++ b/Src/subst.c
@@ -1557,9 +1557,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
      * spbreak, see above; fairly straighforward in use but c.f.
      * the comment for mods.
      *
-     * This ultimately becomes zleparse during lexical analysis, via
-     * the comments argument to bufferwords(). It's got nothing
-     * to do with zle.
+     * This gets set to one of the LEXFLAGS_* values.
      */
     int shsplit = 0;
     /*
@@ -1937,19 +1935,24 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 		    break;
 
 		case 'z':
-		    shsplit = 1;
+		    shsplit = LEXFLAGS_ACTIVE;
 		    if (s[1] == '+') {
 			s += 2;
 			while (*s && *s != '+' && *s != ')' && *s != Outpar) {
 			    switch (*s++) {
 			    case 'c':
 				/* Parse and keep comments */
-				shsplit = 2;
+				shsplit |= LEXFLAGS_COMMENTS_KEEP;
 				break;
 
 			    case 'C':
 				/* Parse and remove comments */
-				shsplit = 3;
+				shsplit |= LEXFLAGS_COMMENTS_STRIP;
+				break;
+
+			    case 'n':
+				/* Treat newlines as whitespace */
+				shsplit |= LEXFLAGS_NEWLINE;
 				break;
 
 			    default:
@@ -3232,10 +3235,10 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 	if (isarr) {
 	    char **ap;
 	    for (ap = aval; *ap; ap++)
-		list = bufferwords(list, *ap, NULL, shsplit-1);
+		list = bufferwords(list, *ap, NULL, shsplit);
 	    isarr = 0;
 	} else
-	    list = bufferwords(NULL, val, NULL, shsplit-1);
+	    list = bufferwords(NULL, val, NULL, shsplit);
 
 	if (!list || !firstnode(list))
 	    val = dupstring("");
diff --git a/Src/zsh.h b/Src/zsh.h
index 1d793741a..cd479462b 100644
--- a/Src/zsh.h
+++ b/Src/zsh.h
@@ -1823,6 +1823,34 @@ struct histent {
 #define HFILE_NO_REWRITE	0x0020
 #define HFILE_USE_OPTIONS	0x8000
 
+/*
+ * Flags argument to bufferwords() used
+ * also by lexflags variable.
+ */
+/*
+ * Kick the lexer into special string-analysis
+ * mode without parsing.  Any bit set in
+ * the flags has this effect, but this
+ * has otherwise all the default effects.
+ */
+#define LEXFLAGS_ACTIVE		0x0001
+/*
+ * Parse comments and treat each comment as a single string
+ */
+#define LEXFLAGS_COMMENTS_KEEP	0x0002
+/*
+ * Parse comments and strip them.
+ */
+#define LEXFLAGS_COMMENTS_STRIP	0x0004
+/*
+ * Either of the above
+ */
+#define LEXFLAGS_COMMENTS (LEXFLAGS_COMMENTS_KEEP|LEXFLAGS_COMMENTS_STRIP)
+/*
+ * Treat newlines as whitespace
+ */
+#define LEXFLAGS_NEWLINE	0x0008
+
 /******************************************/
 /* Definitions for programable completion */
 /******************************************/
diff --git a/Test/D04parameter.ztst b/Test/D04parameter.ztst
index 8791f570a..74c73b360 100644
--- a/Test/D04parameter.ztst
+++ b/Test/D04parameter.ztst
@@ -462,6 +462,17 @@
 >with
 >comment
 
+  line=$'echo one\necho two # with a comment\necho three'
+  print -l ${(z+nc+)line}
+0:Treating zplit newlines as ordinary whitespace
+>echo
+>one
+>echo
+>two
+># with a comment
+>echo
+>three
+
   psvar=(dog)
   setopt promptsubst
   foo='It shouldn'\''t $(happen) to a %1v.'