diff options
| -rw-r--r-- | ChangeLog | 10 | ||||
| -rw-r--r-- | Doc/Zsh/expn.yo | 11 | ||||
| -rw-r--r-- | Src/Zle/compcore.c | 6 | ||||
| -rw-r--r-- | Src/Zle/compctl.c | 10 | ||||
| -rw-r--r-- | Src/Zle/zle_tricky.c | 16 | ||||
| -rw-r--r-- | Src/hist.c | 25 | ||||
| -rw-r--r-- | Src/lex.c | 45 | ||||
| -rw-r--r-- | Src/subst.c | 19 | ||||
| -rw-r--r-- | Src/zsh.h | 28 | ||||
| -rw-r--r-- | Test/D04parameter.ztst | 11 |
10 files changed, 112 insertions, 69 deletions
diff --git a/ChangeLog b/ChangeLog index 05179c468..121da5698 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +2010-12-14 Peter Stephenson <pws@csr.com> + + * 28528: Doc/Zsh/expn.yo, Src/hist.c, Src/lex.c, Src/subst.c, + Src/zsh.h, Src/Zle/compcore.c, Src/Zle/compctl.c, + Src/Zle/zle_tricky.c, Test/D04parameter.ztst: clear up use of + zleparse variable into lexflags; add (z+n+) for splitting with + newline treated as ordinary whitespace. + 2010-12-13 Peter Stephenson <pws@csr.com> * 28526: Src/lex.c, Test/D04parameter.ztst: zplitting @@ -13942,5 +13950,5 @@ ***************************************************** * This is used by the shell to define $ZSH_PATCHLEVEL -* $Revision: 1.5148 $ +* $Revision: 1.5149 $ ***************************************************** diff --git a/Doc/Zsh/expn.yo b/Doc/Zsh/expn.yo index 5768b82df..cf8aee65f 100644 --- a/Doc/Zsh/expn.yo +++ b/Doc/Zsh/expn.yo @@ -1009,14 +1009,17 @@ find the words, i.e. taking into account any quoting in the value. Comments are not treated specially but as ordinary strings, similar to interactive shells with the tt(INTERACTIVE_COMMENTS) option unset. -The flag can take option letters between a following pair of -`tt(PLUS())' characters. tt(LPAR()z+PLUS()c+PLUS()RPAR()) causes -comments to be parsed as a string and retained; any field in the +The flag can take a combination of option letters between a following +pair of `tt(PLUS())' characters. tt(LPAR()z+PLUS()c+PLUS()RPAR()) +causes comments to be parsed as a string and retained; any field in the resulting array beginning with an unquoted comment character is a comment. tt(LPAR()z+PLUS()C+PLUS()RPAR()) causes comments to be parsed and removed. The rule for comments is standard: anything between a word starting with the third charcter of tt($HISTCHARS), default tt(#), up to -the next newline is a comment. +the next newline is a comment. tt(LPAR()z+PLUS()n+PLUS()RPAR()) causes +unquoted newlines to be treated as ordinary whitespace, else they are +treated as if they are shell code delimiters and converted to +semicolons. Note that this is done very late, as for the `tt((s))' flag. So to access single words in the result, one has to use nested expansions as diff --git a/Src/Zle/compcore.c b/Src/Zle/compcore.c index 415f91f79..2e983f62c 100644 --- a/Src/Zle/compcore.c +++ b/Src/Zle/compcore.c @@ -1481,7 +1481,7 @@ set_comp_sep(void) /* Put the string in the lexer buffer and call the lexer to * * get the words we have to expand. */ - zleparse = 1; + lexflags = LEXFLAGS_ACTIVE; ocs = zlemetacs; oll = zlemetall; ol = zlemetaline; @@ -1616,7 +1616,7 @@ set_comp_sep(void) } else p = NULL; - if (!got && !zleparse) { + if (!got && !lexflags) { DPUTS(!p, "no current word in substr"); got = 1; cur = i; @@ -1634,7 +1634,7 @@ set_comp_sep(void) noaliases = ona; strinend(); inpop(); - errflag = zleparse = 0; + errflag = lexflags = 0; noerrs = ne; lexrestore(); wb = owb; diff --git a/Src/Zle/compctl.c b/Src/Zle/compctl.c index f07f80877..4e28fd67c 100644 --- a/Src/Zle/compctl.c +++ b/Src/Zle/compctl.c @@ -2789,7 +2789,7 @@ sep_comp_string(char *ss, char *s, int noffs) /* Put the string in the lexer buffer and call the lexer to * * get the words we have to expand. */ - zleparse = 1; + lexflags = LEXFLAGS_ACTIVE; addedx = 1; noerrs = 1; lexsave(); @@ -2828,7 +2828,7 @@ sep_comp_string(char *ss, char *s, int noffs) addlinknode(foo, (p = ztrdup(tokstr))); else p = NULL; - if (!got && !zleparse) { + if (!got && !lexflags) { DPUTS(!p, "no current word in substr"); got = 1; cur = i; @@ -2843,7 +2843,7 @@ sep_comp_string(char *ss, char *s, int noffs) noaliases = ona; strinend(); inpop(); - errflag = zleparse = 0; + errflag = lexflags = 0; noerrs = ne; lexrestore(); wb = owb; @@ -3703,7 +3703,7 @@ makecomplistflags(Compctl cc, char *s, int incmd, int compadd) /* Put the string in the lexer buffer and call the lexer to * * get the words we have to expand. */ - zleparse = 1; + lexflags = LEXFLAGS_ACTIVE; lexsave(); tmpbuf = (char *)zhalloc(strlen(cc->str) + 5); sprintf(tmpbuf, "foo %s", cc->str); /* KLUDGE! */ @@ -3721,7 +3721,7 @@ makecomplistflags(Compctl cc, char *s, int incmd, int compadd) noaliases = ona; strinend(); inpop(); - errflag = zleparse = 0; + errflag = lexflags = 0; lexrestore(); /* Fine, now do full expansion. */ prefork(foo, 0); diff --git a/Src/Zle/zle_tricky.c b/Src/Zle/zle_tricky.c index 3e2a35171..9a63b2738 100644 --- a/Src/Zle/zle_tricky.c +++ b/Src/Zle/zle_tricky.c @@ -1140,7 +1140,7 @@ get_comp_string(void) zsfree(varname); varname = NULL; insubscr = 0; - zleparse = 1; + lexflags = LEXFLAGS_ACTIVE; clwpos = -1; lexsave(); inpush(dupstrspace(linptr), 0, NULL); @@ -1244,7 +1244,7 @@ get_comp_string(void) if (wordpos != redirpos) wordpos = redirpos = 0; } - if (!zleparse && !tt0) { + if (!lexflags && !tt0) { /* This is done when the lexer reached the word the cursor is on. */ tt = tokstr ? dupstring(tokstr) : NULL; @@ -1345,7 +1345,7 @@ get_comp_string(void) (sl - 1) : (zlemetacs_qsub - wb)]); } } while (tok != LEXERR && tok != ENDINPUT && - (tok != SEPER || (zleparse && !tt0))); + (tok != SEPER || (lexflags && !tt0))); /* Calculate the number of words stored in the clwords array. */ clwnum = (tt || !wordpos) ? wordpos : wordpos - 1; zsfree(clwords[clwnum]); @@ -1360,7 +1360,7 @@ get_comp_string(void) } strinend(); inpop(); - errflag = zleparse = 0; + errflag = lexflags = 0; if (parbegin != -1) { /* We are in command or process substitution if we are not in * a $((...)). */ @@ -2707,7 +2707,7 @@ doexpandhist(void) noaliases = ona; strinend(); inpop(); - zleparse = 0; + lexflags = 0; lexrestore(); expanding = 0; @@ -2807,7 +2807,7 @@ getcurcmd(void) int curlincmd; char *s = NULL; - zleparse = 2; + lexflags = LEXFLAGS_ACTIVE; lexsave(); metafy_line(); inpush(dupstrspace(zlemetaline), 0, NULL); @@ -2825,11 +2825,11 @@ getcurcmd(void) cmdwe = zlemetall + 1 - inbufct; } } - while (tok != ENDINPUT && tok != LEXERR && zleparse); + while (tok != ENDINPUT && tok != LEXERR && lexflags); popheap(); strinend(); inpop(); - errflag = zleparse = 0; + errflag = lexflags = 0; unmetafy_line(); lexrestore(); diff --git a/Src/hist.c b/Src/hist.c index 89db826b3..ae7ada755 100644 --- a/Src/hist.c +++ b/Src/hist.c @@ -2345,7 +2345,8 @@ readhistfile(char *fn, int err, int readflags) /* * Attempt to do this using the lexer. */ - LinkList wordlist = bufferwords(NULL, pt, NULL, 1); + LinkList wordlist = bufferwords(NULL, pt, NULL, + LEXFLAGS_COMMENTS_KEEP); LinkNode wordnode; int nwords_max; nwords_max = 2 * countlinknodes(wordlist); @@ -2905,10 +2906,10 @@ histfileIsLocked(void) /**/ mod_export LinkList -bufferwords(LinkList list, char *buf, int *index, int comments) +bufferwords(LinkList list, char *buf, int *index, int flags) { int num = 0, cur = -1, got = 0, ne = noerrs; - int owb = wb, owe = we, oadx = addedx, ozp = zleparse, onc = nocomments; + int owb = wb, owe = we, oadx = addedx, ozp = lexflags, onc = nocomments; int ona = noaliases, ocs = zlemetacs, oll = zlemetall; int forloop = 0, rcquotes = opts[RCQUOTES]; char *p, *addedspaceptr; @@ -2925,6 +2926,12 @@ bufferwords(LinkList list, char *buf, int *index, int comments) addedx = 0; noerrs = 1; lexsave(); + lexflags = flags | LEXFLAGS_ACTIVE; + /* + * Are we handling comments? + */ + nocomments = !(flags & (LEXFLAGS_COMMENTS_KEEP| + LEXFLAGS_COMMENTS_STRIP)); if (buf) { int l = strlen(buf); @@ -2943,18 +2950,10 @@ bufferwords(LinkList list, char *buf, int *index, int comments) inpush(p, 0, NULL); zlemetall = strlen(p) ; zlemetacs = zlemetall + 1; - - /* - * If comments is non-zero we are handling comments. - * zleparse indicates the mode to the lexer. - */ - zleparse = 1 + comments; - nocomments = !comments; } else { int ll, cs; char *linein; - zleparse = 1; linein = zleentry(ZLE_CMD_GET_LINE, &ll, &cs); zlemetall = ll + 1; /* length of line plus space added below */ zlemetacs = cs; @@ -3096,7 +3095,7 @@ bufferwords(LinkList list, char *buf, int *index, int comments) } forloop--; } - if (!got && !zleparse) { + if (!got && !lexflags) { got = 1; cur = num - 1; } @@ -3121,7 +3120,7 @@ bufferwords(LinkList list, char *buf, int *index, int comments) strinend(); inpop(); errflag = 0; - zleparse = ozp; + lexflags = ozp; nocomments = onc; noerrs = ne; lexrestore(); diff --git a/Src/lex.c b/Src/lex.c index 20511f54e..ca48f3e35 100644 --- a/Src/lex.c +++ b/Src/lex.c @@ -117,15 +117,11 @@ mod_export int wb, we; mod_export int noaliases; /* - * we are parsing a line sent to use by the editor, or some other string - * that's not part of standard command input (e.g. eval is part of - * normal command input). + * If non-zero, we are parsing a line sent to use by the editor, or some + * other string that's not part of standard command input (e.g. eval is + * part of normal command input). * - * zleparse = 1 is the normal case. - * zleparse = 2 is used for word splitting; the difference is we - * preserve comments. - * zleparse = 3 is also for word splitting, here handling comments - * but stripping them. + * Set of bits from LEXFLAGS_*. * * Note that although it is passed into the lexer as an input, the * lexer can set it to zero after finding the word it's searching for. @@ -134,7 +130,7 @@ mod_export int noaliases; */ /**/ -mod_export int zleparse; +mod_export int lexflags; /**/ mod_export int wordbeg; @@ -429,7 +425,7 @@ zshlex(void) isnewlin = 0; else isnewlin = (inbufct) ? -1 : 1; - if (tok == SEMI || tok == NEWLIN) + if (tok == SEMI || (tok == NEWLIN && !(lexflags & LEXFLAGS_NEWLINE))) tok = SEPER; } @@ -588,9 +584,9 @@ add(int c) } } -#define SETPARBEGIN {if (zleparse && !(inbufflags & INP_ALIAS) && zlemetacs >= zlemetall+1-inbufct) parbegin = inbufct;} +#define SETPARBEGIN {if (lexflags && !(inbufflags & INP_ALIAS) && zlemetacs >= zlemetall+1-inbufct) parbegin = inbufct;} #define SETPAREND {\ - if (zleparse && !(inbufflags & INP_ALIAS) && parbegin != -1 && parend == -1) {\ + if (lexflags && !(inbufflags & INP_ALIAS) && parbegin != -1 && parend == -1) {\ if (zlemetacs >= zlemetall + 1 - inbufct)\ parbegin = -1;\ else\ @@ -760,22 +756,17 @@ gettok(void) /* * Handle comments. There are some special cases when this - * is not normal command input: zleparse implies we are examining + * is not normal command input: lexflags implies we are examining * a line lexically without it being used for normal command input. - * If zleparse is 1 we treat comments as normal for interactive - * mode. - * If zleparse is 2 (which has actually got nothing to do with zle) - * we always handle comments and retain them. - * If zleparse is 3 we always handle comments and discard them. */ if (c == hashchar && !nocomments && (isset(INTERACTIVECOMMENTS) || - ((zleparse != 1) && !expanding && + ((!lexflags || (lexflags & LEXFLAGS_COMMENTS)) && !expanding && (!interact || unset(SHINSTDIN) || strin)))) { /* History is handled here to prevent extra * * newlines being inserted into the history. */ - if (zleparse == 2) { + if (lexflags & LEXFLAGS_COMMENTS_KEEP) { len = 0; bptr = tokstr = (char *)hcalloc(bsiz = 32); add(c); @@ -783,14 +774,14 @@ gettok(void) while ((c = ingetc()) != '\n' && !lexstop) { hwaddc(c); addtoline(c); - if (zleparse == 2) + if (lexflags & LEXFLAGS_COMMENTS_KEEP) add(c); } if (errflag) peek = LEXERR; else { - if (zleparse == 2) { + if (lexflags & LEXFLAGS_COMMENTS_KEEP) { *bptr = '\0'; if (!lexstop) hungetc(c); @@ -805,7 +796,7 @@ gettok(void) * we don't want a newline token since it's * treated specially. */ - if (zleparse == 3 && lexstop) + if ((lexflags & LEXFLAGS_COMMENTS_STRIP) && lexstop) peek = ENDINPUT; else peek = NEWLIN; @@ -1778,7 +1769,7 @@ gotword(void) we = zlemetall + 1 - inbufct + (addedx == 2 ? 1 : 0); if (zlemetacs <= we) { wb = zlemetall - wordbeg + addedx; - zleparse = 0; + lexflags = 0; } } @@ -1813,11 +1804,11 @@ exalias(void) } else zshlextext = tokstr; - if (zleparse && !(inbufflags & INP_ALIAS)) { - int zp = zleparse; + if (lexflags && !(inbufflags & INP_ALIAS)) { + int zp = lexflags; gotword(); - if (zp == 1 && !zleparse) { + if (zp == 1 && !lexflags) { if (zshlextext == copy) zshlextext = tokstr; return 0; diff --git a/Src/subst.c b/Src/subst.c index 799682df2..7ad4aecdf 100644 --- a/Src/subst.c +++ b/Src/subst.c @@ -1557,9 +1557,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub) * spbreak, see above; fairly straighforward in use but c.f. * the comment for mods. * - * This ultimately becomes zleparse during lexical analysis, via - * the comments argument to bufferwords(). It's got nothing - * to do with zle. + * This gets set to one of the LEXFLAGS_* values. */ int shsplit = 0; /* @@ -1937,19 +1935,24 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub) break; case 'z': - shsplit = 1; + shsplit = LEXFLAGS_ACTIVE; if (s[1] == '+') { s += 2; while (*s && *s != '+' && *s != ')' && *s != Outpar) { switch (*s++) { case 'c': /* Parse and keep comments */ - shsplit = 2; + shsplit |= LEXFLAGS_COMMENTS_KEEP; break; case 'C': /* Parse and remove comments */ - shsplit = 3; + shsplit |= LEXFLAGS_COMMENTS_STRIP; + break; + + case 'n': + /* Treat newlines as whitespace */ + shsplit |= LEXFLAGS_NEWLINE; break; default: @@ -3232,10 +3235,10 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub) if (isarr) { char **ap; for (ap = aval; *ap; ap++) - list = bufferwords(list, *ap, NULL, shsplit-1); + list = bufferwords(list, *ap, NULL, shsplit); isarr = 0; } else - list = bufferwords(NULL, val, NULL, shsplit-1); + list = bufferwords(NULL, val, NULL, shsplit); if (!list || !firstnode(list)) val = dupstring(""); diff --git a/Src/zsh.h b/Src/zsh.h index 1d793741a..cd479462b 100644 --- a/Src/zsh.h +++ b/Src/zsh.h @@ -1823,6 +1823,34 @@ struct histent { #define HFILE_NO_REWRITE 0x0020 #define HFILE_USE_OPTIONS 0x8000 +/* + * Flags argument to bufferwords() used + * also by lexflags variable. + */ +/* + * Kick the lexer into special string-analysis + * mode without parsing. Any bit set in + * the flags has this effect, but this + * has otherwise all the default effects. + */ +#define LEXFLAGS_ACTIVE 0x0001 +/* + * Parse comments and treat each comment as a single string + */ +#define LEXFLAGS_COMMENTS_KEEP 0x0002 +/* + * Parse comments and strip them. + */ +#define LEXFLAGS_COMMENTS_STRIP 0x0004 +/* + * Either of the above + */ +#define LEXFLAGS_COMMENTS (LEXFLAGS_COMMENTS_KEEP|LEXFLAGS_COMMENTS_STRIP) +/* + * Treat newlines as whitespace + */ +#define LEXFLAGS_NEWLINE 0x0008 + /******************************************/ /* Definitions for programable completion */ /******************************************/ diff --git a/Test/D04parameter.ztst b/Test/D04parameter.ztst index 8791f570a..74c73b360 100644 --- a/Test/D04parameter.ztst +++ b/Test/D04parameter.ztst @@ -462,6 +462,17 @@ >with >comment + line=$'echo one\necho two # with a comment\necho three' + print -l ${(z+nc+)line} +0:Treating zplit newlines as ordinary whitespace +>echo +>one +>echo +>two +># with a comment +>echo +>three + psvar=(dog) setopt promptsubst foo='It shouldn'\''t $(happen) to a %1v.' |