Split tokenize() into shtokenize() and zshtokenize()

to improve SH_GLOB handling, particularly in completion

3 files changed, 22 insertions, 8 deletions

diff --git a/Src/exec.c b/Src/exec.c
index e7b57967b..d3b33e174 100644
--- a/Src/exec.c
+++ b/Src/exec.c
@@ -2797,7 +2797,7 @@ readoutput(int in, int qt)
 
 	while (*words) {
 	    if (isset(GLOBSUBST))
-		tokenize(*words);
+		shtokenize(*words);
 	    addlinknode(ret, *words++);
 	}
     }
diff --git a/Src/glob.c b/Src/glob.c
index 443262dc4..8caa70ba5 100644
--- a/Src/glob.c
+++ b/Src/glob.c
@@ -2448,6 +2448,20 @@ igetmatch(char **sp, Patprog p, int fl, int n, char *replstr)
 mod_export void
 tokenize(char *s)
 {
+    zshtokenize(s, 0);
+}
+
+/**/
+mod_export void
+shtokenize(char *s)
+{
+    zshtokenize(s, isset(SHGLOB));
+}
+
+/**/
+static void
+zshtokenize(char *s, int shglob)
+{
     char *t;
     int bslash = 0;
 
@@ -2463,7 +2477,7 @@ tokenize(char *s)
 	    bslash = 1;
 	    continue;
 	case '<':
-	    if (isset(SHGLOB))
+	    if (shglob)
 		break;
 	    if (bslash) {
 		s[-1] = Bnull;
@@ -2482,7 +2496,7 @@ tokenize(char *s)
 	case '(':
 	case '|':
 	case ')':
-	    if (isset(SHGLOB))
+	    if (shglob)
 		break;
 	case '>':
 	case '^':
diff --git a/Src/subst.c b/Src/subst.c
index 7a0a8b16b..5396cdf6c 100644
--- a/Src/subst.c
+++ b/Src/subst.c
@@ -197,7 +197,7 @@ stringsubst(LinkList list, LinkNode node, int ssub, int asssub)
 		continue;
 	    }
 	    if (!qt && ssub && isset(GLOBSUBST))
-		tokenize(s);
+		shtokenize(s);
 	    l1 = str2 - str3;
 	    l2 = strlen(s);
 	    if (nonempty(pl)) {
@@ -450,14 +450,14 @@ strcatsub(char **d, char *pb, char *pe, char *src, int l, char *s, int glbsub,
     if (!pl && (!s || !*s)) {
 	*d = dest = (copied ? src : dupstring(src));
 	if (glbsub)
-	    tokenize(dest);
+	    shtokenize(dest);
     } else {
 	*d = dest = hcalloc(pl + l + (s ? strlen(s) : 0) + 1);
 	strncpy(dest, pb, pl);
 	dest += pl;
 	strcpy(dest, src);
 	if (glbsub)
-	    tokenize(dest);
+	    shtokenize(dest);
 	dest += l;
 	if (s)
 	    strcpy(dest, s);
@@ -1595,7 +1595,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 		if (!quoteerr) {
 		    errflag = oef;
 		    if (haserr)
-			tokenize(s);
+			shtokenize(s);
 		} else if (haserr || errflag) {
 		    zerr("parse error in ${...%c...} substitution",
 			 NULL, s[-1]);
@@ -2063,7 +2063,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 		else {
 		    y = dupstring(x);
 		    if (globsubst)
-			tokenize(y);
+			shtokenize(y);
 		}
 		insertlinknode(l, n, (void *) y), incnode(n);
 	    }