Updates for ksh array element syntax.

Move detection of key/value pairs down into prefork().

Detect normal array assignment and [key]=val array assignemnt
separately.  Mark key / value pairs with Marker and pass up flag.  Deal
with marked triads specially later on.

5 files changed, 224 insertions, 147 deletions

diff --git a/Src/Zle/zle_tricky.c b/Src/Zle/zle_tricky.c
index 5a9cccb6f..caeef7692 100644
--- a/Src/Zle/zle_tricky.c
+++ b/Src/Zle/zle_tricky.c
@@ -2268,7 +2268,7 @@ doexpansion(char *s, int lst, int olst, int explincmd)
 	int ng = opts[NULLGLOB];
 
 	opts[NULLGLOB] = 1;
-	globlist(vl, 1);
+	globlist(vl, PREFORK_NO_UNTOK);
 	opts[NULLGLOB] = ng;
     }
     if (errflag)
diff --git a/Src/exec.c b/Src/exec.c
index 31edfab55..bd242d140 100644
--- a/Src/exec.c
+++ b/Src/exec.c
@@ -2389,60 +2389,6 @@ addfd(int forked, int *save, struct multio **mfds, int fd1, int fd2, int rflag,
     }
 }
 
-/* Check for array assignent with entries like [key]=val.
- *
- * All entries or none must match this form, else error and return 0.
- *
- * Convert list to alternate key / val form, perform
- * appropriate substitution, and return 1 if found.
- *
- * Caller to check errflag.
- */
-
-/**/
-static int
-keyvalpairarray(LinkList vl, int htok)
-{
-    char *start, *end, *dat;
-    LinkNode ve, next;
-
-    if (vl &&
-	(ve = firstnode(vl)) &&
-	(start = (char *)getdata(ve)) &&
-	start[0] == Inbrack &&
-	(end = strchr(start+1, Outbrack)) &&
-	end[1] == Equals) {
-	for (;;) {
-	    *end = '\0';
-	    next = nextnode(ve);
-
-	    dat = start + 1;
-	    if (htok)
-		singsub(&dat);
-	    untokenize(dat);
-	    setdata(ve, dat);
-	    dat = end + 2;
-	    if (htok)
-		singsub(&dat);
-	    untokenize(dat);
-	    insertlinknode(vl, ve, dat);
-	    ve = next;
-	    if (!ve)
-		break;
-	    if (!(start = (char *)getdata(ve)) ||
-		start[0] != Inbrack ||
-		!(end = strchr(start+1, Outbrack)) ||
-		end[1] != Equals) {
-		zerr("bad array element, expected [key]=value: %s",
-		     start);
-		return 0;
-	    }
-	}
-	return 1;
-    }
-    return 0;
-}
-
 /**/
 static void
 addvars(Estate state, Wordcode pc, int addflags)
@@ -2484,10 +2430,6 @@ addvars(Estate state, Wordcode pc, int addflags)
 	    vl = &svl;
 	} else {
 	    vl = ecgetlist(state, WC_ASSIGN_NUM(ac), EC_DUPTOK, &htok);
-	    if (keyvalpairarray(vl, htok)) {
-		myflags |= ASSPM_KEY_VALUE;
-		htok = 0;
-	    }
 	    if (errflag) {
 		state->pc = opc;
 		return;
@@ -2495,25 +2437,28 @@ addvars(Estate state, Wordcode pc, int addflags)
 	}
 
 	if (vl && htok) {
+	    int prefork_ret = 0;
 	    prefork(vl, (isstr ? (PREFORK_SINGLE|PREFORK_ASSIGN) :
-			 PREFORK_ASSIGN), NULL);
+			 PREFORK_ASSIGN), &prefork_ret);
 	    if (errflag) {
 		state->pc = opc;
 		return;
 	    }
+	    if (prefork_ret & PREFORK_KEY_VALUE)
+		myflags |= ASSPM_KEY_VALUE;
 	    if (!isstr || (isset(GLOBASSIGN) && isstr &&
 			   haswilds((char *)getdata(firstnode(vl))))) {
-		globlist(vl, 0);
+		globlist(vl, prefork_ret);
 		/* Unset the parameter to force it to be recreated
 		 * as either scalar or array depending on how many
 		 * matches were found for the glob.
 		 */
 		if (isset(GLOBASSIGN) && isstr)
-		    unsetparam(name);
-	    }
-	    if (errflag) {
-		state->pc = opc;
-		return;
+			unsetparam(name);
+		if (errflag) {
+		    state->pc = opc;
+		    return;
+		}
 	    }
 	}
 	if (isstr && (empty(vl) || !nextnode(firstnode(vl)))) {
@@ -4030,16 +3975,17 @@ execcmd_exec(Estate state, Execcmd_params eparams,
 					  EC_DUPTOK, &htok);
 			    if (asg->value.array)
 			    {
-				if (keyvalpairarray(asg->value.array, 1))
-				    asg->flags |= ASG_KEY_VALUE;
-				else if (!errflag) {
+				if (!errflag) {
+				    int prefork_ret = 0;
 				    prefork(asg->value.array, PREFORK_ASSIGN,
-					    NULL);
+					    &prefork_ret);
 				    if (errflag) {
 					state->pc = opc;
 					break;
 				    }
-				    globlist(asg->value.array, 0);
+				    if (prefork_ret & PREFORK_KEY_VALUE)
+					asg->flags |= ASG_KEY_VALUE;
+				    globlist(asg->value.array, prefork_ret);
 				}
 				if (errflag) {
 				    state->pc = opc;
diff --git a/Src/params.c b/Src/params.c
index d628ddf37..c7514de8a 100644
--- a/Src/params.c
+++ b/Src/params.c
@@ -2704,7 +2704,7 @@ setarrvalue(Value v, char **val)
 	    v->pm->gsu.a->setfn(v->pm, val);
     } else if (v->start == -1 && v->end == 0 &&
     	    PM_TYPE(v->pm->node.flags) == PM_HASHED) {
-    	arrhashsetfn(v->pm, val, 1);
+    	arrhashsetfn(v->pm, val, ASSPM_AUGMENT);
     } else if ((PM_TYPE(v->pm->node.flags) == PM_HASHED)) {
 	freearray(val);
 	zerr("%s: attempt to set slice of associative array",
@@ -3186,69 +3186,124 @@ assignaparam(char *s, char **val, int flags)
     if (flags & ASSPM_WARN)
 	check_warn_pm(v->pm, "array", created, may_warn_about_nested_vars);
 
-    if ((flags & ASSPM_KEY_VALUE) && (PM_TYPE(v->pm->node.flags) & PM_ARRAY)) {
-	/*
-	 * This is an ordinary array with key / value pairs.
-	 */
-	int maxlen, origlen;
-	char **aptr, **fullval;
-	zlong *subscripts = (zlong *)zhalloc(arrlen(val) * sizeof(zlong));
-	zlong *iptr = subscripts;
-	if (flags & ASSPM_AUGMENT) {
-	    maxlen = origlen = arrlen(v->pm->gsu.a->getfn(v->pm));
-	} else {
-	    maxlen = origlen = 0;
-	}
-	for (aptr = val; *aptr && aptr[1]; aptr += 2) {
-	    *iptr = mathevali(*aptr);
-	    if (*iptr < 0 ||
-		(!isset(KSHARRAYS) && *iptr == 0)) {
-		unqueue_signals();
-		zerr("bad subscript for direct array assignment: %s", *aptr);
+    /*
+     * At this point, we may have array entries consisting of
+     * - a Marker element --- normally allocated array entry but
+     *   with just Marker char and null
+     * - an array index element --- as normal for associative array,
+     *   but non-standard for normal array which we handle now.
+     * - a value for the indexed element.
+     * This only applies if the flag ASSPM_KEY_VALUE is passed in,
+     * indicating prefork() detected this syntax.
+     *
+     * For associative arrays we just junk the Makrer elements.
+     */
+    if (flags & ASSPM_KEY_VALUE) {
+	char **aptr;
+	if (PM_TYPE(v->pm->node.flags) & PM_ARRAY) {
+	    /*
+	     * This is an ordinary array with key / value pairs.
+	     */
+	    int maxlen, origlen, nextind;
+	    char **fullval;
+	    zlong *subscripts = (zlong *)zhalloc(arrlen(val) * sizeof(zlong));
+	    zlong *iptr = subscripts;
+	    if (flags & ASSPM_AUGMENT) {
+		maxlen = origlen = arrlen(v->pm->gsu.a->getfn(v->pm));
+	    } else {
+		maxlen = origlen = 0;
+	    }
+	    nextind = 0;
+	    for (aptr = val; *aptr; ) {
+		if (**aptr == Marker) {
+		    *iptr = mathevali(*++aptr);
+		    if (*iptr < 0 ||
+			(!isset(KSHARRAYS) && *iptr == 0)) {
+			unqueue_signals();
+			zerr("bad subscript for direct array assignment: %s", *aptr);
+			return NULL;
+		    }
+		    if (!isset(KSHARRAYS))
+			--*iptr;
+		    nextind = *iptr + 1;
+		    ++iptr;
+		    aptr += 2;
+		} else {
+		    ++nextind;
+		    ++aptr;
+		}
+		if (nextind > maxlen)
+		    maxlen = nextind;
+	    }
+	    fullval = zshcalloc((maxlen+1) * sizeof(char *));
+	    if (!fullval) {
+		zerr("array too large");
 		return NULL;
 	    }
-	    if (!isset(KSHARRAYS))
-		--*iptr;
-	    if (*iptr + 1 > maxlen)
-		maxlen = *iptr + 1;
-	    ++iptr;
-	}
-	fullval = zshcalloc((maxlen+1) * sizeof(char *));
-	if (!fullval) {
-	    zerr("array too large");
-	    return NULL;
-	}
-	fullval[maxlen] = NULL;
-	if (flags & ASSPM_AUGMENT) {
-	    char **srcptr = v->pm->gsu.a->getfn(v->pm);
-	    for (aptr = fullval; aptr <= fullval + origlen; aptr++) {
-		*aptr = ztrdup(*srcptr); 
-		srcptr++;
+	    fullval[maxlen] = NULL;
+	    if (flags & ASSPM_AUGMENT) {
+		char **srcptr = v->pm->gsu.a->getfn(v->pm);
+		for (aptr = fullval; aptr <= fullval + origlen; aptr++) {
+		    *aptr = ztrdup(*srcptr); 
+		    srcptr++;
+		}
 	    }
-	}
-	iptr = subscripts;
-	for (aptr = val; *aptr && aptr[1]; aptr += 2) {
-	    zsfree(*aptr);
-	    fullval[*iptr] = aptr[1];
-	    ++iptr;
-	}
-	if (*aptr) {		/* Shouldn't be possible */
-	    DPUTS(1, "Extra element in key / value array");
-	    zsfree(*aptr);
-	}
-	free(val);
-	for (aptr = fullval; aptr < fullval + maxlen; aptr++) {
+	    iptr = subscripts;
+	    nextind = 0;
+	    for (aptr = val; *aptr; ++aptr) {
+		if (**aptr == Marker) {
+		    zsfree(*aptr);
+		    zsfree(*++aptr); /* Index, no longer needed */
+		    fullval[*iptr] = *++aptr;
+		    nextind = *iptr + 1;
+		    ++iptr;
+		} else {
+		    fullval[nextind] = *aptr;
+		    ++nextind;
+		}
+		/* aptr now on value in both cases */
+	    }
+	    if (*aptr) {		/* Shouldn't be possible */
+		DPUTS(1, "Extra element in key / value array");
+		zsfree(*aptr);
+	    }
+	    free(val);
+	    for (aptr = fullval; aptr < fullval + maxlen; aptr++) {
+		/*
+		 * Remember we don't have sparse arrays but and they're null
+		 * terminated --- so any value we don't set has to be an
+		 * empty string.
+		 */
+		if (!*aptr)
+		    *aptr = ztrdup("");
+	    }
+	    setarrvalue(v, fullval);
+	    unqueue_signals();
+	    return v->pm;
+	} else if (PM_TYPE(v->pm->node.flags & PM_HASHED)) {
 	    /*
-	     * Remember we don't have sparse arrays but and they're null
-	     * terminated --- so any value we don't set has to be an
-	     * empty string.
+	     * We strictly enforce [key]=value syntax for associative
+	     * arrays.  Marker can only indicate a Marker / key / value
+	     * triad; it cannot be there by accident.
+	     *
+	     * It's too inefficient to strip Markers here, and they
+	     * can't be there in the other form --- so just ignore
+	     * them willy nilly lower down.
 	     */
-	    if (!*aptr)
-		*aptr = ztrdup("");
+	    for (aptr = val; *aptr; aptr += 3) {
+		if (**aptr != Marker) {
+		    unqueue_signals();
+		    freearray(val);
+		    zerr("bad [key]=value syntax for associative array");
+		    return NULL;
+		}
+	    }
+	} else {
+	    unqueue_signals();
+	    freearray(val);
+	    zerr("invalid use of [key]=value assignment syntax");
+	    return NULL;
 	}
-	setarrvalue(v, fullval);
-	unqueue_signals();
-	return v->pm;
     }
 
     if (flags & ASSPM_AUGMENT) {
@@ -3741,30 +3796,37 @@ nullsethashfn(UNUSED(Param pm), HashTable x)
 /* Function to set value of an association parameter using key/value pairs */
 
 /**/
-mod_export void
-arrhashsetfn(Param pm, char **val, int augment)
+static void
+arrhashsetfn(Param pm, char **val, int flags)
 {
     /* Best not to shortcut this by using the existing hash table,   *
      * since that could cause trouble for special hashes.  This way, *
      * it's up to pm->gsu.h->setfn() what to do.                     */
-    int alen = arrlen(val);
+    int alen = 0;
     HashTable opmtab = paramtab, ht = 0;
-    char **aptr = val;
+    char **aptr;
     Value v = (Value) hcalloc(sizeof *v);
     v->end = -1;
 
+    for (aptr = val; *aptr; ++aptr) {
+	if (**aptr != Marker)
+	    ++alen;
+    }
+
     if (alen % 2) {
 	freearray(val);
 	zerr("bad set of key/value pairs for associative array");
 	return;
     }
-    if (augment) {
+    if (flags & ASSPM_AUGMENT) {
 	ht = paramtab = pm->gsu.h->getfn(pm);
     }
-    if (alen && (!augment || !paramtab)) {
+    if (alen && (!(flags & ASSPM_AUGMENT) || !paramtab)) {
 	ht = paramtab = newparamtable(17, pm->node.nam);
     }
-    while (*aptr) {
+    for (aptr = val; *aptr; ) {
+	if (**aptr == Marker)
+	    zsfree(*aptr++);
 	/* The parameter name is ztrdup'd... */
 	v->pm = createparam(*aptr, PM_SCALAR|PM_UNSET);
 	/*
diff --git a/Src/subst.c b/Src/subst.c
index 5df2a8b2d..357dc9168 100644
--- a/Src/subst.c
+++ b/Src/subst.c
@@ -35,6 +35,41 @@
 /**/
 char nulstring[] = {Nularg, '\0'};
 
+/* Check for array assignent with entries like [key]=val.
+ *
+ * Insert Marker node, convert following nodes to list to alternate key
+ * / val form, perform appropriate substitution, and return last
+ * inserted (value) node if found.
+ *
+ * Caller to check errflag.
+ */
+
+/**/
+static LinkNode
+keyvalpairelement(LinkList list, LinkNode node)
+{
+    char *start, *end, *dat;
+
+    if ((start = (char *)getdata(node)) &&
+	start[0] == Inbrack &&
+	(end = strchr(start+1, Outbrack)) &&
+	end[1] == Equals) {
+	static char marker[2] = { Marker, '\0' };
+	*end = '\0';
+
+	dat = start + 1;
+	singsub(&dat);
+	untokenize(dat);
+	setdata(node, marker);
+	node = insertlinknode(list, node, dat);
+	dat = end + 2;
+	singsub(&dat);
+	untokenize(dat);
+	return insertlinknode(list, node, dat);
+    }
+    return NULL;
+}
+
 /* Do substitutions before fork. These are:
  *  - Process substitution: <(...), >(...), =(...)
  *  - Parameter substitution
@@ -46,17 +81,16 @@ char nulstring[] = {Nularg, '\0'};
  *
  * "flag"s contains PREFORK_* flags, defined in zsh.h.
  *
- * "ret_flags" is used to return values from nested parameter
- * substitions.  It may be NULL in which case PREFORK_SUBEXP
- * must not appear in flags; any return value from below
- * will be discarded.
+ * "ret_flags" is used to return PREFORK_* values from nested parameter
+ * substitions.  It may be NULL in which case PREFORK_SUBEXP must not
+ * appear in flags; any return value from below will be discarded.
  */
 
 /**/
 mod_export void
 prefork(LinkList list, int flags, int *ret_flags)
 {
-    LinkNode node, stop = 0;
+    LinkNode node, insnode, stop = 0;
     int keep = 0, asssub = (flags & PREFORK_TYPESET) && isset(KSHTYPESET);
     int ret_flags_local = 0;
     if (!ret_flags)
@@ -64,6 +98,14 @@ prefork(LinkList list, int flags, int *ret_flags)
 
     queue_signals();
     for (node = firstnode(list); node; incnode(node)) {
+	if ((flags & (PREFORK_SINGLE|PREFORK_ASSIGN)) == PREFORK_ASSIGN &&
+	    (insnode = keyvalpairelement(list, node))) {
+	    node = insnode;
+	    *ret_flags |= PREFORK_KEY_VALUE;
+	    continue;
+	}
+	if (errflag)
+	    return;
 	if (isset(SHFILEEXPANSION)) {
 	    /*
 	     * Here and below we avoid taking the address
@@ -400,16 +442,31 @@ quotesubst(char *str)
     return str;
 }
 
+/* Glob entries of a linked list.
+ *
+ * flags are from PREFORK_*, but only two are handled:
+ * - PREFORK_NO_UNTOK: pass into zglob() a flag saying do not untokenise.
+ * - PREFORK_KEY_VALUE: look out for Marker / Key / Value list triads
+ *   and don't glob them.  The key and value should already have
+ *   been untokenised as they are not subject to further expansion.
+ */
+
 /**/
 mod_export void
-globlist(LinkList list, int nountok)
+globlist(LinkList list, int flags)
 {
     LinkNode node, next;
 
     badcshglob = 0;
     for (node = firstnode(list); !errflag && node; node = next) {
 	next = nextnode(node);
-	zglob(list, node, nountok);
+	if ((flags & PREFORK_KEY_VALUE) &&
+	    *(char *)getdata(node) == Marker) {
+	    /* Skip key / value pair */
+	    next = nextnode(nextnode(next));
+	} else {
+	    zglob(list, node, (flags & PREFORK_NO_UNTOK) != 0);
+	}
     }
     if (noerrs)
 	badcshglob = 0;
diff --git a/Src/zsh.h b/Src/zsh.h
index 27642f239..f7242dd34 100644
--- a/Src/zsh.h
+++ b/Src/zsh.h
@@ -223,9 +223,14 @@ struct mathfunc {
  * tokens here.
  */
 /*
- * Marker used in paramsubst for rc_expand_param.
- * Also used in pattern character arrays as guaranteed not to
- * mark a character in a string.
+ * Marker is used in the following special circumstances:
+ * - In paramsubst for rc_expand_param.
+ * - In pattern character arrays as guaranteed not to mark a character in
+ *   a string.
+ * - In assignments with the ASSPM_KEY_VALUE flag set in order to
+ *   mark that there is a key / value pair following.
+ * All the above are local uses --- any case where the Marker has
+ * escaped beyond the context in question is an error.
  */
 #define Marker		((char) 0xa2)
 
@@ -1969,7 +1974,14 @@ enum {
     /* SHWORDSPLIT forced off in nested subst */
     PREFORK_NOSHWORDSPLIT = 0x20,
     /* Prefork is part of a parameter subexpression */
-    PREFORK_SUBEXP        = 0x40
+    PREFORK_SUBEXP        = 0x40,
+    /* Prefork detected an assignment list with [key]=value syntax,
+     * Only used on return from prefork, not meaningful passed down.
+     * Also used as flag to globlist.
+     */
+    PREFORK_KEY_VALUE     = 0x80,
+    /* No untokenise: used only as flag to globlist */
+    PREFORK_NO_UNTOK      = 0x100
 };
 
 /*