fix various bugs in new integer parser framework

1. my interpretation of subject sequence definition was wrong. adjust
parser to conform to the standard.

2. some code for handling tail overflow case was missing (forgot to
finish writing it).

3. typo (= instead of ==) caused ERANGE to wrongly behave like EINVAL

5 files changed, 15 insertions, 10 deletions

diff --git a/src/internal/intparse.c b/src/internal/intparse.c
index 21b07b74..fd403b58 100644
--- a/src/internal/intparse.c
+++ b/src/internal/intparse.c
@@ -35,6 +35,7 @@ int __intparse(struct intparse *v, const void *buf, size_t n)
 	v->cnt += n;
 	for (; n; n--, s++) switch (v->state) {
 	case 0:
+		v->err = EINVAL;
 		v->state++;
 		if (*s=='+' || *s=='-') {
 			v->neg = *s=='-';
@@ -49,6 +50,7 @@ int __intparse(struct intparse *v, const void *buf, size_t n)
 	case 2:
 		v->state++;
 		if ((!b || b==16) && (*s|32) == 'x') {
+			v->err = 0;
 			v->base = b = 16;
 			continue;
 		}
@@ -57,10 +59,11 @@ int __intparse(struct intparse *v, const void *buf, size_t n)
 	case 3:
 	firstdigit:
 		if (digits[*s] >= b) {
-			v->err = EINVAL;
-			return 0;
+			n++;
+			goto finished;
 		}
 	seconddigit:
+		v->err = 0;
 		v->state++;
 	case 4:
 		if (b==10) {
@@ -92,11 +95,11 @@ int __intparse(struct intparse *v, const void *buf, size_t n)
 		if (n && digits[*s]<b) {
 			v->err = ERANGE;
 			v->val = UINTMAX_MAX;
-
 			n--; s++;
+			for (; n && digits[*s]<b; n--, s++);
 		}
-		for (; n && digits[*s]<b; n--, s++);
 		if (!n) return 1;
+		goto finished;
 	}
 	return 1;
 finished:
diff --git a/src/stdlib/strtoimax.c b/src/stdlib/strtoimax.c
index 247f91d4..671aa287 100644
--- a/src/stdlib/strtoimax.c
+++ b/src/stdlib/strtoimax.c
@@ -25,7 +25,7 @@ intmax_t strtoimax(const char *s1, char **p, int base)
 
 	if (ip.err) {
 		errno = ip.err;
-		if (ip.err = EINVAL) return 0;
+		if (ip.err == EINVAL) return 0;
 		return ip.neg ? INTMAX_MIN : INTMAX_MAX;
 	}
 
diff --git a/src/stdlib/strtoumax.c b/src/stdlib/strtoumax.c
index a2bb4d7d..a299dc04 100644
--- a/src/stdlib/strtoumax.c
+++ b/src/stdlib/strtoumax.c
@@ -26,7 +26,7 @@ uintmax_t strtoumax(const char *s1, char **p, int base)
 
 	if (ip.err) {
 		errno = ip.err;
-		if (ip.err = EINVAL) return 0;
+		if (ip.err == EINVAL) return 0;
 		return UINTMAX_MAX;
 	}
 
diff --git a/src/stdlib/wcstoimax.c b/src/stdlib/wcstoimax.c
index 50647f62..344fe3a3 100644
--- a/src/stdlib/wcstoimax.c
+++ b/src/stdlib/wcstoimax.c
@@ -6,6 +6,7 @@
 
 intmax_t wcstoimax(const wchar_t *s, wchar_t **p, int base)
 {
+	const wchar_t *s1 = s;
 	struct intparse ip = {0};
 
 	if (p) *p = (wchar_t *)s;
@@ -21,11 +22,11 @@ intmax_t wcstoimax(const wchar_t *s, wchar_t **p, int base)
 	for (; __intparse(&ip, (char[]){(*s&-(*s<128U))}, 1); s++);
 
 	if (p && ip.err != EINVAL)
-		*p = (wchar_t *)s;
+		*p = (wchar_t *)s1 + ip.cnt;
 
 	if (ip.err) {
 		errno = ip.err;
-		if (ip.err = EINVAL) return 0;
+		if (ip.err == EINVAL) return 0;
 		return ip.neg ? INTMAX_MIN : INTMAX_MAX;
 	}
 
diff --git a/src/stdlib/wcstoumax.c b/src/stdlib/wcstoumax.c
index af7ba3dc..cee5ff7f 100644
--- a/src/stdlib/wcstoumax.c
+++ b/src/stdlib/wcstoumax.c
@@ -7,6 +7,7 @@
 
 uintmax_t wcstoumax(const wchar_t *s, wchar_t **p, int base)
 {
+	const wchar_t *s1 = s;
 	struct intparse ip = {0};
 
 	if (p) *p = (wchar_t *)s;
@@ -22,11 +23,11 @@ uintmax_t wcstoumax(const wchar_t *s, wchar_t **p, int base)
 	for (; __intparse(&ip, (char[]){(*s&-(*s<128U))}, 1); s++);
 
 	if (p && ip.err != EINVAL)
-		*p = (wchar_t *)s;
+		*p = (wchar_t *)s1 + ip.cnt;
 
 	if (ip.err) {
 		errno = ip.err;
-		if (ip.err = EINVAL) return 0;
+		if (ip.err == EINVAL) return 0;
 		return UINTMAX_MAX;
 	}