work around incorrect EPERM from mmap syscall

under some conditions, the mmap syscall wrongly fails with EPERM
instead of ENOMEM when memory is exhausted; this is probably the
result of the kernel trying to fit the allocation somewhere that
crosses into the kernel range or below mmap_min_addr. in any case it's
a conformance bug, so work around it. for now, only handle the case of
anonymous mappings with no requested address; in other cases EPERM may
be a legitimate error.

this indirectly fixes the possibility of malloc failing with the wrong
errno value.

1 files changed, 7 insertions, 2 deletions

diff --git a/src/mman/mmap.c b/src/mman/mmap.c
index 19caadbd..15924033 100644
--- a/src/mman/mmap.c
+++ b/src/mman/mmap.c
@@ -14,6 +14,7 @@ weak_alias(dummy, __vm_wait);
 
 void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off)
 {
+	long ret;
 	if (off & OFF_MASK) {
 		errno = EINVAL;
 		return MAP_FAILED;
@@ -26,10 +27,14 @@ void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off)
 		__vm_wait();
 	}
 #ifdef SYS_mmap2
-	return (void *)syscall(SYS_mmap2, start, len, prot, flags, fd, off/UNIT);
+	ret = __syscall(SYS_mmap2, start, len, prot, flags, fd, off/UNIT);
 #else
-	return (void *)syscall(SYS_mmap, start, len, prot, flags, fd, off);
+	ret = __syscall(SYS_mmap, start, len, prot, flags, fd, off);
 #endif
+	/* Fixup incorrect EPERM from kernel. */
+	if (ret == -EPERM && !start && (flags&MAP_ANON) && !(flags&MAP_FIXED))
+		ret = -ENOMEM;
+	return (void *)__syscall_ret(ret);
 }
 
 weak_alias(__mmap, mmap);