disallow creation of objects larger than PTRDIFF_MAX via mmap

internally, other parts of the library assume sizes don't overflow
ssize_t and/or ptrdiff_t, and the way this assumption is made valid is
by preventing creating of such large objects. malloc already does so,
but the check was missing from mmap.

this is also a quality of implementation issue: even if the
implementation internally could handle such objects, applications
could inadvertently invoke undefined behavior by subtracting pointers
within an object. it is very difficult to guard against this in
applications, so a good implementation should simply ensure that it
does not happen.

1 files changed, 5 insertions, 0 deletions

diff --git a/src/mman/mmap.c b/src/mman/mmap.c
index e99271f7..b56cff86 100644
--- a/src/mman/mmap.c
+++ b/src/mman/mmap.c
@@ -1,6 +1,7 @@
 #include <unistd.h>
 #include <sys/mman.h>
 #include <errno.h>
+#include <stdint.h>
 #include <limits.h>
 #include "syscall.h"
 #include "libc.h"
@@ -20,6 +21,10 @@ void *__mmap(void *start, size_t len, int prot, int flags, int fd, off_t off)
 		errno = EINVAL;
 		return MAP_FAILED;
 	}
+	if (len >= PTRDIFF_MAX) {
+		errno = ENOMEM;
+		return MAP_FAILED;
+	}
 	if (flags & MAP_FIXED) __vm_lock(-1);
 #ifdef SYS_mmap2
 	ret = (void *)syscall(SYS_mmap2, start, len, prot, flags, fd, off>>12);