diff options
| author | Tulio Magno Quites Machado Filho <tuliom@linux.ibm.com> | 2019-11-22 14:23:17 -0300 |
|---|---|---|
| committer | Tulio Magno Quites Machado Filho <tuliom@linux.ibm.com> | 2019-11-22 14:23:17 -0300 |
| commit | b32c2757140d192c0b377c073327573f82190b05 (patch) | |
| tree | 1d577f40aa9bb2387487da32fbf07bd6135086e1 | |
| parent | 22f3c11df1c4dec1ec5310b29f29572805526625 (diff) | |
| parent | cedb3e47c68d319607736a820da2d5b3b8ddff6f (diff) | |
| download | glibc-b32c2757140d192c0b377c073327573f82190b05.tar.gz glibc-b32c2757140d192c0b377c073327573f82190b05.tar.xz glibc-b32c2757140d192c0b377c073327573f82190b05.zip | |
Merge branch release/2.28/master into ibm/2.28/master
61 files changed, 1216 insertions, 963 deletions
diff --git a/ChangeLog b/ChangeLog index 2c82b3063a..74a337eafc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,217 @@ +2019-02-06 Stefan Liebler <stli@linux.ibm.com> + + [BZ #23403] + * nptl/allocatestack.c (allocate_stack): Align pointer pd for + TLS_TCB_AT_TP tls variant. + * nptl/tst-tls1.c: Migrate to support/test-driver.c. + Add alignment checks. + * support/Makefile (libsupport-routines): Add xposix_memalign and + xpthread_setstack. + * support/support.h: Add xposix_memalign. + * support/xthread.h: Add xpthread_attr_setstack. + * support/xposix_memalign.c: New File. + * support/xpthread_attr_setstack.c: Likewise. + +2019-06-18 Florian Weimer <fweimer@redhat.com> + + [BZ #24323] + * include/elf.h (DT_1_SUPPORTED_MASK): Include DF_1_PIE. + * elf/dl-load.c (_dl_map_object_from_fd): Check for DF_1_PIE and + fail when called from dlopen. + +2019-07-10 DJ Delorie <dj@redhat.com> + Sergei Trofimovich <slyfox@inbox.ru> + + [BZ #24696] + [BZ #24695] + * nss/nss_db/db-open.c (internal_endent): Protect against NULL + mappings. + +2019-07-01 H.J. Lu <hongjiu.lu@intel.com> + + [BZ #24259] + * elf/dl-open.c (dl_open_worker): Call _dl_open_check after + relocation. + * sysdeps/x86/Makefile (tests): Add tst-cet-legacy-5a, + tst-cet-legacy-5b, tst-cet-legacy-6a and tst-cet-legacy-6b. + (modules-names): Add tst-cet-legacy-mod-5a, tst-cet-legacy-mod-5b, + tst-cet-legacy-mod-5c, tst-cet-legacy-mod-6a, tst-cet-legacy-mod-6b + and tst-cet-legacy-mod-6c. + (CFLAGS-tst-cet-legacy-5a.c): New. + (CFLAGS-tst-cet-legacy-5b.c): Likewise. + (CFLAGS-tst-cet-legacy-mod-5a.c): Likewise. + (CFLAGS-tst-cet-legacy-mod-5b.c): Likewise. + (CFLAGS-tst-cet-legacy-mod-5c.c): Likewise. + (CFLAGS-tst-cet-legacy-6a.c): Likewise. + (CFLAGS-tst-cet-legacy-6b.c): Likewise. + (CFLAGS-tst-cet-legacy-mod-6a.c): Likewise. + (CFLAGS-tst-cet-legacy-mod-6b.c): Likewise. + (CFLAGS-tst-cet-legacy-mod-6c.c): Likewise. + ($(objpfx)tst-cet-legacy-5a): Likewise. + ($(objpfx)tst-cet-legacy-5a.out): Likewise. + ($(objpfx)tst-cet-legacy-mod-5a.so): Likewise. + ($(objpfx)tst-cet-legacy-mod-5b.so): Likewise. + ($(objpfx)tst-cet-legacy-5b): Likewise. + ($(objpfx)tst-cet-legacy-5b.out): Likewise. + (tst-cet-legacy-5b-ENV): Likewise. + ($(objpfx)tst-cet-legacy-6a): Likewise. + ($(objpfx)tst-cet-legacy-6a.out): Likewise. + ($(objpfx)tst-cet-legacy-mod-6a.so): Likewise. + ($(objpfx)tst-cet-legacy-mod-6b.so): Likewise. + ($(objpfx)tst-cet-legacy-6b): Likewise. + ($(objpfx)tst-cet-legacy-6b.out): Likewise. + (tst-cet-legacy-6b-ENV): Likewise. + * sysdeps/x86/tst-cet-legacy-5.c: New file. + * sysdeps/x86/tst-cet-legacy-5a.c: Likewise. + * sysdeps/x86/tst-cet-legacy-5b.c: Likewise. + * sysdeps/x86/tst-cet-legacy-6.c: Likewise. + * sysdeps/x86/tst-cet-legacy-6a.c: Likewise. + * sysdeps/x86/tst-cet-legacy-6b.c: Likewise. + * sysdeps/x86/tst-cet-legacy-mod-5.c: Likewise. + * sysdeps/x86/tst-cet-legacy-mod-5a.c: Likewise. + * sysdeps/x86/tst-cet-legacy-mod-5b.c: Likewise. + * sysdeps/x86/tst-cet-legacy-mod-5c.c: Likewise. + * sysdeps/x86/tst-cet-legacy-mod-6.c: Likewise. + * sysdeps/x86/tst-cet-legacy-mod-6a.c: Likewise. + * sysdeps/x86/tst-cet-legacy-mod-6b.c: Likewise. + * sysdeps/x86/tst-cet-legacy-mod-6c.c: Likewise. + +2019-08-08 Niklas Hambüchen <mail@nh2.me> + Carlos O'Donell <carlos@redhat.com> + + [BZ #24026] + * malloc/malloc.c (__malloc_info): Account for top chunk. + +2019-08-01 Florian Weimer <fweimer@redhat.com> + + [BZ #24867] + * malloc/malloc.c (__malloc_info): Remove unwanted leading + whitespace. + +2019-08-15 Florian Weimer <fweimer@redhat.com> + + * malloc/Makefile (tests): Only add tst-mxfast for + $(have-tunables). + * malloc/tst-mxfast.c: Fix copyright year. + (do_test): Fix GNU style issues. Use TEST_COMPARE instead of + assert for checks. + +2019-08-09 DJ Delorie <dj@redhat.com> + + * elf/dl-tunables.list: Add glibc.malloc.mxfast. + * manual/tunables.texi: Document it. + * malloc/malloc.c (do_set_mxfast): New. + (__libc_mallopt): Call it. + * malloc/arena.c: Add mxfast tunable. + * malloc/tst-mxfast.c: New. + * malloc/Makefile: Add it. + +2018-12-19 Andreas Schwab <schwab@suse.de> + + * nscd/connections.c (check_use): Don't abort on invalid len. + +2019-05-17 Wilco Dijkstra <wdijkstr@arm.com> + + * malloc/malloc.c (MAX_TCACHE_COUNT): Increase to UINT16_MAX. + (tcache_put): Remove redundant assert. + (tcache_get): Remove redundant asserts. + (__libc_malloc): Check tcache count is not zero. + * manual/tunables.texi (glibc.malloc.tcache_count): Update maximum. + +2019-02-04 Joseph Myers <joseph@codesourcery.com> + + * malloc/malloc.c (tcache_get): Compare tcache->counts[tc_idx] + with 0, not tcache->entries[tc_idx]. + +2019-09-13 Wilco Dijkstra <wdijkstr@arm.com> + + * string/memmem.c (__memmem): Rewrite to improve performance. + +2019-06-12 Wilco Dijkstra <wdijkstr@arm.com> + + * string/str-two-way.h (two_way_short_needle): Add inline to avoid + warning. + (two_way_long_needle): Block inlining. + * string/strstr.c (strstr2): Add new function. + (strstr3): Likewise. + (STRSTR): Completely rewrite strstr to improve performance. + +2019-09-13 Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com> + + * string/memmem.c: Use memcmp for first match. + +2019-09-13 Wilco Dijkstra <wdijkstr@arm.com> + + * string/strcasestr.c (STRCASESTR): Simplify and speedup first match. + * string/strstr.c (AVAILABLE): Likewise. + +2019-09-06 Wilco Dijkstra <wdijkstr@arm.com> + + * manual/tunables.texi (glibc.cpu.name): Add ares tunable. + * sysdeps/aarch64/multiarch/memcpy.c (__libc_memcpy): Use + __memcpy_falkor for ares. + * sysdeps/unix/sysv/linux/aarch64/cpu-features.h (IS_ARES): + Add new define. + * sysdeps/unix/sysv/linux/aarch64/cpu-features.c (cpu_list): + Add ares cpu. + +2019-07-12 Adhemerval Zanella <adhemerval.zanella@linaro.org> + + [BZ #24699] + * posix/tst-mmap-offset.c: Mention BZ #24699. + (do_test_bz21270): Rename to do_test_large_offset and use + mmap64_maximum_offset to check for maximum expected offset value. + * sysdeps/generic/mmap_info.h: New file. + * sysdeps/unix/sysv/linux/mips/mmap_info.h: Likewise. + * sysdeps/unix/sysv/linux/mmap64.c (MMAP_OFF_HIGH_MASK): Define iff + __NR_mmap2 is used. + +2019-07-12 Szabolcs Nagy <szabolcs.nagy@arm.com> + + * sysdeps/aarch64/dl-machine.h (elf_machine_lazy_rel): Check + STO_AARCH64_VARIANT_PCS and bind such symbols at load time. + +2019-06-13 Szabolcs Nagy <szabolcs.nagy@arm.com> + + * elf/elf.h (STO_AARCH64_VARIANT_PCS): Define. + (DT_AARCH64_VARIANT_PCS): Define. + +2019-06-28 Florian Weimer <fweimer@redhat.com> + + [BZ #24744] + io: Remove the copy_file_range emulation. + * sysdeps/unix/sysv/linux/copy_file_range.c (copy_file_range): Do + not define and call copy_file_range_compat. + * io/Makefile (tests-static, tests-internal): Do not add + tst-copy_file_range-compat. + * io/copy_file_range-compat.c: Remove file. + * io/copy_file_range.c (copy_file_range): Define as stub. + * io/tst-copy_file_range-compat.c: Remove file. + * io/tst-copy_file_range.c (xdevfile): Remove variable. + (typical_sizes): Update comment. Remove 16K sizes. + (maximum_offset, maximum_offset_errno, maximum_offset_hard_limit): + Remove variables. + (find_maximum_offset, pipe_as_source, pipe_as_destination) + (delayed_write_failure_beginning, delayed_write_failure_end) + (cross_device_failure, enospc_failure_1, enospc_failure) + (oappend_failure): Remove functions. + (tests): Adjust test case list. + (do_test): Remove file system search code. Check for ENOSYS from + copy_file_range. Do not free xdevfile. + * manual/llio.texi (Copying File Data): Document ENOSYS error from + copy_file_range. Do not document the EXDEV error, which future + kernels may not report. Update the wording to reflect that + further errors are possible. + * sysdeps/unix/sysv/linux/alpha/kernel-features.h + [__LINUX_KERNEL_VERSION < 0x040D00] (__ASSUME_COPY_FILE_RANGE): Do + not undefine. + * sysdeps/unix/sysv/linux/kernel-features.h + [__LINUX_KERNEL_VERSION >= 0x040500] (__ASSUME_COPY_FILE_RANGE): + Remove definition. + * sysdeps/unix/sysv/linux/microblaze/kernel-features.h + [__LINUX_KERNEL_VERSION < 0x040A00] (__ASSUME_COPY_FILE_RANGE): Do + not undefine. + 2019-06-20 Dmitry V. Levin <ldv@altlinux.org> Florian Weimer <fweimer@redhat.com> diff --git a/NEWS b/NEWS index 0f1bff99e8..f249ff690c 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,14 @@ Deprecated and removed features, and other changes affecting compatibility: HTM state is saved and restore lazily (the state being saved even when the process actually does not use HTM). +* The copy_file_range function fails with ENOSYS if the kernel does not + support the system call of the same name. Previously, user space + emulation was performed, but its behavior did not match the kernel + behavior, which was deemed too confusing. Applications which use the + copy_file_range function will have to be run on kernels which implement + the copy_file_range system call. Support for most architectures was added + in version 4.5 of the mainline Linux kernel. + The following bugs are resolved with this release: [18035] Fix pldd hang @@ -60,6 +68,7 @@ The following bugs are resolved with this release: [24161] __run_fork_handlers self-deadlocks in malloc/tst-mallocfork2 [24228] old x86 applications that use legacy libio crash on exit [24476] dlfcn: Guard __dlerror_main_freeres with __libc_once_get (once) + [24744] io: Remove the copy_file_range emulation. Security related changes: diff --git a/elf/dl-load.c b/elf/dl-load.c index c51e4b3718..162a78cb0d 100644 --- a/elf/dl-load.c +++ b/elf/dl-load.c @@ -1173,6 +1173,10 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, goto call_lose; } + /* dlopen of an executable is not valid because it is not possible + to perform proper relocations, handle static TLS, or run the + ELF constructors. For PIE, the check needs the dynamic + section, so there is another check below. */ if (__glibc_unlikely (type != ET_DYN) && __glibc_unlikely ((mode & __RTLD_OPENEXEC) == 0)) { @@ -1209,9 +1213,11 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, elf_get_dynamic_info (l, NULL); /* Make sure we are not dlopen'ing an object that has the - DF_1_NOOPEN flag set. */ - if (__glibc_unlikely (l->l_flags_1 & DF_1_NOOPEN) - && (mode & __RTLD_DLOPEN)) + DF_1_NOOPEN flag set, or a PIE object. */ + if ((__glibc_unlikely (l->l_flags_1 & DF_1_NOOPEN) + && (mode & __RTLD_DLOPEN)) + || (__glibc_unlikely (l->l_flags_1 & DF_1_PIE) + && __glibc_unlikely ((mode & __RTLD_OPENEXEC) == 0))) { /* We are not supposed to load this object. Free all resources. */ _dl_unmap_segments (l); @@ -1222,7 +1228,11 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, if (l->l_phdr_allocated) free ((void *) l->l_phdr); - errstring = N_("shared object cannot be dlopen()ed"); + if (l->l_flags_1 & DF_1_PIE) + errstring + = N_("cannot dynamically load position-independent executable"); + else + errstring = N_("shared object cannot be dlopen()ed"); goto call_lose; } diff --git a/elf/dl-open.c b/elf/dl-open.c index f6c8ef1043..518a6cad69 100644 --- a/elf/dl-open.c +++ b/elf/dl-open.c @@ -292,8 +292,6 @@ dl_open_worker (void *a) _dl_debug_state (); LIBC_PROBE (map_complete, 3, args->nsid, r, new); - _dl_open_check (new); - /* Print scope information. */ if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_SCOPES)) _dl_show_scope (new, 0); @@ -366,6 +364,12 @@ dl_open_worker (void *a) _dl_relocate_object (l, l->l_scope, reloc_mode, 0); } + /* NB: Workaround for [BZ #20839] which doesn't remove the NODELETE + object when _dl_open_check throws an exception. Move it after + relocation to avoid leaving the NODELETE object mapped without + relocation. */ + _dl_open_check (new); + /* If the file is not loaded now as a dependency, add the search list of the newly loaded object to the scope. */ bool any_tls = false; diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list index 1f8ecb8437..1ff6fcb6f2 100644 --- a/elf/dl-tunables.list +++ b/elf/dl-tunables.list @@ -85,6 +85,11 @@ glibc { tcache_unsorted_limit { type: SIZE_T } + mxfast { + type: SIZE_T + minval: 0 + security_level: SXID_IGNORE + } } tune { hwcap_mask { diff --git a/elf/elf.h b/elf/elf.h index 7e2b072a7f..74f7f479ce 100644 --- a/elf/elf.h +++ b/elf/elf.h @@ -2847,6 +2847,13 @@ enum #define R_AARCH64_TLSDESC 1031 /* TLS Descriptor. */ #define R_AARCH64_IRELATIVE 1032 /* STT_GNU_IFUNC relocation. */ +/* AArch64 specific values for the Dyn d_tag field. */ +#define DT_AARCH64_VARIANT_PCS (DT_LOPROC + 5) +#define DT_AARCH64_NUM 6 + +/* AArch64 specific values for the st_other field. */ +#define STO_AARCH64_VARIANT_PCS 0x80 + /* ARM relocs. */ #define R_ARM_NONE 0 /* No reloc */ diff --git a/include/elf.h b/include/elf.h index ab76aafb1e..14ed67ff67 100644 --- a/include/elf.h +++ b/include/elf.h @@ -23,7 +23,7 @@ # endif # define DT_1_SUPPORTED_MASK \ (DF_1_NOW | DF_1_NODELETE | DF_1_INITFIRST | DF_1_NOOPEN \ - | DF_1_ORIGIN | DF_1_NODEFLIB) + | DF_1_ORIGIN | DF_1_NODEFLIB | DF_1_PIE) #endif /* !_ISOMAC */ #endif /* elf.h */ diff --git a/io/Makefile b/io/Makefile index ec5c6d7a2f..043f4b80ea 100644 --- a/io/Makefile +++ b/io/Makefile @@ -73,11 +73,6 @@ tests := test-utime test-stat test-stat2 test-lfs tst-getcwd \ tst-fts tst-fts-lfs tst-open-tmpfile \ tst-copy_file_range tst-getcwd-abspath \ -# This test includes the compat implementation of copy_file_range, -# which uses internal, unexported libc functions. -tests-static += tst-copy_file_range-compat -tests-internal += tst-copy_file_range-compat - # Likewise for statx, but we do not need static linking here. tests-internal += tst-statx diff --git a/io/copy_file_range-compat.c b/io/copy_file_range-compat.c deleted file mode 100644 index 4ab22cad19..0000000000 --- a/io/copy_file_range-compat.c +++ /dev/null @@ -1,160 +0,0 @@ -/* Emulation of copy_file_range. - Copyright (C) 2017-2018 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -/* The following macros should be defined before including this - file: - - COPY_FILE_RANGE_DECL Declaration specifiers for the function below. - COPY_FILE_RANGE Name of the function to define. */ - -#include <errno.h> -#include <fcntl.h> -#include <inttypes.h> -#include <limits.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <unistd.h> - -COPY_FILE_RANGE_DECL -ssize_t -COPY_FILE_RANGE (int infd, __off64_t *pinoff, - int outfd, __off64_t *poutoff, - size_t length, unsigned int flags) -{ - if (flags != 0) - { - __set_errno (EINVAL); - return -1; - } - - { - struct stat64 instat; - struct stat64 outstat; - if (fstat64 (infd, &instat) != 0 || fstat64 (outfd, &outstat) != 0) - return -1; - if (S_ISDIR (instat.st_mode) || S_ISDIR (outstat.st_mode)) - { - __set_errno (EISDIR); - return -1; - } - if (!S_ISREG (instat.st_mode) || !S_ISREG (outstat.st_mode)) - { - /* We need a regular input file so that the we can seek - backwards in case of a write failure. */ - __set_errno (EINVAL); - return -1; - } - if (instat.st_dev != outstat.st_dev) - { - /* Cross-device copies are not supported. */ - __set_errno (EXDEV); - return -1; - } - } - - /* The output descriptor must not have O_APPEND set. */ - { - int flags = __fcntl (outfd, F_GETFL); - if (flags & O_APPEND) - { - __set_errno (EBADF); - return -1; - } - } - - /* Avoid an overflow in the result. */ - if (length > SSIZE_MAX) - length = SSIZE_MAX; - - /* Main copying loop. The buffer size is arbitrary and is a - trade-off between stack size consumption, cache usage, and - amortization of system call overhead. */ - size_t copied = 0; - char buf[8192]; - while (length > 0) - { - size_t to_read = length; - if (to_read > sizeof (buf)) - to_read = sizeof (buf); - - /* Fill the buffer. */ - ssize_t read_count; - if (pinoff == NULL) - read_count = read (infd, buf, to_read); - else - read_count = __libc_pread64 (infd, buf, to_read, *pinoff); - if (read_count == 0) - /* End of file reached prematurely. */ - return copied; - if (read_count < 0) - { - if (copied > 0) - /* Report the number of bytes copied so far. */ - return copied; - return -1; - } - if (pinoff != NULL) - *pinoff += read_count; - - /* Write the buffer part which was read to the destination. */ - char *end = buf + read_count; - for (char *p = buf; p < end; ) - { - ssize_t write_count; - if (poutoff == NULL) - write_count = write (outfd, p, end - p); - else - write_count = __libc_pwrite64 (outfd, p, end - p, *poutoff); - if (write_count < 0) - { - /* Adjust the input read position to match what we have - written, so that the caller can pick up after the - error. */ - size_t written = p - buf; - /* NB: This needs to be signed so that we can form the - negative value below. */ - ssize_t overread = read_count - written; - if (pinoff == NULL) - { - if (overread > 0) - { - /* We are on an error recovery path, so we - cannot deal with failure here. */ - int save_errno = errno; - (void) __libc_lseek64 (infd, -overread, SEEK_CUR); - __set_errno (save_errno); - } - } - else /* pinoff != NULL */ - *pinoff -= overread; - - if (copied + written > 0) - /* Report the number of bytes copied so far. */ - return copied + written; - return -1; - } - p += write_count; - if (poutoff != NULL) - *poutoff += write_count; - } /* Write loop. */ - - copied += read_count; - length -= read_count; - } - return copied; -} diff --git a/io/copy_file_range.c b/io/copy_file_range.c index 98bff8bd26..59fb979773 100644 --- a/io/copy_file_range.c +++ b/io/copy_file_range.c @@ -1,5 +1,5 @@ -/* Generic implementation of copy_file_range. - Copyright (C) 2017-2018 Free Software Foundation, Inc. +/* Stub implementation of copy_file_range. + Copyright (C) 2017-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -16,7 +16,15 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ -#define COPY_FILE_RANGE_DECL -#define COPY_FILE_RANGE copy_file_range +#include <errno.h> +#include <unistd.h> -#include <io/copy_file_range-compat.c> +ssize_t +copy_file_range (int infd, __off64_t *pinoff, + int outfd, __off64_t *poutoff, + size_t length, unsigned int flags) +{ + __set_errno (ENOSYS); + return -1; +} +stub_warning (copy_file_range) diff --git a/io/tst-copy_file_range.c b/io/tst-copy_file_range.c index 3d531a1937..a9237cb384 100644 --- a/io/tst-copy_file_range.c +++ b/io/tst-copy_file_range.c @@ -1,5 +1,5 @@ /* Tests for copy_file_range. - Copyright (C) 2017-2018 Free Software Foundation, Inc. + Copyright (C) 2017-2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -20,22 +20,15 @@ #include <errno.h> #include <fcntl.h> #include <inttypes.h> -#include <libgen.h> -#include <poll.h> -#include <sched.h> #include <stdbool.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <support/check.h> -#include <support/namespace.h> #include <support/support.h> #include <support/temp_file.h> #include <support/test-driver.h> #include <support/xunistd.h> -#ifdef CLONE_NEWNS -# include <sys/mount.h> -#endif /* Boolean flags which indicate whether to use pointers with explicit output flags. */ @@ -49,10 +42,6 @@ static int infd; static char *outfile; static int outfd; -/* Like the above, but on a different file system. xdevfile can be - NULL if no suitable file system has been found. */ -static char *xdevfile; - /* Input and output offsets. Set according to do_inoff and do_outoff before the test. The offsets themselves are always set to zero. */ @@ -61,13 +50,10 @@ static off64_t *pinoff; static off64_t outoff; static off64_t *poutoff; -/* These are a collection of copy sizes used in tests. The selection - takes into account that the fallback implementation uses an - internal buffer of 8192 bytes. */ +/* These are a collection of copy sizes used in tests. */ enum { maximum_size = 99999 }; static const int typical_sizes[] = - { 0, 1, 2, 3, 1024, 2048, 4096, 8191, 8192, 8193, 16383, 16384, 16385, - maximum_size }; + { 0, 1, 2, 3, 1024, 2048, 4096, 8191, 8192, 8193, maximum_size }; /* The random contents of this array can be used as a pattern to check for correct write operations. */ @@ -76,101 +62,6 @@ static unsigned char random_data[maximum_size]; /* The size chosen by the test harness. */ static int current_size; -/* Maximum writable file offset. Updated by find_maximum_offset - below. */ -static off64_t maximum_offset; - -/* Error code when crossing the offset. */ -static int maximum_offset_errno; - -/* If true: Writes which cross the limit will fail. If false: Writes - which cross the limit will result in a partial write. */ -static bool maximum_offset_hard_limit; - -/* Fills maximum_offset etc. above. Truncates outfd as a side - effect. */ -static void -find_maximum_offset (void) -{ - xftruncate (outfd, 0); - if (maximum_offset != 0) - return; - - uint64_t upper = -1; - upper >>= 1; /* Maximum of off64_t. */ - TEST_VERIFY ((off64_t) upper > 0); - TEST_VERIFY ((off64_t) (upper + 1) < 0); - if (lseek64 (outfd, upper, SEEK_SET) >= 0) - { - if (write (outfd, "", 1) == 1) - FAIL_EXIT1 ("created a file larger than the off64_t range"); - } - - uint64_t lower = 1024 * 1024; /* A reasonable minimum file size. */ - /* Loop invariant: writing at lower succeeds, writing at upper fails. */ - while (lower + 1 < upper) - { - uint64_t middle = (lower + upper) / 2; - if (test_verbose > 0) - printf ("info: %s: remaining test range %" PRIu64 " .. %" PRIu64 - ", probe at %" PRIu64 "\n", __func__, lower, upper, middle); - xftruncate (outfd, 0); - if (lseek64 (outfd, middle, SEEK_SET) >= 0 - && write (outfd, "", 1) == 1) - lower = middle; - else - upper = middle; - } - TEST_VERIFY (lower + 1 == upper); - maximum_offset = lower; - printf ("info: maximum writable file offset: %" PRIu64 " (%" PRIx64 ")\n", - lower, lower); - - /* Check that writing at the valid offset actually works. */ - xftruncate (outfd, 0); - xlseek (outfd, lower, SEEK_SET); - TEST_COMPARE (write (outfd, "", 1), 1); - - /* Cross the boundary with a two-byte write. This can either result - in a short write, or a failure. */ - xlseek (outfd, lower, SEEK_SET); - ssize_t ret = write (outfd, " ", 2); - if (ret < 0) - { - maximum_offset_errno = errno; - maximum_offset_hard_limit = true; - } - else - maximum_offset_hard_limit = false; - - /* Check that writing at the next offset actually fails. This also - obtains the expected errno value. */ - xftruncate (outfd, 0); - const char *action; - if (lseek64 (outfd, lower + 1, SEEK_SET) != 0) - { - if (write (outfd, "", 1) != -1) - FAIL_EXIT1 ("write to impossible offset %" PRIu64 " succeeded", - lower + 1); - action = "writing"; - int errno_copy = errno; - if (maximum_offset_hard_limit) - TEST_COMPARE (errno_copy, maximum_offset_errno); - else - maximum_offset_errno = errno_copy; - } - else - { - action = "seeking"; - maximum_offset_errno = errno; - } - printf ("info: %s out of range fails with %m (%d)\n", - action, maximum_offset_errno); - - xftruncate (outfd, 0); - xlseek (outfd, 0, SEEK_SET); -} - /* Perform a copy of a file. */ static void simple_file_copy (void) @@ -247,390 +138,6 @@ simple_file_copy (void) free (bytes); } -/* Test that reading from a pipe willfails. */ -static void -pipe_as_source (void) -{ - int pipefds[2]; - xpipe (pipefds); - - for (int length = 0; length < 2; ++length) - { - if (test_verbose > 0) - printf ("info: %s: length=%d\n", __func__, length); - - /* Make sure that there is something to copy in the pipe. */ - xwrite (pipefds[1], "@", 1); - - TEST_COMPARE (copy_file_range (pipefds[0], pinoff, outfd, poutoff, - length, 0), -1); - /* Linux 4.10 and later return EINVAL. Older kernels return - EXDEV. */ - TEST_VERIFY (errno == EINVAL || errno == EXDEV); - TEST_COMPARE (inoff, 0); - TEST_COMPARE (outoff, 0); - TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), 0); - - /* Make sure that nothing was read. */ - char buf = 'A'; - TEST_COMPARE (read (pipefds[0], &buf, 1), 1); - TEST_COMPARE (buf, '@'); - } - - xclose (pipefds[0]); - xclose (pipefds[1]); -} - -/* Test that writing to a pipe fails. */ -static void -pipe_as_destination (void) -{ - /* Make sure that there is something to read in the input file. */ - xwrite (infd, "abc", 3); - xlseek (infd, 0, SEEK_SET); - - int pipefds[2]; - xpipe (pipefds); - - for (int length = 0; length < 2; ++length) - { - if (test_verbose > 0) - printf ("info: %s: length=%d\n", __func__, length); - - TEST_COMPARE (copy_file_range (infd, pinoff, pipefds[1], poutoff, - length, 0), -1); - /* Linux 4.10 and later return EINVAL. Older kernels return - EXDEV. */ - TEST_VERIFY (errno == EINVAL || errno == EXDEV); - TEST_COMPARE (inoff, 0); - TEST_COMPARE (outoff, 0); - TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 0); - - /* Make sure that nothing was written. */ - struct pollfd pollfd = { .fd = pipefds[0], .events = POLLIN, }; - TEST_COMPARE (poll (&pollfd, 1, 0), 0); - } - - xclose (pipefds[0]); - xclose (pipefds[1]); -} - -/* Test a write failure after (potentially) writing some bytes. - Failure occurs near the start of the buffer. */ -static void -delayed_write_failure_beginning (void) -{ - /* We need to write something to provoke the error. */ - if (current_size == 0) - return; - xwrite (infd, random_data, sizeof (random_data)); - xlseek (infd, 0, SEEK_SET); - - /* Write failure near the start. The actual error code varies among - file systems. */ - find_maximum_offset (); - off64_t where = maximum_offset; - - if (current_size == 1) - ++where; - outoff = where; - if (do_outoff) - xlseek (outfd, 1, SEEK_SET); - else - xlseek (outfd, where, SEEK_SET); - if (maximum_offset_hard_limit || where > maximum_offset) - { - TEST_COMPARE (copy_file_range (infd, pinoff, outfd, poutoff, - sizeof (random_data), 0), -1); - TEST_COMPARE (errno, maximum_offset_errno); - TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 0); - TEST_COMPARE (inoff, 0); - if (do_outoff) - TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), 1); - else - TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), where); - TEST_COMPARE (outoff, where); - struct stat64 st; - xfstat (outfd, &st); - TEST_COMPARE (st.st_size, 0); - } - else - { - /* The offset is not a hard limit. This means we write one - byte. */ - TEST_COMPARE (copy_file_range (infd, pinoff, outfd, poutoff, - sizeof (random_data), 0), 1); - if (do_inoff) - { - TEST_COMPARE (inoff, 1); - TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 0); - } - else - { - TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 1); - TEST_COMPARE (inoff, 0); - } - if (do_outoff) - { - TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), 1); - TEST_COMPARE (outoff, where + 1); - } - else - { - TEST_COMPARE (xlseek (outfd, 0, SEEK_CUR), where + 1); - TEST_COMPARE (outoff, where); - } - struct stat64 st; - xfstat (outfd, &st); - TEST_COMPARE (st.st_size, where + 1); - } -} - -/* Test a write failure after (potentially) writing some bytes. - Failure occurs near the end of the buffer. */ -static void -delayed_write_failure_end (void) -{ - if (current_size <= 1) - /* This would be same as the first test because there is not - enough data to write to make a difference. */ - return; - xwrite (infd, random_data, sizeof (random_data)); - xlseek (infd, 0, SEEK_SET); - - find_maximum_offset (); - off64_t where = maximum_offset - current_size + 1; - if (current_size == sizeof (random_data)) - /* Otherwise we do not reach the non-writable byte. */ - ++where; - outoff = where; - if (do_outoff) - xlseek (outfd, 1, SEEK_SET); - else - xlseek (outfd, where, SEEK_SET); - ssize_t ret = copy_file_range (infd, pinoff, outfd, poutoff, - sizeof (random_data), 0); - if (ret < 0) - { - TEST_COMPARE (ret, -1); - TEST_COMPARE (errno, maximum_offset_errno); - struct stat64 st; - xfstat (outfd, &st); - TEST_COMPARE (st.st_size, 0); - } - else - { - /* The first copy succeeded. This happens in the emulation - because the internal buffer of limited size does not - necessarily cross the off64_t boundary on the first write - operation. */ - if (test_verbose > 0) - printf ("info: copy_file_range (%zu) returned %zd\n", - sizeof (random_data), ret); - TEST_VERIFY (ret > 0); - TEST_VERIFY (ret < maximum_size); - struct stat64 st; - xfstat (outfd, &st); - TEST_COMPARE (st.st_size, where + ret); - if (do_inoff) - { - TEST_COMPARE (inoff, ret); - TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 0); - } - else - TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), ret); - - char *buffer = xmalloc (ret); - TEST_COMPARE (pread64 (outfd, buffer, ret, where), ret); - TEST_VERIFY (memcmp (buffer, random_data, ret) == 0); - free (buffer); - - /* The second copy fails. */ - TEST_COMPARE (copy_file_range (infd, pinoff, outfd, poutoff, - sizeof (random_data), 0), -1); - TEST_COMPARE (errno, maximum_offset_errno); - } -} - -/* Test a write failure across devices. */ -static void -cross_device_failure (void) -{ - if (xdevfile == NULL) - /* Subtest not supported due to missing cross-device file. */ - return; - - /* We need something to write. */ - xwrite (infd, random_data, sizeof (random_data)); - xlseek (infd, 0, SEEK_SET); - - int xdevfd = xopen (xdevfile, O_RDWR | O_LARGEFILE, 0); - TEST_COMPARE (copy_file_range (infd, pinoff, xdevfd, poutoff, - current_size, 0), -1); - TEST_COMPARE (errno, EXDEV); - TEST_COMPARE (xlseek (infd, 0, SEEK_CUR), 0); - struct stat64 st; - xfstat (xdevfd, &st); - TEST_COMPARE (st.st_size, 0); - - xclose (xdevfd); -} - -/* Try to exercise ENOSPC behavior with a tempfs file system (so that - we do not have to fill up a regular file system to get the error). - This function runs in a subprocess, so that we do not change the - mount namespace of the actual test process. */ -static void -enospc_failure_1 (void *closure) -{ -#ifdef CLONE_NEWNS - support_become_root (); - - /* Make sure that we do not alter the file system mounts of the - parents. */ - if (! support_enter_mount_namespace ()) - { - printf ("warning: ENOSPC test skipped\n"); - return; - } - - char *mountpoint = closure; - if (mount ("none", mountpoint, "tmpfs", MS_NODEV | MS_NOEXEC, - "size=500k") != 0) - { - printf ("warning: could not mount tmpfs at %s: %m\n", mountpoint); - return; - } - - /* The source file must reside on the same file system. */ - char *intmpfsfile = xasprintf ("%s/%s", mountpoint, "in"); - int intmpfsfd = xopen (intmpfsfile, O_RDWR | O_CREAT | O_LARGEFILE, 0600); - xwrite (intmpfsfd, random_data, sizeof (random_data)); - xlseek (intmpfsfd, 1, SEEK_SET); - inoff = 1; - - char *outtmpfsfile = xasprintf ("%s/%s", mountpoint, "out"); - int outtmpfsfd = xopen (outtmpfsfile, O_RDWR | O_CREAT | O_LARGEFILE, 0600); - - /* Fill the file with data until ENOSPC is reached. */ - while (true) - { - ssize_t ret = write (outtmpfsfd, random_data, sizeof (random_data)); - if (ret < 0 && errno != ENOSPC) - FAIL_EXIT1 ("write to %s: %m", outtmpfsfile); - if (ret < sizeof (random_data)) - break; - } - TEST_COMPARE (write (outtmpfsfd, "", 1), -1); - TEST_COMPARE (errno, ENOSPC); - off64_t maxsize = xlseek (outtmpfsfd, 0, SEEK_CUR); - TEST_VERIFY_EXIT (maxsize > sizeof (random_data)); - - /* Constructed the expected file contents. */ - char *expected = xmalloc (maxsize); - TEST_COMPARE (pread64 (outtmpfsfd, expected, maxsize, 0), maxsize); - /* Go back a little, so some bytes can be written. */ - enum { offset = 20000 }; - TEST_VERIFY_EXIT (offset < maxsize); - TEST_VERIFY_EXIT (offset < sizeof (random_data)); - memcpy (expected + maxsize - offset, random_data + 1, offset); - - if (do_outoff) - { - outoff = maxsize - offset; - xlseek (outtmpfsfd, 2, SEEK_SET); - } - else - xlseek (outtmpfsfd, -offset, SEEK_CUR); - - /* First call is expected to succeed because we made room for some - bytes. */ - TEST_COMPARE (copy_file_range (intmpfsfd, pinoff, outtmpfsfd, poutoff, - maximum_size, 0), offset); - if (do_inoff) - { - TEST_COMPARE (inoff, 1 + offset); - TEST_COMPARE (xlseek (intmpfsfd, 0, SEEK_CUR), 1); - } - else - TEST_COMPARE (xlseek (intmpfsfd, 0, SEEK_CUR), 1 + offset); - if (do_outoff) - { - TEST_COMPARE (outoff, maxsize); - TEST_COMPARE (xlseek (outtmpfsfd, 0, SEEK_CUR), 2); - } - else - TEST_COMPARE (xlseek (outtmpfsfd, 0, SEEK_CUR), maxsize); - struct stat64 st; - xfstat (outtmpfsfd, &st); - TEST_COMPARE (st.st_size, maxsize); - char *actual = xmalloc (st.st_size); - TEST_COMPARE (pread64 (outtmpfsfd, actual, st.st_size, 0), st.st_size); - TEST_VERIFY (memcmp (expected, actual, maxsize) == 0); - - /* Second call should fail with ENOSPC. */ - TEST_COMPARE (copy_file_range (intmpfsfd, pinoff, outtmpfsfd, poutoff, - maximum_size, 0), -1); - TEST_COMPARE (errno, ENOSPC); - - /* Offsets should be unchanged. */ - if (do_inoff) - { - TEST_COMPARE (inoff, 1 + offset); - TEST_COMPARE (xlseek (intmpfsfd, 0, SEEK_CUR), 1); - } - else - TEST_COMPARE (xlseek (intmpfsfd, 0, SEEK_CUR), 1 + offset); - if (do_outoff) - { - TEST_COMPARE (outoff, maxsize); - TEST_COMPARE (xlseek (outtmpfsfd, 0, SEEK_CUR), 2); - } - else - TEST_COMPARE (xlseek (outtmpfsfd, 0, SEEK_CUR), maxsize); - TEST_COMPARE (xlseek (outtmpfsfd, 0, SEEK_END), maxsize); - TEST_COMPARE (pread64 (outtmpfsfd, actual, maxsize, 0), maxsize); - TEST_VERIFY (memcmp (expected, actual, maxsize) == 0); - - free (actual); - free (expected); - - xclose (intmpfsfd); - xclose (outtmpfsfd); - free (intmpfsfile); - free (outtmpfsfile); - -#else /* !CLONE_NEWNS */ - puts ("warning: ENOSPC test skipped (no mount namespaces)"); -#endif -} - -/* Call enospc_failure_1 in a subprocess. */ -static void -enospc_failure (void) -{ - char *mountpoint - = support_create_temp_directory ("tst-copy_file_range-enospc-"); - support_isolate_in_subprocess (enospc_failure_1, mountpoint); - free (mountpoint); -} - -/* The target file descriptor must have O_APPEND enabled. */ -static void -oappend_failure (void) -{ - /* Add data, to make sure we do not fail because there is - insufficient input data. */ - xwrite (infd, random_data, current_size); - xlseek (infd, 0, SEEK_SET); - - xclose (outfd); - outfd = xopen (outfile, O_RDWR | O_APPEND, 0); - TEST_COMPARE (copy_file_range (infd, pinoff, outfd, poutoff, - current_size, 0), -1); - TEST_COMPARE (errno, EBADF); -} - /* Test that a short input file results in a shortened copy. */ static void short_copy (void) @@ -721,14 +228,6 @@ struct test_case static struct test_case tests[] = { { "simple_file_copy", simple_file_copy, .sizes = true }, - { "pipe_as_source", pipe_as_source, }, - { "pipe_as_destination", pipe_as_destination, }, - { "delayed_write_failure_beginning", delayed_write_failure_beginning, - .sizes = true }, - { "delayed_write_failure_end", delayed_write_failure_end, .sizes = true }, - { "cross_device_failure", cross_device_failure, .sizes = true }, - { "enospc_failure", enospc_failure, }, - { "oappend_failure", oappend_failure, .sizes = true }, { "short_copy", short_copy, .sizes = true }, }; @@ -739,53 +238,18 @@ do_test (void) *p = rand () >> 24; infd = create_temp_file ("tst-copy_file_range-in-", &infile); - xclose (create_temp_file ("tst-copy_file_range-out-", &outfile)); - - /* Try to find a different directory from the default input/output - file. */ + outfd = create_temp_file ("tst-copy_file_range-out-", &outfile); { - struct stat64 instat; - xfstat (infd, &instat); - static const char *const candidates[] = - { NULL, "/var/tmp", "/dev/shm" }; - for (const char *const *c = candidates; c < array_end (candidates); ++c) - { - const char *path = *c; - char *to_free = NULL; - if (path == NULL) - { - to_free = xreadlink ("/proc/self/exe"); - path = dirname (to_free); - } - - struct stat64 cstat; - xstat (path, &cstat); - if (cstat.st_dev == instat.st_dev) - { - free (to_free); - continue; - } - - printf ("info: using alternate temporary files directory: %s\n", path); - xdevfile = xasprintf ("%s/tst-copy_file_range-xdev-XXXXXX", path); - free (to_free); - break; - } - if (xdevfile != NULL) + ssize_t ret = copy_file_range (infd, NULL, outfd, NULL, 0, 0); + if (ret != 0) { - int xdevfd = mkstemp (xdevfile); - if (xdevfd < 0) - FAIL_EXIT1 ("mkstemp (\"%s\"): %m", xdevfile); - struct stat64 xdevst; - xfstat (xdevfd, &xdevst); - TEST_VERIFY (xdevst.st_dev != instat.st_dev); - add_temp_file (xdevfile); - xclose (xdevfd); + if (errno == ENOSYS) + FAIL_UNSUPPORTED ("copy_file_range is not support on this system"); + FAIL_EXIT1 ("copy_file_range probing call: %m"); } - else - puts ("warning: no alternate directory on different file system found"); } xclose (infd); + xclose (outfd); for (do_inoff = 0; do_inoff < 2; ++do_inoff) for (do_outoff = 0; do_outoff < 2; ++do_outoff) @@ -827,7 +291,6 @@ do_test (void) free (infile); free (outfile); - free (xdevfile); return 0; } diff --git a/malloc/Makefile b/malloc/Makefile index 388cf7e9ee..775b8db7e5 100644 --- a/malloc/Makefile +++ b/malloc/Makefile @@ -54,7 +54,7 @@ tests-internal += \ tst-dynarray-at-fail \ ifneq (no,$(have-tunables)) -tests += tst-malloc-usable-tunables +tests += tst-malloc-usable-tunables tst-mxfast tests-static += tst-malloc-usable-static-tunables endif @@ -196,6 +196,8 @@ tst-malloc-usable-static-ENV = $(tst-malloc-usable-ENV) tst-malloc-usable-tunables-ENV = GLIBC_TUNABLES=glibc.malloc.check=3 tst-malloc-usable-static-tunables-ENV = $(tst-malloc-usable-tunables-ENV) +tst-mxfast-ENV = GLIBC_TUNABLES=glibc.malloc.tcache_count=0:glibc.malloc.mxfast=0 + ifeq ($(experimental-malloc),yes) CPPFLAGS-malloc.c += -DUSE_TCACHE=1 else diff --git a/malloc/arena.c b/malloc/arena.c index 497ae475e7..2cad4344ea 100644 --- a/malloc/arena.c +++ b/malloc/arena.c @@ -237,6 +237,7 @@ TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t) TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t) TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t) #endif +TUNABLE_CALLBACK_FNDECL (set_mxfast, size_t) #else /* Initialization routine. */ #include <string.h> @@ -324,6 +325,7 @@ ptmalloc_init (void) TUNABLE_GET (tcache_unsorted_limit, size_t, TUNABLE_CALLBACK (set_tcache_unsorted_limit)); # endif + TUNABLE_GET (mxfast, size_t, TUNABLE_CALLBACK (set_mxfast)); #else const char *s = NULL; if (__glibc_likely (_environ != NULL)) diff --git a/malloc/malloc.c b/malloc/malloc.c index 0e7970001a..63a6cec350 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -321,6 +321,10 @@ __malloc_assert (const char *assertion, const char *file, unsigned int line, /* This is another arbitrary limit, which tunables can change. Each tcache bin will hold at most this number of chunks. */ # define TCACHE_FILL_COUNT 7 + +/* Maximum chunks in tcache bins for tunables. This value must fit the range + of tcache->counts[] entries, else they may overflow. */ +# define MAX_TCACHE_COUNT UINT16_MAX #endif @@ -1624,7 +1628,7 @@ static INTERNAL_SIZE_T global_max_fast; #define set_max_fast(s) \ global_max_fast = (((s) == 0) \ - ? SMALLBIN_WIDTH : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK)) + ? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK)) static inline INTERNAL_SIZE_T get_max_fast (void) @@ -2908,12 +2912,10 @@ typedef struct tcache_entry time), this is for performance reasons. */ typedef struct tcache_perthread_struct { - char counts[TCACHE_MAX_BINS]; + uint16_t counts[TCACHE_MAX_BINS]; tcache_entry *entries[TCACHE_MAX_BINS]; } tcache_perthread_struct; -#define MAX_TCACHE_COUNT 127 /* Maximum value of counts[] entries. */ - static __thread bool tcache_shutting_down = false; static __thread tcache_perthread_struct *tcache = NULL; @@ -2923,7 +2925,6 @@ static __always_inline void tcache_put (mchunkptr chunk, size_t tc_idx) { tcache_entry *e = (tcache_entry *) chunk2mem (chunk); - assert (tc_idx < TCACHE_MAX_BINS); /* Mark this chunk as "in the tcache" so the test in _int_free will detect a double free. */ @@ -2940,8 +2941,6 @@ static __always_inline void * tcache_get (size_t tc_idx) { tcache_entry *e = tcache->entries[tc_idx]; - assert (tc_idx < TCACHE_MAX_BINS); - assert (tcache->entries[tc_idx] > 0); tcache->entries[tc_idx] = e->next; --(tcache->counts[tc_idx]); e->key = NULL; @@ -3046,9 +3045,8 @@ __libc_malloc (size_t bytes) DIAG_PUSH_NEEDS_COMMENT; if (tc_idx < mp_.tcache_bins - /*&& tc_idx < TCACHE_MAX_BINS*/ /* to appease gcc */ && tcache - && tcache->entries[tc_idx] != NULL) + && tcache->counts[tc_idx] > 0) { return tcache_get (tc_idx); } @@ -5142,6 +5140,19 @@ do_set_tcache_unsorted_limit (size_t value) } #endif +static inline int +__always_inline +do_set_mxfast (size_t value) +{ + if (value >= 0 && value <= MAX_FAST_SIZE) + { + LIBC_PROBE (memory_mallopt_mxfast, 2, value, get_max_fast ()); + set_max_fast (value); + return 1; + } + return 0; +} + int __libc_mallopt (int param_number, int value) { @@ -5161,13 +5172,7 @@ __libc_mallopt (int param_number, int value) switch (param_number) { case M_MXFAST: - if (value >= 0 && value <= MAX_FAST_SIZE) - { - LIBC_PROBE (memory_mallopt_mxfast, 2, value, get_max_fast ()); - set_max_fast (value); - } - else - res = 0; + do_set_mxfast (value); break; case M_TRIM_THRESHOLD: @@ -5433,6 +5438,12 @@ __malloc_info (int options, FILE *fp) __libc_lock_lock (ar_ptr->mutex); + /* Account for top chunk. The top-most available chunk is + treated specially and is never in any bin. See "initial_top" + comments. */ + avail = chunksize (ar_ptr->top); + nblocks = 1; /* Top always exists. */ + for (size_t i = 0; i < NFASTBINS; ++i) { mchunkptr p = fastbin (ar_ptr, i); @@ -5518,7 +5529,7 @@ __malloc_info (int options, FILE *fp) for (size_t i = 0; i < nsizes; ++i) if (sizes[i].count != 0 && i != NFASTBINS) - fprintf (fp, " \ + fprintf (fp, "\ <size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n", sizes[i].from, sizes[i].to, sizes[i].total, sizes[i].count); diff --git a/malloc/tst-mxfast.c b/malloc/tst-mxfast.c new file mode 100644 index 0000000000..7a7750bc71 --- /dev/null +++ b/malloc/tst-mxfast.c @@ -0,0 +1,50 @@ +/* Test that glibc.malloc.mxfast tunable works. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* This test verifies that setting the glibc.malloc.mxfast tunable to + zero results in free'd blocks being returned to the small bins, not + the fast bins. */ + +#include <malloc.h> +#include <support/check.h> + +int +do_test (void) +{ + struct mallinfo m; + char *volatile p1; + char *volatile p2; + + /* Arbitrary value; must be in default fastbin range. */ + p1 = malloc (3); + /* Something large so that p1 isn't a "top block" */ + p2 = malloc (512); + free (p1); + + m = mallinfo (); + + /* This will fail if there are any blocks in the fastbins. */ + TEST_COMPARE (m.smblks, 0); + + /* To keep gcc happy. */ + free (p2); + + return 0; +} + +#include <support/test-driver.c> diff --git a/manual/llio.texi b/manual/llio.texi index 2733b9cb73..26f7d2cb3e 100644 --- a/manual/llio.texi +++ b/manual/llio.texi @@ -1404,10 +1404,13 @@ failure occurs. The return value is zero if the end of the input file is encountered immediately. If no bytes can be copied, to report an error, @code{copy_file_range} -returns the value @math{-1} and sets @code{errno}. The following -@code{errno} error conditions are specific to this function: +returns the value @math{-1} and sets @code{errno}. The table below +lists some of the error conditions for this function. @table @code +@item ENOSYS +The kernel does not implement the required functionality. + @item EISDIR At least one of the descriptors @var{inputfd} or @var{outputfd} refers to a directory. @@ -1437,9 +1440,6 @@ reading. The argument @var{outputfd} is not a valid file descriptor open for writing, or @var{outputfd} has been opened with @code{O_APPEND}. - -@item EXDEV -The input and output files reside on different file systems. @end table In addition, @code{copy_file_range} can fail with the error codes diff --git a/manual/tunables.texi b/manual/tunables.texi index 9dccf2ee7f..028868ded3 100644 --- a/manual/tunables.texi +++ b/manual/tunables.texi @@ -188,7 +188,7 @@ per-thread cache. The default (and maximum) value is 1032 bytes on @deftp Tunable glibc.malloc.tcache_count The maximum number of chunks of each size to cache. The default is 7. -The upper limit is 127. If set to zero, the per-thread cache is effectively +The upper limit is 65535. If set to zero, the per-thread cache is effectively disabled. The approximate maximum overhead of the per-thread cache is thus equal @@ -213,6 +213,18 @@ pre-fill the per-thread cache with. The default, or when set to zero, is no limit. @end deftp +@deftp Tunable glibc.malloc.mxfast +One of the optimizations malloc uses is to maintain a series of ``fast +bins'' that hold chunks up to a specific size. The default and +maximum size which may be held this way is 80 bytes on 32-bit systems +or 160 bytes on 64-bit systems. Applications which value size over +speed may choose to reduce the size of requests which are serviced +from fast bins with this tunable. Note that the value specified +includes malloc's internal overhead, which is normally the size of one +pointer, so add 4 on 32-bit systems or 8 on 64-bit systems to the size +passed to @code{malloc} for the largest bin size to enable. +@end deftp + @node Elision Tunables @section Elision Tunables @cindex elision tunables @@ -333,7 +345,7 @@ This tunable is specific to powerpc, powerpc64 and powerpc64le. The @code{glibc.tune.cpu=xxx} tunable allows the user to tell @theglibc{} to assume that the CPU is @code{xxx} where xxx may have one of these values: @code{generic}, @code{falkor}, @code{thunderxt88}, @code{thunderx2t99}, -@code{thunderx2t99p1}. +@code{thunderx2t99p1}, @code{ares}. This tunable is specific to aarch64. @end deftp diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c index 04e3f08465..d0971a97fd 100644 --- a/nptl/allocatestack.c +++ b/nptl/allocatestack.c @@ -572,7 +572,9 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp, /* Place the thread descriptor at the end of the stack. */ #if TLS_TCB_AT_TP - pd = (struct pthread *) ((char *) mem + size) - 1; + pd = (struct pthread *) ((((uintptr_t) mem + size) + - TLS_TCB_SIZE) + & ~__static_tls_align_m1); #elif TLS_DTV_AT_TP pd = (struct pthread *) ((((uintptr_t) mem + size - __static_tls_size) diff --git a/nptl/tst-tls1.c b/nptl/tst-tls1.c index 1295170532..573dd376ca 100644 --- a/nptl/tst-tls1.c +++ b/nptl/tst-tls1.c @@ -19,12 +19,16 @@ #include <pthread.h> #include <stdio.h> #include <stdlib.h> - +#include <stdint.h> +#include <inttypes.h> +#include <support/support.h> +#include <support/check.h> +#include <support/xthread.h> struct test_s { - int a; - int b; + __attribute__ ((aligned(0x20))) int a; + __attribute__ ((aligned(0x200))) int b; }; #define INIT_A 1 @@ -36,15 +40,34 @@ __thread struct test_s s __attribute__ ((tls_model ("initial-exec"))) = .b = INIT_B }; +/* Use noinline in combination with not static to ensure that the + alignment check is really done. Otherwise it was optimized out! */ +__attribute__ ((noinline)) void +check_alignment (const char *thr_name, const char *ptr_name, + int *ptr, int alignment) +{ + uintptr_t offset_aligment = ((uintptr_t) ptr) & (alignment - 1); + if (offset_aligment) + { + FAIL_EXIT1 ("%s (%p) is not 0x%x-byte aligned in %s thread\n", + ptr_name, ptr, alignment, thr_name); + } +} + +static void +check_s (const char *thr_name) +{ + if (s.a != INIT_A || s.b != INIT_B) + FAIL_EXIT1 ("initial value of s in %s thread wrong\n", thr_name); + + check_alignment (thr_name, "s.a", &s.a, 0x20); + check_alignment (thr_name, "s.b", &s.b, 0x200); +} static void * tf (void *arg) { - if (s.a != INIT_A || s.b != INIT_B) - { - puts ("initial value of s in child thread wrong"); - exit (1); - } + check_s ("child"); ++s.a; @@ -55,25 +78,14 @@ tf (void *arg) int do_test (void) { - if (s.a != INIT_A || s.b != INIT_B) - { - puts ("initial value of s in main thread wrong"); - exit (1); - } + check_s ("main"); pthread_attr_t a; - if (pthread_attr_init (&a) != 0) - { - puts ("attr_init failed"); - exit (1); - } + xpthread_attr_init (&a); - if (pthread_attr_setstacksize (&a, 1 * 1024 * 1024) != 0) - { - puts ("attr_setstacksize failed"); - return 1; - } +#define STACK_SIZE (1 * 1024 * 1024) + xpthread_attr_setstacksize (&a, STACK_SIZE); #define N 10 int i; @@ -83,29 +95,25 @@ do_test (void) pthread_t th[M]; int j; for (j = 0; j < M; ++j, ++s.a) - if (pthread_create (&th[j], &a, tf, NULL) != 0) - { - puts ("pthread_create failed"); - exit (1); - } + th[j] = xpthread_create (&a, tf, NULL); for (j = 0; j < M; ++j) - if (pthread_join (th[j], NULL) != 0) - { - puts ("pthread_join failed"); - exit (1); - } + xpthread_join (th[j]); } - if (pthread_attr_destroy (&a) != 0) - { - puts ("attr_destroy failed"); - exit (1); - } + /* Also check the alignment of the tls variables if a misaligned stack is + specified. */ + pthread_t th; + void *thr_stack = NULL; + thr_stack = xposix_memalign (0x200, STACK_SIZE + 1); + xpthread_attr_setstack (&a, thr_stack + 1, STACK_SIZE); + th = xpthread_create (&a, tf, NULL); + xpthread_join (th); + free (thr_stack); + + xpthread_attr_destroy (&a); return 0; } - -#define TEST_FUNCTION do_test () -#include "../test-skeleton.c" +#include <support/test-driver.c> diff --git a/nscd/connections.c b/nscd/connections.c index 47fbb9923a..9818200764 100644 --- a/nscd/connections.c +++ b/nscd/connections.c @@ -304,7 +304,8 @@ static int check_use (const char *data, nscd_ssize_t first_free, uint8_t *usemap, enum usekey use, ref_t start, size_t len) { - assert (len >= 2); + if (len < 2) + return 0; if (start > first_free || start + len > first_free || (start & BLOCK_ALIGN_M1)) diff --git a/nss/nss_db/db-open.c b/nss/nss_db/db-open.c index 8538f8e961..ac430f445a 100644 --- a/nss/nss_db/db-open.c +++ b/nss/nss_db/db-open.c @@ -63,5 +63,9 @@ internal_setent (const char *file, struct nss_db_map *mapping) void internal_endent (struct nss_db_map *mapping) { - munmap (mapping->header, mapping->len); + if (mapping->header != NULL) + { + munmap (mapping->header, mapping->len); + mapping->header = NULL; + } } diff --git a/posix/tst-mmap-offset.c b/posix/tst-mmap-offset.c index 92ea794c5a..cf17ba077c 100644 --- a/posix/tst-mmap-offset.c +++ b/posix/tst-mmap-offset.c @@ -1,4 +1,4 @@ -/* BZ #18877 and #21270 mmap offset test. +/* BZ #18877, BZ #21270, and BZ #24699 mmap offset test. Copyright (C) 2015-2018 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -24,6 +24,7 @@ #include <unistd.h> #include <errno.h> #include <sys/mman.h> +#include <mmap_info.h> #include <support/check.h> @@ -76,7 +77,7 @@ do_test_bz18877 (void) /* Check if invalid offset are handled correctly by mmap. */ static int -do_test_bz21270 (void) +do_test_large_offset (void) { /* For architectures with sizeof (off_t) < sizeof (off64_t) mmap is implemented with __SYS_mmap2 syscall and the offset is represented in @@ -90,7 +91,7 @@ do_test_bz21270 (void) const size_t length = 4096; void *addr = mmap64 (NULL, length, prot, flags, fd, offset); - if (sizeof (off_t) < sizeof (off64_t)) + if (mmap64_maximum_offset (page_shift) < UINT64_MAX) { if ((addr != MAP_FAILED) && (errno != EINVAL)) FAIL_RET ("mmap succeed"); @@ -110,7 +111,7 @@ do_test (void) int ret = 0; ret += do_test_bz18877 (); - ret += do_test_bz21270 (); + ret += do_test_large_offset (); return ret; } diff --git a/string/memmem.c b/string/memmem.c index 43efaa3fb7..7fbe1cb5d6 100644 --- a/string/memmem.c +++ b/string/memmem.c @@ -15,17 +15,13 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ -/* This particular implementation was written by Eric Blake, 2008. */ - #ifndef _LIBC # include <config.h> #endif -/* Specification of memmem. */ #include <string.h> #ifndef _LIBC -# define __builtin_expect(expr, val) (expr) # define __memmem memmem #endif @@ -36,47 +32,98 @@ #undef memmem -/* Return the first occurrence of NEEDLE in HAYSTACK. Return HAYSTACK - if NEEDLE_LEN is 0, otherwise NULL if NEEDLE is not found in - HAYSTACK. */ +/* Hash character pairs so a small shift table can be used. All bits of + p[0] are included, but not all bits from p[-1]. So if two equal hashes + match on p[-1], p[0] matches too. Hash collisions are harmless and result + in smaller shifts. */ +#define hash2(p) (((size_t)(p)[0] - ((size_t)(p)[-1] << 3)) % sizeof (shift)) + +/* Fast memmem algorithm with guaranteed linear-time performance. + Small needles up to size 2 use a dedicated linear search. Longer needles + up to size 256 use a novel modified Horspool algorithm. It hashes pairs + of characters to quickly skip past mismatches. The main search loop only + exits if the last 2 characters match, avoiding unnecessary calls to memcmp + and allowing for a larger skip if there is no match. A self-adapting + filtering check is used to quickly detect mismatches in long needles. + By limiting the needle length to 256, the shift table can be reduced to 8 + bits per entry, lowering preprocessing overhead and minimizing cache effects. + The limit also implies worst-case performance is linear. + Needles larger than 256 characters use the linear-time Two-Way algorithm. */ void * -__memmem (const void *haystack_start, size_t haystack_len, - const void *needle_start, size_t needle_len) +__memmem (const void *haystack, size_t hs_len, + const void *needle, size_t ne_len) { - /* Abstract memory is considered to be an array of 'unsigned char' values, - not an array of 'char' values. See ISO C 99 section 6.2.6.1. */ - const unsigned char *haystack = (const unsigned char *) haystack_start; - const unsigned char *needle = (const unsigned char *) needle_start; - - if (needle_len == 0) - /* The first occurrence of the empty string is deemed to occur at - the beginning of the string. */ - return (void *) haystack; - - /* Sanity check, otherwise the loop might search through the whole - memory. */ - if (__glibc_unlikely (haystack_len < needle_len)) + const unsigned char *hs = (const unsigned char *) haystack; + const unsigned char *ne = (const unsigned char *) needle; + + if (ne_len == 0) + return (void *) hs; + if (ne_len == 1) + return (void *) memchr (hs, ne[0], hs_len); + + /* Ensure haystack length is >= needle length. */ + if (hs_len < ne_len) return NULL; - /* Use optimizations in memchr when possible, to reduce the search - size of haystack using a linear algorithm with a smaller - coefficient. However, avoid memchr for long needles, since we - can often achieve sublinear performance. */ - if (needle_len < LONG_NEEDLE_THRESHOLD) + const unsigned char *end = hs + hs_len - ne_len; + + if (ne_len == 2) + { + uint32_t nw = ne[0] << 16 | ne[1], hw = hs[0] << 16 | hs[1]; + for (hs++; hs <= end && hw != nw; ) + hw = hw << 16 | *++hs; + return hw == nw ? (void *)hs - 1 : NULL; + } + + /* Use Two-Way algorithm for very long needles. */ + if (__builtin_expect (ne_len > 256, 0)) + return two_way_long_needle (hs, hs_len, ne, ne_len); + + uint8_t shift[256]; + size_t tmp, shift1; + size_t m1 = ne_len - 1; + size_t offset = 0; + + memset (shift, 0, sizeof (shift)); + for (int i = 1; i < m1; i++) + shift[hash2 (ne + i)] = i; + /* Shift1 is the amount we can skip after matching the hash of the + needle end but not the full needle. */ + shift1 = m1 - shift[hash2 (ne + m1)]; + shift[hash2 (ne + m1)] = m1; + + for ( ; hs <= end; ) { - haystack = memchr (haystack, *needle, haystack_len); - if (!haystack || __builtin_expect (needle_len == 1, 0)) - return (void *) haystack; - haystack_len -= haystack - (const unsigned char *) haystack_start; - if (haystack_len < needle_len) - return NULL; - return two_way_short_needle (haystack, haystack_len, needle, needle_len); + /* Skip past character pairs not in the needle. */ + do + { + hs += m1; + tmp = shift[hash2 (hs)]; + } + while (tmp == 0 && hs <= end); + + /* If the match is not at the end of the needle, shift to the end + and continue until we match the hash of the needle end. */ + hs -= tmp; + if (tmp < m1) + continue; + + /* Hash of the last 2 characters matches. If the needle is long, + try to quickly filter out mismatches. */ + if (m1 < 15 || memcmp (hs + offset, ne + offset, 8) == 0) + { + if (memcmp (hs, ne, m1) == 0) + return (void *) hs; + + /* Adjust filter offset when it doesn't find the mismatch. */ + offset = (offset >= 8 ? offset : m1) - 8; + } + + /* Skip based on matching the hash of the needle end. */ + hs += shift1; } - else - return two_way_long_needle (haystack, haystack_len, needle, needle_len); + return NULL; } libc_hidden_def (__memmem) weak_alias (__memmem, memmem) libc_hidden_weak (memmem) - -#undef LONG_NEEDLE_THRESHOLD diff --git a/string/str-two-way.h b/string/str-two-way.h index 523d946c59..358959bef0 100644 --- a/string/str-two-way.h +++ b/string/str-two-way.h @@ -221,7 +221,7 @@ critical_factorization (const unsigned char *needle, size_t needle_len, most 2 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching. If AVAILABLE modifies HAYSTACK_LEN (as in strstr), then at most 3 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching. */ -static RETURN_TYPE +static inline RETURN_TYPE two_way_short_needle (const unsigned char *haystack, size_t haystack_len, const unsigned char *needle, size_t needle_len) { @@ -382,8 +382,11 @@ two_way_short_needle (const unsigned char *haystack, size_t haystack_len, and sublinear performance O(HAYSTACK_LEN / NEEDLE_LEN) is possible. If AVAILABLE modifies HAYSTACK_LEN (as in strstr), then at most 3 * HAYSTACK_LEN - NEEDLE_LEN comparisons occur in searching, and - sublinear performance is not possible. */ -static RETURN_TYPE + sublinear performance is not possible. + + Since this function is large and complex, block inlining to avoid + slowing down the common case of small needles. */ +__attribute__((noinline)) static RETURN_TYPE two_way_long_needle (const unsigned char *haystack, size_t haystack_len, const unsigned char *needle, size_t needle_len) { diff --git a/string/strcasestr.c b/string/strcasestr.c index 421764bd1b..8aa76037dc 100644 --- a/string/strcasestr.c +++ b/string/strcasestr.c @@ -59,31 +59,22 @@ case-insensitive comparison. This function gives unspecified results in multibyte locales. */ char * -STRCASESTR (const char *haystack_start, const char *needle_start) +STRCASESTR (const char *haystack, const char *needle) { - const char *haystack = haystack_start; - const char *needle = needle_start; size_t needle_len; /* Length of NEEDLE. */ size_t haystack_len; /* Known minimum length of HAYSTACK. */ - bool ok = true; /* True if NEEDLE is prefix of HAYSTACK. */ - - /* Determine length of NEEDLE, and in the process, make sure - HAYSTACK is at least as long (no point processing all of a long - NEEDLE if HAYSTACK is too short). */ - while (*haystack && *needle) - { - ok &= (TOLOWER ((unsigned char) *haystack) - == TOLOWER ((unsigned char) *needle)); - haystack++; - needle++; - } - if (*needle) + + /* Handle empty NEEDLE special case. */ + if (needle[0] == '\0') + return (char *) haystack; + + /* Ensure HAYSTACK length is at least as long as NEEDLE length. + Since a match may occur early on in a huge HAYSTACK, use strnlen + and read ahead a few cachelines for improved performance. */ + needle_len = strlen (needle); + haystack_len = __strnlen (haystack, needle_len + 256); + if (haystack_len < needle_len) return NULL; - if (ok) - return (char *) haystack_start; - needle_len = needle - needle_start; - haystack = haystack_start + 1; - haystack_len = needle_len - 1; /* Perform the search. Abstract memory is considered to be an array of 'unsigned char' values, not an array of 'char' values. See @@ -91,10 +82,10 @@ STRCASESTR (const char *haystack_start, const char *needle_start) if (needle_len < LONG_NEEDLE_THRESHOLD) return two_way_short_needle ((const unsigned char *) haystack, haystack_len, - (const unsigned char *) needle_start, + (const unsigned char *) needle, needle_len); return two_way_long_needle ((const unsigned char *) haystack, haystack_len, - (const unsigned char *) needle_start, + (const unsigned char *) needle, needle_len); } diff --git a/string/strstr.c b/string/strstr.c index 79ebcc7532..7ffb18ab42 100644 --- a/string/strstr.c +++ b/string/strstr.c @@ -16,29 +16,17 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ -/* This particular implementation was written by Eric Blake, 2008. */ - #ifndef _LIBC # include <config.h> #endif -/* Specification of strstr. */ #include <string.h> -#include <stdbool.h> - -#ifndef _LIBC -# define __builtin_expect(expr, val) (expr) -#endif - #define RETURN_TYPE char * #define AVAILABLE(h, h_l, j, n_l) \ (((j) + (n_l) <= (h_l)) \ || ((h_l) += __strnlen ((void*)((h) + (h_l)), (n_l) + 512), \ (j) + (n_l) <= (h_l))) -#define CHECK_EOL (1) -#define RET0_IF_0(a) if (!a) goto ret0 -#define FASTSEARCH(S,C,N) (void*) strchr ((void*)(S), (C)) #include "str-two-way.h" #undef strstr @@ -47,48 +35,128 @@ #define STRSTR strstr #endif -/* Return the first occurrence of NEEDLE in HAYSTACK. Return HAYSTACK - if NEEDLE is empty, otherwise NULL if NEEDLE is not found in - HAYSTACK. */ +static inline char * +strstr2 (const unsigned char *hs, const unsigned char *ne) +{ + uint32_t h1 = (ne[0] << 16) | ne[1]; + uint32_t h2 = 0; + for (int c = hs[0]; h1 != h2 && c != 0; c = *++hs) + h2 = (h2 << 16) | c; + return h1 == h2 ? (char *)hs - 2 : NULL; +} + +static inline char * +strstr3 (const unsigned char *hs, const unsigned char *ne) +{ + uint32_t h1 = ((uint32_t)ne[0] << 24) | (ne[1] << 16) | (ne[2] << 8); + uint32_t h2 = 0; + for (int c = hs[0]; h1 != h2 && c != 0; c = *++hs) + h2 = (h2 | c) << 8; + return h1 == h2 ? (char *)hs - 3 : NULL; +} + +/* Hash character pairs so a small shift table can be used. All bits of + p[0] are included, but not all bits from p[-1]. So if two equal hashes + match on p[-1], p[0] matches too. Hash collisions are harmless and result + in smaller shifts. */ +#define hash2(p) (((size_t)(p)[0] - ((size_t)(p)[-1] << 3)) % sizeof (shift)) + +/* Fast strstr algorithm with guaranteed linear-time performance. + Small needles up to size 3 use a dedicated linear search. Longer needles + up to size 256 use a novel modified Horspool algorithm. It hashes pairs + of characters to quickly skip past mismatches. The main search loop only + exits if the last 2 characters match, avoiding unnecessary calls to memcmp + and allowing for a larger skip if there is no match. A self-adapting + filtering check is used to quickly detect mismatches in long needles. + By limiting the needle length to 256, the shift table can be reduced to 8 + bits per entry, lowering preprocessing overhead and minimizing cache effects. + The limit also implies worst-case performance is linear. + Needles larger than 256 characters use the linear-time Two-Way algorithm. */ char * -STRSTR (const char *haystack_start, const char *needle_start) +STRSTR (const char *haystack, const char *needle) { - const char *haystack = haystack_start; - const char *needle = needle_start; - size_t needle_len; /* Length of NEEDLE. */ - size_t haystack_len; /* Known minimum length of HAYSTACK. */ - bool ok = true; /* True if NEEDLE is prefix of HAYSTACK. */ - - /* Determine length of NEEDLE, and in the process, make sure - HAYSTACK is at least as long (no point processing all of a long - NEEDLE if HAYSTACK is too short). */ - while (*haystack && *needle) - ok &= *haystack++ == *needle++; - if (*needle) + const unsigned char *hs = (const unsigned char *) haystack; + const unsigned char *ne = (const unsigned char *) needle; + + /* Handle short needle special cases first. */ + if (ne[0] == '\0') + return (char *)hs; + hs = (const unsigned char *)strchr ((const char*)hs, ne[0]); + if (hs == NULL || ne[1] == '\0') + return (char*)hs; + if (ne[2] == '\0') + return strstr2 (hs, ne); + if (ne[3] == '\0') + return strstr3 (hs, ne); + + /* Ensure haystack length is at least as long as needle length. + Since a match may occur early on in a huge haystack, use strnlen + and read ahead a few cachelines for improved performance. */ + size_t ne_len = strlen ((const char*)ne); + size_t hs_len = __strnlen ((const char*)hs, ne_len | 512); + if (hs_len < ne_len) return NULL; - if (ok) - return (char *) haystack_start; - - /* Reduce the size of haystack using strchr, since it has a smaller - linear coefficient than the Two-Way algorithm. */ - needle_len = needle - needle_start; - haystack = strchr (haystack_start + 1, *needle_start); - if (!haystack || __builtin_expect (needle_len == 1, 0)) - return (char *) haystack; - needle -= needle_len; - haystack_len = (haystack > haystack_start + needle_len ? 1 - : needle_len + haystack_start - haystack); - - /* Perform the search. Abstract memory is considered to be an array - of 'unsigned char' values, not an array of 'char' values. See - ISO C 99 section 6.2.6.1. */ - if (needle_len < LONG_NEEDLE_THRESHOLD) - return two_way_short_needle ((const unsigned char *) haystack, - haystack_len, - (const unsigned char *) needle, needle_len); - return two_way_long_needle ((const unsigned char *) haystack, haystack_len, - (const unsigned char *) needle, needle_len); + + /* Check whether we have a match. This improves performance since we + avoid initialization overheads. */ + if (memcmp (hs, ne, ne_len) == 0) + return (char *) hs; + + /* Use Two-Way algorithm for very long needles. */ + if (__glibc_unlikely (ne_len > 256)) + return two_way_long_needle (hs, hs_len, ne, ne_len); + + const unsigned char *end = hs + hs_len - ne_len; + uint8_t shift[256]; + size_t tmp, shift1; + size_t m1 = ne_len - 1; + size_t offset = 0; + + /* Initialize bad character shift hash table. */ + memset (shift, 0, sizeof (shift)); + for (int i = 1; i < m1; i++) + shift[hash2 (ne + i)] = i; + /* Shift1 is the amount we can skip after matching the hash of the + needle end but not the full needle. */ + shift1 = m1 - shift[hash2 (ne + m1)]; + shift[hash2 (ne + m1)] = m1; + + while (1) + { + if (__glibc_unlikely (hs > end)) + { + end += __strnlen ((const char*)end + m1 + 1, 2048); + if (hs > end) + return NULL; + } + + /* Skip past character pairs not in the needle. */ + do + { + hs += m1; + tmp = shift[hash2 (hs)]; + } + while (tmp == 0 && hs <= end); + + /* If the match is not at the end of the needle, shift to the end + and continue until we match the hash of the needle end. */ + hs -= tmp; + if (tmp < m1) + continue; + + /* Hash of the last 2 characters matches. If the needle is long, + try to quickly filter out mismatches. */ + if (m1 < 15 || memcmp (hs + offset, ne + offset, 8) == 0) + { + if (memcmp (hs, ne, m1) == 0) + return (void *) hs; + + /* Adjust filter offset when it doesn't find the mismatch. */ + offset = (offset >= 8 ? offset : m1) - 8; + } + + /* Skip based on matching the hash of the needle end. */ + hs += shift1; + } } libc_hidden_builtin_def (strstr) - -#undef LONG_NEEDLE_THRESHOLD diff --git a/support/Makefile b/support/Makefile index b43fa524a7..3c0d870c96 100644 --- a/support/Makefile +++ b/support/Makefile @@ -93,10 +93,12 @@ libsupport-routines = \ xopen \ xpipe \ xpoll \ + xposix_memalign \ xpthread_attr_destroy \ xpthread_attr_init \ xpthread_attr_setdetachstate \ xpthread_attr_setguardsize \ + xpthread_attr_setstack \ xpthread_attr_setstacksize \ xpthread_barrier_destroy \ xpthread_barrier_init \ diff --git a/support/support.h b/support/support.h index 4ea92e1c21..9d95ebbea1 100644 --- a/support/support.h +++ b/support/support.h @@ -76,6 +76,7 @@ char *support_quote_string (const char *); void *xmalloc (size_t) __attribute__ ((malloc)); void *xcalloc (size_t n, size_t s) __attribute__ ((malloc)); void *xrealloc (void *p, size_t n); +void *xposix_memalign (size_t alignment, size_t n); char *xasprintf (const char *format, ...) __attribute__ ((format (printf, 1, 2), malloc)); char *xstrdup (const char *); diff --git a/io/tst-copy_file_range-compat.c b/support/xposix_memalign.c index 00c109a74d..5501a0846a 100644 --- a/io/tst-copy_file_range-compat.c +++ b/support/xposix_memalign.c @@ -1,5 +1,5 @@ -/* Test the fallback implementation of copy_file_range. - Copyright (C) 2017-2018 Free Software Foundation, Inc. +/* Error-checking wrapper for posix_memalign. + Copyright (C) 2019 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -16,15 +16,20 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ -/* Get the declaration of the official copy_of_range function. */ -#include <unistd.h> +#include <support/support.h> +#include <stdlib.h> +#include <errno.h> -/* Compile a local version of copy_file_range. */ -#define COPY_FILE_RANGE_DECL static -#define COPY_FILE_RANGE copy_file_range_compat -#include <io/copy_file_range-compat.c> +void * +xposix_memalign (size_t alignment, size_t n) +{ + void *p = NULL; -/* Re-use the test, but run it against copy_file_range_compat defined - above. */ -#define copy_file_range copy_file_range_compat -#include "tst-copy_file_range.c" + int ret = posix_memalign (&p, alignment, n); + if (ret) + { + errno = ret; + oom_error ("posix_memalign", n); + } + return p; +} diff --git a/support/xpthread_attr_setstack.c b/support/xpthread_attr_setstack.c new file mode 100644 index 0000000000..c3772e240b --- /dev/null +++ b/support/xpthread_attr_setstack.c @@ -0,0 +1,26 @@ +/* pthread_attr_setstack with error checking. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <support/xthread.h> + +void +xpthread_attr_setstack (pthread_attr_t *attr, void *stackaddr, size_t stacksize) +{ + xpthread_check_return ("pthread_attr_setstack", + pthread_attr_setstack (attr, stackaddr, stacksize)); +} diff --git a/support/xthread.h b/support/xthread.h index 1af7728087..00e2f59737 100644 --- a/support/xthread.h +++ b/support/xthread.h @@ -68,6 +68,8 @@ void xpthread_attr_destroy (pthread_attr_t *attr); void xpthread_attr_init (pthread_attr_t *attr); void xpthread_attr_setdetachstate (pthread_attr_t *attr, int detachstate); +void xpthread_attr_setstack (pthread_attr_t *attr, void *stackaddr, + size_t stacksize); void xpthread_attr_setstacksize (pthread_attr_t *attr, size_t stacksize); void xpthread_attr_setguardsize (pthread_attr_t *attr, diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h index 4935aa7c54..9617cb754f 100644 --- a/sysdeps/aarch64/dl-machine.h +++ b/sysdeps/aarch64/dl-machine.h @@ -388,10 +388,37 @@ elf_machine_lazy_rel (struct link_map *map, /* Check for unexpected PLT reloc type. */ if (__builtin_expect (r_type == AARCH64_R(JUMP_SLOT), 1)) { - if (__builtin_expect (map->l_mach.plt, 0) == 0) - *reloc_addr += l_addr; - else - *reloc_addr = map->l_mach.plt; + if (map->l_mach.plt == 0) + { + /* Prelinking. */ + *reloc_addr += l_addr; + return; + } + + if (1) /* DT_AARCH64_VARIANT_PCS is not available, so always check. */ + { + /* Check the symbol table for variant PCS symbols. */ + const Elf_Symndx symndx = ELFW (R_SYM) (reloc->r_info); + const ElfW (Sym) *symtab = + (const void *)D_PTR (map, l_info[DT_SYMTAB]); + const ElfW (Sym) *sym = &symtab[symndx]; + if (__glibc_unlikely (sym->st_other & STO_AARCH64_VARIANT_PCS)) + { + /* Avoid lazy resolution of variant PCS symbols. */ + const struct r_found_version *version = NULL; + if (map->l_info[VERSYMIDX (DT_VERSYM)] != NULL) + { + const ElfW (Half) *vernum = + (const void *)D_PTR (map, l_info[VERSYMIDX (DT_VERSYM)]); + version = &map->l_versions[vernum[symndx] & 0x7fff]; + } + elf_machine_rela (map, reloc, sym, version, reloc_addr, + skip_ifunc); + return; + } + } + + *reloc_addr = map->l_mach.plt; } else if (__builtin_expect (r_type == AARCH64_R(TLSDESC), 1)) { diff --git a/sysdeps/aarch64/multiarch/memcpy.c b/sysdeps/aarch64/multiarch/memcpy.c index 4a04a63b0f..8f5d4e7df5 100644 --- a/sysdeps/aarch64/multiarch/memcpy.c +++ b/sysdeps/aarch64/multiarch/memcpy.c @@ -36,7 +36,7 @@ extern __typeof (__redirect_memcpy) __memcpy_falkor attribute_hidden; libc_ifunc (__libc_memcpy, (IS_THUNDERX (midr) ? __memcpy_thunderx - : (IS_FALKOR (midr) || IS_PHECDA (midr) + : (IS_FALKOR (midr) || IS_PHECDA (midr) || IS_ARES (midr) ? __memcpy_falkor : (IS_THUNDERX2 (midr) || IS_THUNDERX2PA (midr) ? __memcpy_thunderx2 diff --git a/sysdeps/generic/mmap_info.h b/sysdeps/generic/mmap_info.h new file mode 100644 index 0000000000..b3087df2d3 --- /dev/null +++ b/sysdeps/generic/mmap_info.h @@ -0,0 +1,16 @@ +/* As default architectures with sizeof (off_t) < sizeof (off64_t) the mmap is + implemented with __SYS_mmap2 syscall and the offset is represented in + multiples of page size. For offset larger than + '1 << (page_shift + 8 * sizeof (off_t))' (that is, 1<<44 on system with + page size of 4096 bytes) the system call silently truncates the offset. + For this case, glibc mmap implementation returns EINVAL. */ + +/* Return the maximum value expected as offset argument in mmap64 call. */ +static inline uint64_t +mmap64_maximum_offset (long int page_shift) +{ + if (sizeof (off_t) < sizeof (off64_t)) + return (UINT64_C(1) << (page_shift + (8 * sizeof (off_t)))) - 1; + else + return UINT64_MAX; +} diff --git a/sysdeps/unix/sysv/linux/aarch64/cpu-features.c b/sysdeps/unix/sysv/linux/aarch64/cpu-features.c index 39eba0186f..56076849ca 100644 --- a/sysdeps/unix/sysv/linux/aarch64/cpu-features.c +++ b/sysdeps/unix/sysv/linux/aarch64/cpu-features.c @@ -36,6 +36,7 @@ static struct cpu_list cpu_list[] = { {"thunderx2t99", 0x431F0AF0}, {"thunderx2t99p1", 0x420F5160}, {"phecda", 0x680F0000}, + {"ares", 0x411FD0C0}, {"generic", 0x0} }; diff --git a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h index eb35adfbe9..153d258afe 100644 --- a/sysdeps/unix/sysv/linux/aarch64/cpu-features.h +++ b/sysdeps/unix/sysv/linux/aarch64/cpu-features.h @@ -51,6 +51,8 @@ #define IS_PHECDA(midr) (MIDR_IMPLEMENTOR(midr) == 'h' \ && MIDR_PARTNUM(midr) == 0x000) +#define IS_ARES(midr) (MIDR_IMPLEMENTOR(midr) == 'A' \ + && MIDR_PARTNUM(midr) == 0xd0c) struct cpu_features { diff --git a/sysdeps/unix/sysv/linux/alpha/kernel-features.h b/sysdeps/unix/sysv/linux/alpha/kernel-features.h index 402d2573d7..26344cd610 100644 --- a/sysdeps/unix/sysv/linux/alpha/kernel-features.h +++ b/sysdeps/unix/sysv/linux/alpha/kernel-features.h @@ -48,7 +48,6 @@ /* Support for copy_file_range, statx was added in kernel 4.13. */ #if __LINUX_KERNEL_VERSION < 0x040D00 # undef __ASSUME_MLOCK2 -# undef __ASSUME_COPY_FILE_RANGE # undef __ASSUME_STATX #endif diff --git a/sysdeps/unix/sysv/linux/copy_file_range.c b/sysdeps/unix/sysv/linux/copy_file_range.c index 7b1a50f752..b88b7c9e2e 100644 --- a/sysdeps/unix/sysv/linux/copy_file_range.c +++ b/sysdeps/unix/sysv/linux/copy_file_range.c @@ -20,27 +20,16 @@ #include <sysdep-cancel.h> #include <unistd.h> -/* Include the fallback implementation. */ -#ifndef __ASSUME_COPY_FILE_RANGE -#define COPY_FILE_RANGE_DECL static -#define COPY_FILE_RANGE copy_file_range_compat -#include <io/copy_file_range-compat.c> -#endif - ssize_t copy_file_range (int infd, __off64_t *pinoff, int outfd, __off64_t *poutoff, size_t length, unsigned int flags) { #ifdef __NR_copy_file_range - ssize_t ret = SYSCALL_CANCEL (copy_file_range, infd, pinoff, outfd, poutoff, - length, flags); -# ifndef __ASSUME_COPY_FILE_RANGE - if (ret == -1 && errno == ENOSYS) - ret = copy_file_range_compat (infd, pinoff, outfd, poutoff, length, flags); -# endif - return ret; -#else /* !__NR_copy_file_range */ - return copy_file_range_compat (infd, pinoff, outfd, poutoff, length, flags); + return SYSCALL_CANCEL (copy_file_range, infd, pinoff, outfd, poutoff, + length, flags); +#else + __set_errno (ENOSYS); + return -1; #endif } diff --git a/sysdeps/unix/sysv/linux/kernel-features.h b/sysdeps/unix/sysv/linux/kernel-features.h index 5543d92d7e..7a74835495 100644 --- a/sysdeps/unix/sysv/linux/kernel-features.h +++ b/sysdeps/unix/sysv/linux/kernel-features.h @@ -111,10 +111,6 @@ # define __ASSUME_MLOCK2 1 #endif -#if __LINUX_KERNEL_VERSION >= 0x040500 -# define __ASSUME_COPY_FILE_RANGE 1 -#endif - /* Support for statx was added in kernel 4.11. */ #if __LINUX_KERNEL_VERSION >= 0x040B00 # define __ASSUME_STATX 1 diff --git a/sysdeps/unix/sysv/linux/microblaze/kernel-features.h b/sysdeps/unix/sysv/linux/microblaze/kernel-features.h index e8e2ac6a87..0dab05bedc 100644 --- a/sysdeps/unix/sysv/linux/microblaze/kernel-features.h +++ b/sysdeps/unix/sysv/linux/microblaze/kernel-features.h @@ -58,9 +58,6 @@ # undef __ASSUME_EXECVEAT #endif -/* Support for the copy_file_range syscall was added in 4.10. */ -#if __LINUX_KERNEL_VERSION < 0x040A00 -# undef __ASSUME_COPY_FILE_RANGE #endif /* Support for statx was added in kernel 4.12. */ diff --git a/sysdeps/unix/sysv/linux/mips/Makefile b/sysdeps/unix/sysv/linux/mips/Makefile index 8217f42e75..03044e7365 100644 --- a/sysdeps/unix/sysv/linux/mips/Makefile +++ b/sysdeps/unix/sysv/linux/mips/Makefile @@ -63,14 +63,25 @@ sysdep-dl-routines += dl-static sysdep_routines += dl-vdso endif - -# Supporting non-executable stacks on MIPS requires changes to both -# the Linux kernel and glibc. See -# <https://sourceware.org/ml/libc-alpha/2016-01/msg00567.html> and -# <https://sourceware.org/ml/libc-alpha/2016-01/msg00719.html>. +# If the compiler doesn't use GNU.stack note, +# this test is expected to fail. +ifneq ($(mips-has-gnustack),yes) test-xfail-check-execstack = yes endif +endif ifeq ($(subdir),stdlib) gen-as-const-headers += ucontext_i.sym endif + +ifeq ($(mips-force-execstack),yes) +CFLAGS-.o += -Wa,-execstack +CFLAGS-.os += -Wa,-execstack +CFLAGS-.op += -Wa,-execstack +CFLAGS-.oS += -Wa,-execstack + +ASFLAGS-.o += -Wa,-execstack +ASFLAGS-.os += -Wa,-execstack +ASFLAGS-.op += -Wa,-execstack +ASFLAGS-.oS += -Wa,-execstack +endif diff --git a/sysdeps/unix/sysv/linux/mips/configure b/sysdeps/unix/sysv/linux/mips/configure index 1ee7f41a36..25f98e0c7b 100644 --- a/sysdeps/unix/sysv/linux/mips/configure +++ b/sysdeps/unix/sysv/linux/mips/configure @@ -475,3 +475,44 @@ if test -z "$arch_minimum_kernel"; then arch_minimum_kernel=4.5.0 fi fi + +# Check if we are supposed to run on kernels older than 4.8.0. If so, +# force executable stack to avoid potential runtime problems with fpu +# emulation. +# NOTE: The check below assumes that in absence of user-provided minumum_kernel +# we will default to arch_minimum_kernel which is currently less than 4.8.0 for +# all known configurations. If this changes, the check must be updated. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the compiler must use executable stack" >&5 +$as_echo_n "checking whether the compiler must use executable stack... " >&6; } +if ${libc_cv_mips_force_execstack+:} false; then : + $as_echo_n "(cached) " >&6 +else + libc_cv_mips_force_execstack=no + if test $libc_mips_float = hard; then + if test -n "$minimum_kernel"; then + + min_version=$((`echo "$minimum_kernel.0.0.0" | sed 's/\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\1 \* 65536 + \2 \* 256 + \3/'`)) + + if test $min_version -lt 264192; then + libc_cv_mips_force_execstack=yes + fi + else + libc_cv_mips_force_execstack=yes + fi + fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $libc_cv_mips_force_execstack" >&5 +$as_echo "$libc_cv_mips_force_execstack" >&6; } + +libc_mips_has_gnustack=$libc_cv_as_noexecstack + +if test $libc_cv_mips_force_execstack = yes; then + libc_mips_has_gnustack=no + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: forcing executable stack for pre-4.8.0 Linux kernels" >&5 +$as_echo "$as_me: WARNING: forcing executable stack for pre-4.8.0 Linux kernels" >&2;} +fi + +config_vars="$config_vars +mips-force-execstack = ${libc_cv_mips_force_execstack}" +config_vars="$config_vars +mips-has-gnustack = ${libc_mips_has_gnustack}" diff --git a/sysdeps/unix/sysv/linux/mips/configure.ac b/sysdeps/unix/sysv/linux/mips/configure.ac index 9147aa4582..3db1b32b08 100644 --- a/sysdeps/unix/sysv/linux/mips/configure.ac +++ b/sysdeps/unix/sysv/linux/mips/configure.ac @@ -134,3 +134,35 @@ if test -z "$arch_minimum_kernel"; then arch_minimum_kernel=4.5.0 fi fi + +# Check if we are supposed to run on kernels older than 4.8.0. If so, +# force executable stack to avoid potential runtime problems with fpu +# emulation. +# NOTE: The check below assumes that in absence of user-provided minumum_kernel +# we will default to arch_minimum_kernel which is currently less than 4.8.0 for +# all known configurations. If this changes, the check must be updated. +AC_CACHE_CHECK([whether the compiler must use executable stack], + libc_cv_mips_force_execstack, [dnl +libc_cv_mips_force_execstack=no + if test $libc_mips_float = hard; then + if test -n "$minimum_kernel"; then + changequote(,) + min_version=$((`echo "$minimum_kernel.0.0.0" | sed 's/\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\1 \* 65536 + \2 \* 256 + \3/'`)) + changequote([,]) + if test $min_version -lt 264192; then + libc_cv_mips_force_execstack=yes + fi + else + libc_cv_mips_force_execstack=yes + fi + fi]) + +libc_mips_has_gnustack=$libc_cv_as_noexecstack + +if test $libc_cv_mips_force_execstack = yes; then + libc_mips_has_gnustack=no + AC_MSG_WARN([forcing executable stack for pre-4.8.0 Linux kernels]) +fi + +LIBC_CONFIG_VAR([mips-force-execstack],[${libc_cv_mips_force_execstack}]) +LIBC_CONFIG_VAR([mips-has-gnustack],[${libc_mips_has_gnustack}]) diff --git a/sysdeps/unix/sysv/linux/mips/mmap_info.h b/sysdeps/unix/sysv/linux/mips/mmap_info.h new file mode 100644 index 0000000000..07c9e3a044 --- /dev/null +++ b/sysdeps/unix/sysv/linux/mips/mmap_info.h @@ -0,0 +1,13 @@ +/* mips64n32 uses __NR_mmap for mmap64 while still having sizeof (off_t) + smaller than sizeof (off64_t). So it allows mapping large offsets + using mmap64 than 32-bit archs which uses __NR_mmap2. */ + +static inline uint64_t +mmap64_maximum_offset (long int page_shift) +{ +#if _MIPS_SIM == _ABIN32 || _MIPS_SIM == _ABI64 + return UINT64_MAX; +#else + return (UINT64_C(1) << (page_shift + (8 * sizeof (off_t)))) - 1; +#endif +} diff --git a/sysdeps/unix/sysv/linux/mmap64.c b/sysdeps/unix/sysv/linux/mmap64.c index 118624185e..5d7598b4ba 100644 --- a/sysdeps/unix/sysv/linux/mmap64.c +++ b/sysdeps/unix/sysv/linux/mmap64.c @@ -23,11 +23,18 @@ #include <sysdep.h> #include <mmap_internal.h> +#ifdef __NR_mmap2 /* To avoid silent truncation of offset when using mmap2, do not accept offset larger than 1 << (page_shift + off_t bits). For archictures with 32 bits off_t and page size of 4096 it would be 1^44. */ -#define MMAP_OFF_HIGH_MASK \ +# define MMAP_OFF_HIGH_MASK \ ((-(MMAP2_PAGE_UNIT << 1) << (8 * sizeof (off_t) - 1))) +#else +/* Some ABIs might use __NR_mmap while having sizeof (off_t) smaller than + sizeof (off64_t) (currently only MIPS64n32). For this case just set + zero the higher bits so mmap with large offset does not fail. */ +# define MMAP_OFF_HIGH_MASK 0x0 +#endif #define MMAP_OFF_MASK (MMAP_OFF_HIGH_MASK | MMAP_OFF_LOW_MASK) diff --git a/sysdeps/x86/Makefile b/sysdeps/x86/Makefile index 337b0b63dc..43ad4a79ff 100644 --- a/sysdeps/x86/Makefile +++ b/sysdeps/x86/Makefile @@ -19,12 +19,17 @@ ifeq ($(subdir),elf) sysdep-dl-routines += dl-cet tests += tst-cet-legacy-1 tst-cet-legacy-2 tst-cet-legacy-2a \ - tst-cet-legacy-3 tst-cet-legacy-4 + tst-cet-legacy-3 tst-cet-legacy-4 \ + tst-cet-legacy-5a tst-cet-legacy-6a ifneq (no,$(have-tunables)) -tests += tst-cet-legacy-4a tst-cet-legacy-4b tst-cet-legacy-4c +tests += tst-cet-legacy-4a tst-cet-legacy-4b tst-cet-legacy-4c \ + tst-cet-legacy-5b tst-cet-legacy-6b endif modules-names += tst-cet-legacy-mod-1 tst-cet-legacy-mod-2 \ - tst-cet-legacy-mod-4 + tst-cet-legacy-mod-4 tst-cet-legacy-mod-5a \ + tst-cet-legacy-mod-5b tst-cet-legacy-mod-5c \ + tst-cet-legacy-mod-6a tst-cet-legacy-mod-6b \ + tst-cet-legacy-mod-6c CFLAGS-tst-cet-legacy-2.c += -fcf-protection=branch CFLAGS-tst-cet-legacy-2a.c += -fcf-protection @@ -35,6 +40,16 @@ CFLAGS-tst-cet-legacy-4.c += -fcf-protection=branch CFLAGS-tst-cet-legacy-4a.c += -fcf-protection CFLAGS-tst-cet-legacy-4b.c += -fcf-protection CFLAGS-tst-cet-legacy-mod-4.c += -fcf-protection=none +CFLAGS-tst-cet-legacy-5a.c += -fcf-protection +CFLAGS-tst-cet-legacy-5b.c += -fcf-protection +CFLAGS-tst-cet-legacy-mod-5a.c += -fcf-protection=none +CFLAGS-tst-cet-legacy-mod-5b.c += -fcf-protection +CFLAGS-tst-cet-legacy-mod-5c.c += -fcf-protection +CFLAGS-tst-cet-legacy-6a.c += -fcf-protection +CFLAGS-tst-cet-legacy-6b.c += -fcf-protection +CFLAGS-tst-cet-legacy-mod-6a.c += -fcf-protection=none +CFLAGS-tst-cet-legacy-mod-6b.c += -fcf-protection +CFLAGS-tst-cet-legacy-mod-6c.c += -fcf-protection $(objpfx)tst-cet-legacy-1: $(objpfx)tst-cet-legacy-mod-1.so \ $(objpfx)tst-cet-legacy-mod-2.so @@ -44,6 +59,17 @@ $(objpfx)tst-cet-legacy-2a: $(objpfx)tst-cet-legacy-mod-2.so $(libdl) $(objpfx)tst-cet-legacy-2a.out: $(objpfx)tst-cet-legacy-mod-1.so $(objpfx)tst-cet-legacy-4: $(libdl) $(objpfx)tst-cet-legacy-4.out: $(objpfx)tst-cet-legacy-mod-4.so +$(objpfx)tst-cet-legacy-5a: $(libdl) +$(objpfx)tst-cet-legacy-5a.out: $(objpfx)tst-cet-legacy-mod-5a.so \ + $(objpfx)tst-cet-legacy-mod-5b.so +$(objpfx)tst-cet-legacy-mod-5a.so: $(objpfx)tst-cet-legacy-mod-5c.so +$(objpfx)tst-cet-legacy-mod-5b.so: $(objpfx)tst-cet-legacy-mod-5c.so +$(objpfx)tst-cet-legacy-6a: $(libdl) +$(objpfx)tst-cet-legacy-6a.out: $(objpfx)tst-cet-legacy-mod-6a.so \ + $(objpfx)tst-cet-legacy-mod-6b.so +$(objpfx)tst-cet-legacy-mod-6a.so: $(objpfx)tst-cet-legacy-mod-6c.so +$(objpfx)tst-cet-legacy-mod-6b.so: $(objpfx)tst-cet-legacy-mod-6c.so +LDFLAGS-tst-cet-legacy-mod-6c.so = -Wl,--enable-new-dtags,-z,nodelete ifneq (no,$(have-tunables)) $(objpfx)tst-cet-legacy-4a: $(libdl) $(objpfx)tst-cet-legacy-4a.out: $(objpfx)tst-cet-legacy-mod-4.so @@ -54,6 +80,14 @@ tst-cet-legacy-4b-ENV = GLIBC_TUNABLES=glibc.tune.x86_shstk=on $(objpfx)tst-cet-legacy-4c: $(libdl) $(objpfx)tst-cet-legacy-4c.out: $(objpfx)tst-cet-legacy-mod-4.so tst-cet-legacy-4c-ENV = GLIBC_TUNABLES=glibc.tune.x86_shstk=off +$(objpfx)tst-cet-legacy-5b: $(libdl) +$(objpfx)tst-cet-legacy-5b.out: $(objpfx)tst-cet-legacy-mod-5a.so \ + $(objpfx)tst-cet-legacy-mod-5b.so +tst-cet-legacy-5b-ENV = GLIBC_TUNABLES=glibc.tune.x86_ibt=off:glibc.tune.x86_shstk=off +$(objpfx)tst-cet-legacy-6b: $(libdl) +$(objpfx)tst-cet-legacy-6b.out: $(objpfx)tst-cet-legacy-mod-6a.so \ + $(objpfx)tst-cet-legacy-mod-6b.so +tst-cet-legacy-6b-ENV = GLIBC_TUNABLES=glibc.tune.x86_ibt=off:glibc.tune.x86_shstk=off endif endif diff --git a/sysdeps/x86/tst-cet-legacy-5.c b/sysdeps/x86/tst-cet-legacy-5.c new file mode 100644 index 0000000000..fbf640f664 --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-5.c @@ -0,0 +1,76 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <dlfcn.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> + +static void +do_test_1 (const char *modname, bool fail) +{ + int (*fp) (void); + void *h; + + h = dlopen (modname, RTLD_LAZY); + if (h == NULL) + { + if (fail) + { + const char *err = dlerror (); + if (strstr (err, "shadow stack isn't enabled") == NULL) + { + printf ("incorrect dlopen '%s' error: %s\n", modname, + dlerror ()); + exit (1); + } + + return; + } + + printf ("cannot open '%s': %s\n", modname, dlerror ()); + exit (1); + } + + fp = dlsym (h, "test"); + if (fp == NULL) + { + printf ("cannot get symbol 'test': %s\n", dlerror ()); + exit (1); + } + + if (fp () != 0) + { + puts ("test () != 0"); + exit (1); + } + + dlclose (h); +} + +static int +do_test (void) +{ + do_test_1 ("tst-cet-legacy-mod-5a.so", true); + do_test_1 ("tst-cet-legacy-mod-5b.so", false); + return 0; +} + +#include <support/test-driver.c> diff --git a/sysdeps/x86/tst-cet-legacy-5a.c b/sysdeps/x86/tst-cet-legacy-5a.c new file mode 100644 index 0000000000..fc5a609dff --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-5a.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-5.c" diff --git a/sysdeps/x86/tst-cet-legacy-5b.c b/sysdeps/x86/tst-cet-legacy-5b.c new file mode 100644 index 0000000000..fc5a609dff --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-5b.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-5.c" diff --git a/sysdeps/x86/tst-cet-legacy-6.c b/sysdeps/x86/tst-cet-legacy-6.c new file mode 100644 index 0000000000..9151225264 --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-6.c @@ -0,0 +1,76 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <dlfcn.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> + +static void +do_test_1 (const char *modname, bool fail) +{ + int (*fp) (void); + void *h; + + h = dlopen (modname, RTLD_LAZY); + if (h == NULL) + { + if (fail) + { + const char *err = dlerror (); + if (strstr (err, "shadow stack isn't enabled") == NULL) + { + printf ("incorrect dlopen '%s' error: %s\n", modname, + dlerror ()); + exit (1); + } + + return; + } + + printf ("cannot open '%s': %s\n", modname, dlerror ()); + exit (1); + } + + fp = dlsym (h, "test"); + if (fp == NULL) + { + printf ("cannot get symbol 'test': %s\n", dlerror ()); + exit (1); + } + + if (fp () != 0) + { + puts ("test () != 0"); + exit (1); + } + + dlclose (h); +} + +static int +do_test (void) +{ + do_test_1 ("tst-cet-legacy-mod-6a.so", true); + do_test_1 ("tst-cet-legacy-mod-6b.so", false); + return 0; +} + +#include <support/test-driver.c> diff --git a/sysdeps/x86/tst-cet-legacy-6a.c b/sysdeps/x86/tst-cet-legacy-6a.c new file mode 100644 index 0000000000..2d1546d36b --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-6a.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-6.c" diff --git a/sysdeps/x86/tst-cet-legacy-6b.c b/sysdeps/x86/tst-cet-legacy-6b.c new file mode 100644 index 0000000000..2d1546d36b --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-6b.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-6.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-5.c b/sysdeps/x86/tst-cet-legacy-mod-5.c new file mode 100644 index 0000000000..3c1071c2ef --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-5.c @@ -0,0 +1,31 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <error.h> +#include <stdio.h> +#include <stdlib.h> + +extern void foo (void); + +int +test (void) +{ + foo (); + return 0; +} diff --git a/sysdeps/x86/tst-cet-legacy-mod-5a.c b/sysdeps/x86/tst-cet-legacy-mod-5a.c new file mode 100644 index 0000000000..daa43e4e8d --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-5a.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-5.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-5b.c b/sysdeps/x86/tst-cet-legacy-mod-5b.c new file mode 100644 index 0000000000..daa43e4e8d --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-5b.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-5.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-5c.c b/sysdeps/x86/tst-cet-legacy-mod-5c.c new file mode 100644 index 0000000000..e529a42ac0 --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-5c.c @@ -0,0 +1,36 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <stdlib.h> + +static int called = 0; + +static void +__attribute__ ((constructor)) +init (void) +{ + called = 1; +} + +void +foo (void) +{ + if (!called) + abort (); +} diff --git a/sysdeps/x86/tst-cet-legacy-mod-6.c b/sysdeps/x86/tst-cet-legacy-mod-6.c new file mode 100644 index 0000000000..3c1071c2ef --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6.c @@ -0,0 +1,31 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <error.h> +#include <stdio.h> +#include <stdlib.h> + +extern void foo (void); + +int +test (void) +{ + foo (); + return 0; +} diff --git a/sysdeps/x86/tst-cet-legacy-mod-6a.c b/sysdeps/x86/tst-cet-legacy-mod-6a.c new file mode 100644 index 0000000000..c89b8fe8ff --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6a.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-6.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-6b.c b/sysdeps/x86/tst-cet-legacy-mod-6b.c new file mode 100644 index 0000000000..c89b8fe8ff --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6b.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-6.c" diff --git a/sysdeps/x86/tst-cet-legacy-mod-6c.c b/sysdeps/x86/tst-cet-legacy-mod-6c.c new file mode 100644 index 0000000000..e529a42ac0 --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6c.c @@ -0,0 +1,36 @@ +/* Check compatibility of CET-enabled executable with dlopened legacy + shared object. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <stdlib.h> + +static int called = 0; + +static void +__attribute__ ((constructor)) +init (void) +{ + called = 1; +} + +void +foo (void) +{ + if (!called) + abort (); +} diff --git a/sysdeps/x86/tst-cet-legacy-mod-6d.c b/sysdeps/x86/tst-cet-legacy-mod-6d.c new file mode 100644 index 0000000000..eb233a1d10 --- /dev/null +++ b/sysdeps/x86/tst-cet-legacy-mod-6d.c @@ -0,0 +1 @@ +#include "tst-cet-legacy-mod-6c.c" |