diff options
Diffstat (limited to 'elf')
| -rw-r--r-- | elf/dl-misc.c | 18 | ||||
| -rw-r--r-- | elf/dl-reloc.c | 26 | ||||
| -rw-r--r-- | elf/dl-support.c | 3 | ||||
| -rw-r--r-- | elf/dl-tls.c | 139 | ||||
| -rw-r--r-- | elf/rtld.c | 24 |
5 files changed, 180 insertions, 30 deletions
diff --git a/elf/dl-misc.c b/elf/dl-misc.c index b3c56ab6b0..ec70298d42 100644 --- a/elf/dl-misc.c +++ b/elf/dl-misc.c @@ -530,12 +530,21 @@ __signal_safe_memalign (size_t boundary, size_t size) void * weak_function __signal_safe_malloc (size_t size) { + if (!GLRO(dl_async_signal_safe)) + return malloc (size); + return __signal_safe_memalign (1, size); } void weak_function __signal_safe_free (void *ptr) { + if (!GLRO(dl_async_signal_safe)) + { + free (ptr); + return; + } + if (ptr == NULL) return; @@ -549,6 +558,9 @@ __signal_safe_free (void *ptr) void * weak_function __signal_safe_realloc (void *ptr, size_t size) { + if (!GLRO(dl_async_signal_safe)) + return realloc (ptr, size); + if (size == 0) { __signal_safe_free (ptr); @@ -567,7 +579,8 @@ __signal_safe_realloc (void *ptr, size_t size) if (new_ptr == NULL) return NULL; - memcpy (new_ptr, ptr, old_size); + /* Copy over the old block (but not its header). */ + memcpy (new_ptr, ptr, old_size - sizeof (*header)); __signal_safe_free (ptr); return new_ptr; @@ -576,6 +589,9 @@ __signal_safe_realloc (void *ptr, size_t size) void * weak_function __signal_safe_calloc (size_t nmemb, size_t size) { + if (!GLRO(dl_async_signal_safe)) + return calloc (nmemb, size); + void *ptr = __signal_safe_malloc (nmemb * size); if (ptr == NULL) return NULL; diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c index 8696453566..1625f35d1b 100644 --- a/elf/dl-reloc.c +++ b/elf/dl-reloc.c @@ -72,6 +72,9 @@ _dl_try_allocate_static_tls (struct link_map *map) size_t offset = GL(dl_tls_static_used) + (freebytes - n * map->l_tls_align - map->l_tls_firstbyte_offset); + + if (!GLRO(dl_async_signal_safe)) + map->l_tls_offset = GL(dl_tls_static_used) = offset; #elif TLS_DTV_AT_TP /* dl_tls_static_used includes the TCB at the beginning. */ size_t offset = (ALIGN_UP(GL(dl_tls_static_used) @@ -83,9 +86,15 @@ _dl_try_allocate_static_tls (struct link_map *map) if (used > GL(dl_tls_static_size)) goto fail; + if (!GLRO(dl_async_signal_safe)) { + map->l_tls_offset = offset; + map->l_tls_firstbyte_offset = GL(dl_tls_static_used); + GL(dl_tls_static_used) = used; + } #else # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" #endif + if (GLRO(dl_async_signal_safe)) { /* We've computed the new value we want, now try to install it. */ ptrdiff_t val; if ((val = map->l_tls_offset) == NO_TLS_OFFSET) @@ -110,14 +119,15 @@ _dl_try_allocate_static_tls (struct link_map *map) goto fail; } /* We installed the value; now update the globals. */ -#if TLS_TCB_AT_TP +#if TLS_TCB_AT_TP // second GL(dl_tls_static_used) = offset; -#elif TLS_DTV_AT_TP +#elif TLS_DTV_AT_TP // second map->l_tls_firstbyte_offset = GL(dl_tls_static_used); GL(dl_tls_static_used) = used; -#else -# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" -#endif +#else // second +# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" // second +#endif // second + } /* If the object is not yet relocated we cannot initialize the static TLS region. Delay it. */ @@ -147,12 +157,14 @@ _dl_allocate_static_tls (struct link_map *map) (including this one) and update this map's TLS entry. A signal handler accessing TLS would try to do the same update and break. */ sigset_t old; - _dl_mask_all_signals (&old); + if (GLRO(dl_async_signal_safe)) + _dl_mask_all_signals (&old); int err = -1; if (map->l_tls_offset != FORCED_DYNAMIC_TLS_OFFSET) err = _dl_try_allocate_static_tls (map); - _dl_unmask_signals (&old); + if (GLRO(dl_async_signal_safe)) + _dl_unmask_signals (&old); if (err != 0) { _dl_signal_error (0, map->l_name, NULL, N_("\ diff --git a/elf/dl-support.c b/elf/dl-support.c index febc5e2f70..9349134fa0 100644 --- a/elf/dl-support.c +++ b/elf/dl-support.c @@ -73,6 +73,9 @@ const char *_google_exec_origin_dir; /* Nonzero if runtime lookup should not update the .got/.plt. */ int _dl_bind_not; +/* Nonzero if TLS handling should be async-signal-safe. */ +int _dl_async_signal_safe; + /* A dummy link map for the executable, used by dlopen to access the global scope. We don't export any symbols ourselves, so this can be minimal. */ static struct link_map _dl_main_map = diff --git a/elf/dl-tls.c b/elf/dl-tls.c index 5659adbba5..47242097fc 100644 --- a/elf/dl-tls.c +++ b/elf/dl-tls.c @@ -409,14 +409,14 @@ _dl_resize_dtv (dtv_t *dtv) dl-minimal.c malloc instead of the real malloc. We can't free it, we have to abandon the old storage. */ - newp = malloc ((2 + newsize) * sizeof (dtv_t)); + newp = __signal_safe_malloc ((2 + newsize) * sizeof (dtv_t)); if (newp == NULL) oom (); memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t)); } else { - newp = realloc (&dtv[-1], + newp = __signal_safe_realloc (&dtv[-1], (2 + newsize) * sizeof (dtv_t)); if (newp == NULL) oom (); @@ -534,13 +534,15 @@ _dl_allocate_tls (void *mem) } rtld_hidden_def (_dl_allocate_tls) +/* Clear the given dtv. (We have this here because __signal_safe_free is + not visible to nptl/allocatestack.c.) */ + void _dl_clear_dtv (dtv_t *dtv) { for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) - if (/*! dtv[1 + cnt].pointer.is_static */ 1 - && dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED) - __signal_safe_free (dtv[1 + cnt].pointer.val); + __signal_safe_free (dtv[1 + cnt].pointer.to_free); + memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t)); } @@ -584,8 +586,53 @@ rtld_hidden_def (_dl_deallocate_tls) # define GET_ADDR_OFFSET ti->ti_offset # endif +/* Allocate one DTV entry. */ +static struct dtv_pointer +allocate_dtv_entry (size_t alignment, size_t size) +{ + if (powerof2 (alignment) && alignment <= _Alignof (max_align_t)) + { + /* The alignment is supported by malloc. */ + void *ptr = malloc (size); + return (struct dtv_pointer) { ptr, ptr }; + } + + /* Emulate memalign to by manually aligning a pointer returned by + malloc. First compute the size with an overflow check. */ + size_t alloc_size = size + alignment; + if (alloc_size < size) + return (struct dtv_pointer) {}; + + /* Perform the allocation. This is the pointer we need to free + later. */ + void *start = malloc (alloc_size); + if (start == NULL) + return (struct dtv_pointer) {}; + + /* Find the aligned position within the larger allocation. */ + void *aligned = (void *) roundup ((uintptr_t) start, alignment); + + return (struct dtv_pointer) { .val = aligned, .to_free = start }; +} + +static struct dtv_pointer +allocate_and_init (struct link_map *map) +{ + struct dtv_pointer result = allocate_dtv_entry + (map->l_tls_align, map->l_tls_blocksize); + if (result.val == NULL) + oom (); + + /* Initialize the memory. */ + memset (__mempcpy (result.val, map->l_tls_initimage, + map->l_tls_initimage_size), + '\0', map->l_tls_blocksize - map->l_tls_initimage_size); + + return result; +} + static void -allocate_and_init (dtv_t *dtv, struct link_map *map) +signal_safe_allocate_and_init (dtv_t *dtv, struct link_map *map) { void *newp; newp = __signal_safe_memalign (map->l_tls_align, map->l_tls_blocksize); @@ -642,13 +689,18 @@ _dl_update_slotinfo (unsigned long int req_modid) size_t total = 0; sigset_t old; - _dl_mask_all_signals (&old); - /* We use the signal mask as a lock against reentrancy here. - Check that a signal taken before the lock didn't already - update us. */ - dtv = THREAD_DTV (); - if (dtv[0].counter >= listp->slotinfo[idx].gen) - goto out; + if (GLRO(dl_async_signal_safe)) { + _dl_mask_all_signals (&old); + /* We use the signal mask as a lock against reentrancy here. + Check that a signal taken before the lock didn't already + update us. */ + dtv = THREAD_DTV (); + if (dtv[0].counter >= listp->slotinfo[idx].gen) + { + _dl_unmask_signals (&old); + return the_map; + } + } /* We have to look through the entire dtv slotinfo list. */ listp = GL(dl_tls_dtv_slotinfo_list); do @@ -674,13 +726,13 @@ _dl_update_slotinfo (unsigned long int req_modid) struct link_map *map = listp->slotinfo[cnt].map; if (map == NULL) { - if (dtv[-1].counter >= total + cnt) + if (dtv[-1].counter >= modid) { /* If this modid was used at some point the memory might still be allocated. */ - __signal_safe_free (dtv[total + cnt].pointer.to_free); - dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED; - dtv[total + cnt].pointer.to_free = NULL; + __signal_safe_free (dtv[modid].pointer.to_free); + dtv[modid].pointer.val = TLS_DTV_UNALLOCATED; + dtv[modid].pointer.to_free = NULL; } continue; @@ -718,8 +770,8 @@ _dl_update_slotinfo (unsigned long int req_modid) /* This will be the new maximum generation counter. */ dtv[0].counter = new_gen; - out: - _dl_unmask_signals (&old); + if (GLRO(dl_async_signal_safe)) + _dl_unmask_signals (&old); } return the_map; @@ -745,6 +797,50 @@ tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) the_map = listp->slotinfo[idx].map; } + + if (!GLRO(dl_async_signal_safe)) { + + /* Make sure that, if a dlopen running in parallel forces the + variable into static storage, we'll wait until the address in the + static TLS block is set up, and use that. If we're undecided + yet, make sure we make the decision holding the lock as well. */ + if (__glibc_unlikely (the_map->l_tls_offset + != FORCED_DYNAMIC_TLS_OFFSET)) + { + __rtld_lock_lock_recursive (GL(dl_load_lock)); + if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET)) + { + the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET; + __rtld_lock_unlock_recursive (GL(dl_load_lock)); + } + else if (__glibc_likely (the_map->l_tls_offset + != FORCED_DYNAMIC_TLS_OFFSET)) + { +#if TLS_TCB_AT_TP + void *p = (char *) THREAD_SELF - the_map->l_tls_offset; +#elif TLS_DTV_AT_TP + void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE; +#else +# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" +#endif + __rtld_lock_unlock_recursive (GL(dl_load_lock)); + + dtv[GET_ADDR_MODULE].pointer.to_free = NULL; + dtv[GET_ADDR_MODULE].pointer.val = p; + + return (char *) p + GET_ADDR_OFFSET; + } + else + __rtld_lock_unlock_recursive (GL(dl_load_lock)); + } + struct dtv_pointer result = allocate_and_init (the_map); + dtv[GET_ADDR_MODULE].pointer = result; + assert (result.to_free != NULL); + + return (char *) result.val + GET_ADDR_OFFSET; + + } else { + sigset_t old; _dl_mask_all_signals (&old); @@ -778,11 +874,11 @@ tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) if (offset == FORCED_DYNAMIC_TLS_OFFSET) { - allocate_and_init (&dtv[GET_ADDR_MODULE], the_map); + signal_safe_allocate_and_init (&dtv[GET_ADDR_MODULE], the_map); } else { - void **pp = &dtv[GET_ADDR_MODULE].pointer.val; + void ** volatile pp = &dtv[GET_ADDR_MODULE].pointer.val; while (atomic_forced_read (*pp) == TLS_DTV_UNALLOCATED) { /* for lack of a better (safe) thing to do, just spin. @@ -803,6 +899,7 @@ tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) _dl_unmask_signals (&old); return (char *) dtv[GET_ADDR_MODULE].pointer.val + GET_ADDR_OFFSET; + } } diff --git a/elf/rtld.c b/elf/rtld.c index 9250cf4f94..f4e382d9c7 100644 --- a/elf/rtld.c +++ b/elf/rtld.c @@ -2502,6 +2502,9 @@ process_envvars (enum mode *modep) enum mode mode = normal; char *debug_output = NULL; + /* Enable async-signal-safe TLS by default. */ + GLRO(dl_async_signal_safe) = 1; + /* This is the default place for profiling data file. */ GLRO(dl_profile_output) = &"/var/tmp\0/var/profile"[__libc_enable_secure ? 9 : 0]; @@ -2525,6 +2528,10 @@ process_envvars (enum mode *modep) /* Warning level, verbose or not. */ if (memcmp (envline, "WARN", 4) == 0) GLRO(dl_verbose) = envline[5] != '\0'; +#if 0 /* enable to get runtime control over async signal safety */ + if (memcmp (envline, "SAFE", 4) == 0) + GLRO(dl_async_signal_safe) = 1; +#endif break; case 5: @@ -2537,7 +2544,12 @@ process_envvars (enum mode *modep) if (memcmp (envline, "AUDIT", 5) == 0) audit_list_string = &envline[6]; break; - +#if 0 /* enable to get runtime control over async signal safety */ + case 6: + if (memcmp (envline, "UNSAFE", 6) == 0) + GLRO(dl_async_signal_safe) = 0; + break; +#endif case 7: /* Print information about versions. */ if (memcmp (envline, "VERBOSE", 7) == 0) @@ -2666,6 +2678,16 @@ process_envvars (enum mode *modep) /* The caller wants this information. */ *modep = mode; +#if 0 /* enable this to help debug async-safe TLS */ + if (GLRO(dl_debug_mask)) + { + if (GLRO(dl_async_signal_safe)) + _dl_printf ("TLS is async-signal-safe\n"); + else + _dl_printf ("TLS is NOT async-signal-safe\n"); + } +#endif /* for async-safe TLS */ + /* Extra security for SUID binaries. Remove all dangerous environment variables. */ if (__builtin_expect (__libc_enable_secure, 0)) |