WIP delayed IFUNC relocation fw/bug21041

8 files changed, 393 insertions, 22 deletions

diff --git a/elf/Makefile b/elf/Makefile
index c7a29693bd..04c0013380 100644
--- a/elf/Makefile
+++ b/elf/Makefile
@@ -29,7 +29,7 @@ routines	= $(all-dl-routines) dl-support dl-iteratephdr \
 # The core dynamic linking functions are in libc for the static and
 # profiled libraries.
 dl-routines	= $(addprefix dl-,load lookup object reloc deps hwcaps \
-				  runtime init fini debug misc \
+				  runtime init fini debug misc ifunc \
 				  version profile conflict tls origin scope \
 				  execstack caller open close trampoline)
 ifeq (yes,$(use-ldconfig))
diff --git a/elf/dl-ifunc.c b/elf/dl-ifunc.c
new file mode 100644
index 0000000000..96e4d077d8
--- /dev/null
+++ b/elf/dl-ifunc.c
@@ -0,0 +1,192 @@
+/* Delayed IFUNC processing.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifdef HAVE_IFUNC
+
+# include <dl-ifunc.h>
+# include <errno.h>
+# include <ldsodefs.h>
+# include <sys/mman.h>
+# include <unistd.h>
+
+/* Machine-specific definitions.  */
+# include <dl-ifunc-reloc.h>
+
+/* This struct covers a whole page containing individual struct
+   dl_ifunc_relocation elements, which are allocated individually by
+   allocate_relocation below.  */
+struct dl_ifunc_relocation_array
+{
+  struct dl_ifunc_relocation_array *next;
+  struct dl_ifunc_relocation data[];
+};
+
+/* Global list of allocated arrays of struct dl_ifunc_relocation
+   elements.  */
+static struct dl_ifunc_relocation_array *array_list;
+
+/* Position of the next allocation in the current array used as the
+   source of struct dl_ifunc_relocation allocations.  */
+static struct dl_ifunc_relocation *next_alloc;
+static unsigned int remaining_alloc;
+
+/* Allocate a new struct dl_ifunc_relocation_array object.  Return the
+   first data object therein.  Update array_list, next_alloc,
+   remaining_alloc.  This function assumes taht remaining_alloc is
+   zero. */
+static struct dl_ifunc_relocation *
+allocate_array (void)
+{
+  size_t page_size = GLRO(dl_pagesize);
+  void *ptr = __mmap (NULL, page_size, PROT_READ | PROT_WRITE,
+                      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (ptr == MAP_FAILED)
+    _dl_signal_error (ENOMEM, NULL, NULL,
+                      "cannot allocate IFUNC resolver information");
+  struct dl_ifunc_relocation_array *new_head = ptr;
+  new_head->next = array_list;
+  array_list = new_head;
+  next_alloc = &new_head->data[1];
+  /* The function returns the first element of the new array.  */
+  remaining_alloc
+    = (page_size - offsetof (struct dl_ifunc_relocation_array, data))
+    / sizeof (new_head->data[0]) - 1;
+  return &new_head->data[0];
+}
+
+/* Allocate whone struct dl_ifunc_relocation element from the active
+   array.  Updated next_alloc, remaining_alloc, and potentially
+   array_list.  */
+static struct dl_ifunc_relocation *
+allocate_ifunc (void)
+{
+  if (remaining_alloc == 0)
+    return allocate_array ();
+  --remaining_alloc;
+  return next_alloc++;
+}
+
+/* Deallocate the list of array allocations starting at
+   array_list.  */
+static void
+free_allocations (void)
+{
+  size_t page_size = GLRO(dl_pagesize);
+  struct dl_ifunc_relocation_array *p = array_list;
+  while (p != NULL)
+    {
+      struct dl_ifunc_relocation_array *next = p->next;
+      munmap (p, page_size);
+      p = next;
+    }
+  /* Restart from scratch if _dl_ifunc_record_reloc is called
+     again.  */
+  array_list = NULL;
+  next_alloc = NULL;
+  remaining_alloc = 0;
+}
+
+/* Process all delayed IFUNC resolutions for IFUNC_MAP alone.  */
+static void
+apply_relocations (struct link_map *ifunc_map)
+{
+  if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS)
+      && ifunc_map->l_name != NULL)
+    _dl_debug_printf ("applying delayed IFUNC relocations against %s\n",
+                      ifunc_map->l_name);
+
+  struct dl_ifunc_relocation *p = ifunc_map->l_ifunc_relocations;
+  unsigned int count = 0;
+  while (p != NULL)
+    {
+      if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS)
+          && ifunc_map->l_name != NULL)
+        {
+          if (p->ifunc_sym == NULL)
+            _dl_debug_printf ("processing relative IFUNC relocation for %s\n",
+                              ifunc_map->l_name);
+          else
+            {
+              const char *strtab = (const char *) D_PTR (ifunc_map, l_info[DT_STRTAB]);
+              _dl_debug_printf ("binding IFUNC symbol %s in %s against %s\n",
+                                strtab + p->ifunc_sym->st_name,
+                                p->reloc_map->l_name, ifunc_map->l_name);
+            }
+        }
+      _dl_ifunc_process_relocation (p, ifunc_map);
+      p = p->next;
+      ++count;
+    }
+
+  if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS)
+      && ifunc_map->l_name != NULL)
+    _dl_debug_printf ("%u delayed IFUNC relocations performed against %s\n",
+                      count, ifunc_map->l_name);
+
+  /* No deallocation takes place here.  All allocated objects are
+     freed in bulk by a later call to free_allocations.  */
+  ifunc_map->l_ifunc_relocations = NULL;
+}
+
+void internal_function
+_dl_ifunc_record_reloc (struct link_map *reloc_map,
+                        const ElfW(Rela) *reloc,
+                        ElfW(Addr) *reloc_addr,
+                        struct link_map *ifunc_map,
+                        const ElfW(Sym) *ifunc_sym)
+{
+  /* Add the delayed IFUNC relocation to the list for ifunc_map.  */
+  struct dl_ifunc_relocation *ifunc = allocate_ifunc ();
+  *ifunc = (struct dl_ifunc_relocation)
+    {
+      .reloc_map = reloc_map,
+      .reloc = reloc,
+      .reloc_addr = reloc_addr,
+      .ifunc_sym = ifunc_sym,
+      .next = ifunc_map->l_ifunc_relocations,
+    };
+  ifunc_map->l_ifunc_relocations = ifunc;
+}
+
+void internal_function
+_dl_ifunc_apply_relocations (struct link_map *new)
+{
+  /* Traverse the search list in reverse order, in case an IFUNC
+     resolver calls into one its dependencies, and those dependency
+     needs IFUNC resolution as well.  */
+  struct link_map **r_list = new->l_searchlist.r_list;
+  for (unsigned int i = new->l_searchlist.r_nlist; i-- > 0; )
+    {
+      struct link_map *l = r_list[i];
+      if (l->l_ifunc_relocations != NULL)
+        apply_relocations (l);
+    }
+  free_allocations ();
+}
+
+void internal_function
+_dl_ifunc_clear_relocations (struct link_map *map)
+{
+  Lmid_t nsid = map->l_ns;
+  struct link_namespaces *ns = &GL(dl_ns)[nsid];
+  for (struct link_map *l = ns->_ns_loaded; l != NULL; l = l->l_next)
+    l->l_ifunc_relocations = NULL;
+  free_allocations ();
+}
+
+#endif  /* HAVE_IFUNC */
diff --git a/elf/dl-ifunc.h b/elf/dl-ifunc.h
new file mode 100644
index 0000000000..39a5c12c73
--- /dev/null
+++ b/elf/dl-ifunc.h
@@ -0,0 +1,88 @@
+/* Private declarations for delayed IFUNC processing.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef DL_IFUNC_H
+#define DL_IFUNC_H
+
+#include <link.h>
+
+/* All functions declared in this file must be called while the global
+   rtld lock is acquired.  */
+
+/* Apply all pending IFUNC relocations in the search scope of the new
+   link map.  Deallocate all auxiliary allocations.  */
+void _dl_ifunc_apply_relocations (struct link_map *new)
+  attribute_hidden internal_function;
+
+/* Clear all allocated delayed IFUNC relocations in the namespace of
+   the link map.  Deallocate all auxiliary allocations.  This function
+   is intended for clearing up after a dlopen failure.  */
+void _dl_ifunc_clear_relocations (struct link_map *map)
+  attribute_hidden internal_function;
+
+/* The no-op implementatoin is only available if IFUNCs are
+   supported.  */
+#ifdef HAVE_IFUNC
+
+/* Delayed IFUNC relocation.  For each link map with a referenced
+   IFUNC symbol, a separate single-linked list of delayed IFUNC
+   relocations exists while the dynamic linker is running.  */
+struct dl_ifunc_relocation
+{
+  /* Information about the relocation.  */
+  struct link_map *reloc_map;
+  const ElfW(Rela) *reloc;
+  ElfW(Addr) *reloc_addr;
+
+  /* Information about the IFUNC resolver.  The corresponding symbol
+     map is implied.  */
+  const ElfW(Sym) *ifunc_sym;
+
+  /* Pointer to the next element in the list of IFUNC relocations for
+     the symbol map.  */
+  struct dl_ifunc_relocation *next;
+};
+
+/* Record a delayed IFUNC relocation for IFUNC_SYM at *RELOC_ADDR.
+   The relocation is associated with IFUNC_MAP.  Can raise an error
+   using _dl_signal_error.  IFUNC_SYM can be NULL if the relocation is
+   a relative IFUNC relocation.  */
+void _dl_ifunc_record_reloc (struct link_map *reloc_map,
+                             const ElfW(Rela) *reloc,
+                             ElfW(Addr) *reloc_addr,
+                             struct link_map *ifunc_map,
+                             const ElfW(Sym) *ifunc_sym)
+  attribute_hidden internal_function;
+
+#else /* !HAVE_IFUNC  */
+
+/* Dummy implementations for targets without IFUNC support.  */
+
+static inline void
+_dl_ifunc_apply_relocations (struct link_map *new)
+{
+}
+
+static inline void
+_dl_ifunc_clear_relocations (struct link_map *map)
+{
+}
+
+#endif /* !HAVE_IFUNC */
+
+#endif /* DL_IFUNC_H */
diff --git a/elf/dl-open.c b/elf/dl-open.c
index d238fb2cc5..e40fb3b7e8 100644
--- a/elf/dl-open.c
+++ b/elf/dl-open.c
@@ -35,7 +35,7 @@
 #include <atomic.h>
 
 #include <dl-dst.h>
-
+#include <dl-ifunc.h>
 
 extern int __libc_multiple_libcs;	/* Defined in init-first.c.  */
 
@@ -552,6 +552,7 @@ TLS generation counter wrapped!  Please report this."));
 	}
     }
 
+  _dl_ifunc_apply_relocations (new);
   _dl_relocate_apply_relro (new);
 
   /* Notify the debugger all new objects have been relocated.  */
@@ -673,6 +674,7 @@ no more namespaces available for dlmopen()"));
 	  if ((mode & __RTLD_AUDIT) == 0)
 	    GL(dl_tls_dtv_gaps) = true;
 
+	  _dl_ifunc_clear_relocations (args.map);
 	  _dl_close_worker (args.map, true);
 	}
 
diff --git a/elf/rtld.c b/elf/rtld.c
index 16119e8c31..cb9d330d66 100644
--- a/elf/rtld.c
+++ b/elf/rtld.c
@@ -41,6 +41,7 @@
 #include <tls.h>
 #include <stap-probe.h>
 #include <stackinfo.h>
+#include <dl-ifunc.h>
 
 #include <assert.h>
 
@@ -2114,10 +2115,11 @@ ERROR: ld.so: object '%s' cannot be loaded as audit interface: %s; ignored.\n",
       HP_TIMING_ACCUM_NT (relocate_time, add);
     }
 
-  /* Activate RELRO protection.  In the prelink case, this was already
-     done earlier.  */
+  /* Perform delayed IFUNC relocations and activate RELRO protection.
+     In the prelink case, this was already done earlier.  */
   if (! prelinked)
     {
+      _dl_ifunc_apply_relocations (main_map);
       /* Make sure that this covers the dynamic linker as well.
 	 TODO: rtld_multiple_ref is always true because libc.so needs
 	 the dynamic linker internally.  */
diff --git a/include/link.h b/include/link.h
index eeb5f4d53b..a8d85c9f37 100644
--- a/include/link.h
+++ b/include/link.h
@@ -60,6 +60,7 @@ struct r_search_path_elem;
 
 /* Forward declaration.  */
 struct link_map;
+struct dl_ifunc_relocation;
 
 /* Structure to describe a single list of scope elements.  The lookup
    functions get passed an array of pointers to such structures.  */
@@ -318,6 +319,11 @@ struct link_map
     ElfW(Addr) l_relro_addr;
     size_t l_relro_size;
 
+#ifdef HAVE_IFUNC
+    /* Deferred IFUNC relocations.  Only used during relocation.  */
+    struct dl_ifunc_relocation *l_ifunc_relocations;
+#endif
+
     unsigned long long int l_serial;
 
     /* Audit information.  This array apparent must be the last in the
diff --git a/sysdeps/x86_64/dl-ifunc-reloc.h b/sysdeps/x86_64/dl-ifunc-reloc.h
new file mode 100644
index 0000000000..bf436bf71c
--- /dev/null
+++ b/sysdeps/x86_64/dl-ifunc-reloc.h
@@ -0,0 +1,91 @@
+/* IFUNC relocation processing, x86-64 version.
+   Copyright (C) 2001-2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Andreas Jaeger <aj@suse.de>.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef DL_IFUNC_RELOC_H
+#define DL_IFUNC_RELOC_H
+
+static inline void
+_dl_ifunc_process_relocation (const struct dl_ifunc_relocation *ifunc,
+                              struct link_map *sym_map)
+{
+  const ElfW(Rela) *reloc = ifunc->reloc;
+  const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info);
+  ElfW(Addr) *const reloc_addr = ifunc->reloc_addr;
+
+  /* Special case: A relative IFUNC relocation does not have an
+     associated symbol.  */
+  if (r_type == R_X86_64_IRELATIVE)
+    {
+      ElfW(Addr) value = sym_map->l_addr + reloc->r_addend;
+      value = ((ElfW(Addr) (*) (void)) value) ();
+      *reloc_addr = value;
+      return;
+    }
+
+  ElfW(Addr) value = (ElfW(Addr)) sym_map->l_addr + ifunc->ifunc_sym->st_value;
+  value = ((ElfW(Addr) (*) (void)) value) ();
+
+  /* This switch statement needs to be kept in sync with the switch
+     statement in elf_machine_rela.  */
+  switch (r_type)
+    {
+    case R_X86_64_GLOB_DAT:
+    case R_X86_64_JUMP_SLOT:
+      *reloc_addr = value + reloc->r_addend;
+      break;
+
+    case R_X86_64_64:
+      /* value + r_addend may be > 0xffffffff and R_X86_64_64
+         relocation updates the whole 64-bit entry.  */
+      *(Elf64_Addr *) reloc_addr = (Elf64_Addr) value + reloc->r_addend;
+      break;
+    case R_X86_64_32:
+      value += reloc->r_addend;
+      *(unsigned int *) reloc_addr = value;
+
+      const char *fmt;
+      if (__glibc_unlikely (value > UINT_MAX))
+        {
+          const char *strtab;
+
+          fmt = "\
+%s: Symbol `%s' causes overflow in R_X86_64_32 relocation\n";
+        print_err:
+          strtab = (const char *) D_PTR (sym_map, l_info[DT_STRTAB]);
+
+          _dl_error_printf (fmt, RTLD_PROGNAME,
+                            strtab + ifunc->ifunc_sym->st_name);
+        }
+      break;
+    case R_X86_64_PC32:
+      value += reloc->r_addend - (ElfW(Addr)) reloc_addr;
+      *(unsigned int *) reloc_addr = value;
+      if (__glibc_unlikely (value != (int) value))
+        {
+          fmt = "\
+%s: Symbol `%s' causes overflow in R_X86_64_PC32 relocation\n";
+          goto print_err;
+        }
+      break;
+    default:
+      _dl_reloc_bad_type (ifunc->reloc_map, r_type, 0);
+    }
+}
+
+#endif  /* DL_IFUNC_RELOC_H */
diff --git a/sysdeps/x86_64/dl-machine.h b/sysdeps/x86_64/dl-machine.h
index 3e7ae22c67..02b6703b2e 100644
--- a/sysdeps/x86_64/dl-machine.h
+++ b/sysdeps/x86_64/dl-machine.h
@@ -27,6 +27,7 @@
 #include <tls.h>
 #include <dl-tlsdesc.h>
 #include <cpu-features.c>
+#include <dl-ifunc.h>
 
 /* Return nonzero iff ELF header is compatible with the running host.  */
 static inline int __attribute__ ((unused))
@@ -326,29 +327,20 @@ elf_machine_rela (struct link_map *map, const ElfW(Rela) *reloc,
       ElfW(Addr) value = (sym == NULL ? 0
 			  : (ElfW(Addr)) sym_map->l_addr + sym->st_value);
 
+# ifndef RTLD_BOOTSTRAP
       if (sym != NULL
 	  && __builtin_expect (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC,
 			       0)
 	  && __builtin_expect (sym->st_shndx != SHN_UNDEF, 1)
 	  && __builtin_expect (!skip_ifunc, 1))
 	{
-# ifndef RTLD_BOOTSTRAP
-	  if (sym_map != map
-	      && sym_map->l_type != lt_executable
-	      && !sym_map->l_relocated)
-	    {
-	      const char *strtab
-		= (const char *) D_PTR (map, l_info[DT_STRTAB]);
-	      _dl_fatal_printf ("\
-%s: Relink `%s' with `%s' for IFUNC symbol `%s'\n",
-				RTLD_PROGNAME, map->l_name,
-				sym_map->l_name,
-				strtab + refsym->st_name);
-	    }
-# endif
-	  value = ((ElfW(Addr) (*) (void)) value) ();
+	  _dl_ifunc_record_reloc (map, reloc, reloc_addr, sym_map, sym);
+	  return;
 	}
+# endif	/* !RTLD_BOOTSTRAP */
 
+      /* This switch statement needs to be kept in sync with the
+	 switch statement in _dl_ifunc_process_relocation.  */
       switch (r_type)
 	{
 # ifndef RTLD_BOOTSTRAP
@@ -528,9 +520,7 @@ elf_machine_rela (struct link_map *map, const ElfW(Rela) *reloc,
 	  break;
 #  endif
 	case R_X86_64_IRELATIVE:
-	  value = map->l_addr + reloc->r_addend;
-	  value = ((ElfW(Addr) (*) (void)) value) ();
-	  *reloc_addr = value;
+	  _dl_ifunc_record_reloc (map, reloc, reloc_addr, map, NULL);
 	  break;
 	default:
 	  _dl_reloc_bad_type (map, r_type, 0);