For b/2471323, implement multi-level cache search for shared libraries.

7 files changed, 176 insertions, 41 deletions

diff --git a/README.google b/README.google
index 50900b9c3c..dfa4bfd71c 100644
--- a/README.google
+++ b/README.google
@@ -163,3 +163,17 @@ sysdeps/unix/readdir_r.c
   Forward-ported from cl/51430993 (from cl/45000-p2).
   (ppluzhnikov, google-local)
 
+elf/Makefile
+elf/dl-cache.c
+elf/dl-support.c
+elf/rtld.c
+sysdeps/generic/ldsodefs.h
+sysdeps/generic/unsecvars.h
+  For b/2471323, implement multi-level cache search for shared libraries.
+  Currently, two-level: $prefix/etc/ld.so.conf and /etc/ld.so.conf.
+  Allow build-time configuration override via GOOGLE_LD_SO_CACHE_LIST.
+  Allow run-time override via LD_GOOGLE_LD_SO_CACHE_LIST environment
+  variable.
+  Forward-ported from cl/51433387 (from cl/38694-p2, cl/38725-p2, and
+  cl/38741-p2).
+  (ppluzhnikov, google-local)
diff --git a/elf/Makefile b/elf/Makefile
index 4c58fc9c24..d2250cb813 100644
--- a/elf/Makefile
+++ b/elf/Makefile
@@ -422,6 +422,7 @@ CFLAGS-ldconfig.c = $(SYSCONF-FLAGS) -D'LIBDIR="$(libdir)"' \
 CFLAGS-dl-cache.c = $(SYSCONF-FLAGS)
 CFLAGS-cache.c = $(SYSCONF-FLAGS)
 CFLAGS-rtld.c = $(SYSCONF-FLAGS)
+CFLAGS-dl-support.c = $(SYSCONF-FLAGS)
 
 CPPFLAGS-.os += $(if $(filter $(@F),$(patsubst %,%.os,$(all-rtld-routines))),\
 		     -DNOT_IN_libc=1 -DIS_IN_rtld=1 -DIN_LIB=rtld)
diff --git a/elf/dl-cache.c b/elf/dl-cache.c
index d36623f09e..34de935000 100644
--- a/elf/dl-cache.c
+++ b/elf/dl-cache.c
@@ -29,10 +29,19 @@
 # define _DL_PLATFORMS_COUNT 0
 #endif
 
-/* This is the starting address and the size of the mmap()ed file.  */
-static struct cache_file *cache;
-static struct cache_file_new *cache_new;
-static size_t cachesize;
+/* This struct describes a single ld.so.cache (there could be several,
+   as we want to look in $prefix/etc/ld.so.cache and in the system default
+   /etc/ld.so.cache). See b/2471323.  */
+struct cache_info {
+  const char *ld_so_cache;
+  /* This is the starting address and the size of the mmap()ed file.  */
+  struct cache_file *cache;
+  struct cache_file_new *cache_new;
+  size_t cachesize;
+};
+
+/* Dynamically allocated; last entry is sentinel with ld_so_cache == NULL.  */
+static struct cache_info *cache_info;
 
 /* 1 if cache_data + PTR points into the cache.  */
 #define _dl_cache_verify_ptr(ptr) (ptr < cache_data_size)
@@ -172,13 +181,14 @@ _dl_cache_libcmp (const char *p1, const char *p2)
 }
 
 
-/* Look up NAME in ld.so.cache and return the file name stored there, or null
-   if none is found.  The cache is loaded if it was not already.  If loading
-   the cache previously failed there will be no more attempts to load it.  */
-
+/* Look up NAME in ld.so.cache described by INFO and return the file name
+   stored there, or null if none is found.  The cache is loaded if it was not
+   already.  If loading the cache previously failed there will be no more
+   attempts to load it.  */
+static
 const char *
 internal_function
-_dl_load_cache_lookup (const char *name)
+_dl_load_cache_lookup_2 (const char *name, struct cache_info *info)
 {
   int left, right, middle;
   int cmpres;
@@ -188,12 +198,13 @@ _dl_load_cache_lookup (const char *name)
 
   /* Print a message if the loading of libs is traced.  */
   if (__builtin_expect (GLRO(dl_debug_mask) & DL_DEBUG_LIBS, 0))
-    _dl_debug_printf (" search cache=%s\n", LD_SO_CACHE);
+    _dl_debug_printf (" search cache=%s\n", info->ld_so_cache);
 
-  if (cache == NULL)
+  if (info->cache == NULL)
     {
       /* Read the contents of the file.  */
-      void *file = _dl_sysdep_read_whole_file (LD_SO_CACHE, &cachesize,
+      void *file = _dl_sysdep_read_whole_file (info->ld_so_cache,
+					       &info->cachesize,
 					       PROT_READ);
 
       /* We can handle three different cache file formats here:
@@ -201,55 +212,58 @@ _dl_load_cache_lookup (const char *name)
 	 - the old format with the new format in it
 	 - only the new format
 	 The following checks if the cache contains any of these formats.  */
-      if (file != MAP_FAILED && cachesize > sizeof *cache
+      if (file != MAP_FAILED && info->cachesize > sizeof *info->cache
 	  && memcmp (file, CACHEMAGIC, sizeof CACHEMAGIC - 1) == 0)
 	{
 	  size_t offset;
 	  /* Looks ok.  */
-	  cache = file;
+	  info->cache = file;
 
 	  /* Check for new version.  */
-	  offset = ALIGN_CACHE (sizeof (struct cache_file)
-				+ cache->nlibs * sizeof (struct file_entry));
-
-	  cache_new = (struct cache_file_new *) ((void *) cache + offset);
-	  if (cachesize < (offset + sizeof (struct cache_file_new))
-	      || memcmp (cache_new->magic, CACHEMAGIC_VERSION_NEW,
+	  offset =
+	    ALIGN_CACHE (sizeof (struct cache_file)
+			 + info->cache->nlibs * sizeof (struct file_entry));
+
+	  info->cache_new =
+	    (struct cache_file_new *) ((void *) info->cache + offset);
+	  if (info->cachesize < (offset + sizeof (struct cache_file_new))
+	      || memcmp (info->cache_new->magic, CACHEMAGIC_VERSION_NEW,
 			 sizeof CACHEMAGIC_VERSION_NEW - 1) != 0)
-	    cache_new = (void *) -1;
+	    info->cache_new = (void *) -1;
 	}
-      else if (file != MAP_FAILED && cachesize > sizeof *cache_new
+      else if (file != MAP_FAILED && info->cachesize > sizeof *info->cache_new
 	       && memcmp (file, CACHEMAGIC_VERSION_NEW,
 			  sizeof CACHEMAGIC_VERSION_NEW - 1) == 0)
 	{
-	  cache_new = file;
-	  cache = file;
+	  info->cache_new = file;
+	  info->cache = file;
 	}
       else
 	{
 	  if (file != MAP_FAILED)
-	    __munmap (file, cachesize);
-	  cache = (void *) -1;
+	    __munmap (file, info->cachesize);
+	  info->cache = (void *) -1;
 	}
 
-      assert (cache != NULL);
+      assert (info->cache != NULL);
     }
 
-  if (cache == (void *) -1)
+  if (info->cache == (void *) -1)
     /* Previously looked for the cache file and didn't find it.  */
     return NULL;
 
   best = NULL;
 
-  if (cache_new != (void *) -1)
+  if (info->cache_new != (void *) -1)
     {
       uint64_t platform;
 
       /* This is where the strings start.  */
-      cache_data = (const char *) cache_new;
+      cache_data = (const char *) info->cache_new;
 
       /* Now we can compute how large the string table is.  */
-      cache_data_size = (const char *) cache + cachesize - cache_data;
+      cache_data_size =
+	(const char *) info->cache + info->cachesize - cache_data;
 
       platform = _dl_string_platform (GLRO(dl_platform));
       if (platform != (uint64_t) -1)
@@ -269,19 +283,20 @@ _dl_load_cache_lookup (const char *name)
 	  && (lib->hwcap & _DL_HWCAP_PLATFORM) != 0			      \
 	  && (lib->hwcap & _DL_HWCAP_PLATFORM) != platform)		      \
 	continue
-      SEARCH_CACHE (cache_new);
+      SEARCH_CACHE (info->cache_new);
     }
   else
     {
       /* This is where the strings start.  */
-      cache_data = (const char *) &cache->libs[cache->nlibs];
+      cache_data = (const char *) &info->cache->libs[info->cache->nlibs];
 
       /* Now we can compute how large the string table is.  */
-      cache_data_size = (const char *) cache + cachesize - cache_data;
+      cache_data_size =
+	(const char *) info->cache + info->cachesize - cache_data;
 
 #undef HWCAP_CHECK
 #define HWCAP_CHECK do {} while (0)
-      SEARCH_CACHE (cache);
+      SEARCH_CACHE (info->cache);
     }
 
   /* Print our result if wanted.  */
@@ -292,18 +307,100 @@ _dl_load_cache_lookup (const char *name)
   return best;
 }
 
+/* Parse CACHE_LIST, and build cache_info array.  */
+static
+struct cache_info *
+_dl_alloc_cache_info (const char *const cache_list)
+{
+  struct cache_info *info;
+  const char *begin;
+  int num_caches, i;
+
+  begin = cache_list;
+  num_caches = 0;
+
+  /* cache_list is a colon-separated list of absolute pathnames of
+     ld.so.cache files.  We stop at first non-absolute path (if any).  */
+  while (begin[0] == '/')
+    {
+      ++num_caches;
+      begin = strchr (begin, ':');
+      if (begin == NULL)
+	break;
+      ++begin;
+    }
+
+  /* Allocate one extra for the sentinel.  */
+  info = (struct cache_info *) calloc (num_caches + 1, sizeof (*cache_info));
+
+  begin = cache_list;
+  for (i = 0; i < num_caches; ++i)
+    {
+      const char *const end = strchr(begin, ':');
+
+      if (end == NULL)
+	info[i].ld_so_cache = strdup (begin);
+      else
+	{
+	  info[i].ld_so_cache = strndup (begin, end - begin);
+	  begin = end + 1;
+	}
+    }
+  return info;
+}
+
+/* Look up NAME in each of ld.so.cache caches in turn, and return the
+   file name stored there, or null if none is found.  */
+const char *
+internal_function
+_dl_load_cache_lookup (const char *name)
+{
+  struct cache_info *info;
+
+  /* This runs at startup, or during dlopen with loader lock held.  */
+  if (cache_info == NULL)
+    /* Caches have not been initialized yet.  */
+    cache_info = _dl_alloc_cache_info (GLRO(google_ld_so_cache_list));
+
+  for (info = cache_info; info->ld_so_cache != NULL; ++info)
+    {
+      const char *result;
+
+      result = _dl_load_cache_lookup_2 (name, info);
+      if (result != NULL)
+	return result;
+    }
+
+  return NULL;
+}
+
 #ifndef MAP_COPY
 /* If the system does not support MAP_COPY we cannot leave the file open
    all the time since this would create problems when the file is replaced.
-   Therefore we provide this function to close the file and open it again
-   once needed.  */
+   Therefore we provide this function to close the file described by INFO
+   and open it again once needed.  */
+static
 void
-_dl_unload_cache (void)
+_dl_unload_cache_2 (struct cache_info *info)
 {
-  if (cache != NULL && cache != (struct cache_file *) -1)
+  if (info->cache != NULL && info->cache != (struct cache_file *) -1)
     {
-      __munmap (cache, cachesize);
-      cache = NULL;
+      __munmap (info->cache, info->cachesize);
+      info->cache = NULL;
     }
 }
+
+/* Unload all loaded ld.so.cache caches.  */
+void
+_dl_unload_cache (void)
+{
+  struct cache_info *info;
+
+  if (cache_info == NULL)
+    /* No caches loaded yet.  */
+    return;
+
+  for (info = cache_info; info->ld_so_cache != NULL; ++info)
+    _dl_unload_cache_2 (info);
+}
 #endif
diff --git a/elf/dl-support.c b/elf/dl-support.c
index e435436c31..4cb94529b8 100644
--- a/elf/dl-support.c
+++ b/elf/dl-support.c
@@ -222,6 +222,8 @@ __rtld_lock_define_initialized_recursive (, _dl_load_lock)
    that list.  */
 __rtld_lock_define_initialized_recursive (, _dl_load_write_lock)
 
+/* Colon-separated list of absolute paths to ld.so.cache files we'll load.  */
+const char *_google_ld_so_cache_list = GOOGLE_LD_SO_CACHE_LIST;
 
 #ifdef HAVE_AUX_VECTOR
 int _dl_clktck;
diff --git a/elf/rtld.c b/elf/rtld.c
index 55299c3386..92d6e0ade8 100644
--- a/elf/rtld.c
+++ b/elf/rtld.c
@@ -2494,6 +2494,8 @@ process_envvars (enum mode *modep)
   GLRO(dl_profile_output)
     = &"/var/tmp\0/var/profile"[INTUSE(__libc_enable_secure) ? 9 : 0];
 
+  GLRO(google_ld_so_cache_list) = GOOGLE_LD_SO_CACHE_LIST;
+
   while ((envline = _dl_next_ld_env_entry (&runp)) != NULL)
     {
       size_t len = 0;
@@ -2641,6 +2643,13 @@ process_envvars (enum mode *modep)
 	    mode = trace;
 	  break;
 
+	  /* Google local change.  */
+	case 23:
+	  if (!INTUSE(__libc_enable_secure)
+	      && memcmp (envline, "GOOGLE_LD_SO_CACHE_LIST", 23) == 0)
+	    GLRO(google_ld_so_cache_list) = &envline[24];
+	  break;
+
 	  /* We might have some extra environment variable to handle.  This
 	     is tricky due to the pre-processing of the length of the name
 	     in the switch statement here.  The code here assumes that added
diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
index 72de344b9c..5afa846859 100644
--- a/sysdeps/generic/ldsodefs.h
+++ b/sysdeps/generic/ldsodefs.h
@@ -129,6 +129,13 @@ typedef struct link_map *lookup_t;
    | ((PROT_WRITE | PROT_EXEC) << (PF_W | PF_X) * 4)			      \
    | ((PROT_READ | PROT_WRITE | PROT_EXEC) << ((PF_R | PF_W | PF_X) * 4)))
 
+#ifndef GOOGLE_LD_SO_CACHE_LIST
+/* List of absolute paths to ld.so.cache files we'll load.
+   By default we use "$prefix/etc/ld.so.cache:/etc/ld.so.cache". The second
+   is needed so we can find system libraries not included in GRTE.  */
+#define GOOGLE_LD_SO_CACHE_LIST LD_SO_CACHE ":/etc/ld.so.cache"
+#endif
+
 /* The filename itself, or the main program name, if available.  */
 #define DSO_FILENAME(name) ((name)[0] ? (name)				      \
 			    : (rtld_progname ?: "<main program>"))
@@ -540,6 +547,10 @@ struct rtld_global_ro
   /* All search directories defined at startup.  */
   EXTERN struct r_search_path_elem *_dl_init_all_dirs;
 
+  /* Colon-separated list of absolute paths to ld.so.cache files
+     we'll load.  */
+  EXTERN const char *_google_ld_so_cache_list;
+
 #if HP_TIMING_AVAIL || HP_SMALL_TIMING_AVAIL
   /* Overhead of a high-precision timing measurement.  */
   EXTERN hp_timing_t _dl_hp_timing_overhead;
diff --git a/sysdeps/generic/unsecvars.h b/sysdeps/generic/unsecvars.h
index d5b8119c9c..fb277438f0 100644
--- a/sysdeps/generic/unsecvars.h
+++ b/sysdeps/generic/unsecvars.h
@@ -9,6 +9,7 @@
   "LD_DEBUG\0"								      \
   "LD_DEBUG_OUTPUT\0"							      \
   "LD_DYNAMIC_WEAK\0"							      \
+  "LD_GOOGLE_LD_SO_CACHE_LIST\0"					      \
   "LD_LIBRARY_PATH\0"							      \
   "LD_ORIGIN_PATH\0"							      \
   "LD_PRELOAD\0"							      \