Update.

2001-06-07  David Mosberger  <davidm@hpl.hp.com>

	* sysdeps/ia64/dl-symaddr.c (_dl_symbol_address): Update call to
	__ia64_make_fptr() for new interface.
	(_dl_function_address): Remove.

	* sysdeps/ia64/dl-machine.h (IA64_BOOT_FPTR_TABLE_LEN): Rename
	from IA64_BOOT_FPTR_SIZE and reduce its value from 256 to 200.
	(struct ia64_fdesc): Rename from struct ia64_fptr and drop "next"
	pointer.  Rename member "func" to "ip".
	(struct ia64_fdesc_table): New type.
	(__ia64_make_fptr): Change prototype.
	(__ia64_init_bootstrap_fdesc_table): New function.
	(ELF_MACHINE_BEFORE_RTLD_RELOC): New macro.
	(__boot_ldso_fptr): Removed.
	(__fptr_next): Removed.
	(__fptr_root): Removed.
	(__fptr_count): Removed.
	(TRAMPOLINE_TEMPLATE): Fix typo in comment.  Don't use multi-line
	strings to avoid warning from gcc3.
	(RTLD_START): Don't use multi-line strings.
	(ELF_MACHINE_START_ADDRESS): Use DL_STATIC_FUNCTION_ADDRESS.
	(elf_machine_fixup_plt): Update the code entry point in the
	function descriptor through a "volatile" pointer to ensure proper
	write ordering.
	(elf_machine_rela): Simplify handling of FPTR relocs: always call
	__ia64_make_fptr().

	* sysdeps/ia64/dl-lookupcfg.h (DL_AUTO_FUNCTION_ADDRESS): New macro.
	(DL_STATIC_FUNCTION_ADDRESS): Likewise.
	(DL_DT_INIT_ADDRESS): Use DL_AUTO_FUNCTION_ADDRESS.
	(DL_DT_FINI_ADDRESS): Likewise.

	* sysdeps/ia64/dl-fptr.c: Rewrite for better scalability and to
	avoid deadlocks.

	* include/link.h: Add member machine specific "l_mach" member to
	link_map.

	* sysdeps/generic/bits/link.h: New file.
	* sysdeps/ia64/bits/link.h: Likewise.

	* elf/rtld.c (_dl_start): Fix typo in comment.
	(_dl_start_final): Copy new "l_mach" member from temporary to
	final bootstrap map.

1 files changed, 216 insertions, 141 deletions

diff --git a/sysdeps/ia64/dl-fptr.c b/sysdeps/ia64/dl-fptr.c
index 588bb9d599..c31de86079 100644
--- a/sysdeps/ia64/dl-fptr.c
+++ b/sysdeps/ia64/dl-fptr.c
@@ -1,5 +1,5 @@
-/* Unmap a loaded object.  IA-64 version.
-   Copyright (C) 1999, 2000 Free Software Foundation, Inc.
+/* Manage function descriptors.  IA-64 version.
+   Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -17,6 +17,7 @@
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
    02111-1307 USA.  */
 
+#include <ia64intrin.h>
 #include <unistd.h>
 #include <string.h>
 #include <sys/param.h>
@@ -27,186 +28,260 @@
 #include <dl-machine.h>
 #ifdef _LIBC_REENTRANT
 # include <pt-machine.h>
-
-static int __ia64_fptr_lock = 0;
+# include <signal.h>
+# include <time.h>
 #endif
 
-/* Because ld.so is now versioned, these functions can be in their own
-   file; no relocations need to be done to call them.  Of course, if
-   ld.so is not versioned...  */
-#if 0
-#ifndef DO_VERSIONING
-# error "This will not work with versioning turned off, sorry."
-#endif
+Elf64_Addr __ia64_boot_fptr_table[IA64_BOOT_FPTR_TABLE_LEN];
+
+static struct local
+  {
+    struct ia64_fdesc_table *root;
+    struct ia64_fdesc *free_list;
+    unsigned int npages;		/* # of pages to allocate */
+#ifdef _LIBC_REENTRANT
+    volatile int lock;
+    sigset_t full_sigset;
 #endif
+    /* the next to members MUST be consecutive! */
+    struct ia64_fdesc_table boot_table;
+    struct ia64_fdesc boot_fdescs[1024];
+  }
+local =
+  {
+    root: &local.boot_table,
+    npages: 2,
+    boot_table:
+      {
+	len: sizeof (local.boot_fdescs) / sizeof (local.boot_fdescs[0]),
+	first_unused: 0
+      }
+  };
 
-#ifdef MAP_ANON
-/* The fd is not examined when using MAP_ANON.  */
-#define ANONFD -1
+/* Locking is tricky: we may get a signal while holding the lock and
+   the signal handler may end up calling into the dynamic loader
+   again.  Also, if a real-time process spins on the lock, a
+   non-realtime process may never get the chance to release it's lock,
+   unless the realtime process relinquishes the CPU from time to time.
+   Hence we (a) block signals before acquiring the lock and (b) do a
+   nanosleep() when we detect prolongued contention.  */
+#ifdef _LIBC_REENTRANT
+# define lock(l)						\
+{								\
+  sigset_t _saved_set;						\
+  int i = 10000;						\
+  if (!__sigismember (&(l)->full_sigset, SIGINT))		\
+    __sigfillset (&(l)->full_sigset);				\
+								\
+  while (testandset ((int *) &(l)->lock))			\
+    {								\
+      struct timespec ts;					\
+      if (i > 0)						\
+	{							\
+	  --i;							\
+	  continue;						\
+	}							\
+      ts.tv_sec = 0;						\
+      ts.tv_nsec = 1*1000*1000;					\
+      __nanosleep (&ts, NULL);					\
+    }								\
+  __sigprocmask (SIG_BLOCK, &(l)->full_sigset, &_saved_set);
+# define unlock(l)						\
+  __sigprocmask (SIG_SETMASK, &_saved_set, NULL);		\
+  (l)->lock = 0;						\
+}
 #else
-extern int _dl_zerofd;
-#define ANONFD _dl_zerofd
+# define lock(l)
+# define unlock(l)
 #endif
 
+/* Create a new fdesc table and return a pointer to the first fdesc
+   entry.  The fdesc lock must have been acquired already.  */
 
-/* ld.so currently has 14 FPTR relocs, we take 256 and use them for
-   the relocs for the dynamic app itself.  */
-struct ia64_fptr __boot_ldso_fptr[IA64_BOOT_FPTR_SIZE];
-struct ia64_fptr *__fptr_root = NULL;
-struct ia64_fptr *__fptr_next = __boot_ldso_fptr;
-static struct ia64_fptr *__fptr_free = NULL;
-int __fptr_count = IA64_BOOT_FPTR_SIZE;
-
-Elf64_Addr
-__ia64_make_fptr (const struct link_map *sym_map, Elf64_Addr value,
-		  struct ia64_fptr **root, struct ia64_fptr *mem)
+static struct ia64_fdesc *
+new_fdesc_table (struct local *l)
 {
-  struct ia64_fptr **loc;
-  struct ia64_fptr *f;
+  size_t size = l->npages * _dl_pagesize;
+  struct ia64_fdesc_table *new_table;
+  struct ia64_fdesc *fdesc;
 
-#ifdef _LIBC_REENTRANT
-  /* Make sure we are alone. We don't need a lock during bootstrap. */
-  if (mem == NULL)
-    while (testandset (&__ia64_fptr_lock));
-#endif
+  l->npages += l->npages;
+  new_table = __mmap (0, size, PROT_READ | PROT_WRITE,
+		      MAP_ANON | MAP_PRIVATE, -1, 0);
+  if (new_table == MAP_FAILED)
+    _dl_signal_error (errno, NULL, "cannot map pages for fdesc table");
 
-  /* Search the sorted linked list for an existing entry for this
-     symbol.  */
-  loc = root;
-  f = *loc;
-  while (f != NULL && f->func <= value)
-    {
-      if (f->func == value)
-	goto found;
-      loc = &f->next;
-      f = *loc;
-    }
+  new_table->len = (size - sizeof (*new_table)) / sizeof (struct ia64_fdesc);
+  fdesc = &new_table->fdesc[0];
+  new_table->first_unused = 1;
+  new_table->next = l->root;
+  l->root = new_table;
+  return fdesc;
+}
 
-  /* Not found.  Create a new one.  */
-  if (mem != NULL)
-    f = mem;
-  else if (__fptr_free != NULL)
-    {
-      f = __fptr_free;
-      __fptr_free = f->next;
-    }
-  else
+static Elf64_Addr
+make_fdesc (Elf64_Addr ip, Elf64_Addr gp)
+{
+  struct ia64_fdesc *fdesc = NULL;
+  struct ia64_fdesc_table *t;
+  unsigned int old;
+  struct local *l;
+
+  asm ("addl %0 = @gprel (local), gp" : "=r" (l));
+
+  t = l->root;
+  while (1)
     {
-      if (__fptr_count == 0)
+      old = t->first_unused;
+      if (old >= t->len)
+	break;
+      else if (__sync_bool_compare_and_swap (&t->first_unused, old, old + 1))
 	{
-#ifndef MAP_ANON
-# define MAP_ANON 0
-	  if (_dl_zerofd == -1)
-	    {
-	      _dl_zerofd = _dl_sysdep_open_zero_fill ();
-	      if (_dl_zerofd == -1)
-		{
-		  __close (fd);
-		  _dl_signal_error (errno, NULL,
-				    "cannot open zero fill device");
-		}
-	    }
-#endif
-
-	  __fptr_next = __mmap (0, _dl_pagesize, PROT_READ | PROT_WRITE,
-				MAP_ANON | MAP_PRIVATE, ANONFD, 0);
-	  if (__fptr_next == MAP_FAILED)
-	    _dl_signal_error(errno, NULL, "cannot map page for fptr");
-	  __fptr_count = _dl_pagesize / sizeof (struct ia64_fptr);
+	  fdesc = &t->fdesc[old];
+	  goto install;
 	}
-      f = __fptr_next++;
-      __fptr_count--;
     }
 
-  f->func = value;
-  /* GOT has already been relocated in elf_get_dynamic_info - don't
-     try to relocate it again.  */
-  f->gp = sym_map->l_info[DT_PLTGOT]->d_un.d_ptr;
-  f->next = *loc;
-  *loc = f;
+  lock (l);
+  {
+    if (l->free_list)
+      {
+	fdesc = l->free_list;		/* get it from free-list */
+	l->free_list = (struct ia64_fdesc *) fdesc->ip;
+      }
+    else
+      fdesc = new_fdesc_table (l);	/* create new fdesc table */
+  }
+  unlock (l);
 
-found:
-#ifdef _LIBC_REENTRANT
-  /* Release the lock.  */
-  if (mem == NULL)
-    __ia64_fptr_lock = 0;
+ install:
+  fdesc->ip = ip;
+  fdesc->gp = gp;
+
+  return (Elf64_Addr) fdesc;
+}
+
+static inline Elf64_Addr *
+make_fptr_table (struct link_map *map)
+{
+  const Elf64_Sym *symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]);
+  const char *strtab = (const void *) D_PTR (map, l_info[DT_STRTAB]);
+  Elf64_Addr *fptr_table;
+  size_t size;
+  size_t len;
+
+  /* XXX Apparently the only way to find out the size of the dynamic
+     symbol section is to assume that the string table follows right
+     afterwards...  */
+  len = ((strtab - (char *) symtab) / map->l_info[DT_SYMENT]->d_un.d_val);
+  size = ((len * sizeof (fptr_table[0]) + _dl_pagesize - 1) & -_dl_pagesize);
+  /* XXX We don't support here in the moment systems without MAP_ANON.
+     There probably are none for IA-64.  In case this is proven wrong
+     we will have to open /dev/null here and use the file descriptor
+     instead of the hard-coded -1.  */
+  fptr_table = __mmap (NULL, size, PROT_READ | PROT_WRITE,
+		       MAP_ANON | MAP_PRIVATE, -1, 0);
+  if (fptr_table == MAP_FAILED)
+    _dl_signal_error (errno, NULL, "cannot map pages for fptr table");
+
+  map->l_mach.fptr_table_len = len;
+  map->l_mach.fptr_table = fptr_table;
+  return fptr_table;
+}
+
+Elf64_Addr
+__ia64_make_fptr (struct link_map *map, const Elf64_Sym *sym, Elf64_Addr ip)
+{
+  Elf64_Addr *ftab = map->l_mach.fptr_table;
+  const Elf64_Sym *symtab;
+  Elf_Symndx symidx;
+
+  if (__builtin_expect (!map->l_mach.fptr_table, 0))
+    ftab = make_fptr_table (map);
+
+  symtab = (const void *) D_PTR (map, l_info[DT_SYMTAB]);
+  symidx = sym - symtab;
+
+  if (symidx >= map->l_mach.fptr_table_len)
+    _dl_signal_error (0, NULL,
+		      "internal error: symidx out of range of fptr table");
+
+  if (!ftab[symidx])
+    {
+      /* GOT has already been relocated in elf_get_dynamic_info -
+	 don't try to relocate it again.  */
+      ftab[symidx] = make_fdesc (ip, map->l_info[DT_PLTGOT]->d_un.d_ptr);
+#if 0
+      {
+	const char *strtab = (const void *) D_PTR (map, l_info[DT_STRTAB]);
+	struct local *l;
+
+	asm ("addl %0 = @gprel (local), gp" : "=r" (l));
+	if (l->root != &l->boot_table || l->boot_table.first_unused > 20)
+	  _dl_debug_printf ("created fdesc symbol `%s' at %lx\n",
+			    strtab + sym->st_name, ftab[symidx]);
+      }
 #endif
+    }
 
-  return (Elf64_Addr) f;
+  return ftab[symidx];
 }
 
 void
 _dl_unmap (struct link_map *map)
 {
-  struct ia64_fptr **floc;
-  struct ia64_fptr *f;
-  struct ia64_fptr **lloc;
-  struct ia64_fptr *l;
+  Elf64_Addr *ftab = map->l_mach.fptr_table;
+  struct ia64_fdesc *head = NULL, *tail = NULL;
+  size_t i;
 
   __munmap ((void *) map->l_map_start, map->l_map_end - map->l_map_start);
 
-#ifdef _LIBC_REENTRANT
-  /* Make sure we are alone.  */
-  while (testandset (&__ia64_fptr_lock));
-#endif
+  if (!ftab)
+    return;
 
-  /* Search the sorted linked list for the first entry for this object.  */
-  floc = &__fptr_root;
-  f = *floc;
-  while (f != NULL && f->func < map->l_map_start)
+  /* String together the fdesc structures that are being freed.  */
+  for (i = 0; i < map->l_mach.fptr_table_len; ++i)
     {
-      floc = &f->next;
-      f = *floc;
-    }
-
-  /* We found one.  */
-  if (f != NULL && f->func < map->l_map_end)
-    {
-      /* Get the last entry.  */
-      lloc = floc;
-      l = f;
-      while (l && l->func < map->l_map_end)
+      if (ftab[i])
 	{
-	  lloc = &l->next;
-	  l = *lloc;
+	  *(struct ia64_fdesc **) ftab[i] = head;
+	  head = (struct ia64_fdesc *) ftab[i];
+	  if (!tail)
+	    tail = head;
 	}
+    }
 
-      /* Updated FPTR.  */
-      *floc = l;
-
-      /* Prepend them to the free list.  */
-      *lloc = __fptr_free;
-      __fptr_free = f;
+  /* Prepend the new list to the free_list: */
+  if (tail)
+    {
+      lock (&local);
+      {
+	*(struct ia64_fdesc **) tail = local.free_list;
+	local.free_list = head;
+      }
+      unlock (&local);
     }
 
-#ifdef _LIBC_REENTRANT
-  /* Release the lock.   */
-  __ia64_fptr_lock = 0;
-#endif
+  __munmap (ftab,
+	    map->l_mach.fptr_table_len * sizeof (map->l_mach.fptr_table[0]));
+  map->l_mach.fptr_table = NULL;
 }
 
 Elf64_Addr
 _dl_lookup_address (const void *address)
 {
   Elf64_Addr addr = (Elf64_Addr) address;
-  struct ia64_fptr *f;
-
-#ifdef _LIBC_REENTRANT
-  /* Make sure we are alone.  */
-  while (testandset (&__ia64_fptr_lock));
-#endif
-
-  for (f = __fptr_root; f != NULL; f = f->next)
-    if (f == address)
-      {
-	addr = f->func;
-	break;
-      }
-
-#ifdef _LIBC_REENTRANT
-  /* Release the lock.   */
-  __ia64_fptr_lock = 0;
-#endif
+  struct ia64_fdesc_table *t;
+  unsigned long int i;
 
+  for (t = local.root; t != NULL; t = t->next)
+    {
+      i = (struct ia64_fdesc *) addr - &t->fdesc[0];
+      if (i < t->first_unused && addr == (Elf64_Addr) &t->fdesc[i])
+	{
+	  addr = t->fdesc[i].ip;
+	  break;
+	}
+    }
   return addr;
 }