1999-11-18 Roland McGrath <roland@baalperazim.frob.com>

        * hurd/hurdsig.c (_hurdsig_init): If __hurd_threadvar_stack_mask is
        nonzero, use cthread_fork to create the signal thread.
        * hurd/msgportdemux.c (_hurd_msgport_receive): Initialize
        _hurd_msgport_thread here (to self).
        * sysdeps/mach/hurd/fork.c (__fork): When __hurd_sigthread_stack_end
        is zero, instead compute child signal thread's starting SP from parent
        signal thread's current SP and the threadvar_stack variables.
        * hurd/Versions (GLIBC_2.1.3): Add cthread_fork, cthread_detach.
        These are now referenced weakly by _hurdsig_init.

        * hurd/report-wait.c (_S_msg_report_wait): Fix typo:
        &_hurd_itimer_thread not &_hurd_msgport_thread.

1999-10-01  Roland McGrath  <roland@baalperazim.frob.com>

        * hurd/hurdfchdir.c (_hurd_change_directory_port_from_fd): Rewrite
        without HURD_DPORT_USE to clean up warnings.
        * hurd/dtable.c (get_dtable_port): Likewise.

        * hurd/hurdioctl.c (rectty_dtable): Renamed to install_ctty.
        (install_ctty): Do the changing of the cttyid port cell here, inside
        the critical section while we holding the dtable lock.
        (_hurd_setcttyid, tiocsctty, tiocnotty): Use that instead of changing
        the port cell and calling rectty_dtable.
        (_hurd_locked_install_cttyid): New function, split out of install_ctty.
        (install_ctty): Use it inside a critical section, with the lock held.
        * sysdeps/mach/hurd/setsid.c (__setsid): Use
        _hurd_locked_install_cttyid to effect the cttyid and dtable changes
        after proc_setsid, having held the dtable lock throughout.
        * hurd/dtable.c (ctty_new_pgrp): With the dtable lock held, check the
        cttyid port for null and bail out early if so.  The dtable lock
        serializes us after any cttyid change and its associated dtable update.

10 files changed, 265 insertions, 111 deletions

diff --git a/ChangeLog b/ChangeLog
index 9c3459ab28..98636dedcd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,38 @@
+1999-11-18  Roland McGrath  <roland@baalperazim.frob.com>
+
+        * hurd/hurdsig.c (_hurdsig_init): If __hurd_threadvar_stack_mask is
+        nonzero, use cthread_fork to create the signal thread.
+        * hurd/msgportdemux.c (_hurd_msgport_receive): Initialize
+        _hurd_msgport_thread here (to self).
+        * sysdeps/mach/hurd/fork.c (__fork): When __hurd_sigthread_stack_end
+        is zero, instead compute child signal thread's starting SP from parent
+        signal thread's current SP and the threadvar_stack variables.
+        * hurd/Versions (GLIBC_2.1.3): Add cthread_fork, cthread_detach.
+        These are now referenced weakly by _hurdsig_init.
+
+        * hurd/report-wait.c (_S_msg_report_wait): Fix typo:
+        &_hurd_itimer_thread not &_hurd_msgport_thread.
+
+1999-10-01  Roland McGrath  <roland@baalperazim.frob.com>
+
+        * hurd/hurdfchdir.c (_hurd_change_directory_port_from_fd): Rewrite
+        without HURD_DPORT_USE to clean up warnings.
+        * hurd/dtable.c (get_dtable_port): Likewise.
+
+        * hurd/hurdioctl.c (rectty_dtable): Renamed to install_ctty.
+        (install_ctty): Do the changing of the cttyid port cell here, inside
+        the critical section while we holding the dtable lock.
+        (_hurd_setcttyid, tiocsctty, tiocnotty): Use that instead of changing
+        the port cell and calling rectty_dtable.
+        (_hurd_locked_install_cttyid): New function, split out of install_ctty.
+        (install_ctty): Use it inside a critical section, with the lock held.
+        * sysdeps/mach/hurd/setsid.c (__setsid): Use
+        _hurd_locked_install_cttyid to effect the cttyid and dtable changes
+        after proc_setsid, having held the dtable lock throughout.
+        * hurd/dtable.c (ctty_new_pgrp): With the dtable lock held, check the
+        cttyid port for null and bail out early if so.  The dtable lock
+        serializes us after any cttyid change and its associated dtable update.
+
 1999-11-14  Roland McGrath  <roland@baalperazim.frob.com>
 
 	* sysdeps/mach/hurd/nfs/nfs.h: New file, empty but for comments.
diff --git a/hurd/Versions b/hurd/Versions
index 5dcdc355d9..a81bc551b2 100644
--- a/hurd/Versions
+++ b/hurd/Versions
@@ -108,6 +108,9 @@ libc {
     seteuids;
   }
   GLIBC_2.1.3 {
+    # c*
+    cthread_fork; cthread_detach;
+
     # d*
     directory_name_split;
 
diff --git a/hurd/dtable.c b/hurd/dtable.c
index 30f6a5ee82..f3eb88fb3a 100644
--- a/hurd/dtable.c
+++ b/hurd/dtable.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 1991, 92, 93, 94, 95, 96, 97 Free Software Foundation, Inc.
+/* Copyright (C) 1991,92,93,94,95,96,97,99 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -107,18 +107,35 @@ text_set_element (_hurd_subinit, init_dtable);
 static file_t
 get_dtable_port (int fd)
 {
+  struct hurd_fd *d = _hurd_fd_get (fd);
   file_t dport;
-  int err = HURD_DPORT_USE (fd, __mach_port_mod_refs (__mach_task_self (),
-						      (dport = port),
-						      MACH_PORT_RIGHT_SEND,
-						      1));
-  if (err)
-    {
-      errno = err;
-      return MACH_PORT_NULL;
-    }
-  else
-    return dport;
+
+  if (!d)
+    return __hurd_fail (EBADF), MACH_PORT_NULL;
+
+  HURD_CRITICAL_BEGIN;
+
+  dport = HURD_PORT_USE (&d->port,
+			 ({
+			   error_t err;
+			   mach_port_t outport;
+			   err = __mach_port_mod_refs (__mach_task_self (),
+						       port,
+						       MACH_PORT_RIGHT_SEND,
+						       1);
+			   if (err)
+			     {
+			       errno = err;
+			       outport = MACH_PORT_NULL;
+			     }
+			   else
+			     outport = port;
+			   outport;
+			 }));
+
+  HURD_CRITICAL_END;
+
+  return dport;
 }
 
 file_t (*_hurd_getdport_fn) (int fd) = get_dtable_port;
@@ -176,32 +193,46 @@ ctty_new_pgrp (void)
   HURD_CRITICAL_BEGIN;
   __mutex_lock (&_hurd_dtable_lock);
 
-  for (i = 0; i < _hurd_dtablesize; ++i)
+  if (__USEPORT (CTTYID, port == MACH_PORT_NULL))
     {
-      struct hurd_fd *const d = _hurd_dtable[i];
-      struct hurd_userlink ulink, ctty_ulink;
-      io_t port, ctty;
-
-      if (d == NULL)
-	/* Nothing to do for an unused descriptor cell.  */
-	continue;
-
-      port = _hurd_port_get (&d->port, &ulink);
-      ctty = _hurd_port_get (&d->ctty, &ctty_ulink);
-
-      if (ctty != MACH_PORT_NULL)
-	{
-	  /* This fd has a ctty-special port.  We need a new one, to tell
-             the io server of our different process group.  */
-	  io_t new;
-	  if (__term_open_ctty (port, _hurd_pid, _hurd_pgrp, &new))
-	    new = MACH_PORT_NULL;
-	  _hurd_port_set (&d->ctty, new);
-	}
-
-      _hurd_port_free (&d->port, &ulink, port);
-      _hurd_port_free (&d->ctty, &ctty_ulink, ctty);
+      /* We have no controlling terminal.  If we haven't had one recently,
+	 but our pgrp is being pointlessly diddled anyway, then we will
+	 have nothing to do in the loop below because no fd will have a
+	 ctty port at all.
+
+	 More likely, a setsid call is responsible both for the change
+	 in pgrp and for clearing the cttyid port.  In that case, setsid
+	 held the dtable lock while updating the dtable to clear all the
+	 ctty ports, and ergo must have finished doing so before we run here.
+	 So we can be sure, again, that the loop below has no work to do.  */
     }
+  else
+    for (i = 0; i < _hurd_dtablesize; ++i)
+      {
+	struct hurd_fd *const d = _hurd_dtable[i];
+	struct hurd_userlink ulink, ctty_ulink;
+	io_t port, ctty;
+
+	if (d == NULL)
+	  /* Nothing to do for an unused descriptor cell.  */
+	  continue;
+
+	port = _hurd_port_get (&d->port, &ulink);
+	ctty = _hurd_port_get (&d->ctty, &ctty_ulink);
+
+	if (ctty != MACH_PORT_NULL)
+	  {
+	    /* This fd has a ctty-special port.  We need a new one, to tell
+	       the io server of our different process group.  */
+	    io_t new;
+	    if (__term_open_ctty (port, _hurd_pid, _hurd_pgrp, &new))
+	      new = MACH_PORT_NULL;
+	    _hurd_port_set (&d->ctty, new);
+	  }
+
+	_hurd_port_free (&d->port, &ulink, port);
+	_hurd_port_free (&d->ctty, &ctty_ulink, ctty);
+      }
 
   __mutex_unlock (&_hurd_dtable_lock);
   HURD_CRITICAL_END;
diff --git a/hurd/hurdfchdir.c b/hurd/hurdfchdir.c
index efb705393e..4a01f0719c 100644
--- a/hurd/hurdfchdir.c
+++ b/hurd/hurdfchdir.c
@@ -27,17 +27,30 @@
 int
 _hurd_change_directory_port_from_fd (struct hurd_port *portcell, int fd)
 {
-  error_t err;
-  file_t dir;
-
-  err = HURD_DPORT_USE (fd,
-			({
-			  dir = __file_name_lookup_under (port, ".", 0, 0);
-			  dir == MACH_PORT_NULL ? errno : 0;
-			}));
-
-  if (! err)
-    _hurd_port_set (portcell, dir);
-
-  return err ? __hurd_fail (err) : 0;
+  int ret;
+  struct hurd_fd *d = _hurd_fd_get (fd);
+
+  if (!d)
+    return __hurd_fail (EBADF);
+
+  HURD_CRITICAL_BEGIN;
+
+  ret = HURD_PORT_USE (&d->port,
+		       ({
+			 int ret;
+			 file_t dir = __file_name_lookup_under (port, ".",
+								0, 0);
+			 if (dir == MACH_PORT_NULL)
+			   ret = -1;
+			 else
+			   {
+			     _hurd_port_set (portcell, dir);
+			     ret = 0;
+			   }
+			 ret;
+		       }));
+
+  HURD_CRITICAL_END;
+
+  return ret;
 }
diff --git a/hurd/hurdioctl.c b/hurd/hurdioctl.c
index 8482571299..073c15edcc 100644
--- a/hurd/hurdioctl.c
+++ b/hurd/hurdioctl.c
@@ -1,5 +1,5 @@
 /* ioctl commands which must be done in the C library.
-   Copyright (C) 1994, 1995, 1996, 1997 Free Software Foundation, Inc.
+   Copyright (C) 1994, 1995, 1996, 1997, 1999 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -132,13 +132,39 @@ _HURD_HANDLE_IOCTLS (fioclex, FIOCLEX, FIONCLEX);
 #include <hurd/term.h>
 #include <hurd/tioctl.h>
 
-static void
-rectty_dtable (mach_port_t cttyid)
+/* Install a new CTTYID port, atomically updating the dtable appropriately.
+   This consumes the send right passed in.  */
+
+void
+_hurd_locked_install_cttyid (mach_port_t cttyid)
 {
+  mach_port_t old;
+  struct hurd_port *const port = &_hurd_ports[INIT_PORT_CTTYID];
+  struct hurd_userlink ulink;
   int i;
 
-  HURD_CRITICAL_BEGIN;
-  __mutex_lock (&_hurd_dtable_lock);
+  /* Install the new cttyid port, and preserve it with a ulink.
+     We unroll the _hurd_port_set + _hurd_port_get here so that
+     there is no window where the cell is unlocked and CTTYID could
+     be changed by another thread.  (We also delay the deallocation
+     of the old port until the end, to minimize the duration of the
+     critical section.)
+
+     It is important that changing the cttyid port is only ever done by
+     holding the dtable lock continuously while updating the port cell and
+     re-ctty'ing the dtable; dtable.c assumes we do this.  Otherwise, the
+     pgrp-change notification code in dtable.c has to worry about racing
+     against us here in odd situations.  The one exception to this is
+     setsid, which holds the dtable lock while changing the pgrp and
+     clearing the cttyid port, and then unlocks the dtable lock to allow
+
+
+  */
+
+  __spin_lock (&port->lock);
+  old = _hurd_userlink_clear (&port->users) ? port->port : MACH_PORT_NULL;
+  port->port = cttyid;
+  cttyid = _hurd_port_locked_get (port, &ulink);
 
   for (i = 0; i < _hurd_dtablesize; ++i)
     {
@@ -178,6 +204,18 @@ rectty_dtable (mach_port_t cttyid)
     }
 
   __mutex_unlock (&_hurd_dtable_lock);
+
+  if (old != MACH_PORT_NULL)
+    __mach_port_deallocate (__mach_task_self (), old);
+  _hurd_port_free (port, &ulink, cttyid);
+}
+
+static void
+install_ctty (mach_port_t cttyid)
+{
+  HURD_CRITICAL_BEGIN;
+  __mutex_lock (&_hurd_dtable_lock);
+  _hurd_locked_install_cttyid (cttyid);
   HURD_CRITICAL_END;
 }
 
@@ -199,10 +237,7 @@ _hurd_setcttyid (mach_port_t cttyid)
     }
 
   /* Install the port, consuming the reference we just created.  */
-  _hurd_port_set (&_hurd_ports[INIT_PORT_CTTYID], cttyid);
-
-  /* Reset all the ctty ports in all the descriptors.  */
-  __USEPORT (CTTYID, (rectty_dtable (cttyid), 0));
+  install_ctty (cttyid);
 
   return 0;
 }
@@ -233,10 +268,7 @@ tiocsctty (int fd,
     return __hurd_fail (err);
 
   /* Make it our own.  */
-  _hurd_port_set (&_hurd_ports[INIT_PORT_CTTYID], cttyid);
-
-  /* Reset all the ctty ports in all the descriptors.  */
-  __USEPORT (CTTYID, (rectty_dtable (cttyid), 0));
+  install_ctty (cttyid);
 
   return 0;
 }
@@ -262,12 +294,8 @@ tiocnotty (int fd,
   if (err)
     return __hurd_fail (err);
 
-  /* Clear our cttyid port cell.  */
-  _hurd_port_set (&_hurd_ports[INIT_PORT_CTTYID], MACH_PORT_NULL);
-
-  /* Reset all the ctty ports in all the descriptors.  */
-
-  __USEPORT (CTTYID, (rectty_dtable (MACH_PORT_NULL), 0));
+  /* Clear our cttyid port.  */
+  install_ctty (MACH_PORT_NULL);
 
   return 0;
 }
diff --git a/hurd/hurdsig.c b/hurd/hurdsig.c
index 2c9625b474..47f5fbf0a4 100644
--- a/hurd/hurdsig.c
+++ b/hurd/hurdsig.c
@@ -1227,28 +1227,47 @@ _hurdsig_init (const int *intarray, size_t intarraysize)
 
   /* Start the signal thread listening on the message port.  */
 
-  err = __thread_create (__mach_task_self (), &_hurd_msgport_thread);
-  assert_perror (err);
+  if (__hurd_threadvar_stack_mask == 0)
+    {
+      err = __thread_create (__mach_task_self (), &_hurd_msgport_thread);
+      assert_perror (err);
 
-  stacksize = __vm_page_size * 8; /* Small stack for signal thread.  */
-  err = __mach_setup_thread (__mach_task_self (), _hurd_msgport_thread,
-			     _hurd_msgport_receive,
-			     (vm_address_t *) &__hurd_sigthread_stack_base,
-			     &stacksize);
-  assert_perror (err);
+      stacksize = ~__hurd_threadvar_stack_mask + 1;
+      stacksize = __vm_page_size * 8; /* Small stack for signal thread.  */
+      err = __mach_setup_thread (__mach_task_self (), _hurd_msgport_thread,
+				 _hurd_msgport_receive,
+				 (vm_address_t *) &__hurd_sigthread_stack_base,
+				 &stacksize);
+      assert_perror (err);
 
-  __hurd_sigthread_stack_end = __hurd_sigthread_stack_base + stacksize;
-  __hurd_sigthread_variables =
-    malloc (__hurd_threadvar_max * sizeof (unsigned long int));
-  if (__hurd_sigthread_variables == NULL)
-    __libc_fatal ("hurd: Can't allocate thread variables for signal thread\n");
+      __hurd_sigthread_stack_end = __hurd_sigthread_stack_base + stacksize;
+      __hurd_sigthread_variables =
+	malloc (__hurd_threadvar_max * sizeof (unsigned long int));
+      if (__hurd_sigthread_variables == NULL)
+	__libc_fatal ("hurd: Can't allocate threadvars for signal thread\n");
 
-  /* Reinitialize the MiG support routines so they will use a per-thread
-     variable for the cached reply port.  */
-  __mig_init ((void *) __hurd_sigthread_stack_base);
+      /* Reinitialize the MiG support routines so they will use a per-thread
+	 variable for the cached reply port.  */
+      __mig_init ((void *) __hurd_sigthread_stack_base);
 
-  err = __thread_resume (_hurd_msgport_thread);
-  assert_perror (err);
+      err = __thread_resume (_hurd_msgport_thread);
+      assert_perror (err);
+    }
+  else
+    {
+      /* When cthreads is being used, we need to make the signal thread a
+         proper cthread.  Otherwise it cannot use mutex_lock et al, which
+         will be the cthreads versions.  Various of the message port RPC
+         handlers need to take locks, so we need to be able to call into
+         cthreads code and meet its assumptions about how our thread and
+         its stack are arranged.  Since cthreads puts it there anyway,
+         we'll let the signal thread's per-thread variables be found as for
+         any normal cthread, and just leave the magic __hurd_sigthread_*
+         values all zero so they'll be ignored.  */
+#pragma weak cthread_fork
+#pragma weak cthread_detach
+      cthread_detach (cthread_fork ((cthread_fn_t) &_hurd_msgport_receive, 0));
+    }
 
   /* Receive exceptions on the signal port.  */
   __task_set_special_port (__mach_task_self (),
diff --git a/hurd/msgportdemux.c b/hurd/msgportdemux.c
index 4250affd0b..7f61758cc3 100644
--- a/hurd/msgportdemux.c
+++ b/hurd/msgportdemux.c
@@ -1,5 +1,5 @@
 /* Demux messages sent on the signal port.
-   Copyright (C) 1991, 1992, 1994, 1995, 1997 Free Software Foundation, Inc.
+   Copyright (C) 1991,92,94,95,97,99 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -57,8 +57,12 @@ _hurd_msgport_receive (void)
 {
   /* Get our own sigstate cached so we never again have to take a lock to
      fetch it.  There is much code in hurdsig.c that operates with some
-     sigstate lock held, which will deadlock with _hurd_thread_sigstate.  */
-  (void) _hurd_self_sigstate ();
+     sigstate lock held, which will deadlock with _hurd_thread_sigstate.
+
+     Furthermore, in the cthreads case this is the convenient spot
+     to initialize _hurd_msgport_thread (see hurdsig.c:_hurdsig_init).  */
+
+  _hurd_msgport_thread = _hurd_self_sigstate ()->thread;
 
   while (1)
     (void) __mach_msg_server (msgport_server, __vm_page_size, _hurd_msgport);
diff --git a/hurd/report-wait.c b/hurd/report-wait.c
index 41fdca1619..d0e3d47ad9 100644
--- a/hurd/report-wait.c
+++ b/hurd/report-wait.c
@@ -112,7 +112,7 @@ _S_msg_report_wait (mach_port_t msgport, thread_t thread,
   if (thread == _hurd_msgport_thread)
     /* Cute.  */
     strcpy (description, "msgport");
-  else if (&_hurd_msgport_thread && thread == _hurd_itimer_thread)
+  else if (&_hurd_itimer_thread && thread == _hurd_itimer_thread)
     strcpy (description, "itimer");
   else
     {
diff --git a/sysdeps/mach/hurd/fork.c b/sysdeps/mach/hurd/fork.c
index 3981ed2087..b28f5cb5d4 100644
--- a/sysdeps/mach/hurd/fork.c
+++ b/sysdeps/mach/hurd/fork.c
@@ -480,9 +480,27 @@ __fork (void)
 				    (natural_t *) &state, &statecount))
 	LOSE;
 #if STACK_GROWTH_UP
-      state.SP = __hurd_sigthread_stack_base;
+#define THREADVAR_SPACE (__hurd_threadvar_max \
+			 * sizeof *__hurd_sightread_variables)
+      if (__hurd_sigthread_stack_base == 0)
+	{
+	  state.SP &= __hurd_threadvar_stack_mask;
+	  state.SP += __hurd_threadvar_stack_offset + THREADVAR_SPACE;
+	}
+      else
+	state.SP = __hurd_sigthread_stack_base;
 #else
-      state.SP = __hurd_sigthread_stack_end;
+      if (__hurd_sigthread_stack_end == 0)
+	{
+	  /* The signal thread has a normal stack assigned by cthreads.
+	     The threadvar_stack variables conveniently tell us how
+	     to get to the highest address in the stack, just below
+	     the per-thread variables.  */
+	  state.SP &= __hurd_threadvar_stack_mask;
+	  state.SP += __hurd_threadvar_stack_offset;
+	}
+      else
+	state.SP = __hurd_sigthread_stack_end;
 #endif
       MACHINE_THREAD_STATE_SET_PC (&state,
 				   (unsigned long int) _hurd_msgport_receive);
diff --git a/sysdeps/mach/hurd/setsid.c b/sysdeps/mach/hurd/setsid.c
index 6653b81164..a1e84b0a1d 100644
--- a/sysdeps/mach/hurd/setsid.c
+++ b/sysdeps/mach/hurd/setsid.c
@@ -38,29 +38,32 @@ __setsid (void)
 
   /* Tell the proc server we want to start a new session.  */
   err = __USEPORT (PROC, __proc_setsid (port));
-  if (!err)
-    /* Punt our current ctty.  We hold the dtable lock from before the
-       proc_setsid call through clearing the cttyid port so that we can be
-       sure that it's been cleared by the time the signal thread attempts
-       to re-ctty the dtable in response to the pgrp change notification.  */
-    _hurd_port_set (&_hurd_ports[INIT_PORT_CTTYID], MACH_PORT_NULL);
+  if (err)
+    __mutex_unlock (&_hurd_dtable_lock);
+  else
+    {
+      /* Punt our current ctty, and update the dtable accordingly.  We hold
+	 the dtable lock from before the proc_setsid call through clearing
+	 the cttyid port and processing the dtable, so that we can be sure
+	 that it's all done by the time the signal thread processes the
+	 pgrp change notification.  */
+      _hurd_locked_install_cttyid (MACH_PORT_NULL);
 
-  __mutex_unlock (&_hurd_dtable_lock);
-
-  if (!err)
-    /* Synchronize with the signal thread to make sure we have
-       received and processed proc_newids before returning to the user.
-       This both updates _hurd_pgrp, and
-    */
-    while (_hurd_pids_changed_stamp == stamp)
-      {
+      /* Synchronize with the signal thread to make sure we have received
+	 and processed proc_newids before returning to the user.
+	 This is necessary to ensure that _hurd_pgrp (and thus the value
+	 returned by `getpgrp ()' in other threads) has been updated before
+	 we return.  */
+      while (_hurd_pids_changed_stamp == stamp)
+	{
 #ifdef noteven
-	/* XXX we have no need for a mutex, but cthreads demands one.  */
-	__condition_wait (&_hurd_pids_changed_sync, NULL);
+	  /* XXX we have no need for a mutex, but cthreads demands one.  */
+	  __condition_wait (&_hurd_pids_changed_sync, NULL);
 #else
-	__swtch_pri (0);
+	  __swtch_pri (0);
 #endif
-      }
+	}
+    }
 
   HURD_CRITICAL_END;