Forward port of rtkaio.

40 files changed, 369 insertions, 122 deletions

diff --git a/rtkaio/sysdeps/pthread/Makefile b/rtkaio/sysdeps/pthread/Makefile
index 99561b7659..7ced3eb5a9 100644
--- a/rtkaio/sysdeps/pthread/Makefile
+++ b/rtkaio/sysdeps/pthread/Makefile
@@ -14,6 +14,11 @@ $(objpfx)tst-timer: $(objpfx)librtkaio.so $(shared-thread-library)
 else
 $(objpfx)tst-timer: $(objpfx)librtkaio.a $(static-thread-library)
 endif
+
+ifeq ($(have-forced-unwind),yes)
+tests += tst-mqueue8x
+CFLAGS-tst-mqueue8x.c += -fexceptions
+endif
 endif
 
 endif
diff --git a/rtkaio/sysdeps/pthread/Versions b/rtkaio/sysdeps/pthread/Versions
new file mode 100644
index 0000000000..7677b76687
--- /dev/null
+++ b/rtkaio/sysdeps/pthread/Versions
@@ -0,0 +1,7 @@
+%ifdef HAVE_FORCED_UNWIND
+librtkaio {
+  GLIBC_2.4 {
+    lio_listio; lio_listio64;
+  }
+}
+%endif
diff --git a/rtkaio/sysdeps/pthread/tst-mqueue8x.c b/rtkaio/sysdeps/pthread/tst-mqueue8x.c
new file mode 100644
index 0000000000..1259ebdf50
--- /dev/null
+++ b/rtkaio/sysdeps/pthread/tst-mqueue8x.c
@@ -0,0 +1 @@
+#include_next <tst-mqueue8x.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_cancel.c b/rtkaio/sysdeps/rtkaio/kaio_cancel.c
deleted file mode 100644
index f1ea4b7ccd..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_cancel.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_cancel.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_error.c b/rtkaio/sysdeps/rtkaio/kaio_error.c
deleted file mode 100644
index bad0a22a1e..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_error.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <rt/aio_error.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_fsync.c b/rtkaio/sysdeps/rtkaio/kaio_fsync.c
deleted file mode 100644
index f3483aef9e..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_fsync.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_fsync.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_misc.c b/rtkaio/sysdeps/rtkaio/kaio_misc.c
deleted file mode 100644
index 16eaa21cf5..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_misc.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_misc.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_notify.c b/rtkaio/sysdeps/rtkaio/kaio_notify.c
deleted file mode 100644
index 6861e7fd78..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_notify.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_notify.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_read.c b/rtkaio/sysdeps/rtkaio/kaio_read.c
deleted file mode 100644
index 5d7774b818..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_read.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_read.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_read64.c b/rtkaio/sysdeps/rtkaio/kaio_read64.c
deleted file mode 100644
index becd1bee8e..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_read64.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_read64.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_return.c b/rtkaio/sysdeps/rtkaio/kaio_return.c
deleted file mode 100644
index 7cff9dcc40..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_return.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <rt/aio_return.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_sigqueue.c b/rtkaio/sysdeps/rtkaio/kaio_sigqueue.c
deleted file mode 100644
index 6e49f01d61..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_sigqueue.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_sigqueue.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_suspend.c b/rtkaio/sysdeps/rtkaio/kaio_suspend.c
deleted file mode 100644
index 145420bd12..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_suspend.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_suspend.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_write.c b/rtkaio/sysdeps/rtkaio/kaio_write.c
deleted file mode 100644
index 68e47ca1f3..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_write.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_write.c>
diff --git a/rtkaio/sysdeps/rtkaio/kaio_write64.c b/rtkaio/sysdeps/rtkaio/kaio_write64.c
deleted file mode 100644
index 316e55dcf3..0000000000
--- a/rtkaio/sysdeps/rtkaio/kaio_write64.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <aio_write64.c>
diff --git a/rtkaio/sysdeps/rtkaio/klio_listio.c b/rtkaio/sysdeps/rtkaio/klio_listio.c
deleted file mode 100644
index 08378bd22d..0000000000
--- a/rtkaio/sysdeps/rtkaio/klio_listio.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <lio_listio.c>
diff --git a/rtkaio/sysdeps/rtkaio/klio_listio64.c b/rtkaio/sysdeps/rtkaio/klio_listio64.c
deleted file mode 100644
index 3a087e41b1..0000000000
--- a/rtkaio/sysdeps/rtkaio/klio_listio64.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <lio_listio64.c>
diff --git a/rtkaio/sysdeps/unix/alpha/Makefile b/rtkaio/sysdeps/unix/alpha/Makefile
new file mode 100644
index 0000000000..223ec37222
--- /dev/null
+++ b/rtkaio/sysdeps/unix/alpha/Makefile
@@ -0,0 +1,3 @@
+ifeq ($(subdir),rtkaio)
+librtkaio-sysdep_routines += rtkaio-sysdep
+endif
diff --git a/rtkaio/sysdeps/unix/alpha/rtkaio-sysdep.S b/rtkaio/sysdeps/unix/alpha/rtkaio-sysdep.S
new file mode 100644
index 0000000000..11ee214b23
--- /dev/null
+++ b/rtkaio/sysdeps/unix/alpha/rtkaio-sysdep.S
@@ -0,0 +1 @@
+#include <rt-sysdep.S>
diff --git a/rtkaio/sysdeps/unix/sysv/linux/Makefile b/rtkaio/sysdeps/unix/sysv/linux/Makefile
new file mode 100644
index 0000000000..9ed3d7551a
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/Makefile
@@ -0,0 +1,4 @@
+ifeq ($(subdir),rtkaio)
+CFLAGS-kaio_mq_send.c += -fexceptions
+CFLAGS-kaio_mq_receive.c += -fexceptions
+endif
diff --git a/rtkaio/sysdeps/unix/sysv/linux/hppa/Versions b/rtkaio/sysdeps/unix/sysv/linux/hppa/Versions
deleted file mode 100644
index b2d59a9140..0000000000
--- a/rtkaio/sysdeps/unix/sysv/linux/hppa/Versions
+++ /dev/null
@@ -1,6 +0,0 @@
-librtkaio {
-  GLIBC_2.3 {
-    # AIO functions.
-    aio_cancel; aio_cancel64;
-  }
-}
diff --git a/rtkaio/sysdeps/unix/sysv/linux/hppa/kaio_cancel.c b/rtkaio/sysdeps/unix/sysv/linux/hppa/kaio_cancel.c
deleted file mode 100644
index 6e345e1643..0000000000
--- a/rtkaio/sysdeps/unix/sysv/linux/hppa/kaio_cancel.c
+++ /dev/null
@@ -1,33 +0,0 @@
-#include <shlib-compat.h>
-
-#define aio_cancel64 XXX
-#include <aio.h>
-#undef aio_cancel64
-#include <errno.h>
-
-extern __typeof (aio_cancel) __new_aio_cancel;
-extern __typeof (aio_cancel) __old_aio_cancel;
-
-#define aio_cancel	__new_aio_cancel
-
-#include <sysdeps/unix/sysv/linux/kaio_cancel.c>
-
-#undef aio_cancel
-strong_alias (__new_aio_cancel, __new_aio_cancel64);
-versioned_symbol (librt, __new_aio_cancel, aio_cancel, GLIBC_2_3);
-versioned_symbol (librt, __new_aio_cancel64, aio_cancel64, GLIBC_2_3);
-
-#if SHLIB_COMPAT (librt, GLIBC_2_1, GLIBC_2_3)
-
-#undef ECANCELED
-#define aio_cancel	__old_aio_cancel
-#define ECANCELED	125
-
-#include <sysdeps/unix/sysv/linux/kaio_cancel.c>
-
-#undef aio_cancel
-strong_alias (__old_aio_cancel, __old_aio_cancel64);
-compat_symbol (librt, __old_aio_cancel, aio_cancel, GLIBC_2_1);
-compat_symbol (librt, __old_aio_cancel64, aio_cancel64, GLIBC_2_1);
-
-#endif
diff --git a/rtkaio/sysdeps/unix/sysv/linux/ia64/Makefile b/rtkaio/sysdeps/unix/sysv/linux/ia64/Makefile
new file mode 100644
index 0000000000..ead21fb111
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/ia64/Makefile
@@ -0,0 +1,3 @@
+ifeq ($(subdir),rtkaio)
+librtkaio-routines += rtkaio-sysdep
+endif
diff --git a/rtkaio/sysdeps/unix/sysv/linux/ia64/rtkaio-sysdep.S b/rtkaio/sysdeps/unix/sysv/linux/ia64/rtkaio-sysdep.S
new file mode 100644
index 0000000000..11ee214b23
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/ia64/rtkaio-sysdep.S
@@ -0,0 +1 @@
+#include <rt-sysdep.S>
diff --git a/rtkaio/sysdeps/unix/sysv/linux/kaio_cancel.c b/rtkaio/sysdeps/unix/sysv/linux/kaio_cancel.c
index 19d7e1b16d..7d2738d5bc 100644
--- a/rtkaio/sysdeps/unix/sysv/linux/kaio_cancel.c
+++ b/rtkaio/sysdeps/unix/sysv/linux/kaio_cancel.c
@@ -1,5 +1,5 @@
 /* Cancel requests associated with given file descriptor.
-   Copyright (C) 1997, 1998, 2000, 2002, 2003 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 2000, 2002, 2003, 2005 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
diff --git a/rtkaio/sysdeps/unix/sysv/linux/kaio_error.c b/rtkaio/sysdeps/unix/sysv/linux/kaio_error.c
index 509913d149..23859c363a 100644
--- a/rtkaio/sysdeps/unix/sysv/linux/kaio_error.c
+++ b/rtkaio/sysdeps/unix/sysv/linux/kaio_error.c
@@ -31,7 +31,7 @@
 #include <kaio_misc.h>
 
 #ifndef USE_KAIO
-#include <rt/aio_error.c>
+#include <aio_error.c>
 #else
 
 #include <errno.h>
diff --git a/rtkaio/sysdeps/unix/sysv/linux/kaio_misc.c b/rtkaio/sysdeps/unix/sysv/linux/kaio_misc.c
index 77c55b3e8b..76e0c430b7 100644
--- a/rtkaio/sysdeps/unix/sysv/linux/kaio_misc.c
+++ b/rtkaio/sysdeps/unix/sysv/linux/kaio_misc.c
@@ -1,5 +1,5 @@
 /* Handle general operations.
-   Copyright (C) 1997,1998,1999,2000,2001,2002,2003
+   Copyright (C) 1997,1998,1999,2000,2001,2002,2003,2006
    Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -27,6 +27,7 @@
 
 #include <aio.h>
 #include <assert.h>
+#include <atomic.h>
 #include <errno.h>
 #include <limits.h>
 #include <pthread.h>
@@ -34,6 +35,28 @@
 #include <unistd.h>
 #include <sys/stat.h>
 #include <sys/time.h>
+#include <sys/sysmacros.h>
+
+#ifndef aio_create_helper_thread
+# define aio_create_helper_thread __aio_create_helper_thread
+
+extern inline int
+__aio_create_helper_thread (pthread_t *threadp, void *(*tf) (void *), void *arg)
+{
+  pthread_attr_t attr;
+
+  /* Make sure the thread is created detached.  */
+  pthread_attr_init (&attr);
+  pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
+
+  int ret = pthread_create (threadp, &attr, tf, arg);
+
+  (void) pthread_attr_destroy (&attr);
+  return ret;
+}
+
+#endif
+
 
 static void add_request_to_runlist (struct requestlist *newrequest)
 	internal_function;
@@ -359,14 +382,16 @@ static void
 kernel_callback (kctx_t ctx, struct kiocb *kiocb, long res, long res2)
 {
   struct requestlist *req = (struct requestlist *)kiocb;
+  long errcode = 0;
 
-  req->aiocbp->aiocb.__error_code = 0;
-  req->aiocbp->aiocb.__return_value = res;
   if (res < 0 && res > -1000)
     {
-      req->aiocbp->aiocb.__error_code = -res;
-      req->aiocbp->aiocb.__return_value = -1;
+      errcode = -res;
+      res = -1;
     }
+  req->aiocbp->aiocb.__return_value = res;
+  atomic_write_barrier ();
+  req->aiocbp->aiocb.__error_code = errcode;
   __aio_notify (req);
   assert (req->running == allocated);
   req->running = done;
@@ -421,7 +446,7 @@ __aio_wait_for_events (kctx_t kctx, const struct timespec *timespec)
   ts.tv_nsec = 0;
   do
     {
-      ret = INTERNAL_SYSCALL (io_getevents, err, 5, kctx, 0, 10, ev,
+      ret = INTERNAL_SYSCALL (io_getevents, err, 5, kctx, 1, 10, ev,
 			      timespec);
       if (INTERNAL_SYSCALL_ERROR_P (ret, err) || ret == 0)
 	break;
@@ -453,16 +478,11 @@ internal_function
 __aio_create_kernel_thread (void)
 {
   pthread_t thid;
-  pthread_attr_t attr;
 
   if (__kernel_thread_started)
     return 0;
 
-  /* Make sure the thread is created detached.  */
-  pthread_attr_init (&attr);
-  pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
-
-  if (pthread_create (&thid, &attr, handle_kernel_aio, NULL) != 0)
+  if (aio_create_helper_thread (&thid, handle_kernel_aio, NULL) != 0)
     return -1;
   __kernel_thread_started = 1;
   return 0;
@@ -477,7 +497,7 @@ handle_kernel_aio (void *arg __attribute__((unused)))
 
   for (;;)
     {
-      ret = INTERNAL_SYSCALL (io_getevents, err, 5, __aio_kioctx, 0, 10, ev,
+      ret = INTERNAL_SYSCALL (io_getevents, err, 5, __aio_kioctx, 1, 10, ev,
 			      NULL);
       if (INTERNAL_SYSCALL_ERROR_P (ret, err) || ret == 0)
         continue;
@@ -593,16 +613,11 @@ __aio_enqueue_user_request (struct requestlist *newp)
       if (nthreads < optim.aio_threads && idle_thread_count == 0)
 	{
 	  pthread_t thid;
-	  pthread_attr_t attr;
-
-	  /* Make sure the thread is created detached.  */
-	  pthread_attr_init (&attr);
-	  pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
 
 	  running = newp->running = allocated;
 
 	  /* Now try to start a thread.  */
-	  if (pthread_create (&thid, &attr, handle_fildes_io, newp) == 0)
+	  if (aio_create_helper_thread (&thid, handle_fildes_io, newp) == 0)
 	    /* We managed to enqueue the request.  All errors which can
 	       happen now can be recognized by calls to `aio_return' and
 	       `aio_error'.  */
@@ -653,6 +668,7 @@ __aio_enqueue_request_ctx (aiocb_union *aiocbp, int operation, kctx_t kctx)
       aiocbp->aiocb.aio_reqprio = 0;
       /* FIXME: Kernel doesn't support sync yet.  */
       operation &= ~LIO_KTHREAD;
+      kctx = KCTX_NONE;
     }
   else if (aiocbp->aiocb.aio_reqprio < 0
 	   || aiocbp->aiocb.aio_reqprio > AIO_PRIO_DELTA_MAX)
@@ -664,6 +680,23 @@ __aio_enqueue_request_ctx (aiocb_union *aiocbp, int operation, kctx_t kctx)
       return NULL;
     }
 
+  if ((operation & LIO_KTHREAD) || kctx != KCTX_NONE)
+    {
+      /* io_* is only really asynchronous for O_DIRECT or /dev/raw*.  */
+      int fl = __fcntl (aiocbp->aiocb.aio_fildes, F_GETFL);
+      if (fl < 0 || (fl & O_DIRECT) == 0)
+	{
+	  struct stat64 st;
+	  if (__fxstat64 (_STAT_VER, aiocbp->aiocb.aio_fildes, &st) < 0
+	      || ! S_ISCHR (st.st_mode)
+	      || major (st.st_rdev) != 162)
+	    {
+	      operation &= ~LIO_KTHREAD;
+	      kctx = KCTX_NONE;
+	    }
+	}
+    }
+
   /* Compute priority for this request.  */
   pthread_getschedparam (pthread_self (), &policy, &param);
   prio = param.sched_priority - aiocbp->aiocb.aio_reqprio;
@@ -799,7 +832,9 @@ wait_for_kernel_requests (int fildes)
 	  return -1;
 	}
 
+#ifndef DONT_NEED_AIO_MISC_COND
       pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+#endif
       struct waitlist waitlist[nent];
       int cnt = 0;
 
@@ -807,7 +842,10 @@ wait_for_kernel_requests (int fildes)
 	{
 	  if (kreq->running == allocated)
 	    {
+#ifndef DONT_NEED_AIO_MISC_COND
 	      waitlist[cnt].cond = &cond;
+#endif
+	      waitlist[cnt].result = NULL;
 	      waitlist[cnt].next = kreq->waiting;
 	      waitlist[cnt].counterp = &nent;
 	      waitlist[cnt].sigevp = NULL;
@@ -819,11 +857,15 @@ wait_for_kernel_requests (int fildes)
 	  kreq = kreq->next_prio;
 	}
 
+#ifdef DONT_NEED_AIO_MISC_COND
+      AIO_MISC_WAIT (ret, nent, NULL, 0);
+#else
       do
 	pthread_cond_wait (&cond, &__aio_requests_mutex);
       while (nent);
 
       pthread_cond_destroy (&cond);
+#endif
     }
 
   pthread_mutex_unlock (&__aio_requests_mutex);
@@ -832,7 +874,6 @@ wait_for_kernel_requests (int fildes)
 
 
 static void *
-__attribute__ ((noreturn))
 handle_fildes_io (void *arg)
 {
   pthread_t self = pthread_self ();
@@ -1026,16 +1067,11 @@ handle_fildes_io (void *arg)
 	      else if (nthreads < optim.aio_threads)
 		{
 		  pthread_t thid;
-		  pthread_attr_t attr;
-
-		  /* Make sure the thread is created detached.  */
-		  pthread_attr_init (&attr);
-		  pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
 
 		  /* Now try to start a thread. If we fail, no big deal,
 		     because we know that there is at least one thread (us)
 		     that is working on AIO operations. */
-		  if (pthread_create (&thid, &attr, handle_fildes_io, NULL)
+		  if (aio_create_helper_thread (&thid, handle_fildes_io, NULL)
 		      == 0)
 		    ++nthreads;
 		}
@@ -1047,7 +1083,7 @@ handle_fildes_io (void *arg)
     }
   while (runp != NULL);
 
-  pthread_exit (NULL);
+  return NULL;
 }
 
 
diff --git a/rtkaio/sysdeps/unix/sysv/linux/kaio_misc.h b/rtkaio/sysdeps/unix/sysv/linux/kaio_misc.h
index bee75f3f2b..5e0ca19c31 100644
--- a/rtkaio/sysdeps/unix/sysv/linux/kaio_misc.h
+++ b/rtkaio/sysdeps/unix/sysv/linux/kaio_misc.h
@@ -1,4 +1,5 @@
-/* Copyright (C) 1997,1999,2000,2001,2002,2003 Free Software Foundation, Inc.
+/* Copyright (C) 1997,1999,2000,2001,2002,2003,2006
+   Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -34,6 +35,68 @@
 #include <aio.h>
 #include <pthread.h>
 #include <stdint.h>
+#include <signal.h>
+#include <sysdep.h>
+#include <limits.h>
+
+#ifdef HAVE_FORCED_UNWIND
+
+/* We define a special synchronization primitive for AIO.  POSIX
+   conditional variables would be ideal but the pthread_cond_*wait
+   operations do not return on EINTR.  This is a requirement for
+   correct aio_suspend and lio_listio implementations.  */
+
+#include <assert.h>
+#include <pthreadP.h>
+#include <lowlevellock.h>
+
+# define DONT_NEED_AIO_MISC_COND	1
+
+# define AIO_MISC_NOTIFY(waitlist) \
+  do {									      \
+    if (*waitlist->counterp > 0 && --*waitlist->counterp == 0)		      \
+      lll_futex_wake (waitlist->counterp, 1);				      \
+  } while (0)
+
+# define AIO_MISC_WAIT(result, futex, timeout, cancel)			      \
+  do {									      \
+    volatile int *futexaddr = &futex;					      \
+    int oldval = futex;							      \
+									      \
+    if (oldval != 0)							      \
+      {									      \
+	pthread_mutex_unlock (&__aio_requests_mutex);			      \
+									      \
+	int oldtype;							      \
+	if (cancel)							      \
+	  oldtype = LIBC_CANCEL_ASYNC ();				      \
+									      \
+	int status;							      \
+	do								      \
+	  {								      \
+	    status = lll_futex_timed_wait (futexaddr, oldval, timeout);	      \
+	    if (status != -EWOULDBLOCK)					      \
+	      break;							      \
+									      \
+	    oldval = *futexaddr;					      \
+	  }								      \
+	while (oldval != 0);						      \
+									      \
+	if (cancel)							      \
+	  LIBC_CANCEL_RESET (oldtype);					      \
+									      \
+	if (status == -EINTR)						      \
+	  result = EINTR;						      \
+	else if (status == -ETIMEDOUT)					      \
+	  result = EAGAIN;						      \
+	else								      \
+	  assert (status == 0 || status == -EWOULDBLOCK);		      \
+									      \
+	pthread_mutex_lock (&__aio_requests_mutex);			      \
+      }									      \
+  } while (0)
+
+#endif
 
 typedef unsigned long kctx_t;
 #define KCTX_NONE ~0UL
@@ -95,7 +158,12 @@ struct waitlist
   {
     struct waitlist *next;
 
+    /* The next two fields is used in synchronous io_listio' operations.  */
+#ifndef DONT_NEED_AIO_MISC_COND
     pthread_cond_t *cond;
+#endif
+    int *result;
+
     volatile int *counterp;
     /* The next field is used in asynchronous `lio_listio' operations.  */
     struct sigevent *sigevp;
@@ -212,5 +280,49 @@ extern int __aio_create_kernel_thread (void)
 extern int __have_no_kernel_aio attribute_hidden;
 extern int __kernel_thread_started attribute_hidden;
 
+#ifndef BROKEN_THREAD_SIGNALS
+# define aio_start_notify_thread __aio_start_notify_thread
+# define aio_create_helper_thread __aio_create_helper_thread
+
+extern inline void
+__aio_start_notify_thread (void)
+{
+  sigset_t ss;
+  sigemptyset (&ss);
+  INTERNAL_SYSCALL_DECL (err);
+  INTERNAL_SYSCALL (rt_sigprocmask, err, 4, SIG_SETMASK, &ss, NULL, _NSIG / 8);
+}
+
+extern inline int
+__aio_create_helper_thread (pthread_t *threadp, void *(*tf) (void *), void *arg)
+{
+  pthread_attr_t attr;
+
+  /* Make sure the thread is created detached.  */
+  pthread_attr_init (&attr);
+  pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
+
+  /* The helper thread needs only very little resources.  */
+  (void) pthread_attr_setstacksize (&attr, PTHREAD_STACK_MIN);
+
+  /* Block all signals in the helper thread.  To do this thoroughly we
+     temporarily have to block all signals here.  */
+  sigset_t ss;
+  sigset_t oss;
+  sigfillset (&ss);
+  INTERNAL_SYSCALL_DECL (err);
+  INTERNAL_SYSCALL (rt_sigprocmask, err, 4, SIG_SETMASK, &ss, &oss, _NSIG / 8);
+
+  int ret = pthread_create (threadp, &attr, tf, arg);
+
+  /* Restore the signal mask.  */
+  INTERNAL_SYSCALL (rt_sigprocmask, err, 4, SIG_SETMASK, &oss, NULL,
+		    _NSIG / 8);
+
+  (void) pthread_attr_destroy (&attr);
+  return ret;
+}                                                                                 
+#endif
+
 #endif
 #endif /* aio_misc.h */
diff --git a/rtkaio/sysdeps/unix/sysv/linux/kaio_return.c b/rtkaio/sysdeps/unix/sysv/linux/kaio_return.c
index 462514b477..1d9f6cbabb 100644
--- a/rtkaio/sysdeps/unix/sysv/linux/kaio_return.c
+++ b/rtkaio/sysdeps/unix/sysv/linux/kaio_return.c
@@ -32,7 +32,7 @@
 #include <kaio_misc.h>
 
 #ifndef USE_KAIO
-#include <rt/aio_return.c>
+#include <aio_return.c>
 #else
 
 #include <errno.h>
diff --git a/rtkaio/sysdeps/unix/sysv/linux/kaio_suspend.c b/rtkaio/sysdeps/unix/sysv/linux/kaio_suspend.c
index 0b3380f98a..2400c5223c 100644
--- a/rtkaio/sysdeps/unix/sysv/linux/kaio_suspend.c
+++ b/rtkaio/sysdeps/unix/sysv/linux/kaio_suspend.c
@@ -1,5 +1,6 @@
 /* Suspend until termination of a requests.
-   Copyright (C) 1997,1998,1999,2000,2002,2003 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2000, 2002, 2003, 2006
+   Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -37,8 +38,10 @@
 
 #include <assert.h>
 #include <errno.h>
+#include <stdbool.h>
 #include <stdlib.h>
 #include <sys/time.h>
+
 #include <bits/libc-lock.h>
 #include <sysdep-cancel.h>
 
@@ -48,7 +51,9 @@ struct clparam
   const struct aiocb *const *list;
   struct waitlist *waitlist;
   struct requestlist **requestlist;
+#ifndef DONT_NEED_AIO_MISC_COND
   pthread_cond_t *cond;
+#endif
   int nent;
 };
 
@@ -56,6 +61,12 @@ struct clparam
 static void
 cleanup (void *arg)
 {
+#ifdef DONT_NEED_AIO_MISC_COND
+  /* Acquire the mutex.  If pthread_cond_*wait is used this would
+     happen implicitly.  */
+  pthread_mutex_lock (&__aio_requests_mutex);
+#endif
+
   const struct clparam *param = (const struct clparam *) arg;
 
   /* Now remove the entry in the waiting list for all requests
@@ -79,8 +90,10 @@ cleanup (void *arg)
 	  *listp = (*listp)->next;
       }
 
+#ifndef DONT_NEED_AIO_MISC_COND
   /* Release the conditional variable.  */
   (void) pthread_cond_destroy (param->cond);
+#endif
 
   /* Release the mutex.  */
   pthread_mutex_unlock (&__aio_requests_mutex);
@@ -93,11 +106,20 @@ aio_suspend (list, nent, timeout)
      int nent;
      const struct timespec *timeout;
 {
+  if (__builtin_expect (nent < 0, 0))
+    {
+      __set_errno (EINVAL);
+      return -1;
+    }
+
   struct waitlist waitlist[nent];
   struct requestlist *requestlist[nent];
+#ifndef DONT_NEED_AIO_MISC_COND
   pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
+#endif
   int cnt;
   int result = 0;
+  int cntr = 1;
   int total = 0, ktotal = 0;
 
   /* Request the mutex.  */
@@ -114,9 +136,12 @@ aio_suspend (list, nent, timeout)
 
 	    if (requestlist[cnt] != NULL)
 	      {
+#ifndef DONT_NEED_AIO_MISC_COND
 		waitlist[cnt].cond = &cond;
+#endif
+		waitlist[cnt].result = NULL;
 		waitlist[cnt].next = requestlist[cnt]->waiting;
-		waitlist[cnt].counterp = &total;
+		waitlist[cnt].counterp = &cntr;
 		waitlist[cnt].sigevp = NULL;
 #ifdef BROKEN_THREAD_SIGNALS
 		waitlist[cnt].caller_pid = 0;	/* Not needed.  */
@@ -144,13 +169,15 @@ aio_suspend (list, nent, timeout)
 	  .list = list,
 	  .waitlist = waitlist,
 	  .requestlist = requestlist,
+#ifndef DONT_NEED_AIO_MISC_COND
 	  .cond = &cond,
+#endif
 	  .nent = nent
 	};
 
       pthread_cleanup_push (cleanup, &clparam);
 
-      if (!__kernel_thread_started)
+      if (!__kernel_thread_started && ktotal)
 	{
 	  /* If the kernel aio thread was not started yet all requests
 	     are served by the kernel and there are no other threads running,
@@ -160,7 +187,7 @@ aio_suspend (list, nent, timeout)
 	    {
 	      if (timeout == NULL)
 		{
-		  while (total == ktotal)
+		  while (cntr == 1)
 		    __aio_wait_for_events (__aio_kioctx, NULL);
 		}
 	      else
@@ -180,7 +207,7 @@ aio_suspend (list, nent, timeout)
 		  for (;;)
 		    {
 		      result = __aio_wait_for_events (__aio_kioctx, timeout);
-		      if (total < ktotal)
+		      if (cntr < 1)
 			break;
 		      if (result == ETIMEDOUT)
 			break;
@@ -201,7 +228,7 @@ aio_suspend (list, nent, timeout)
 		      timeout = &ts;
 		    }
 
-		  if (total < ktotal)
+		  if (cntr < 1)
 		    result = 0;
 		  else
 		    result = ETIMEDOUT;
@@ -219,6 +246,10 @@ aio_suspend (list, nent, timeout)
       if (total == 0)
 	/* Suspending was handled above.  */
 	;
+#ifdef DONT_NEED_AIO_MISC_COND
+      else
+	AIO_MISC_WAIT (result, cntr, timeout, 1);
+#else
       else if (timeout == NULL)
 	result = pthread_cond_wait (&cond, &__aio_requests_mutex);
       else
@@ -240,6 +271,7 @@ aio_suspend (list, nent, timeout)
 	  result = pthread_cond_timedwait (&cond, &__aio_requests_mutex,
 					   &abstime);
 	}
+#endif
 
       pthread_cleanup_pop (0);
     }
@@ -263,19 +295,23 @@ aio_suspend (list, nent, timeout)
 	  *listp = (*listp)->next;
       }
 
+#ifndef DONT_NEED_AIO_MISC_COND
   /* Release the conditional variable.  */
   if (__builtin_expect (pthread_cond_destroy (&cond) != 0, 0))
     /* This must never happen.  */
     abort ();
+#endif
 
   if (result != 0)
     {
-      /* An error occurred.  Possibly it's EINTR.  We have to translate
+#ifndef DONT_NEED_AIO_MISC_COND
+      /* An error occurred.  Possibly it's ETIMEDOUT.  We have to translate
 	 the timeout error report of `pthread_cond_timedwait' to the
 	 form expected from `aio_suspend'.  */
       if (result == ETIMEDOUT)
 	__set_errno (EAGAIN);
       else
+#endif
 	__set_errno (result);
 
       result = -1;
diff --git a/rtkaio/sysdeps/unix/sysv/linux/klio_listio.c b/rtkaio/sysdeps/unix/sysv/linux/klio_listio.c
index 73cb0d547b..b7676c9f03 100644
--- a/rtkaio/sysdeps/unix/sysv/linux/klio_listio.c
+++ b/rtkaio/sysdeps/unix/sysv/linux/klio_listio.c
@@ -1,5 +1,5 @@
 /* Enqueue and list of read or write requests.
-   Copyright (C) 1997,1998,1999,2000,2001,2002,2003
+   Copyright (C) 1997,1998,1999,2000,2001,2002,2003,2005,2006
    Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -25,16 +25,19 @@
 #include <lio_listio.c>
 #else
 
+#ifndef lio_listio
 #include <aio.h>
 #include <assert.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <unistd.h>
 
-#ifndef lio_listio
 #define LIO_OPCODE_BASE 0
 #endif
 
+#include <shlib-compat.h>
+
+
 /* We need this special structure to handle asynchronous I/O.  */
 struct async_waitlist
   {
@@ -43,12 +46,23 @@ struct async_waitlist
     struct waitlist list[0];
   };
 
-int
-lio_listio (mode, list, nent, sig)
-     int mode;
-     struct aiocb *const list[];
-     int nent;
-     struct sigevent *sig;
+
+/* The code in glibc 2.1 to glibc 2.4 issued only one event when all
+   requests submitted with lio_listio finished.  The existing practice
+   is to issue events for the individual requests as well.  This is
+   what the new code does.  */
+#if SHLIB_COMPAT (librt, GLIBC_2_1, GLIBC_2_4)
+# define LIO_MODE(mode) ((mode) & 127)
+# define NO_INDIVIDUAL_EVENT_P(mode) ((mode) & 128)
+#else
+# define LIO_MODE(mode) mode
+# define NO_INDIVIDUAL_EVENT_P(mode) 0
+#endif
+
+
+static int
+lio_listio_internal (int mode, struct aiocb *const list[], int nent,
+		     struct sigevent *sig)
 {
   struct sigevent defsigev;
   struct requestlist *requests[nent];
@@ -57,13 +71,6 @@ lio_listio (mode, list, nent, sig)
   int result = 0, op = 0;
   kctx_t kctx = KCTX_NONE;
 
-  /* Check arguments.  */
-  if (mode != LIO_WAIT && mode != LIO_NOWAIT)
-    {
-      __set_errno (EINVAL);
-      return -1;
-    }
-
   if (sig == NULL)
     {
       defsigev.sigev_notify = SIGEV_NONE;
@@ -73,7 +80,7 @@ lio_listio (mode, list, nent, sig)
   /* Request the mutex.  */
   pthread_mutex_lock (&__aio_requests_mutex);
 
-  if (mode == LIO_WAIT && ! __have_no_kernel_aio && nent > 0)
+  if (LIO_MODE (mode) == LIO_WAIT && ! __have_no_kernel_aio && nent > 0)
     {
       int res;
       INTERNAL_SYSCALL_DECL (err);
@@ -90,7 +97,7 @@ lio_listio (mode, list, nent, sig)
 	    __have_no_kernel_aio = 1;
 	}
     }
-  else if (mode == LIO_NOWAIT)
+  else if (LIO_MODE (mode) == LIO_NOWAIT)
     {
       op = LIO_KTHREAD;
       if (sig->sigev_notify != SIGEV_NONE)
@@ -103,7 +110,8 @@ lio_listio (mode, list, nent, sig)
   for (cnt = 0; cnt < nent; ++cnt)
     if (list[cnt] != NULL && list[cnt]->aio_lio_opcode != LIO_NOP)
       {
-	list[cnt]->aio_sigevent.sigev_notify = SIGEV_NONE;
+	if (NO_INDIVIDUAL_EVENT_P (mode))
+	  list[cnt]->aio_sigevent.sigev_notify = SIGEV_NONE;
 	requests[cnt]
 	  = __aio_enqueue_request_ctx ((aiocb_union *) list[cnt],
 				       list[cnt]->aio_lio_opcode | op,
@@ -136,7 +144,7 @@ lio_listio (mode, list, nent, sig)
 	 locked forever.  */
       pthread_mutex_unlock (&__aio_requests_mutex);
 
-      if (mode == LIO_NOWAIT)
+      if (LIO_MODE (mode) == LIO_NOWAIT)
 	{
 #ifdef BROKEN_THREAD_SIGNALS
 	  __aio_notify_only (sig,
@@ -148,11 +156,13 @@ lio_listio (mode, list, nent, sig)
 
       return result;
     }
-  else if (mode == LIO_WAIT)
+  else if (LIO_MODE (mode) == LIO_WAIT)
     {
+#ifndef DONT_NEED_AIO_MISC_COND
       pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
-      struct waitlist waitlist[nent];
       int oldstate;
+#endif
+      struct waitlist waitlist[nent];
       volatile int ktotal = 0;
 
       total = 0;
@@ -173,7 +183,10 @@ lio_listio (mode, list, nent, sig)
 		  waitlist[cnt].counterp = &total;
 	  	  ++total;
 		}
+#ifndef DONT_NEED_AIO_MISC_COND
 	      waitlist[cnt].cond = &cond;
+#endif
+	      waitlist[cnt].result = &result;
 	      waitlist[cnt].next = requests[cnt]->waiting;
 	      waitlist[cnt].sigevp = NULL;
 #ifdef BROKEN_THREAD_SIGNALS
@@ -183,29 +196,40 @@ lio_listio (mode, list, nent, sig)
 	    }
 	}
 
-      /* Since `pthread_cond_wait'/`pthread_cond_timedwait' are cancelation
+      while (ktotal > 0)
+	__aio_wait_for_events (kctx, NULL);
+#ifdef DONT_NEED_AIO_MISC_COND
+      AIO_MISC_WAIT (result, total, NULL, 0);
+#else
+      /* Since `pthread_cond_wait'/`pthread_cond_timedwait' are cancellation
 	 points we must be careful.  We added entries to the waiting lists
-	 which we must remove.  So defer cancelation for now.  */
+	 which we must remove.  So defer cancellation for now.  */
       pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, &oldstate);
 
-      while (ktotal > 0)
-	__aio_wait_for_events (kctx, NULL);
       while (total > 0)
 	pthread_cond_wait (&cond, &__aio_requests_mutex);
 
-      /* Now it's time to restore the cancelation state.  */
+      /* Now it's time to restore the cancellation state.  */
       pthread_setcancelstate (oldstate, NULL);
 
+      /* Release the conditional variable.  */
+      if (pthread_cond_destroy (&cond) != 0)
+	/* This must never happen.  */
+	abort ();
+#endif
+
       if (kctx != KCTX_NONE)
 	{
 	  INTERNAL_SYSCALL_DECL (err);
 	  INTERNAL_SYSCALL (io_destroy, err, 1, kctx);
 	}
 
-      /* Release the conditional variable.  */
-      if (pthread_cond_destroy (&cond) != 0)
-	/* This must never happen.  */
-	abort ();
+      /* If any of the I/O requests failed, return -1 and set errno.  */
+      if (result != 0)
+	{
+	  __set_errno (result == EINTR ? EINTR : EIO);
+	  result = -1;
+	}
     }
   else if (sig->sigev_notify != SIGEV_NONE)
     {
@@ -234,7 +258,10 @@ lio_listio (mode, list, nent, sig)
 	      if (requests[cnt] != NULL
 		  && list[cnt]->aio_lio_opcode != LIO_NOP)
 		{
+#ifndef DONT_NEED_AIO_MISC_COND
 		  waitlist->list[cnt].cond = NULL;
+#endif
+		  waitlist->list[cnt].result = NULL;
 		  waitlist->list[cnt].next = requests[cnt]->waiting;
 		  waitlist->list[cnt].counterp = &waitlist->counter;
 		  waitlist->list[cnt].sigevp = &waitlist->sigev;
@@ -256,4 +283,40 @@ lio_listio (mode, list, nent, sig)
 
   return result;
 }
+
+
+#if SHLIB_COMPAT (librt, GLIBC_2_1, GLIBC_2_4)
+int
+attribute_compat_text_section
+__lio_listio_21 (int mode, struct aiocb *const list[], int nent,
+		 struct sigevent *sig)
+{
+  /* Check arguments.  */
+  if (mode != LIO_WAIT && mode != LIO_NOWAIT)
+    {
+      __set_errno (EINVAL);
+      return -1;
+    }
+
+  return lio_listio_internal (mode | LIO_NO_INDIVIDUAL_EVENT, list, nent, sig);
+}
+compat_symbol (librt, __lio_listio_21, lio_listio, GLIBC_2_1);
+#endif
+
+
+int
+__lio_listio_item_notify (int mode, struct aiocb *const list[], int nent,
+			  struct sigevent *sig)
+{
+    /* Check arguments.  */
+  if (mode != LIO_WAIT && mode != LIO_NOWAIT)
+    {
+      __set_errno (EINVAL);
+      return -1;
+    }
+
+  return lio_listio_internal (mode, list, nent, sig);
+}
+versioned_symbol (librt, __lio_listio_item_notify, lio_listio, GLIBC_2_4);
+
 #endif
diff --git a/rtkaio/sysdeps/unix/sysv/linux/klio_listio64.c b/rtkaio/sysdeps/unix/sysv/linux/klio_listio64.c
index 364d6623f6..937cf1a9f0 100644
--- a/rtkaio/sysdeps/unix/sysv/linux/klio_listio64.c
+++ b/rtkaio/sysdeps/unix/sysv/linux/klio_listio64.c
@@ -1,5 +1,5 @@
 /* Enqueue and list of read or write requests, 64bit offset version.
-   Copyright (C) 1997, 1998, 1999, 2003 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2003, 2006 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -31,6 +31,8 @@
 #include <unistd.h>
 
 #define lio_listio lio_listio64
+#define __lio_listio_21 __lio_listio64_21
+#define __lio_listio_item_notify __lio_listio64_item_notify
 #define aiocb aiocb64
 #define LIO_OPCODE_BASE 128
 #include <klio_listio.c>
diff --git a/rtkaio/sysdeps/unix/sysv/linux/powerpc/Makefile b/rtkaio/sysdeps/unix/sysv/linux/powerpc/Makefile
new file mode 100644
index 0000000000..ead21fb111
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/powerpc/Makefile
@@ -0,0 +1,3 @@
+ifeq ($(subdir),rtkaio)
+librtkaio-routines += rtkaio-sysdep
+endif
diff --git a/rtkaio/sysdeps/unix/sysv/linux/powerpc/rtkaio-sysdep.c b/rtkaio/sysdeps/unix/sysv/linux/powerpc/rtkaio-sysdep.c
new file mode 100644
index 0000000000..ff0440aa83
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/powerpc/rtkaio-sysdep.c
@@ -0,0 +1 @@
+#include <rt-sysdep.c>
diff --git a/rtkaio/sysdeps/unix/sysv/linux/s390/Makefile b/rtkaio/sysdeps/unix/sysv/linux/s390/Makefile
new file mode 100644
index 0000000000..ead21fb111
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/s390/Makefile
@@ -0,0 +1,3 @@
+ifeq ($(subdir),rtkaio)
+librtkaio-routines += rtkaio-sysdep
+endif
diff --git a/rtkaio/sysdeps/unix/sysv/linux/s390/rtkaio-sysdep.S b/rtkaio/sysdeps/unix/sysv/linux/s390/rtkaio-sysdep.S
new file mode 100644
index 0000000000..11ee214b23
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/s390/rtkaio-sysdep.S
@@ -0,0 +1 @@
+#include <rt-sysdep.S>
diff --git a/rtkaio/sysdeps/unix/sysv/linux/sparc/Makefile b/rtkaio/sysdeps/unix/sysv/linux/sparc/Makefile
new file mode 100644
index 0000000000..ead21fb111
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/sparc/Makefile
@@ -0,0 +1,3 @@
+ifeq ($(subdir),rtkaio)
+librtkaio-routines += rtkaio-sysdep
+endif
diff --git a/rtkaio/sysdeps/unix/sysv/linux/sparc/rtkaio-sysdep.c b/rtkaio/sysdeps/unix/sysv/linux/sparc/rtkaio-sysdep.c
new file mode 100644
index 0000000000..ff0440aa83
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/sparc/rtkaio-sysdep.c
@@ -0,0 +1 @@
+#include <rt-sysdep.c>
diff --git a/rtkaio/sysdeps/unix/sysv/linux/sparc/sparc64/Versions b/rtkaio/sysdeps/unix/sysv/linux/sparc/sparc64/Versions
new file mode 100644
index 0000000000..7443c81d6a
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/sparc/sparc64/Versions
@@ -0,0 +1,9 @@
+%ifdef HAVE_FORCED_UNWIND
+librtkaio {
+  GLIBC_2.3.3 {
+    # Changed timer_t.
+    timer_create; timer_delete; timer_getoverrun; timer_gettime;
+    timer_settime;
+  }
+}
+%endif
diff --git a/rtkaio/sysdeps/unix/sysv/linux/syscalls.list b/rtkaio/sysdeps/unix/sysv/linux/syscalls.list
new file mode 100644
index 0000000000..686b8d5d2e
--- /dev/null
+++ b/rtkaio/sysdeps/unix/sysv/linux/syscalls.list
@@ -0,0 +1,5 @@
+# File name		Caller	Syscall name	Args		Strong name		Weak names
+
+kaio_mq_timedsend	-	mq_timedsend	Ci:ipiip	__GI_mq_timedsend	mq_timedsend
+kaio_mq_timedreceive	-	mq_timedreceive	Ci:ipipp	__GI_mq_timedreceive	mq_timedreceive
+kaio_mq_setattr		-	mq_getsetattr	i:ipp		__GI_mq_setattr		mq_setattr