nptl: Set sem_open as a non cancellation point (BZ #15765)

This patch changes sem_open to not act as a cancellation point.
Cancellation is disable at start and reenable in function exit.
It fixes BZ #15765.

Tested on x86_64 and i686.

	[BZ #15765]
	* nptl/Makefile (tests): Add tst-sem16.
	* nptl/tst-sem16.c: New file.
	* nptl/sem_open.c (sem_open): Disable asynchronous cancellation.

3 files changed, 152 insertions, 5 deletions

diff --git a/nptl/Makefile b/nptl/Makefile
index e8de1bc7e1..e9485dfdb3 100644
--- a/nptl/Makefile
+++ b/nptl/Makefile
@@ -246,7 +246,7 @@ tests = tst-typesizes \
 	tst-key1 tst-key2 tst-key3 tst-key4 \
 	tst-sem1 tst-sem2 tst-sem3 tst-sem4 tst-sem5 tst-sem6 tst-sem7 \
 	tst-sem8 tst-sem9 tst-sem10 tst-sem11 tst-sem12 tst-sem13 tst-sem14 \
-	tst-sem15 \
+	tst-sem15 tst-sem16 \
 	tst-barrier1 tst-barrier2 tst-barrier3 tst-barrier4 tst-barrier5 \
 	tst-align tst-align3 \
 	tst-basic1 tst-basic2 tst-basic3 tst-basic4 tst-basic5 tst-basic6 \
diff --git a/nptl/sem_open.c b/nptl/sem_open.c
index 5b45fadd71..d10828637a 100644
--- a/nptl/sem_open.c
+++ b/nptl/sem_open.c
@@ -31,7 +31,7 @@
 #include "semaphoreP.h"
 #include <shm-directory.h>
 #include <futex-internal.h>
-
+#include <libc-lock.h>
 
 /* Comparison function for search of existing mapping.  */
 int
@@ -153,6 +153,13 @@ sem_open (const char *name, int oflag, ...)
   /* Create the name of the final file in local variable SHM_NAME.  */
   SHM_GET_NAME (EINVAL, SEM_FAILED, SEM_SHM_PREFIX);
 
+  /* Disable asynchronous cancellation.  */
+#ifdef __libc_ptf_call
+  int state;
+  __libc_ptf_call (__pthread_setcancelstate,
+                   (PTHREAD_CANCEL_DISABLE, &state), 0);
+#endif
+
   /* If the semaphore object has to exist simply open it.  */
   if ((oflag & O_CREAT) == 0 || (oflag & O_EXCL) == 0)
     {
@@ -193,7 +200,8 @@ sem_open (const char *name, int oflag, ...)
       if (value > SEM_VALUE_MAX)
 	{
 	  __set_errno (EINVAL);
-	  return SEM_FAILED;
+	  result = SEM_FAILED;
+	  goto out;
 	}
 
       /* Create the initial file content.  */
@@ -233,7 +241,10 @@ sem_open (const char *name, int oflag, ...)
 	     mode cannot later be set since then we cannot apply the
 	     file create mask.  */
 	  if (__mktemp (tmpfname) == NULL)
-	    return SEM_FAILED;
+	    {
+	      result = SEM_FAILED;
+	      goto out;
+	    }
 
 	  /* Open the file.  Make sure we do not overwrite anything.  */
 	  fd = __libc_open (tmpfname, O_RDWR | O_CREAT | O_EXCL, mode);
@@ -247,7 +258,8 @@ sem_open (const char *name, int oflag, ...)
 		  __set_errno (EAGAIN);
 		}
 
-	      return SEM_FAILED;
+	      result = SEM_FAILED;
+	      goto out;
 	    }
 
 	  /* We got a file.  */
@@ -308,5 +320,10 @@ sem_open (const char *name, int oflag, ...)
       errno = save;
     }
 
+out:
+#ifdef __libc_ptf_call
+  __libc_ptf_call (__pthread_setcancelstate, (state, NULL), 0);
+#endif
+
   return result;
 }
diff --git a/nptl/tst-sem16.c b/nptl/tst-sem16.c
new file mode 100644
index 0000000000..70067540d3
--- /dev/null
+++ b/nptl/tst-sem16.c
@@ -0,0 +1,130 @@
+/* Test for sem_open cancellation handling: BZ #15765.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <pthread.h>
+#include <sys/mman.h>
+#include <semaphore.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdlib.h>
+
+static sem_t sem;	/* Use to sync with thread start.  */
+static const char pipe_name[] = "/glibc-tst-sem16";
+
+static void
+remove_sem (int status, void *arg)
+{
+  sem_unlink (arg);
+}
+
+static void *
+tf (void *arg)
+{
+  pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, 0);
+
+  if (sem_wait (&sem) != 0)
+    {
+      printf ("error: sem_wait failed: %m");
+      exit (1);
+    }
+
+  if (pthread_setcancelstate (PTHREAD_CANCEL_ENABLE, 0) != 0)
+    {
+      printf ("error: pthread_setcancelstate failed: %m");
+      exit (1);
+    }
+
+  /* Neither sem_unlink or sem_open should act on thread cancellation.  */
+  sem_unlink (pipe_name);
+  on_exit (remove_sem, (void *) pipe_name);
+
+  sem_t *s = sem_open (pipe_name, O_CREAT, 0600, 1);
+  if (s == SEM_FAILED)
+    {
+      int exit_code;
+      if (errno == ENOSYS || errno == EACCES)
+	exit_code = 77;
+      else
+	exit_code = 1;
+      exit (exit_code);
+    }
+
+  if (pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, 0) != 0)
+    {
+      printf ("error: pthread_setcancelstate failed: %m");
+      exit (1);
+    }
+
+  if (sem_close (s) != 0)
+    {
+      printf ("error: sem_close failed: %m");
+      exit (1);
+    }
+
+  return NULL;
+}
+
+static int
+do_test (void)
+{
+  pthread_t td;
+
+  if (sem_init (&sem, 0, 0))
+    {
+      printf ("error: sem_init failed: %m\n");
+      exit (1);
+    }
+
+  if (pthread_create (&td, NULL, tf, NULL) != 0)
+    {
+      printf ("error: pthread_create failed: %m\n");
+      exit (1);
+    }
+
+  if (pthread_cancel (td) != 0)
+    {
+      printf ("error: pthread_cancel failed: %m\n");
+      exit (1);
+    }
+
+  if (sem_post (&sem) != 0)
+    {
+      printf ("error: sem_post failed: %m\n");
+      exit (1);
+    }
+
+  void *r;
+  if (pthread_join (td, &r) != 0)
+    {
+      printf ("error: pthread_join failed: %m\n");
+      exit (1);
+    }
+
+  if (r == PTHREAD_CANCELED)
+    {
+      puts ("error: pthread_join returned PTHREAD_CANCELED");
+      exit (1);
+    }
+
+  return 0;
+}
+
+#define TEST_FUNCTION do_test ()
+#include <test-skeleton.c>