linux: open and openat ignore 'mode' with O_TMPFILE in flags

Both open and openat load their last argument 'mode' lazily, using
va_arg() only if O_CREAT is found in oflag. This is wrong, mode is also
necessary if O_TMPFILE is in oflag.

By chance on x86_64, the problem wasn't evident when using O_TMPFILE
with open, as the 3rd argument of open, even when not loaded with
va_arg, is left untouched in RDX, where the syscall expects it.

However, openat was not so lucky, and O_TMPFILE couldn't be used: mode
is the 4th argument, in RCX, but the syscall expects its 4th argument in
a different register than the glibc wrapper, in R10.

Introduce a macro __OPEN_NEEDS_MODE (oflag) to test if either O_CREAT or
O_TMPFILE is set in oflag.

Tested on Linux x86_64.

	[BZ #17523]
	* io/fcntl.h (__OPEN_NEEDS_MODE): New macro.
	* io/bits/fcntl2.h (open): Use it.
	(openat): Likewise.
	* io/open.c (__libc_open): Likewise.
	* io/open64.c (__libc_open64): Likewise.
	* io/open64_2.c (__open64_2): Likewise.
	* io/open_2.c (__open_2): Likewise.
	* io/openat.c (__openat): Likewise.
	* io/openat64.c (__openat64): Likewise.
	* io/openat64_2.c (__openat64_2): Likewise.
	* io/openat_2.c (__openat_2): Likewise.
	* sysdeps/mach/hurd/open.c (__libc_open): Likewise.
	* sysdeps/mach/hurd/openat.c (__openat): Likewise.
	* sysdeps/posix/open64.c (__libc_open64): Likewise.
	* sysdeps/unix/sysv/linux/dl-openat64.c (openat64): Likewise.
	* sysdeps/unix/sysv/linux/generic/open.c (__libc_open): Likewise.
	(__open_nocancel): Likewise.
	* sysdeps/unix/sysv/linux/generic/open64.c (__libc_open64): Likewise.
	* sysdeps/unix/sysv/linux/open64.c (__libc_open64): Likewise.
	* sysdeps/unix/sysv/linux/openat.c (__OPENAT): Likewise.

8 files changed, 17 insertions, 17 deletions

diff --git a/sysdeps/mach/hurd/open.c b/sysdeps/mach/hurd/open.c
index 2d1415af6e..c497a7cb44 100644
--- a/sysdeps/mach/hurd/open.c
+++ b/sysdeps/mach/hurd/open.c
@@ -22,7 +22,7 @@
 #include <hurd.h>
 #include <hurd/fd.h>
 
-/* Open FILE with access OFLAG.  If OFLAG includes O_CREAT,
+/* Open FILE with access OFLAG.  If O_CREAT or O_TMPFILE is in OFLAG,
    a third argument is the file protection.  */
 int
 __libc_open (const char *file, int oflag, ...)
@@ -30,7 +30,7 @@ __libc_open (const char *file, int oflag, ...)
   mode_t mode;
   io_t port;
 
-  if (oflag & O_CREAT)
+  if (__OPEN_NEEDS_MODE (oflag))
     {
       va_list arg;
       va_start (arg, oflag);
diff --git a/sysdeps/mach/hurd/openat.c b/sysdeps/mach/hurd/openat.c
index 0928dff624..0d3e138890 100644
--- a/sysdeps/mach/hurd/openat.c
+++ b/sysdeps/mach/hurd/openat.c
@@ -26,7 +26,7 @@
 #include <hurd/fd.h>
 
 /* Open FILE with access OFLAG.  Interpret relative paths relative to
-   the directory associated with FD.  If OFLAG includes O_CREAT, a
+   the directory associated with FD.  If O_CREAT or O_TMPFILE is in OFLAG, a
    third argument is the file protection.  */
 int
 __openat (fd, file, oflag)
@@ -37,7 +37,7 @@ __openat (fd, file, oflag)
   mode_t mode;
   io_t port;
 
-  if (oflag & O_CREAT)
+  if (__OPEN_NEEDS_MODE (oflag))
     {
       va_list arg;
       va_start (arg, oflag);
diff --git a/sysdeps/posix/open64.c b/sysdeps/posix/open64.c
index 8f8dc60e24..74f669f408 100644
--- a/sysdeps/posix/open64.c
+++ b/sysdeps/posix/open64.c
@@ -19,14 +19,14 @@
 #include <stdarg.h>
 #include <sysdep-cancel.h>
 
-/* Open FILE with access OFLAG.  If OFLAG includes O_CREAT,
+/* Open FILE with access OFLAG.  If O_CREAT or O_TMPFILE is in OFLAG,
    a third argument is the file protection.  */
 int
 __libc_open64 (const char *file, int oflag, ...)
 {
   int mode = 0;
 
-  if (oflag & O_CREAT)
+  if (__OPEN_NEEDS_MODE (oflag))
     {
       va_list arg;
       va_start (arg, oflag);
diff --git a/sysdeps/unix/sysv/linux/dl-openat64.c b/sysdeps/unix/sysv/linux/dl-openat64.c
index 4dd5ed5c3b..732097dd92 100644
--- a/sysdeps/unix/sysv/linux/dl-openat64.c
+++ b/sysdeps/unix/sysv/linux/dl-openat64.c
@@ -28,7 +28,7 @@ openat64 (dfd, file, oflag)
      const char *file;
      int oflag;
 {
-  assert ((oflag & O_CREAT) == 0);
+  assert (!__OPEN_NEEDS_MODE (oflag));
 
 #ifdef __NR_openat
   return INLINE_SYSCALL (openat, 3, dfd, file, oflag | O_LARGEFILE);
diff --git a/sysdeps/unix/sysv/linux/generic/open.c b/sysdeps/unix/sysv/linux/generic/open.c
index f15144cc2a..289f57aada 100644
--- a/sysdeps/unix/sysv/linux/generic/open.c
+++ b/sysdeps/unix/sysv/linux/generic/open.c
@@ -22,14 +22,14 @@
 #include <stdio.h>
 #include <sysdep-cancel.h>
 
-/* Open FILE with access OFLAG.  If OFLAG includes O_CREAT,
+/* Open FILE with access OFLAG.  If O_CREAT or O_TMPFILE is in OFLAG,
    a third argument is the file protection.  */
 int
 __libc_open (const char *file, int oflag, ...)
 {
   int mode = 0;
 
-  if (oflag & O_CREAT)
+  if (__OPEN_NEEDS_MODE (oflag))
     {
       va_list arg;
       va_start (arg, oflag);
@@ -59,7 +59,7 @@ __open_nocancel (const char *file, int oflag, ...)
 {
   int mode = 0;
 
-  if (oflag & O_CREAT)
+  if (__OPEN_NEEDS_MODE (oflag))
     {
       va_list arg;
       va_start (arg, oflag);
diff --git a/sysdeps/unix/sysv/linux/generic/open64.c b/sysdeps/unix/sysv/linux/generic/open64.c
index 4feedbaff5..374e41cf3a 100644
--- a/sysdeps/unix/sysv/linux/generic/open64.c
+++ b/sysdeps/unix/sysv/linux/generic/open64.c
@@ -22,14 +22,14 @@
 #include <stdio.h>
 #include <sysdep-cancel.h>
 
-/* Open FILE with access OFLAG.  If OFLAG includes O_CREAT,
+/* Open FILE with access OFLAG.  If O_CREAT or O_TMPFILE is in OFLAG,
    a third argument is the file protection.  */
 int
 __libc_open64 (const char *file, int oflag, ...)
 {
   int mode = 0;
 
-  if (oflag & O_CREAT)
+  if (__OPEN_NEEDS_MODE (oflag))
     {
       va_list arg;
       va_start (arg, oflag);
diff --git a/sysdeps/unix/sysv/linux/open64.c b/sysdeps/unix/sysv/linux/open64.c
index 0067601283..d3ab813ab6 100644
--- a/sysdeps/unix/sysv/linux/open64.c
+++ b/sysdeps/unix/sysv/linux/open64.c
@@ -21,14 +21,14 @@
 #include <stdio.h>
 #include <sysdep-cancel.h>
 
-/* Open FILE with access OFLAG.  If OFLAG includes O_CREAT,
+/* Open FILE with access OFLAG.  If O_CREAT or O_TMPFILE is in OFLAG,
    a third argument is the file protection.  */
 int
 __libc_open64 (const char *file, int oflag, ...)
 {
   int mode = 0;
 
-  if (oflag & O_CREAT)
+  if (__OPEN_NEEDS_MODE (oflag))
     {
       va_list arg;
       va_start (arg, oflag);
diff --git a/sysdeps/unix/sysv/linux/openat.c b/sysdeps/unix/sysv/linux/openat.c
index 5145823b8e..30d8e6e1ac 100644
--- a/sysdeps/unix/sysv/linux/openat.c
+++ b/sysdeps/unix/sysv/linux/openat.c
@@ -54,13 +54,13 @@ OPENAT_NOT_CANCEL (int fd, const char *file, int oflag, mode_t mode)
 
 
 /* Open FILE with access OFLAG.  Interpret relative paths relative to
-   the directory associated with FD.  If OFLAG includes O_CREAT, a
-   third argument is the file protection.  */
+   the directory associated with FD.  If OFLAG includes O_CREAT or
+   O_TMPFILE, a fourth argument is the file protection.  */
 int
 __OPENAT (int fd, const char *file, int oflag, ...)
 {
   mode_t mode = 0;
-  if (oflag & O_CREAT)
+  if (__OPEN_NEEDS_MODE (oflag))
     {
       va_list arg;
       va_start (arg, oflag);