linux: open and openat ignore 'mode' with O_TMPFILE in flags

Both open and openat load their last argument 'mode' lazily, using
va_arg() only if O_CREAT is found in oflag. This is wrong, mode is also
necessary if O_TMPFILE is in oflag.

By chance on x86_64, the problem wasn't evident when using O_TMPFILE
with open, as the 3rd argument of open, even when not loaded with
va_arg, is left untouched in RDX, where the syscall expects it.

However, openat was not so lucky, and O_TMPFILE couldn't be used: mode
is the 4th argument, in RCX, but the syscall expects its 4th argument in
a different register than the glibc wrapper, in R10.

Introduce a macro __OPEN_NEEDS_MODE (oflag) to test if either O_CREAT or
O_TMPFILE is set in oflag.

Tested on Linux x86_64.

	[BZ #17523]
	* io/fcntl.h (__OPEN_NEEDS_MODE): New macro.
	* io/bits/fcntl2.h (open): Use it.
	(openat): Likewise.
	* io/open.c (__libc_open): Likewise.
	* io/open64.c (__libc_open64): Likewise.
	* io/open64_2.c (__open64_2): Likewise.
	* io/open_2.c (__open_2): Likewise.
	* io/openat.c (__openat): Likewise.
	* io/openat64.c (__openat64): Likewise.
	* io/openat64_2.c (__openat64_2): Likewise.
	* io/openat_2.c (__openat_2): Likewise.
	* sysdeps/mach/hurd/open.c (__libc_open): Likewise.
	* sysdeps/mach/hurd/openat.c (__openat): Likewise.
	* sysdeps/posix/open64.c (__libc_open64): Likewise.
	* sysdeps/unix/sysv/linux/dl-openat64.c (openat64): Likewise.
	* sysdeps/unix/sysv/linux/generic/open.c (__libc_open): Likewise.
	(__open_nocancel): Likewise.
	* sysdeps/unix/sysv/linux/generic/open64.c (__libc_open64): Likewise.
	* sysdeps/unix/sysv/linux/open64.c (__libc_open64): Likewise.
	* sysdeps/unix/sysv/linux/openat.c (__OPENAT): Likewise.

1 files changed, 9 insertions, 9 deletions

diff --git a/io/bits/fcntl2.h b/io/bits/fcntl2.h
index 745bb0b9c1..2ac54b200a 100644
--- a/io/bits/fcntl2.h
+++ b/io/bits/fcntl2.h
@@ -20,7 +20,7 @@
 # error "Never include <bits/fcntl2.h> directly; use <fcntl.h> instead."
 #endif
 
-/* Check that calls to open and openat with O_CREAT set have an
+/* Check that calls to open and openat with O_CREAT or O_TMPFILE set have an
    appropriate third/fourth parameter.  */
 #ifndef __USE_FILE_OFFSET64
 extern int __open_2 (const char *__path, int __oflag) __nonnull ((1));
@@ -35,7 +35,7 @@ extern int __REDIRECT (__open_alias, (const char *__path, int __oflag, ...),
 __errordecl (__open_too_many_args,
 	     "open can be called either with 2 or 3 arguments, not more");
 __errordecl (__open_missing_mode,
-	     "open with O_CREAT in second argument needs 3 arguments");
+	     "open with O_CREAT or O_TMPFILE in second argument needs 3 arguments");
 
 __fortify_function int
 open (const char *__path, int __oflag, ...)
@@ -45,7 +45,7 @@ open (const char *__path, int __oflag, ...)
 
   if (__builtin_constant_p (__oflag))
     {
-      if ((__oflag & O_CREAT) != 0 && __va_arg_pack_len () < 1)
+      if (__OPEN_NEEDS_MODE (__oflag) && __va_arg_pack_len () < 1)
 	{
 	  __open_missing_mode ();
 	  return __open_2 (__path, __oflag);
@@ -67,7 +67,7 @@ extern int __REDIRECT (__open64_alias, (const char *__path, int __oflag,
 __errordecl (__open64_too_many_args,
 	     "open64 can be called either with 2 or 3 arguments, not more");
 __errordecl (__open64_missing_mode,
-	     "open64 with O_CREAT in second argument needs 3 arguments");
+	     "open64 with O_CREAT or O_TMPFILE in second argument needs 3 arguments");
 
 __fortify_function int
 open64 (const char *__path, int __oflag, ...)
@@ -77,7 +77,7 @@ open64 (const char *__path, int __oflag, ...)
 
   if (__builtin_constant_p (__oflag))
     {
-      if ((__oflag & O_CREAT) != 0 && __va_arg_pack_len () < 1)
+      if (__OPEN_NEEDS_MODE (__oflag) && __va_arg_pack_len () < 1)
 	{
 	  __open64_missing_mode ();
 	  return __open64_2 (__path, __oflag);
@@ -111,7 +111,7 @@ extern int __REDIRECT (__openat_alias, (int __fd, const char *__path,
 __errordecl (__openat_too_many_args,
 	     "openat can be called either with 3 or 4 arguments, not more");
 __errordecl (__openat_missing_mode,
-	     "openat with O_CREAT in third argument needs 4 arguments");
+	     "openat with O_CREAT or O_TMPFILE in third argument needs 4 arguments");
 
 __fortify_function int
 openat (int __fd, const char *__path, int __oflag, ...)
@@ -121,7 +121,7 @@ openat (int __fd, const char *__path, int __oflag, ...)
 
   if (__builtin_constant_p (__oflag))
     {
-      if ((__oflag & O_CREAT) != 0 && __va_arg_pack_len () < 1)
+      if (__OPEN_NEEDS_MODE (__oflag) && __va_arg_pack_len () < 1)
 	{
 	  __openat_missing_mode ();
 	  return __openat_2 (__fd, __path, __oflag);
@@ -145,7 +145,7 @@ extern int __REDIRECT (__openat64_alias, (int __fd, const char *__path,
 __errordecl (__openat64_too_many_args,
 	     "openat64 can be called either with 3 or 4 arguments, not more");
 __errordecl (__openat64_missing_mode,
-	     "openat64 with O_CREAT in third argument needs 4 arguments");
+	     "openat64 with O_CREAT or O_TMPFILE in third argument needs 4 arguments");
 
 __fortify_function int
 openat64 (int __fd, const char *__path, int __oflag, ...)
@@ -155,7 +155,7 @@ openat64 (int __fd, const char *__path, int __oflag, ...)
 
   if (__builtin_constant_p (__oflag))
     {
-      if ((__oflag & O_CREAT) != 0 && __va_arg_pack_len () < 1)
+      if (__OPEN_NEEDS_MODE (__oflag) && __va_arg_pack_len () < 1)
 	{
 	  __openat64_missing_mode ();
 	  return __openat64_2 (__fd, __path, __oflag);