linux ttyname{_r}: Make tty checks consistent

In the ttyname and ttyname_r routines on Linux, at several points it needs to
check if a given TTY is the TTY we are looking for. It used to be that this
check was (to see if `maybe` is `mytty`):

       __xstat64(_STAT_VER, maybe_filename, &maybe) == 0
    #ifdef _STATBUF_ST_RDEV
       && S_ISCHR(maybe.st_mode) && maybe.st_rdev == mytty.st_rdev
    #else
       && maybe.st_ino == mytty.st_ino && maybe.st_dev == mytty.st_dev
    #endif

This check appears in several places.

Then, one of the changes made in commit 15e9a4f378c8607c2ae1aa465436af4321db0e23
was to change that check to:

       __xstat64(_STAT_VER, maybe_filename, &maybe) == 0
    #ifdef _STATBUF_ST_RDEV
       && S_ISCHR(maybe.st_mode) && maybe.st_rdev == mytty.st_rdev
    #endif
       && maybe.st_ino == mytty.st_ino && maybe.st_dev == mytty.st_dev

That is, it made the st_ino and st_dev parts of the check happen even if we have
the st_rdev member. This is an important change, because the kernel allows
multiple devpts filesystem instances to be created; a device file in one devpts
instance may share the same st_rdev with a file in another devpts instance, but
they aren't the same file.

This check appears twice in each file (ttyname.c and ttyname_r.c), once (in
ttyname and __ttyname_r) to check if a candidate file found by inspecting /proc
is the desired TTY, and once (in getttyname and getttyname_r) to check if a
candidate file found by searching /dev is the desired TTY. However, 15e9a4f
only updated the checks for files found via /proc; but the concern about
collisions between devpts instances is just as valid for files found via /dev.

So, update all 4 occurrences the check to be consistent with the version of the
check introduced in 15e9a4f. Make it easy to keep all 4 occurrences of the
check consistent by pulling it in to a static inline function, is_mytty.

Reviewed-by: Christian Brauner <christian.brauner@ubuntu.com>
(cherry picked from commit 2fbce9c2031e70b6bd67876accfc34b0ec492878)

5 files changed, 42 insertions, 69 deletions

diff --git a/ChangeLog b/ChangeLog
index 91e12d0f20..4c65ba2839 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,14 @@
 2017-11-15  Luke Shumaker  <lukeshu@parabola.nu>
 
+	[BZ #22145]
+	* sysdeps/unix/sysv/linux/ttyname.h (is_mytty): New function.
+	* sysdeps/unix/sysv/linux/ttyname.c (getttyname): Call is_mytty.
+	(ttyname): Likewise.
+	* sysdeps/unix/sysv/linux/ttyname_r.c (getttyname_r): Likewise.
+	(__ttyname_r): Likewise.
+
+2017-11-15  Luke Shumaker  <lukeshu@parabola.nu>
+
 	* sysdeps/unix/sysv/linux/ttyname.h (is_pty): Change return type from
 	int to bool.
 
diff --git a/NEWS b/NEWS
index 2c49212cb5..c93bf0493b 100644
--- a/NEWS
+++ b/NEWS
@@ -80,6 +80,7 @@ The following bugs are resolved with this release:
   [22095] resolv: Fix memory leak with OOM during resolv.conf parsing
   [22096] resolv: __resolv_conf_attach must not free passed conf object
   [22111] malloc: per thread cache is not returned when thread exits
+  [22145] ttyname gives up too early in the face of namespaces
   [22146] Let fpclassify use the builtin when optimizing for size in C++ mode
   [22225] math: nearbyint arithmetic moved before feholdexcept
   [22235] Add C++ versions of iscanonical for ldbl-96 and ldbl-128ibm
diff --git a/sysdeps/unix/sysv/linux/ttyname.c b/sysdeps/unix/sysv/linux/ttyname.c
index 5909cb765f..6e97d2d455 100644
--- a/sysdeps/unix/sysv/linux/ttyname.c
+++ b/sysdeps/unix/sysv/linux/ttyname.c
@@ -35,16 +35,14 @@
 char *__ttyname;
 #endif
 
-static char *getttyname (const char *dev, dev_t mydev,
-			 ino64_t myino, int save, int *dostat)
-     internal_function;
-
+static char *getttyname (const char *dev, const struct stat64 *mytty,
+			 int save, int *dostat);
 
 libc_freeres_ptr (static char *getttyname_name);
 
 static char *
-internal_function attribute_compat_text_section
-getttyname (const char *dev, dev_t mydev, ino64_t myino, int save, int *dostat)
+attribute_compat_text_section
+getttyname (const char *dev, const struct stat64 *mytty, int save, int *dostat)
 {
   static size_t namelen;
   struct stat64 st;
@@ -65,7 +63,7 @@ getttyname (const char *dev, dev_t mydev, ino64_t myino, int save, int *dostat)
     *((char *) __mempcpy (getttyname_name, dev, devlen - 1)) = '/';
 
   while ((d = __readdir64 (dirstream)) != NULL)
-    if ((d->d_fileno == myino || *dostat)
+    if ((d->d_fileno == mytty->st_ino || *dostat)
 	&& strcmp (d->d_name, "stdin")
 	&& strcmp (d->d_name, "stdout")
 	&& strcmp (d->d_name, "stderr"))
@@ -87,12 +85,7 @@ getttyname (const char *dev, dev_t mydev, ino64_t myino, int save, int *dostat)
 	  }
 	memcpy (&getttyname_name[devlen], d->d_name, dlen);
 	if (__xstat64 (_STAT_VER, getttyname_name, &st) == 0
-#ifdef _STATBUF_ST_RDEV
-	    && S_ISCHR (st.st_mode) && st.st_rdev == mydev
-#else
-	    && d->d_fileno == myino && st.st_dev == mydev
-#endif
-	   )
+	    && is_mytty (mytty, &st))
 	  {
 	    (void) __closedir (dirstream);
 #if 0
@@ -169,12 +162,7 @@ ttyname (int fd)
       /* Verify readlink result, fall back on iterating through devices.  */
       if (ttyname_buf[0] == '/'
 	  && __xstat64 (_STAT_VER, ttyname_buf, &st1) == 0
-#ifdef _STATBUF_ST_RDEV
-	  && S_ISCHR (st1.st_mode)
-	  && st1.st_rdev == st.st_rdev
-#endif
-	  && st1.st_ino == st.st_ino
-	  && st1.st_dev == st.st_dev)
+	  && is_mytty (&st, &st1))
 	return ttyname_buf;
 
       /* If the link doesn't exist, then it points to a device in another
@@ -188,11 +176,7 @@ ttyname (int fd)
 
   if (__xstat64 (_STAT_VER, "/dev/pts", &st1) == 0 && S_ISDIR (st1.st_mode))
     {
-#ifdef _STATBUF_ST_RDEV
-      name = getttyname ("/dev/pts", st.st_rdev, st.st_ino, save, &dostat);
-#else
-      name = getttyname ("/dev/pts", st.st_dev, st.st_ino, save, &dostat);
-#endif
+      name = getttyname ("/dev/pts", &st, save, &dostat);
     }
   else
     {
@@ -202,21 +186,13 @@ ttyname (int fd)
 
   if (!name && dostat != -1)
     {
-#ifdef _STATBUF_ST_RDEV
-      name = getttyname ("/dev", st.st_rdev, st.st_ino, save, &dostat);
-#else
-      name = getttyname ("/dev", st.st_dev, st.st_ino, save, &dostat);
-#endif
+      name = getttyname ("/dev", &st, save, &dostat);
     }
 
   if (!name && dostat != -1)
     {
       dostat = 1;
-#ifdef _STATBUF_ST_RDEV
-      name = getttyname ("/dev", st.st_rdev, st.st_ino, save, &dostat);
-#else
-      name = getttyname ("/dev", st.st_dev, st.st_ino, save, &dostat);
-#endif
+      name = getttyname ("/dev", &st, save, &dostat);
     }
 
   return name;
diff --git a/sysdeps/unix/sysv/linux/ttyname.h b/sysdeps/unix/sysv/linux/ttyname.h
index cdaf60fb87..48181330a9 100644
--- a/sysdeps/unix/sysv/linux/ttyname.h
+++ b/sysdeps/unix/sysv/linux/ttyname.h
@@ -34,3 +34,15 @@ is_pty (struct stat64 *sb)
   return false;
 #endif
 }
+
+static inline bool
+is_mytty (const struct stat64 *mytty, const struct stat64 *maybe)
+{
+  return (maybe->st_ino == mytty->st_ino
+	  && maybe->st_dev == mytty->st_dev
+#ifdef _STATBUF_ST_RDEV
+	  && S_ISCHR (maybe->st_mode)
+	  && maybe->st_rdev == mytty->st_rdev
+#endif
+	  );
+}
diff --git a/sysdeps/unix/sysv/linux/ttyname_r.c b/sysdeps/unix/sysv/linux/ttyname_r.c
index dc863526ba..58eb919c3f 100644
--- a/sysdeps/unix/sysv/linux/ttyname_r.c
+++ b/sysdeps/unix/sysv/linux/ttyname_r.c
@@ -31,12 +31,12 @@
 #include "ttyname.h"
 
 static int getttyname_r (char *buf, size_t buflen,
-			 dev_t mydev, ino64_t myino, int save,
-			 int *dostat) internal_function;
+			 const struct stat64 *mytty, int save,
+			 int *dostat);
 
 static int
-internal_function attribute_compat_text_section
-getttyname_r (char *buf, size_t buflen, dev_t mydev, ino64_t myino,
+attribute_compat_text_section
+getttyname_r (char *buf, size_t buflen, const struct stat64 *mytty,
 	      int save, int *dostat)
 {
   struct stat64 st;
@@ -52,7 +52,7 @@ getttyname_r (char *buf, size_t buflen, dev_t mydev, ino64_t myino,
     }
 
   while ((d = __readdir64 (dirstream)) != NULL)
-    if ((d->d_fileno == myino || *dostat)
+    if ((d->d_fileno == mytty->st_ino || *dostat)
 	&& strcmp (d->d_name, "stdin")
 	&& strcmp (d->d_name, "stdout")
 	&& strcmp (d->d_name, "stderr"))
@@ -72,12 +72,7 @@ getttyname_r (char *buf, size_t buflen, dev_t mydev, ino64_t myino,
 	cp[0] = '\0';
 
 	if (__xstat64 (_STAT_VER, buf, &st) == 0
-#ifdef _STATBUF_ST_RDEV
-	    && S_ISCHR (st.st_mode) && st.st_rdev == mydev
-#else
-	    && d->d_fileno == myino && st.st_dev == mydev
-#endif
-	   )
+	    && is_mytty (mytty, &st))
 	  {
 	    (void) __closedir (dirstream);
 	    __set_errno (save);
@@ -151,12 +146,7 @@ __ttyname_r (int fd, char *buf, size_t buflen)
       /* Verify readlink result, fall back on iterating through devices.  */
       if (buf[0] == '/'
 	  && __xstat64 (_STAT_VER, buf, &st1) == 0
-#ifdef _STATBUF_ST_RDEV
-	  && S_ISCHR (st1.st_mode)
-	  && st1.st_rdev == st.st_rdev
-#endif
-	  && st1.st_ino == st.st_ino
-	  && st1.st_dev == st.st_dev)
+	  && is_mytty (&st, &st1))
 	return 0;
 
       /* If the link doesn't exist, then it points to a device in another
@@ -175,13 +165,8 @@ __ttyname_r (int fd, char *buf, size_t buflen)
 
   if (__xstat64 (_STAT_VER, buf, &st1) == 0 && S_ISDIR (st1.st_mode))
     {
-#ifdef _STATBUF_ST_RDEV
-      ret = getttyname_r (buf, buflen, st.st_rdev, st.st_ino, save,
+      ret = getttyname_r (buf, buflen, &st, save,
 			  &dostat);
-#else
-      ret = getttyname_r (buf, buflen, st.st_dev, st.st_ino, save,
-			  &dostat);
-#endif
     }
   else
     {
@@ -193,26 +178,16 @@ __ttyname_r (int fd, char *buf, size_t buflen)
     {
       buf[sizeof ("/dev/") - 1] = '\0';
       buflen += sizeof ("pts/") - 1;
-#ifdef _STATBUF_ST_RDEV
-      ret = getttyname_r (buf, buflen, st.st_rdev, st.st_ino, save,
-			  &dostat);
-#else
-      ret = getttyname_r (buf, buflen, st.st_dev, st.st_ino, save,
+      ret = getttyname_r (buf, buflen, &st, save,
 			  &dostat);
-#endif
     }
 
   if (ret && dostat != -1)
     {
       buf[sizeof ("/dev/") - 1] = '\0';
       dostat = 1;
-#ifdef _STATBUF_ST_RDEV
-      ret = getttyname_r (buf, buflen, st.st_rdev, st.st_ino,
-			  save, &dostat);
-#else
-      ret = getttyname_r (buf, buflen, st.st_dev, st.st_ino,
+      ret = getttyname_r (buf, buflen, &st,
 			  save, &dostat);
-#endif
     }
 
   return ret;