[vte] pty: Prefer using TIOCGPTPEER ioctl

[Christian Persch <chpe src gnome org>]

commit 118ad1caab2256bf731404c467c08b98e14b8ccc
Author: Christian Persch <chpe src gnome org>
Date:   Sat Aug 3 19:01:13 2019 +0200

    pty: Prefer using TIOCGPTPEER ioctl

 src/pty.cc | 38 +++++++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 11 deletions(-)
---
diff --git a/src/pty.cc b/src/pty.cc
index f5e227fb..6dd60345 100644
--- a/src/pty.cc
+++ b/src/pty.cc
@@ -33,6 +33,7 @@
 #include "vtetypes.hh"
 #include "vtespawn.hh"
 
+#include <assert.h>
 #include <sys/types.h>
 #include <sys/ioctl.h>
 #include <sys/socket.h>
@@ -159,24 +160,39 @@ vte_pty_child_setup (VtePty *pty)
         }
 
         /* Note: *not* O_CLOEXEC! */
-        auto const fd_flags = O_RDWR | ((priv->flags & VTE_PTY_NO_CTTY) ? O_NOCTTY : 0);
+        auto const fd_flags = int{O_RDWR | ((priv->flags & VTE_PTY_NO_CTTY) ? O_NOCTTY : 0)};
+        auto fd = int{-1};
 
-       char *name = ptsname(masterfd);
-        if (name == nullptr) {
-               _vte_debug_print(VTE_DEBUG_PTY, "%s failed: %m\n", "ptsname");
+#ifdef __linux__
+        fd = ioctl(masterfd, TIOCGPTPEER, fd_flags);
+        if (fd == -1 && errno != EINVAL) {
+               _vte_debug_print(VTE_DEBUG_PTY, "%s failed: %m\n", "ioctl(TIOCGPTPEER)");
                _exit(127);
-       }
+        }
 
-        _vte_debug_print (VTE_DEBUG_PTY,
-                          "Setting up child pty: master FD = %d name = %s\n",
-                          masterfd, name);
+        /* EINVAL means the kernel doesn't support this ioctl; fall back to ptsname + open */
+#endif
 
-        int fd = open(name, fd_flags);
         if (fd == -1) {
-                _vte_debug_print (VTE_DEBUG_PTY, "Failed to open PTY: %m\n");
-                _exit(127);
+                auto const name = ptsname(masterfd);
+                if (name == nullptr) {
+                        _vte_debug_print(VTE_DEBUG_PTY, "%s failed: %m\n", "ptsname");
+                        _exit(127);
+                }
+
+                _vte_debug_print (VTE_DEBUG_PTY,
+                                  "Setting up child pty: master FD = %d name = %s\n",
+                                  masterfd, name);
+
+                fd = open(name, fd_flags);
+                if (fd == -1) {
+                        _vte_debug_print (VTE_DEBUG_PTY, "Failed to open PTY: %m\n");
+                        _exit(127);
+                }
         }
 
+        assert(fd != -1);
+
 #if defined(HAVE_SETSID) && defined(HAVE_SETPGID)
         if (!(priv->flags & VTE_PTY_NO_SESSION)) {
                 /* Start a new session and become process-group leader. */