[vte/vte-0-58] pty: Fix error check for ioctl(TIOCGPTPEER) failure

[Christian Persch <chpe src gnome org>]

commit af0d039707da5d47eb7bca0ebabd38af62d1e007
Author: Christian Persch <chpe src gnome org>
Date:   Thu Oct 10 21:25:47 2019 +0200

    pty: Fix error check for ioctl(TIOCGPTPEER) failure
    
    While the kernel's own tests say that EINVAL would be returned when the
    running kernel doesn't support the ioctl, it appears that actually it
    returns ENOTTY.
    
    Fixes: https://gitlab.gnome.org/GNOME/vte/issues/182
    (cherry picked from commit 9dd455a3f80657e17afabdc86c9cef0bb1d0a7e6)

 src/pty.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)
---
diff --git a/src/pty.cc b/src/pty.cc
index 6dd60345..babb0f8d 100644
--- a/src/pty.cc
+++ b/src/pty.cc
@@ -165,12 +165,19 @@ vte_pty_child_setup (VtePty *pty)
 
 #ifdef __linux__
         fd = ioctl(masterfd, TIOCGPTPEER, fd_flags);
-        if (fd == -1 && errno != EINVAL) {
+        /* Note: According to the kernel's own tests (tools/testing/selftests/filesystems/devpts_pts.c),
+         * the error returned when the running kernel does not support this ioctl should be EINVAL.
+         * However it appears that the actual error returned is ENOTTY. So we check for both of them.
+         * See issue#182.
+         */
+        if (fd == -1 &&
+            errno != EINVAL &&
+            errno != ENOTTY) {
                _vte_debug_print(VTE_DEBUG_PTY, "%s failed: %m\n", "ioctl(TIOCGPTPEER)");
                _exit(127);
         }
 
-        /* EINVAL means the kernel doesn't support this ioctl; fall back to ptsname + open */
+        /* Fall back to ptsname + open */
 #endif
 
         if (fd == -1) {