[glib: 1/2] improve console check in gspawn-win32

[Philip Withnall <pwithnall src gnome org>]

commit 30ed2ea2630d6130100ac815c5ff075a3761f363
Author: Princeton Ferro <princetonf optonline net>
Date:   Mon Apr 19 10:25:06 2021 +0000

    improve console check in gspawn-win32
    
    We use the return value of AllocConsole (GetCurrentProcessId ()) to
    determine whether the spawning process is attached to a console.

 glib/gspawn-win32.c | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)
---
diff --git a/glib/gspawn-win32.c b/glib/gspawn-win32.c
index 0f6579eab..dc7219acd 100644
--- a/glib/gspawn-win32.c
+++ b/glib/gspawn-win32.c
@@ -530,6 +530,28 @@ do_spawn_directly (gint                 *exit_status,
   return TRUE;
 }
 
+static gboolean
+might_be_console_process (void)
+{
+  // we should always fail to attach ourself to a console (because we're
+  // either already attached, or we do not have a console)
+  gboolean attached_to_self = AttachConsole (GetCurrentProcessId ());
+  g_return_val_if_fail (!attached_to_self, TRUE);
+
+  switch (GetLastError ())
+    {
+    // current process is already attached to a console
+    case ERROR_ACCESS_DENIED:
+      return TRUE;
+    // current process does not have a console
+    case ERROR_INVALID_HANDLE:
+      return FALSE;
+    // we should not get ERROR_INVALID_PARAMETER
+    }
+
+  g_return_val_if_reached (FALSE);
+}
+
 static gboolean
 fork_exec (gint                  *exit_status,
            gboolean               do_return_handle,
@@ -625,7 +647,7 @@ fork_exec (gint                  *exit_status,
     goto cleanup_and_fail;
   
   new_argv = g_new (char *, argc + 1 + ARG_COUNT);
-  if (GetConsoleWindow () != NULL)
+  if (might_be_console_process ())
     helper_process = HELPER_PROCESS "-console.exe";
   else
     helper_process = HELPER_PROCESS ".exe";