[gtk/wip/otte/for-master] x11: Be more careful with NVIDIA workaround

[Benjamin Otte <otte src gnome org>]

commit 5af6c37f8c407ea1e7c12956b1bab21758fb1f22
Author: Benjamin Otte <otte redhat com>
Date:   Sat May 22 01:43:31 2021 +0200

    x11: Be more careful with NVIDIA workaround
    
    Check that we are indeed running inside an Xorg server before enabling
    the workaround.
    
    XWayland or other nested X servers deadlock when that workaround is
    applied.

 gdk/x11/gdkglcontext-glx.c | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)
---
diff --git a/gdk/x11/gdkglcontext-glx.c b/gdk/x11/gdkglcontext-glx.c
index 4181168262..9325d5d9b6 100644
--- a/gdk/x11/gdkglcontext-glx.c
+++ b/gdk/x11/gdkglcontext-glx.c
@@ -1271,6 +1271,12 @@ gdk_x11_screen_init_glx (GdkX11Screen *screen)
 
   if (g_strcmp0 (glXGetClientString (dpy, GLX_VENDOR), "NVIDIA Corporation") == 0)
     {
+      Atom type;
+      int format;
+      gulong nitems;
+      gulong bytes_after;
+      guchar *data = NULL;
+
       /* With the mesa based drivers, we can safely assume the compositor can
        * access the updated surface texture immediately after glXSwapBuffers is
        * run, because the kernel ensures there is an implicit synchronization
@@ -1280,8 +1286,24 @@ gdk_x11_screen_init_glx (GdkX11Screen *screen)
        * in that case, to defer telling the compositor our latest frame is
        * ready until after the GPU has completed all issued commands related
        * to the frame, and that the X server says the frame has been drawn.
+       *
+       * As this can cause deadlocks, we want to make sure to only enable it for Xorg,
+       * but not for XWayland, Xnest or whatever other X servers exist.
        */
-      display_x11->has_async_glx_swap_buffers = TRUE;
+
+      gdk_x11_display_error_trap_push (display);
+      if (XGetWindowProperty (dpy, DefaultRootWindow (dpy),
+                              gdk_x11_get_xatom_by_name_for_display (display, "XFree86_VT"),
+                              0, 1, False, AnyPropertyType,
+                              &type, &format, &nitems, &bytes_after, &data) == Success)
+        {
+          if (type != None)
+            display_x11->has_async_glx_swap_buffers = TRUE;
+        }
+      gdk_x11_display_error_trap_pop_ignored (display);
+
+      if (data)
+        XFree (data);
     }
 
   GDK_DISPLAY_NOTE (display, OPENGL,