[gtk/wip/otte/gleanup: 5/9] x11: Get Visual from EGL directly

[Benjamin Otte <otte src gnome org>]

commit 52e83ab037b33136f569f08e081f70aa06462551
Author: Benjamin Otte <otte redhat com>
Date:   Fri Jun 18 14:46:24 2021 +0200

    x11: Get Visual from EGL directly
    
    Query the EGL_VISUAL_ID from the egl Config and select a config with the
    matching Visual.
    
    This is currently broken on Mesa because it does not expose any RGBA
    X Visuals in any EGL config, so we always end up with opaque Windows.
    
    https://gitlab.freedesktop.org/mesa/mesa/-/issues/149

 gdk/x11/gdkdisplay-x11.c   |   2 +-
 gdk/x11/gdkdisplay-x11.h   |   3 --
 gdk/x11/gdkglcontext-egl.c | 131 ++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 124 insertions(+), 12 deletions(-)
---
diff --git a/gdk/x11/gdkdisplay-x11.c b/gdk/x11/gdkdisplay-x11.c
index 334f33eb6f..6e9a46ec12 100644
--- a/gdk/x11/gdkdisplay-x11.c
+++ b/gdk/x11/gdkdisplay-x11.c
@@ -1339,7 +1339,7 @@ set_sm_client_id (GdkDisplay  *display,
                      gdk_x11_get_xatom_by_name_for_display (display, "SM_CLIENT_ID"));
 }
 
-void
+static void
 gdk_x11_display_query_default_visual (GdkX11Display  *self,
                                       Visual        **out_visual,
                                       int            *out_depth)
diff --git a/gdk/x11/gdkdisplay-x11.h b/gdk/x11/gdkdisplay-x11.h
index cf4f1dd418..4f27d28f09 100644
--- a/gdk/x11/gdkdisplay-x11.h
+++ b/gdk/x11/gdkdisplay-x11.h
@@ -176,9 +176,6 @@ struct _GdkX11DisplayClass
                                                                  const XEvent           *event);
 };
 
-void            gdk_x11_display_query_default_visual            (GdkX11Display          *self,
-                                                                 Visual                **out_visual,
-                                                                 int                    *out_depth);
 void            _gdk_x11_display_error_event                    (GdkDisplay             *display,
                                                                  XErrorEvent            *error);
 gsize           gdk_x11_display_get_max_request_size            (GdkDisplay             *display);
diff --git a/gdk/x11/gdkglcontext-egl.c b/gdk/x11/gdkglcontext-egl.c
index 61cabceb16..3cfdf7310b 100644
--- a/gdk/x11/gdkglcontext-egl.c
+++ b/gdk/x11/gdkglcontext-egl.c
@@ -101,14 +101,55 @@ gdk_x11_display_create_egl_display (GdkX11Display *self)
   self->egl_display = eglGetDisplay ((EGLNativeDisplayType) dpy);
 }
 
+static XVisualInfo *
+gdk_x11_display_get_visual_info_for_visual (GdkX11Display  *self,
+                                            VisualID        visualid)
+{
+  XVisualInfo template, *visinfo;
+  int nvisuals;
+
+  template.screen = self->screen->screen_num;
+  template.visualid = visualid;
+
+  visinfo = XGetVisualInfo (gdk_x11_display_get_xdisplay (GDK_DISPLAY (self)),
+                            VisualScreenMask | VisualIDMask,
+                            &template,
+                            &nvisuals);
+  g_warn_if_fail (nvisuals == 1);
+  
+  return visinfo;
+}
+
+static gboolean
+visual_is_rgba (XVisualInfo *visinfo)
+{
+  return
+    visinfo->depth == 32 &&
+    visinfo->visual->red_mask   == 0xff0000 &&
+    visinfo->visual->green_mask == 0x00ff00 &&
+    visinfo->visual->blue_mask  == 0x0000ff;
+}
+
 #define MAX_EGL_ATTRS   30
 
 static void
-gdk_x11_display_create_egl_config (GdkX11Display *display)
+gdk_x11_display_create_egl_config (GdkX11Display  *display,
+                                   Visual        **out_visual,
+                                   int            *out_depth)
 {
   GdkX11Display *self = GDK_X11_DISPLAY (display);
   EGLint attrs[MAX_EGL_ATTRS];
-  EGLint count;
+  EGLConfig *configs;
+  EGLint count, alloced;
+  enum {
+    NO_VISUAL_FOUND,
+    WITH_MULTISAMPLING,
+    WITH_STENCIL_AND_DEPTH_BUFFER,
+    NO_ALPHA,
+    NO_ALPHA_VISUAL,
+    PERFECT
+  } best_features;
+
   int i = 0;
 
   attrs[i++] = EGL_SURFACE_TYPE;
@@ -129,9 +170,85 @@ gdk_x11_display_create_egl_config (GdkX11Display *display)
   attrs[i++] = EGL_NONE;
   g_assert (i < MAX_EGL_ATTRS);
 
-  /* Pick first valid configuration that the driver returns us */
-  if (!eglChooseConfig (self->egl_display, attrs, &display->egl_config, 1, &count) && count >= 1)
-    display->egl_config = NULL;
+  if (!eglChooseConfig (self->egl_display, attrs, NULL, -1, &alloced))
+    return;
+
+  configs = g_new (EGLConfig, alloced);
+  if (!eglChooseConfig (self->egl_display, attrs, configs, alloced, &count))
+    {
+      g_free (configs);
+      return;
+    }
+  g_warn_if_fail (alloced == count);
+
+  best_features = NO_VISUAL_FOUND;
+
+  for (i = 0; i < count; i++)
+    {
+      XVisualInfo *visinfo;
+      int tmp, visualid;
+
+      if (!eglGetConfigAttrib (self->egl_display, configs[i], EGL_NATIVE_VISUAL_ID, &visualid))
+        continue;
+
+      visinfo = gdk_x11_display_get_visual_info_for_visual (self, visualid);
+      if (visinfo == NULL)
+        continue;
+
+      if (!eglGetConfigAttrib (self->egl_display, configs[i], EGL_SAMPLE_BUFFERS, &tmp) || tmp != 0)
+        {
+          if (best_features < WITH_MULTISAMPLING)
+            {
+              GDK_NOTE (OPENGL, g_message ("Best config is %u for visual 0x%lX with multisampling\n", i, 
visinfo->visualid));
+              best_features = WITH_MULTISAMPLING;
+              *out_visual = visinfo->visual;
+              *out_depth = visinfo->depth;
+              self->egl_config = configs[i];
+            }
+          XFree (visinfo);
+          continue;
+        }
+
+      if (!eglGetConfigAttrib (self->egl_display, configs[i], EGL_DEPTH_SIZE, &tmp) || tmp != 0 ||
+          !eglGetConfigAttrib (self->egl_display, configs[i], EGL_STENCIL_SIZE, &tmp) || tmp != 0)
+        {
+          GDK_NOTE (OPENGL, g_message ("Best config is %u for visual 0x%lX with stencil or depth buffer\n", 
i, visinfo->visualid));
+          if (best_features < WITH_STENCIL_AND_DEPTH_BUFFER)
+            {
+              best_features = WITH_STENCIL_AND_DEPTH_BUFFER;
+              *out_visual = visinfo->visual;
+              *out_depth = visinfo->depth;
+              self->egl_config = configs[i];
+            }
+          XFree (visinfo);
+          continue;
+        }
+    
+      if (!visual_is_rgba (visinfo))
+        {
+          GDK_NOTE (OPENGL, g_message ("Best config is %u for visual 0x%lX without RGBA Visual\n", i, 
visinfo->visualid));
+          if (best_features < NO_ALPHA_VISUAL)
+            {
+              best_features = NO_ALPHA_VISUAL;
+              *out_visual = visinfo->visual;
+              *out_depth = visinfo->depth;
+              self->egl_config = configs[i];
+            }
+          XFree (visinfo);
+          continue;
+        }
+
+      GDK_NOTE (OPENGL, g_message ("Config %u for visual 0x%lX is the perfect choice\n", i, 
visinfo->visualid));
+      *out_visual = visinfo->visual;
+      *out_depth = visinfo->depth;
+      self->egl_config = configs[i];
+      XFree (visinfo);
+      /* everything is perfect */
+      best_features = PERFECT;
+      break;
+    }
+
+  g_free (configs);
 }
 
 #undef MAX_EGL_ATTRS
@@ -478,7 +595,7 @@ gdk_x11_display_init_egl (GdkX11Display  *self,
       return FALSE;
     }
 
-  gdk_x11_display_create_egl_config (self);
+  gdk_x11_display_create_egl_config (self, out_visual, out_depth);
   if (self->egl_config == NULL)
     {
       eglTerminate (self->egl_display);
@@ -515,8 +632,6 @@ gdk_x11_display_init_egl (GdkX11Display  *self,
                                self->has_egl_swap_buffers_with_damage ? "yes" : "no",
                                self->has_egl_surfaceless_context ? "yes" : "no"));
 
-  gdk_x11_display_query_default_visual (self, out_visual, out_depth);
-
   return TRUE;
 }