[gtk+] Win32: Disable layered windows for GL

[Chun-wei Fan <fanchunwei src gnome org>]

commit 3f190e0fa6c2c6e00cc9e0de2627b711539dc1e8
Author: Chun-wei Fan <fanchunwei src gnome org>
Date:   Sat Mar 5 11:34:16 2016 +0800

    Win32: Disable layered windows for GL
    
    Layered windows and GL do not work well together, so disable layered
    windows when initiating a GdkGLContext, so that GtkGLArea programs can run
    properly.
    
    Also based on patch by LRN to address the issue.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=763080

 gdk/win32/gdkglcontext-win32.c |   29 +++++++++++++++++++++++++++++
 gdk/win32/gdkwindow-win32.c    |   14 +++++++++++---
 gdk/win32/gdkwindow-win32.h    |    3 +++
 3 files changed, 43 insertions(+), 3 deletions(-)
---
diff --git a/gdk/win32/gdkglcontext-win32.c b/gdk/win32/gdkglcontext-win32.c
index 2961b11..af82903 100644
--- a/gdk/win32/gdkglcontext-win32.c
+++ b/gdk/win32/gdkglcontext-win32.c
@@ -49,6 +49,7 @@ _gdk_win32_gl_context_dispose (GObject *gobject)
   GdkGLContext *context = GDK_GL_CONTEXT (gobject);
   GdkWin32GLContext *context_win32 = GDK_WIN32_GL_CONTEXT (gobject);
   GdkWin32Display *display_win32 = GDK_WIN32_DISPLAY (gdk_gl_context_get_display (context));
+  GdkWindow *window = gdk_gl_context_get_window (context);
 
   if (context_win32->hglrc != NULL)
     {
@@ -63,6 +64,20 @@ _gdk_win32_gl_context_dispose (GObject *gobject)
       ReleaseDC (display_win32->gl_hwnd, context_win32->gl_hdc);
     }
 
+  if (window != NULL && window->impl != NULL)
+    {
+      GdkWindowImplWin32 *impl = GDK_WINDOW_IMPL_WIN32 (window->impl);
+
+      if (impl->suppress_layered > 0)
+        impl->suppress_layered--;
+
+      /* If we don't have any window that forces layered windows off,
+       * trigger update_style_bits() to enable layered windows again
+       */
+      if (impl->suppress_layered == 0)
+        gdk_window_set_type_hint (window, gdk_window_get_type_hint (window));
+    }
+
   G_OBJECT_CLASS (gdk_win32_gl_context_parent_class)->dispose (gobject);
 }
 
@@ -455,6 +470,9 @@ _gdk_win32_gl_context_realize (GdkGLContext *context,
   gint glver_major = 0;
   gint glver_minor = 0;
 
+  GdkWindow *window = gdk_gl_context_get_window (context);
+  GdkWindowImplWin32 *impl = GDK_WINDOW_IMPL_WIN32 (window->impl);
+
   if (!_set_pixformat_for_hdc (context_win32->gl_hdc,
                                &pixel_format,
                                context_win32->need_alpha_bits))
@@ -500,6 +518,17 @@ _gdk_win32_gl_context_realize (GdkGLContext *context,
 
   context_win32->hglrc = hglrc;
 
+  /* OpenGL does not work with WS_EX_LAYERED enabled, so we need to
+   * disable WS_EX_LAYERED when we acquire a valid HGLRC
+   */
+  impl->suppress_layered++;
+
+  /* if this is the first time a GL context is acquired for the window,
+   * disable layered windows by triggering update_style_bits()
+   */
+  if (impl->suppress_layered == 1)
+    gdk_window_set_type_hint (window, gdk_window_get_type_hint (window));
+
   return TRUE;
 }
 
diff --git a/gdk/win32/gdkwindow-win32.c b/gdk/win32/gdkwindow-win32.c
index 2669283..0bdb3c3 100644
--- a/gdk/win32/gdkwindow-win32.c
+++ b/gdk/win32/gdkwindow-win32.c
@@ -241,7 +241,6 @@ gdk_win32_window_end_paint (GdkWindow *window)
   window_rect.right -= _gdk_offset_x;
   window_rect.top -= _gdk_offset_y;
   window_rect.bottom -= _gdk_offset_y;
-
   if (!impl->layered)
     {
       GDK_NOTE (EVENTS, g_print ("Setting window position ... "));
@@ -2626,9 +2625,18 @@ update_style_bits (GdkWindow *window)
   /* We can get away with using layered windows
    * only when no decorations are needed. It can mean
    * CSD or borderless non-CSD windows (tooltips?).
+   *
+   * If this window cannot use layered windows, disable it always.
+   * This currently applies to windows using OpenGL, which
+   * does not work with layered windows.
    */
-  if (_gdk_win32_window_lacks_wm_decorations (window))
-    impl->layered = g_strcmp0 (g_getenv ("GDK_WIN32_LAYERED"), "0") != 0;
+  if (impl->suppress_layered == 0)
+    {
+      if (_gdk_win32_window_lacks_wm_decorations (window))
+        impl->layered = g_strcmp0 (g_getenv ("GDK_WIN32_LAYERED"), "0") != 0;
+    }
+  else
+    impl->layered = FALSE;
 
   if (impl->layered)
     new_exstyle |= WS_EX_LAYERED;
diff --git a/gdk/win32/gdkwindow-win32.h b/gdk/win32/gdkwindow-win32.h
index 24e7d18..a337e10 100644
--- a/gdk/win32/gdkwindow-win32.h
+++ b/gdk/win32/gdkwindow-win32.h
@@ -170,6 +170,9 @@ struct _GdkWindowImplWin32
 
   /* Decorations set by gdk_window_set_decorations() or NULL if unset */
   GdkWMDecoration* decorations;
+
+  /* No. of windows to force layered windows off */
+  guint suppress_layered;
 };
 
 struct _GdkWindowImplWin32Class