[gtk-vnc-devel] PATCH: Fix opengl rendering z-buffer depth

["Daniel P. Berrange" <berrange redhat com>]

When using the OpenGL scaling on a Intel graphics card the VNC display 
renders on top of all other windows. None of the gtkglext examples show
this problem, and after a little debugging & comparison I narrowed it
down to the missing GDK_GL_DEPTH_SIZE attribute. This specifies something
related to the z-buffer depth, though its undocumented. All the other
example programs use  gdk_gl_config_new_by_mode() and pass in a mode of

  GDK_GL_MODE_RGB | GDK_GL_MODE_DEPTH

instead of using the low level gdk_gl_config_new() and an attribute list.
So while I could set the GDK_GL_DEPTH_SIZE attribute, I chose to switch
us over to the simpler higher level API & set GDK_GL_MODE_DEPTH. This
fixes the rendering artifacts I saw.

While debugging this I found a couple of other minor bugs. When deciding
whether to scale or not, we had a typo tested width == height, so we always
used scaling even when the display was 1:1. Second, we were setting a pixel
format of BGRA when doing rendering instead of RGBA. This breaks BGR X
displays - I had previously 'fixed' this by  this

-       visual = gdk_drawable_get_visual(GTK_WIDGET(obj)->window);
+       visual = gdk_screen_get_system_visual(gdk_screen_get_default());

but I found that broke one of my normal X servers. So the patch reverts
that previous change and fixes the texture pixel format. Finally I enable
scaling by default in gvncviewer

Dan.

diff -r 6f18ca44fbe3 examples/gvncviewer.c
--- a/examples/gvncviewer.c	Thu Apr 03 12:57:25 2008 -0300
+++ b/examples/gvncviewer.c	Thu Apr 03 14:23:16 2008 -0400
@@ -339,7 +339,8 @@
 	submenu = gtk_menu_new();
 
 	fullscreen = gtk_check_menu_item_new_with_mnemonic("_Full Screen");
-	scaling = gtk_check_menu_item_new_with_mnemonic("OpenGL _Scaling");
+	scaling = gtk_check_menu_item_new_with_mnemonic("Scaled display");
+	gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(scaling), TRUE);
 
 	gtk_menu_append(GTK_MENU(submenu), fullscreen);
 	gtk_menu_append(GTK_MENU(submenu), scaling);
@@ -369,6 +370,7 @@
 	vnc_display_open_host(VNC_DISPLAY(vnc), hostname, port);
 	vnc_display_set_keyboard_grab(VNC_DISPLAY(vnc), TRUE);
 	vnc_display_set_pointer_grab(VNC_DISPLAY(vnc), TRUE);
+	vnc_display_set_scaling(VNC_DISPLAY(vnc), TRUE);
 
 	gtk_signal_connect(GTK_OBJECT(window), "delete-event",
 			   GTK_SIGNAL_FUNC(gtk_main_quit), NULL);
diff -r 6f18ca44fbe3 src/vncdisplay.c
--- a/src/vncdisplay.c	Thu Apr 03 12:57:25 2008 -0300
+++ b/src/vncdisplay.c	Thu Apr 03 14:23:16 2008 -0400
@@ -325,7 +325,7 @@
 		glPixelStorei(GL_UNPACK_ROW_LENGTH, priv->fb.width);
 		glTexSubImage2D(GL_TEXTURE_2D, 0,
 				x, y, w, h,
-				GL_BGRA_EXT /* GL_RGB */,
+				GL_RGBA,
 				GL_UNSIGNED_BYTE,
 				priv->gl_tex_data +
 				y * 4 * priv->fb.width +
@@ -340,7 +340,7 @@
 		glTexCoord2f(0,0);  glVertex3f(wx, wy+wh, 0);
 		glTexCoord2f(rx,0);  glVertex3f(wx+ww, wy+wh, 0);
 		glTexCoord2f(rx,ry);  glVertex3f(wx+ww, wy, 0);
-		glEnd();
+		glEnd();		
 		glDisable(GL_TEXTURE_2D);
 		glFlush();
 		gdk_gl_drawable_gl_end(priv->gl_drawable);
@@ -853,8 +853,8 @@
 	VncDisplayPrivate *priv = obj->priv;
 	GdkVisual *visual;
 
-	visual = gdk_screen_get_system_visual(gdk_screen_get_default());
-	
+	visual = gdk_drawable_get_visual(GTK_WIDGET(obj)->window);
+
 	priv->image = gdk_image_new(GDK_IMAGE_FASTEST, visual, width, height);
 	GVNC_DEBUG("Visual mask: %3d %3d %3d\n      shift: %3d %3d %3d\n",
 		   visual->red_mask,
@@ -1170,8 +1170,8 @@
 {
 	VncDisplayPrivate *priv = obj->priv;
 
-	if (priv->allow_scaling &&
-	    (priv->fb.height != width ||
+	if (priv->allow_scaling && 
+	    (priv->fb.width != width ||
 	     priv->fb.height != height))
 		scale_display(obj, width, height);
 	else if (priv->gl_enabled) {
@@ -2010,13 +2010,6 @@
 	GtkObject *obj = GTK_OBJECT(display);
 	GtkWidget *widget = GTK_WIDGET(display);
 	VncDisplayPrivate *priv;
-#if WITH_GTKGLEXT
-	static const int attrib[] = { GDK_GL_RGBA,
-				      GDK_GL_RED_SIZE, 1,
-				      GDK_GL_GREEN_SIZE, 1,
-				      GDK_GL_BLUE_SIZE, 1,
-				      GDK_GL_ATTRIB_LIST_NONE };
-#endif
 
 	g_signal_connect(obj, "expose-event",
 			 G_CALLBACK(expose_event), NULL);
@@ -2073,7 +2066,8 @@
 
 #if WITH_GTKGLEXT
 	if (gtk_gl_init_check(NULL, NULL)) {
-		priv->gl_config = gdk_gl_config_new(attrib);
+		priv->gl_config = gdk_gl_config_new_by_mode(GDK_GL_MODE_RGB |
+							    GDK_GL_MODE_DEPTH);
 		if (!gtk_widget_set_gl_capability(widget,
 						  priv->gl_config,
 						  NULL,


-- 
|: Red Hat, Engineering, Boston   -o-   http://people.redhat.com/berrange/ :|
|: http://libvirt.org  -o-  http://virt-manager.org  -o-  http://ovirt.org :|
|: http://autobuild.org       -o-         http://search.cpan.org/~danberr/ :|
|: GnuPG: 7D3B9505  -o-  F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|