[gtk-vnc-devel] PATCH: Implement cursor encoding

["Daniel P. Berrange" <berrange redhat com>]

The OSX-VNC server violates the RFB spec by never drawing a remote cursor
itself. So if you connect to it with GTK-VNC you never see a cursor on the
remote framebuffer. The server stupidly assumes that all clients will 
implement the rich cursor extension, even if the client doesn't advertise
this fact. I can't find anyway around this, so I implemented the rich
cursor extension

Dan.
-- 
|=- Red Hat, Engineering, Emerging Technologies, Boston.  +1 978 392 2496 -=|
|=-           Perl modules: http://search.cpan.org/~danberr/              -=|
|=-               Projects: http://freshmeat.net/~danielpb/               -=|
|=-  GnuPG: 7D3B9505   F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505  -=| 

diff -r b1c48ddc01d9 examples/gvncviewer.c
--- a/examples/gvncviewer.c	Wed Aug 22 15:10:12 2007 -0400
+++ b/examples/gvncviewer.c	Thu Aug 30 14:21:36 2007 -0400
@@ -259,8 +259,9 @@ int main(int argc, char **argv)
 		snprintf(port, sizeof(port), "%d", 5900);
 
 	vnc_display_open_host(VNC_DISPLAY(vnc), hostname, port);
+	//vnc_display_open_tunnel(VNC_DISPLAY(vnc), "root", hostname, port);
 	vnc_display_set_keyboard_grab(VNC_DISPLAY(vnc), TRUE);
-	vnc_display_set_pointer_grab(VNC_DISPLAY(vnc), TRUE);
+	//vnc_display_set_pointer_grab(VNC_DISPLAY(vnc), TRUE);
 	//vnc_display_set_pointer_local(VNC_DISPLAY(vnc), TRUE);
 
 	gtk_signal_connect(GTK_OBJECT(window), "delete-event",
diff -r b1c48ddc01d9 src/gvnc.c
--- a/src/gvnc.c	Wed Aug 22 15:10:12 2007 -0400
+++ b/src/gvnc.c	Thu Aug 30 15:45:19 2007 -0400
@@ -1057,6 +1057,62 @@ static void gvnc_shared_memory_rmid(stru
 	if (!gvnc->ops.shared_memory_rmid(gvnc->ops_data, shmid))
 		gvnc->has_error = TRUE;
 }
+
+#define CURSOR_BLIT(gvnc, pixbuf, image, mask, pitch, width, height, src_pixel_t) \
+	do {								\
+		int x, y;						\
+		uint8_t *src = image;					\
+		uint32_t *dst = (uint32_t*)pixbuf;			\
+		uint8_t *alpha = mask;					\
+		for (y = 0; y < height; y++) {				\
+			uint32_t *dp = (uint32_t *)dst;			\
+			src_pixel_t *sp = (src_pixel_t *)src;		\
+			uint8_t *mp = alpha;				\
+			for (x = 0; x < width; x++) {			\
+				*dp = (((mp[x/8] >> (7-(x % 8))) &1) ? (255 << 24) : 0) \
+					| (((*sp >> gvnc->fmt.red_shift) & (gvnc->fmt.red_max)) << 16) \
+					| (((*sp >> gvnc->fmt.green_shift) & (gvnc->fmt.green_max)) << 8) \
+					| (((*sp >> gvnc->fmt.blue_shift) & (gvnc->fmt.blue_max)) << 0); \
+				dp++;					\
+				sp++;					\
+			}						\
+			src += pitch;					\
+			dst += width;					\
+			alpha += ((width+7)/8);				\
+		}							\
+	} while(0)
+
+static void gvnc_rich_cursor(struct gvnc *gvnc, int x, int y, int width, int height)
+{
+	uint8_t *image, *mask, *pixbuf;
+	int imagelen, masklen;
+	imagelen = width * height * (gvnc->fmt.bits_per_pixel / 8);
+	masklen = ((width + 7)/8) * height;
+
+	image = malloc(imagelen);
+	mask = malloc(masklen);
+	pixbuf = malloc(width * height * 4); /* RGB-A 8bit */
+	gvnc_read(gvnc, image, imagelen);
+	gvnc_read(gvnc, mask, masklen);
+
+	if (gvnc->fmt.bits_per_pixel == 8) {
+		CURSOR_BLIT(gvnc, pixbuf, image, mask, width * (gvnc->fmt.bits_per_pixel/8), width, height, uint8_t);
+	} else if (gvnc->fmt.bits_per_pixel == 16) {
+		CURSOR_BLIT(gvnc, pixbuf, image, mask, width * (gvnc->fmt.bits_per_pixel/8), width, height, uint16_t);
+	} else if (gvnc->fmt.bits_per_pixel == 24 || gvnc->fmt.bits_per_pixel == 32) {
+		CURSOR_BLIT(gvnc, pixbuf, image, mask, width * (gvnc->fmt.bits_per_pixel/8), width, height, uint32_t);
+	}
+
+	if (gvnc->has_error || !gvnc->ops.rich_cursor)
+		return;
+	if (!gvnc->ops.rich_cursor(gvnc->ops_data, x, y, width, height, pixbuf))
+		gvnc->has_error = TRUE;
+
+	free(image);
+	free(mask);
+	free(pixbuf);
+}
+
 
 static void gvnc_framebuffer_update(struct gvnc *gvnc, int32_t etype,
 				    uint16_t x, uint16_t y,
@@ -1096,6 +1152,9 @@ static void gvnc_framebuffer_update(stru
 			break;
 		}
 		break;
+	case GVNC_ENCODING_RICH_CURSOR:
+		gvnc_rich_cursor(gvnc, x, y, width, height);
+		break;
 	default:
 		gvnc->has_error = TRUE;
 		break;
diff -r b1c48ddc01d9 src/gvnc.h
--- a/src/gvnc.h	Wed Aug 22 15:10:12 2007 -0400
+++ b/src/gvnc.h	Thu Aug 30 13:11:44 2007 -0400
@@ -18,6 +18,7 @@ struct gvnc_ops
 	gboolean (*resize)(void *, int, int);
 	gboolean (*pointer_type_change)(void *, int);
 	gboolean (*shared_memory_rmid)(void *, int);
+	gboolean (*rich_cursor)(void *, int, int, int, int, uint8_t *);
 };
 
 struct gvnc_pixel_format
@@ -68,7 +69,7 @@ typedef enum {
 	GVNC_ENCODING_DESKTOP_RESIZE = -223,
 	GVNC_ENCODING_CURSOR_POS = -232,
 	GVNC_ENCODING_RICH_CURSOR = -239,
-	GVNC_ENCODING_XCUSOR = -240,
+	GVNC_ENCODING_XCURSOR = -240,
 
 	GVNC_ENCODING_POINTER_CHANGE = -257,
 	GVNC_ENCODING_SHARED_MEMORY = -258,
diff -r b1c48ddc01d9 src/vncdisplay.c
--- a/src/vncdisplay.c	Wed Aug 22 15:10:12 2007 -0400
+++ b/src/vncdisplay.c	Thu Aug 30 15:45:56 2007 -0400
@@ -35,6 +35,7 @@ struct _VncDisplayPrivate
 	GdkGC *gc;
 	VncShmImage *shm_image;
 	GdkCursor *null_cursor;
+	GdkCursor *remote_cursor;
 
 	struct gvnc_framebuffer fb;
 	struct coroutine coroutine;
@@ -178,7 +179,7 @@ static void do_pointer_grab(VncDisplay *
 			 GDK_BUTTON_MOTION_MASK |
 			 GDK_SCROLL_MASK,
 			 GTK_WIDGET(obj)->window,
-			 priv->null_cursor,
+			 priv->remote_cursor ? priv->remote_cursor : priv->null_cursor,
 			 GDK_CURRENT_TIME);
 	priv->in_pointer_grab = TRUE;
 	g_signal_emit(obj, signals[VNC_POINTER_GRAB], 0);
@@ -201,13 +202,14 @@ static void do_pointer_hide(VncDisplay *
 {
 	VncDisplayPrivate *priv = obj->priv;
 	gdk_window_set_cursor(GTK_WIDGET(obj)->window,
-			      priv->null_cursor);
+			      priv->remote_cursor ? priv->remote_cursor : priv->null_cursor);
 }
 
 static void do_pointer_show(VncDisplay *obj)
 {
+	VncDisplayPrivate *priv = obj->priv;
 	gdk_window_set_cursor(GTK_WIDGET(obj)->window,
-			      NULL);
+			      priv->remote_cursor);
 }
 
 
@@ -572,6 +574,23 @@ static gboolean on_auth_subtype(void *op
 	return TRUE;
 }
 
+static gboolean on_rich_cursor(void *opaque, int x, int y, int width, int height, uint8_t *image)
+{
+	VncDisplay *obj = VNC_DISPLAY(opaque);
+	VncDisplayPrivate *priv = obj->priv;
+
+	GdkPixbuf *pixbuf = gdk_pixbuf_new_from_data(image, GDK_COLORSPACE_RGB,
+						     TRUE, 8, width, height, width*4,NULL, NULL);
+
+	GdkCursor *c = gdk_cursor_new_from_pixbuf(gdk_drawable_get_display(GDK_DRAWABLE(GTK_WIDGET(obj)->window)),
+						  pixbuf, x, y);
+
+	if (priv->remote_cursor)
+		gdk_cursor_unref(priv->remote_cursor);
+	priv->remote_cursor = c;
+	do_pointer_hide(obj);
+}
+
 
 static const struct gvnc_ops vnc_display_ops = {
 	.auth_cred = on_auth_cred,
@@ -581,6 +600,7 @@ static const struct gvnc_ops vnc_display
 	.resize = on_resize,
 	.pointer_type_change = on_pointer_type_change,
 	.shared_memory_rmid = on_shared_memory_rmid,
+	.rich_cursor = on_rich_cursor,
 };
 
 static void *vnc_coroutine(void *opaque)
@@ -588,6 +608,7 @@ static void *vnc_coroutine(void *opaque)
 	VncDisplay *obj = VNC_DISPLAY(opaque);
 	VncDisplayPrivate *priv = obj->priv;
 	int32_t encodings[] = { GVNC_ENCODING_DESKTOP_RESIZE,
+				GVNC_ENCODING_RICH_CURSOR,
 				GVNC_ENCODING_SHARED_MEMORY,
 				GVNC_ENCODING_POINTER_CHANGE,
 				GVNC_ENCODING_HEXTILE,