[gtk-vnc-devel] [PATCH] Fix RichCursor encoding for server depths of < 24 bpp

[Anthony Liguori <anthony codemonkey ws>]

I've committed this since it's a bug fix and adds no new functionality.  
I'm posting here though as an FYI.

Regards,

Anthony Liguori

# HG changeset patch
# User Anthony Liguori <anthony codemonkey ws>
# Date 1198868930 21600
# Node ID df452cd30b8cc46e315d54fe275498a4bd096c17
# Parent  be60cc1bd95feb06e2af1c09262c2bcab1553049
Fix rich cursor encoding.

If the depth of the server is less than 24, when we built the cursor, we were
expanding the smaller pixels to the lower bits of the new pixel instead of the
upper bits.  The result was that colors in the cursor were showing up as dark
grays.

This patch refactors the cursor blitting function to be more similar to the
rest of the blitting functions and ensures that the proper shift values are
used.

diff -r be60cc1bd95f -r df452cd30b8c src/blt.h
--- a/src/blt.h	Wed Dec 26 09:09:48 2007 -0600
+++ b/src/blt.h	Fri Dec 28 13:08:50 2007 -0600
@@ -9,6 +9,7 @@
 #define FAST_FILL SPLICE(gvnc_fill_fast_, SUFFIX())
 #define HEXTILE SPLICE(gvnc_hextile_, SUFFIX())
 #define RRE SPLICE(gvnc_rre_, SUFFIX())
+#define RICH_CURSOR_BLIT SPLICE(gvnc_rich_cursor_blt_, SUFFIX())
 
 static void FAST_FILL(struct gvnc *gvnc, src_pixel_t *sp,
 		      int x, int y, int width, int height)
@@ -142,6 +143,42 @@ static void HEXTILE(struct gvnc *gvnc, u
 	}
 }
 
+/* We need to convert to a GdkPixbuf which is always 32-bit */
+#if DST == 32
+static void RICH_CURSOR_BLIT(struct gvnc *gvnc, uint8_t *pixbuf,
+			     uint8_t *image, uint8_t *mask, int pitch,
+			     uint16_t width, uint16_t height)
+{
+	int x1, y1;
+	uint32_t *dst = (uint32_t *)pixbuf;
+	uint8_t *src = image;
+	uint8_t *alpha = mask;
+	int rs, gs, bs;
+
+	rs = 24 - ((sizeof(src_pixel_t) * 8) - gvnc->fmt.red_shift);
+	gs = 16 - (gvnc->fmt.red_shift - gvnc->fmt.green_shift);
+	bs = 8 - (gvnc->fmt.green_shift - gvnc->fmt.blue_shift);
+
+	for (y1 = 0; y1 < height; y1++) {
+		src_pixel_t *sp = (src_pixel_t *)src;
+		uint8_t *mp = alpha;
+		for (x1 = 0; x1 < width; x1++) {
+			*dst = (((*sp >> gvnc->fmt.red_shift) & gvnc->fmt.red_max) << rs)
+				| (((*sp >> gvnc->fmt.green_shift) & gvnc->fmt.green_max) << gs)
+				| (((*sp >> gvnc->fmt.blue_shift) & gvnc->fmt.blue_max) << bs);
+
+			if ((mp[x1 / 8] >> (7 - (x1 % 8))) & 1)
+				*dst |= 0xFF000000;
+
+			dst++;
+			sp++;
+		}
+		src += pitch;
+		alpha += ((width + 7) / 8);
+	}
+}
+#endif
+
 #undef HEXTILE
 #undef FILL
 #undef FAST_FILL
diff -r be60cc1bd95f -r df452cd30b8c src/gvnc.c
--- a/src/gvnc.c	Wed Dec 26 09:09:48 2007 -0600
+++ b/src/gvnc.c	Fri Dec 28 13:08:50 2007 -0600
@@ -51,6 +51,9 @@ typedef void gvnc_hextile_func(struct gv
 			       uint16_t width, uint16_t height,
 			       uint8_t *fg, uint8_t *bg);
 
+typedef void gvnc_rich_cursor_blt_func(struct gvnc *, uint8_t *, uint8_t *,
+				       uint8_t *, int, uint16_t, uint16_t);
+
 /*
  * A special GSource impl which allows us to wait on a certain
  * condition to be satisified. This is effectively a boolean test
@@ -113,6 +116,7 @@ struct gvnc
 	gvnc_blt_func *blt;
 	gvnc_fill_func *fill;
 	gvnc_hextile_func *hextile;
+	gvnc_rich_cursor_blt_func *rich_cursor_blt;
 
 	struct gvnc_ops ops;
 	gpointer ops_data;
@@ -866,7 +870,13 @@ static gvnc_fill_func *gvnc_fill_table[3
 	{ (gvnc_fill_func *)gvnc_fill_32x8,
 	  (gvnc_fill_func *)gvnc_fill_32x16,
 	  (gvnc_fill_func *)gvnc_fill_32x32 },
-};	
+};
+
+static gvnc_rich_cursor_blt_func *gvnc_rich_cursor_blt_table[3] = {
+	gvnc_rich_cursor_blt_8x32,
+	gvnc_rich_cursor_blt_16x32,
+	gvnc_rich_cursor_blt_32x32,
+};
 
 /* a fast blit for the perfect match scenario */
 static void gvnc_blt_fast(struct gvnc *gvnc, uint8_t *src, int pitch,
@@ -1055,26 +1065,12 @@ static void gvnc_pointer_type_change(str
 		gvnc->has_error = TRUE;
 }
 
-#define RICH_CURSOR_BLIT(gvnc, pixbuf, image, mask, pitch, width, height, src_pixel_t) \
-	do {								\
-		int x1, y1;						\
-		uint8_t *src = image;					\
-		uint32_t *dst = (uint32_t*)pixbuf;			\
-		uint8_t *alpha = mask;					\
-		for (y1 = 0; y1 < height; y1++) {			\
-			src_pixel_t *sp = (src_pixel_t *)src;		\
-			uint8_t *mp = alpha;				\
-			for (x1 = 0; x1 < width; x1++) {		\
-				*dst++ = (((mp[x1 / 8] >> (7 - (x1 % 8))) & 1) ? (255 << 24) : 0) \
-					| (((*sp >> gvnc->fmt.red_shift) & (gvnc->fmt.red_max)) << 16) \
-					| (((*sp >> gvnc->fmt.green_shift) & (gvnc->fmt.green_max)) << 8) \
-					| (((*sp >> gvnc->fmt.blue_shift) & (gvnc->fmt.blue_max)) << 0); \
-				sp++;					\
-			}						\
-			src += pitch;					\
-			alpha += ((width + 7) / 8);			\
-		}							\
-	} while(0)
+static void gvnc_rich_cursor_blt(struct gvnc *gvnc, uint8_t *pixbuf,
+				 uint8_t *image, uint8_t *mask,
+				 int pitch, uint16_t width, uint16_t height)
+{
+	gvnc->rich_cursor_blt(gvnc, pixbuf, image, mask, pitch, width, height);
+}
 
 static void gvnc_rich_cursor(struct gvnc *gvnc, int x, int y, int width, int height)
 {
@@ -1108,13 +1104,10 @@ static void gvnc_rich_cursor(struct gvnc
 		gvnc_read(gvnc, image, imagelen);
 		gvnc_read(gvnc, mask, masklen);
 
-		if (gvnc->fmt.bits_per_pixel == 8) {
-			RICH_CURSOR_BLIT(gvnc, pixbuf, image, mask, width * (gvnc->fmt.bits_per_pixel/8), width, height, uint8_t);
-		} else if (gvnc->fmt.bits_per_pixel == 16) {
-			RICH_CURSOR_BLIT(gvnc, pixbuf, image, mask, width * (gvnc->fmt.bits_per_pixel/8), width, height, uint16_t);
-		} else if (gvnc->fmt.bits_per_pixel == 24 || gvnc->fmt.bits_per_pixel == 32) {
-			RICH_CURSOR_BLIT(gvnc, pixbuf, image, mask, width * (gvnc->fmt.bits_per_pixel/8), width, height, uint32_t);
-		}
+		gvnc_rich_cursor_blt(gvnc, pixbuf, image, mask,
+				     width * (gvnc->fmt.bits_per_pixel/8),
+				     width, height);
+
 		free(image);
 		free(mask);
 	}
@@ -1171,8 +1164,6 @@ static void gvnc_xcursor(struct gvnc *gv
 		free(data);
 		free(mask);
 	}
-
-
 
 	if (gvnc->has_error || !gvnc->ops.local_cursor)
 		return;
@@ -2310,6 +2301,7 @@ gboolean gvnc_set_local(struct gvnc *gvn
 	gvnc->blt = gvnc_blt_table[i - 1][j - 1];
 	gvnc->fill = gvnc_fill_table[i - 1][j - 1];
 	gvnc->hextile = gvnc_hextile_table[i - 1][j - 1];
+	gvnc->rich_cursor_blt = gvnc_rich_cursor_blt_table[i - 1];
 
 	if (gvnc->perfect_match)
 		gvnc->blt = gvnc_blt_fast;