[gtk-vnc-devel] [PATCH] Fix little to big endian conversion

[Hiroyuki Kaguchi <fj7025cf aa jp fujitsu com>]

When the endian between VNC server and X server is different, the displayed color is abnormal.
This is because the endian conversion of the pixel data is not done .

X server that uses big endian cannot be used.
The reason is that most Linux vncserver sends data by little endian.

Of course, Fedora8(Linux) and Windows(Xming) works fine, since it uses
little endian for X protocol.

This patch applies follows:
(a)The endian conversion function is called by the SET_PIXEL function.
(b)It is checked whether there is difference in endian between X server and VNC Server.
(c)The byte_order variable is added to the gvnc_framebuffer structure.

Sign-off-by: Hiroyuki Kaguchi <fj7025cf aa jp fujitsu com>


diff -r c008e589a12b src/blt.h
--- a/src/blt.h	Mon Feb 04 19:38:17 2008 -0600
+++ b/src/blt.h	Wed Feb 06 15:42:54 2008 +0900
@@ -16,7 +16,7 @@
 #define RGB24_BLIT SPLICE(gvnc_rgb24_blt_, SUFFIX())
 #define TIGHT_COMPUTE_PREDICTED SPLICE(gvnc_tight_compute_predicted_, SUFFIX())
 #define TIGHT_SUM_PIXEL SPLICE(gvnc_tight_sum_pixel_, SUFFIX())
-#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, SRC)(gvnc, pixel)
+#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, DST)(gvnc, pixel)
 #define COMPONENT(color, pixel) ((SWAP(gvnc, pixel) >> gvnc->fmt.SPLICE(color, _shift) & gvnc->fmt.SPLICE(color, _max)))
 
 static void FAST_FILL(struct gvnc *gvnc, src_pixel_t *sp,
@@ -43,9 +43,9 @@ static void FAST_FILL(struct gvnc *gvnc,
 
 static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t *sp)
 {
-	*dp = ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
+	*dp = SWAP(gvnc, ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
 		| ((*sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
-		| ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls;
+		| ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls);
 }
 
 static void SET_PIXEL_AT(struct gvnc *gvnc, int x, int y, src_pixel_t *sp)
diff -r c008e589a12b src/gvnc.c
--- a/src/gvnc.c	Mon Feb 04 19:38:17 2008 -0600
+++ b/src/gvnc.c	Wed Feb 06 15:42:54 2008 +0900
@@ -953,7 +953,7 @@ static uint8_t gvnc_swap_8(struct gvnc *
 
 static uint16_t gvnc_swap_16(struct gvnc *gvnc, uint16_t pixel)
 {
-	if (gvnc->fmt.byte_order != __BYTE_ORDER)
+	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
 		return  (((pixel >> 8) & 0xFF) << 0) |
 			(((pixel >> 0) & 0xFF) << 8);
 	else
@@ -962,7 +962,7 @@ static uint16_t gvnc_swap_16(struct gvnc
 
 static uint32_t gvnc_swap_32(struct gvnc *gvnc, uint32_t pixel)
 {
-	if (gvnc->fmt.byte_order != __BYTE_ORDER)
+	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
 		return  (((pixel >> 24) & 0xFF) <<  0) |
 			(((pixel >> 16) & 0xFF) <<  8) |
 			(((pixel >>  8) & 0xFF) << 16) |
@@ -2953,6 +2953,7 @@ gboolean gvnc_set_local(struct gvnc *gvn
 	    fb->red_shift == gvnc->fmt.red_shift &&
 	    fb->green_shift == gvnc->fmt.green_shift &&
 	    fb->blue_shift == gvnc->fmt.blue_shift &&
+	    fb->byte_order == gvnc->fmt.byte_order &&
 	    __BYTE_ORDER == gvnc->fmt.byte_order)
 		gvnc->perfect_match = TRUE;
 	else
@@ -3001,13 +3002,6 @@ gboolean gvnc_set_local(struct gvnc *gvn
 	for (n = gvnc->fmt.blue_max; n > gvnc->local.blue_mask ; n>>= 1)
 		gvnc->brs++;
 
-	/* This adjusts for remote having less bpp than remote */
-	for (n = gvnc->local.red_mask ; n > gvnc->fmt.red_max ; n>>= 1)
-		gvnc->rls++;
-	for (n = gvnc->local.green_mask ; n > gvnc->fmt.green_max ; n>>= 1)
-		gvnc->gls++;
-	for (n = gvnc->local.blue_mask ; n > gvnc->fmt.blue_max ; n>>= 1)
-		gvnc->bls++;
 	GVNC_DEBUG("Pixel shifts\n   right: %3d %3d %3d\n    left: %3d %3d %3d\n",
 		   gvnc->rrs, gvnc->grs, gvnc->brs,
 		   gvnc->rls, gvnc->gls, gvnc->bls);
diff -r c008e589a12b src/gvnc.h
--- a/src/gvnc.h	Mon Feb 04 19:38:17 2008 -0600
+++ b/src/gvnc.h	Wed Feb 06 15:42:54 2008 +0900
@@ -49,6 +49,7 @@ struct gvnc_framebuffer
 
 	int linesize;
 
+	uint16_t byte_order;
 	int depth;
 	int bpp;
 
diff -r c008e589a12b src/vncdisplay.c
--- a/src/vncdisplay.c	Mon Feb 04 19:38:17 2008 -0600
+++ b/src/vncdisplay.c	Wed Feb 06 15:42:54 2008 +0900
@@ -735,6 +735,7 @@ static void setup_gdk_image(VncDisplay *
 	priv->fb.height = priv->image->height;
 	priv->fb.linesize = priv->image->bpl;
 	priv->fb.data = (uint8_t *)priv->image->mem;
+	priv->fb.byte_order = priv->image->byte_order == GDK_LSB_FIRST ? __LITTLE_ENDIAN : __BIG_ENDIAN;
 
 	gtk_widget_set_size_request(GTK_WIDGET(obj), width, height);
 }