Re: [gtk-vnc-devel] [PATCH] Fix little to big endian conversion

[Hiroyuki Kaguchi <fj7025cf aa jp fujitsu com>]

On 2008/02/07 12:02, Daniel P. Berrange wrote:
> Here's the problem:
>
> On Wed, Feb 06, 2008 at 10:49:29PM -0300, Jonh Wendell wrote:
> > Pixel format BPP: 16,  Depth: 16, Byte order: 1234, True color: 1
> >              Mask  red:  31, green:  63, blue:  31
> >              Shift red:  11, green:   5, blue:   0
> > Display name 'mac_cpd_wendell ( 200.100.100.191 )'
> > Mask local: 255 255 255
> >     remote:  31  63  31
> >     merged:  31  63  31
>
> The remote server is 16 bpp, while your local desktop is 24 (or 32?) bpp.  
> The bogus code removal I pointed out will most likely be the cause of
> your problems.
>
> Dan.

Sorry.
This patch is corrected.

diff -r c008e589a12b src/blt.h
--- a/src/blt.h	Mon Feb 04 19:38:17 2008 -0600
+++ b/src/blt.h	Wed Feb 13 17:34:06 2008 +0900
@@ -16,7 +16,7 @@
 #define RGB24_BLIT SPLICE(gvnc_rgb24_blt_, SUFFIX())
 #define TIGHT_COMPUTE_PREDICTED SPLICE(gvnc_tight_compute_predicted_, SUFFIX())
 #define TIGHT_SUM_PIXEL SPLICE(gvnc_tight_sum_pixel_, SUFFIX())
-#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, SRC)(gvnc, pixel)
+#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, DST)(gvnc, pixel)
 #define COMPONENT(color, pixel) ((SWAP(gvnc, pixel) >> gvnc->fmt.SPLICE(color, _shift) & gvnc->fmt.SPLICE(color, _max)))
 
 static void FAST_FILL(struct gvnc *gvnc, src_pixel_t *sp,
@@ -43,9 +43,9 @@ static void FAST_FILL(struct gvnc *gvnc,
 
 static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t *sp)
 {
-	*dp = ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
+	*dp = SWAP(gvnc, ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
 		| ((*sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
-		| ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls;
+		| ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls);
 }
 
 static void SET_PIXEL_AT(struct gvnc *gvnc, int x, int y, src_pixel_t *sp)
diff -r c008e589a12b src/gvnc.c
--- a/src/gvnc.c	Mon Feb 04 19:38:17 2008 -0600
+++ b/src/gvnc.c	Wed Feb 13 17:54:46 2008 +0900
@@ -953,7 +953,7 @@ static uint8_t gvnc_swap_8(struct gvnc *
 
 static uint16_t gvnc_swap_16(struct gvnc *gvnc, uint16_t pixel)
 {
-	if (gvnc->fmt.byte_order != __BYTE_ORDER)
+	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
 		return  (((pixel >> 8) & 0xFF) << 0) |
 			(((pixel >> 0) & 0xFF) << 8);
 	else
@@ -962,7 +962,7 @@ static uint16_t gvnc_swap_16(struct gvnc
 
 static uint32_t gvnc_swap_32(struct gvnc *gvnc, uint32_t pixel)
 {
-	if (gvnc->fmt.byte_order != __BYTE_ORDER)
+	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
 		return  (((pixel >> 24) & 0xFF) <<  0) |
 			(((pixel >> 16) & 0xFF) <<  8) |
 			(((pixel >>  8) & 0xFF) << 16) |
@@ -2953,7 +2953,7 @@ gboolean gvnc_set_local(struct gvnc *gvn
 	    fb->red_shift == gvnc->fmt.red_shift &&
 	    fb->green_shift == gvnc->fmt.green_shift &&
 	    fb->blue_shift == gvnc->fmt.blue_shift &&
-	    __BYTE_ORDER == gvnc->fmt.byte_order)
+	    fb->byte_order == gvnc->fmt.byte_order)
 		gvnc->perfect_match = TRUE;
 	else
 		gvnc->perfect_match = FALSE;
@@ -2981,7 +2981,6 @@ gboolean gvnc_set_local(struct gvnc *gvn
 	gvnc->gls = gvnc->local.green_shift;
 	gvnc->bls = gvnc->local.blue_shift;
 
-
 	/* This adjusts for server/client endianness mismatch */
 	if (__BYTE_ORDER != gvnc->fmt.byte_order) {
 		gvnc->rrs = gvnc->fmt.bits_per_pixel - gvnc->rrs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
@@ -2991,7 +2990,6 @@ gboolean gvnc_set_local(struct gvnc *gvn
 		GVNC_DEBUG("Flipped shifts red: %3d, green: %3d, blue: %3d\n",
 			   gvnc->rrs, gvnc->grs, gvnc->brs);
 	}
-
 
 	/* This adjusts for remote having more bpp than local */
 	for (n = gvnc->fmt.red_max; n > gvnc->local.red_mask ; n>>= 1)
diff -r c008e589a12b src/gvnc.h
--- a/src/gvnc.h	Mon Feb 04 19:38:17 2008 -0600
+++ b/src/gvnc.h	Wed Feb 13 17:34:06 2008 +0900
@@ -49,6 +49,7 @@ struct gvnc_framebuffer
 
 	int linesize;
 
+	uint16_t byte_order;
 	int depth;
 	int bpp;
 
diff -r c008e589a12b src/vncdisplay.c
--- a/src/vncdisplay.c	Mon Feb 04 19:38:17 2008 -0600
+++ b/src/vncdisplay.c	Wed Feb 13 17:34:06 2008 +0900
@@ -735,6 +735,7 @@ static void setup_gdk_image(VncDisplay *
 	priv->fb.height = priv->image->height;
 	priv->fb.linesize = priv->image->bpl;
 	priv->fb.data = (uint8_t *)priv->image->mem;
+	priv->fb.byte_order = priv->image->byte_order == GDK_LSB_FIRST ? __LITTLE_ENDIAN : __BIG_ENDIAN;
 
 	gtk_widget_set_size_request(GTK_WIDGET(obj), width, height);
 }