Re: [gtk-vnc-devel] [PATCH] Fix little to big endian conversion

[Hiroyuki Kaguchi <fj7025cf aa jp fujitsu com>]

On 2008/02/14 11:00, Hiroyuki Kaguchi wrote:
> Hi, Anthony
> 
> I add describing more detail on posted patch.
> This patch is endian problem on X-server protocol.
> 
> And from my posting patch,
> http://sourceforge.net/mailarchive/message.php?msg_name=47AA4EFD.3080400%40aa.jp.fujitsu.com
> I fix the variable color depth problem.
> (Of course, same color depth on client and server works on previous patch.
> This fix adds support different color depth on client and server.)
> 
> I checked following conditions.
> You should show the matrix of your test set.
> 
> gtk-vnc (little endian)          | VNC server    |
>           +----------------------+---------------+
>           | byte order           | little endian |
>           |        +-------------+-------+-------+
>           |        | color depth | 16bit | 24bit |
> ----------+--------+-------------+-------+-------+
>  X server | little |       16bit |  OK   |  OK   |
>           | endian +-------------+-------+-------+
>           |        |       24bit |  OK   |  OK   |
>           +--------+-------------+-------+-------+
>           | big    |       16bit |  OK   |  OK   |
>           | endian +-------------+-------+-------+
>           |        |       24bit |  OK   |  OK   |
> ----------+--------+-------------+-------+-------+
> 
> Thanks
> Hiroyuki Kaguchi
> 
I posted the old patch by mistake.
The new patch is this.

Thanks
Hiroyuki Kaguchi


diff -r 777ee6d6a72d src/blt.h
--- a/src/blt.h	Fri Feb 08 21:30:38 2008 -0300
+++ b/src/blt.h	Thu Feb 14 11:50:10 2008 +0900
@@ -16,7 +16,7 @@
 #define RGB24_BLIT SPLICE(gvnc_rgb24_blt_, SUFFIX())
 #define TIGHT_COMPUTE_PREDICTED SPLICE(gvnc_tight_compute_predicted_, SUFFIX())
 #define TIGHT_SUM_PIXEL SPLICE(gvnc_tight_sum_pixel_, SUFFIX())
-#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, SRC)(gvnc, pixel)
+#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, DST)(gvnc, pixel)
 #define COMPONENT(color, pixel) ((SWAP(gvnc, pixel) >> gvnc->fmt.SPLICE(color, _shift) & gvnc->fmt.SPLICE(color, _max)))
 
 static void FAST_FILL(struct gvnc *gvnc, src_pixel_t *sp,
@@ -43,9 +43,9 @@ static void FAST_FILL(struct gvnc *gvnc,
 
 static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t *sp)
 {
-	*dp = ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
+	*dp = SWAP(gvnc, ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
 		| ((*sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
-		| ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls;
+		| ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls);
 }
 
 static void SET_PIXEL_AT(struct gvnc *gvnc, int x, int y, src_pixel_t *sp)
diff -r 777ee6d6a72d src/gvnc.c
--- a/src/gvnc.c	Fri Feb 08 21:30:38 2008 -0300
+++ b/src/gvnc.c	Thu Feb 14 11:50:37 2008 +0900
@@ -953,7 +953,7 @@ static uint8_t gvnc_swap_8(struct gvnc *
 
 static uint16_t gvnc_swap_16(struct gvnc *gvnc, uint16_t pixel)
 {
-	if (gvnc->fmt.byte_order != __BYTE_ORDER)
+	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
 		return  (((pixel >> 8) & 0xFF) << 0) |
 			(((pixel >> 0) & 0xFF) << 8);
 	else
@@ -962,7 +962,7 @@ static uint16_t gvnc_swap_16(struct gvnc
 
 static uint32_t gvnc_swap_32(struct gvnc *gvnc, uint32_t pixel)
 {
-	if (gvnc->fmt.byte_order != __BYTE_ORDER)
+	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
 		return  (((pixel >> 24) & 0xFF) <<  0) |
 			(((pixel >> 16) & 0xFF) <<  8) |
 			(((pixel >>  8) & 0xFF) << 16) |
@@ -2953,7 +2953,7 @@ gboolean gvnc_set_local(struct gvnc *gvn
 	    fb->red_shift == gvnc->fmt.red_shift &&
 	    fb->green_shift == gvnc->fmt.green_shift &&
 	    fb->blue_shift == gvnc->fmt.blue_shift &&
-	    __BYTE_ORDER == gvnc->fmt.byte_order)
+	    fb->byte_order == gvnc->fmt.byte_order)
 		gvnc->perfect_match = TRUE;
 	else
 		gvnc->perfect_match = FALSE;
diff -r 777ee6d6a72d src/gvnc.h
--- a/src/gvnc.h	Fri Feb 08 21:30:38 2008 -0300
+++ b/src/gvnc.h	Thu Feb 14 11:51:38 2008 +0900
@@ -49,6 +49,7 @@ struct gvnc_framebuffer
 
 	int linesize;
 
+	uint16_t byte_order;
 	int depth;
 	int bpp;
 
diff -r 777ee6d6a72d src/vncdisplay.c
--- a/src/vncdisplay.c	Fri Feb 08 21:30:38 2008 -0300
+++ b/src/vncdisplay.c	Thu Feb 14 11:58:21 2008 +0900
@@ -775,6 +775,7 @@ static void setup_gdk_image(VncDisplay *
 	priv->fb.height = priv->image->height;
 	priv->fb.linesize = priv->image->bpl;
 	priv->fb.data = (uint8_t *)priv->image->mem;
+	priv->fb.byte_order = priv->image->byte_order == GDK_LSB_FIRST ? __LITTLE_ENDIAN : __BIG_ENDIAN;
 
 	gtk_widget_set_size_request(GTK_WIDGET(obj), width, height);
 }