[gtk-vnc] src: optimize to avoid recreating framebuffer if size/format is unchanged

[Daniel P. Berrange <dberrange src gnome org>]

commit 61c088e3360316a85c22b1605145951eb674ec76
Author: Daniel P. Berrangé <dan berrange com>
Date:   Thu Dec 10 18:45:04 2020 +0000

    src: optimize to avoid recreating framebuffer if size/format is unchanged
    
    The QEMU VNC server will send a desktop size rectangle immediately
    following any vmware desktop size change rectangle. This is redundant as
    the latter already reports the size change. This leads to the display
    needlessly re-creating the framebuffer despite nothing having changed.
    
    Signed-off-by: Daniel P. Berrangé <berrange redhat com>

 src/libgvnc_sym.version |  1 +
 src/vncdisplay.c        | 16 ++++++++++++++++
 src/vncpixelformat.c    | 15 +++++++++++++++
 src/vncpixelformat.h    |  3 +++
 4 files changed, 35 insertions(+)
---
diff --git a/src/libgvnc_sym.version b/src/libgvnc_sym.version
index 85b90ee..83bc44f 100644
--- a/src/libgvnc_sym.version
+++ b/src/libgvnc_sym.version
@@ -102,6 +102,7 @@
        vnc_pixel_format_copy;
        vnc_pixel_format_free;
        vnc_pixel_format_get_type;
+       vnc_pixel_format_match;
 
     local:
        *;
diff --git a/src/vncdisplay.c b/src/vncdisplay.c
index 229c1f3..12ba2a5 100644
--- a/src/vncdisplay.c
+++ b/src/vncdisplay.c
@@ -1211,14 +1211,30 @@ static void do_framebuffer_init(VncDisplay *obj,
                                 int width, int height, gboolean quiet)
 {
     VncDisplayPrivate *priv = obj->priv;
+    const VncPixelFormat *oldformat;
+    int oldwidth, oldheight;
 
     if (priv->conn == NULL || !vnc_connection_is_initialized(priv->conn))
         return;
 
     if (priv->fb) {
+        oldformat = vnc_framebuffer_get_remote_format(VNC_FRAMEBUFFER(priv->fb));
+        oldwidth = vnc_framebuffer_get_width(VNC_FRAMEBUFFER(priv->fb));
+        oldheight = vnc_framebuffer_get_height(VNC_FRAMEBUFFER(priv->fb));
+
+        if (oldwidth == width &&
+            oldheight == height &&
+            vnc_pixel_format_match(remoteFormat, oldformat)) {
+            VNC_DEBUG("Framebuffer size / format unchanged, skipping recreate");
+            return;
+        }
+
         g_object_unref(priv->fb);
         priv->fb = NULL;
     }
+    VNC_DEBUG("Re-creating framebuffer %dx%d after size/format change",
+              width, height);
+
     if (priv->fbCache) {
         cairo_surface_destroy(priv->fbCache);
         priv->fbCache = NULL;
diff --git a/src/vncpixelformat.c b/src/vncpixelformat.c
index 09e89b4..4afffd6 100644
--- a/src/vncpixelformat.c
+++ b/src/vncpixelformat.c
@@ -88,6 +88,21 @@ void vnc_pixel_format_free(VncPixelFormat *format)
     g_slice_free(VncPixelFormat, format);
 }
 
+gboolean vnc_pixel_format_match(const VncPixelFormat *format,
+                                const VncPixelFormat *other)
+{
+    return format->bits_per_pixel == other->bits_per_pixel &&
+        format->depth == other->depth &&
+        format->byte_order == other->byte_order &&
+        format->true_color_flag == other->true_color_flag &&
+        format->red_max == other->red_max &&
+        format->green_max == other->green_max &&
+        format->blue_max == other->blue_max &&
+        format->red_shift == other->red_shift &&
+        format->green_shift == other->green_shift &&
+        format->blue_shift == other->blue_shift;
+}
+
 /*
  * Local variables:
  *  c-indent-level: 4
diff --git a/src/vncpixelformat.h b/src/vncpixelformat.h
index 62e308a..63fb661 100644
--- a/src/vncpixelformat.h
+++ b/src/vncpixelformat.h
@@ -52,6 +52,9 @@ VncPixelFormat *vnc_pixel_format_new(void);
 VncPixelFormat *vnc_pixel_format_copy(VncPixelFormat *format);
 void vnc_pixel_format_free(VncPixelFormat *format);
 
+gboolean vnc_pixel_format_match(const VncPixelFormat *format,
+                                const VncPixelFormat *other);
+
 G_END_DECLS
 
 #endif /* VNC_PIXEL_FORMAT_H */