[gtk-vnc] src: add support for alpha cursor encoding

[Daniel P. Berrange <dberrange src gnome org>]

commit ff615a4ad70e30aa3c9690ea33966ca6335e19ec
Author: Daniel P. Berrangé <dan berrange com>
Date:   Fri Dec 4 12:35:52 2020 +0000

    src: add support for alpha cursor encoding
    
    The alpha cursor encoding sends the cursor in RGBA format.
    
    Signed-off-by: Daniel P. Berrangé <berrange redhat com>

 src/vncconnection.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/vncconnection.h |   2 +
 2 files changed, 122 insertions(+)
---
diff --git a/src/vncconnection.c b/src/vncconnection.c
index 0ba8093..be3f265 100644
--- a/src/vncconnection.c
+++ b/src/vncconnection.c
@@ -24,6 +24,7 @@
 #include "vncconnection.h"
 #include "vncconnectionenums.h"
 #include "vncmarshal.h"
+#include "vncbaseframebuffer.h"
 #include "vncutil.h"
 
 #include <string.h>
@@ -3105,6 +3106,121 @@ static void vnc_connection_xcursor(VncConnection *conn, guint16 x, guint16 y, gu
     vnc_connection_emit_main_context(conn, VNC_CURSOR_CHANGED, &sigdata);
 }
 
+static void vnc_connection_alpha_cursor(VncConnection *conn, guint16 x, guint16 y, guint16 width, guint16 
height)
+{
+    VncConnectionPrivate *priv = conn->priv;
+    struct signal_data sigdata;
+
+    if (priv->cursor)
+        g_clear_object(&priv->cursor);
+
+    if (width && height) {
+        guint32 encoding;
+        VncFramebuffer *fb;
+        guint8 *pixels;
+        VncPixelFormat localFmt = {
+            .red_max = 255,
+            .green_max = 255,
+            .blue_max = 255,
+            .red_shift = 16,
+            .green_shift = 8,
+            .blue_shift = 0,
+            .depth = 32,
+            .bits_per_pixel = 32,
+            .byte_order = G_BYTE_ORDER,
+        };
+
+        pixels = g_new0(guint8, width * height * 4);
+        fb = VNC_FRAMEBUFFER(vnc_base_framebuffer_new(pixels,
+                                                      width,
+                                                      height,
+                                                      width * 4,
+                                                      &localFmt,
+                                                      &priv->fmt));
+
+        encoding = vnc_connection_read_u32(conn);
+        if (vnc_connection_has_error(conn)) {
+            g_free(pixels);
+            g_object_unref(fb);
+            return;
+        }
+        switch (encoding) {
+        case VNC_CONNECTION_ENCODING_RAW:
+            if (!vnc_connection_validate_boundary(conn, fb,
+                                                  0, 0, width, height))
+                break;
+            vnc_connection_raw_update(conn, fb, 0, 0, width, height);
+            vnc_connection_update(conn, 0, 0, width, height);
+            break;
+        case VNC_CONNECTION_ENCODING_COPY_RECT:
+            if (!vnc_connection_validate_boundary(conn, fb,
+                                                  0, 0, width, height))
+                break;
+            vnc_connection_copyrect_update(conn, fb, 0, 0, width, height);
+            vnc_connection_update(conn, 0, 0, width, height);
+            break;
+        case VNC_CONNECTION_ENCODING_RRE:
+            if (!vnc_connection_validate_boundary(conn, fb,
+                                                  0, 0, width, height))
+                break;
+            vnc_connection_rre_update(conn, fb, 0, 0, width, height);
+            vnc_connection_update(conn, 0, 0, width, height);
+            break;
+        case VNC_CONNECTION_ENCODING_HEXTILE:
+            if (!vnc_connection_validate_boundary(conn, fb,
+                                                  0, 0, width, height))
+                break;
+            vnc_connection_hextile_update(conn, fb, 0, 0, width, height);
+            vnc_connection_update(conn, 0, 0, width, height);
+            break;
+        case VNC_CONNECTION_ENCODING_ZRLE:
+            if (!vnc_connection_validate_boundary(conn, fb,
+                                                  0, 0, width, height))
+                break;
+            vnc_connection_zrle_update(conn, fb, 0, 0, width, height);
+            vnc_connection_update(conn, 0, 0, width, height);
+            break;
+        case VNC_CONNECTION_ENCODING_TIGHT:
+            if (!vnc_connection_validate_boundary(conn, fb,
+                                                  0, 0, width, height))
+                break;
+            vnc_connection_tight_update(conn, fb, 0, 0, width, height);
+            vnc_connection_update(conn, 0, 0, width, height);
+            break;
+        default:
+            vnc_connection_set_error(conn,
+                                     "Unsupported encoding %u for alpha cursor",
+                                     encoding);
+            break;
+        }
+
+        if (vnc_connection_has_error(conn)) {
+            g_free(pixels);
+            g_object_unref(fb);
+            return;
+        }
+
+        /* Some broken VNC servers send hot-pixel outside
+         * bounds of the cursor. We could disconnect and
+         * report an error, but it is more user friendly
+         * to just clamp the hot pixel coords
+         */
+        if (x >= width)
+            x = width - 1;
+        if (y >= height)
+            y = height - 1;
+
+        priv->cursor = vnc_cursor_new(vnc_framebuffer_get_buffer(fb),
+                                      x, y, width, height);
+        /* pixels is owned by priv->cursor now */
+        g_object_unref(fb);
+    }
+
+    sigdata.params.cursor = priv->cursor;
+
+    vnc_connection_emit_main_context(conn, VNC_CURSOR_CHANGED, &sigdata);
+}
+
 static void vnc_connection_ext_key_event(VncConnection *conn)
 {
     VncConnectionPrivate *priv = conn->priv;
@@ -3207,6 +3323,10 @@ static gboolean vnc_connection_framebuffer_update(VncConnection *conn, gint32 et
         if (priv->audio_enable_pending)
             vnc_connection_audio_enable(conn);
         break;
+    case VNC_CONNECTION_ENCODING_ALPHA_CURSOR:
+        vnc_connection_alpha_cursor(conn, x, y, width, height);
+        vnc_connection_resend_framebuffer_update_request(conn);
+        break;
     default:
         vnc_connection_set_error(conn, "Received an unknown encoding type: %d", etype);
         break;
diff --git a/src/vncconnection.h b/src/vncconnection.h
index 5a4b85e..f203caf 100644
--- a/src/vncconnection.h
+++ b/src/vncconnection.h
@@ -122,6 +122,8 @@ typedef enum {
     VNC_CONNECTION_ENCODING_EXT_KEY_EVENT = -258,
     VNC_CONNECTION_ENCODING_AUDIO = -259,
     VNC_CONNECTION_ENCODING_LED_STATE = -261,
+
+    VNC_CONNECTION_ENCODING_ALPHA_CURSOR = -314,
 } VncConnectionEncoding;
 
 typedef enum {