[gtk-vnc] src: pull challenge encryption out into separate helper method

[Daniel P. Berrange <dberrange src gnome org>]

commit 6b5e767cabfdaf9ad138150995f8c59a8cd57c85
Author: Daniel P. Berrangé <dan berrange com>
Date:   Thu Nov 11 16:58:41 2021 +0000

    src: pull challenge encryption out into separate helper method
    
    Signed-off-by: Daniel P. Berrangé <berrange redhat com>

 src/vncconnection.c | 67 +++++++++++++++++++++++++++++++----------------------
 1 file changed, 39 insertions(+), 28 deletions(-)
---
diff --git a/src/vncconnection.c b/src/vncconnection.c
index cb59fa9..dd32349 100644
--- a/src/vncconnection.c
+++ b/src/vncconnection.c
@@ -3898,33 +3898,14 @@ vnc_munge_des_key(unsigned char *key, unsigned char *newkey)
     }
 }
 
-static gboolean vnc_connection_perform_auth_vnc(VncConnection *conn)
+
+static gboolean vnc_connection_encrypt_challenge(VncConnection *conn,
+                                                 guint8 *challenge,
+                                                 guint8 *key)
 {
-    VncConnectionPrivate *priv = conn->priv;
-    guint8 challenge[16];
-    guint8 key[8];
-    gsize keylen;
     gcry_cipher_hd_t c;
     gcry_error_t error;
 
-    VNC_DEBUG("Do Challenge");
-    priv->want_cred_password = TRUE;
-    priv->want_cred_username = FALSE;
-    priv->want_cred_x509 = FALSE;
-    if (!vnc_connection_gather_credentials(conn))
-        return FALSE;
-
-    if (!priv->cred_password)
-        return FALSE;
-
-    vnc_connection_read(conn, challenge, 16);
-
-    memset(key, 0, sizeof(key));
-    keylen = strlen(priv->cred_password);
-    if (keylen > sizeof(key))
-        keylen = sizeof(key);
-    memcpy(key, priv->cred_password, keylen);
-
     vnc_munge_des_key(key, key);
 
     error = gcry_cipher_open(&c, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB, 0);
@@ -3956,17 +3937,47 @@ static gboolean vnc_connection_perform_auth_vnc(VncConnection *conn)
         goto error;
     }
     gcry_cipher_close(c);
+    return TRUE;
 
-    vnc_connection_write(conn, challenge, 16);
-    vnc_connection_flush(conn);
-    return vnc_connection_check_auth_result(conn);
-
-error:
+ error:
     vnc_connection_set_error(conn, "Unknown authentication failure: %s",
                              gcry_strerror(error));
     return FALSE;
 }
 
+static gboolean vnc_connection_perform_auth_vnc(VncConnection *conn)
+{
+    VncConnectionPrivate *priv = conn->priv;
+    guint8 challenge[16];
+    guint8 key[8];
+    gsize keylen;
+
+    VNC_DEBUG("Do Challenge");
+    priv->want_cred_password = TRUE;
+    priv->want_cred_username = FALSE;
+    priv->want_cred_x509 = FALSE;
+    if (!vnc_connection_gather_credentials(conn))
+        return FALSE;
+
+    if (!priv->cred_password)
+        return FALSE;
+
+    vnc_connection_read(conn, challenge, 16);
+
+    memset(key, 0, sizeof(key));
+    keylen = strlen(priv->cred_password);
+    if (keylen > sizeof(key))
+        keylen = sizeof(key);
+    memcpy(key, priv->cred_password, keylen);
+
+    if (!vnc_connection_encrypt_challenge(conn, challenge, key))
+        return FALSE;
+
+    vnc_connection_write(conn, challenge, 16);
+    vnc_connection_flush(conn);
+    return vnc_connection_check_auth_result(conn);
+}
+
 static gboolean vnc_connection_perform_auth_plain(VncConnection *conn)
 {
     VncConnectionPrivate *priv = conn->priv;