[gtk-vnc] vncEncryptBytes2: set connection error and cleanup

[Daniel P. Berrange <dberrange src gnome org>]

commit a44684adde1e958153090391c8ee095440af89ce
Author: Jakub Janků <jjanku redhat com>
Date:   Wed May 5 22:03:26 2021 +0200

    vncEncryptBytes2: set connection error and cleanup
    
    If vncEncryptBytes2 fails, the false return value propagates
    upwards, but no error is set. The connection ends up stuck in
    the mainloop with nothing happening.
    
    Additionally, the mpis and dh aren't freed.
    
    To fix that, return the gcry error from vncEncryptBytes2
    and set the connection error accordingly.
    
    Signed-off-by: Jakub Janků <jjanku redhat com>

 src/vncconnection.c | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)
---
diff --git a/src/vncconnection.c b/src/vncconnection.c
index 731307a..2d64642 100644
--- a/src/vncconnection.c
+++ b/src/vncconnection.c
@@ -3995,12 +3995,11 @@ static gboolean vnc_connection_perform_auth_plain(VncConnection *conn)
  *   Encrypt bytes[length] in memory using key.
  *   Key has to be 8 bytes, length a multiple of 8 bytes.
  */
-static gboolean
+static gcry_error_t
 vncEncryptBytes2(unsigned char *where, const int length, unsigned char *key)
 {
     gcry_cipher_hd_t c;
     int i, j;
-    gboolean ret = FALSE;
     gcry_error_t error;
     unsigned char newkey[8];
 
@@ -4009,7 +4008,7 @@ vncEncryptBytes2(unsigned char *where, const int length, unsigned char *key)
     error = gcry_cipher_open(&c, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB, 0);
     if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
         VNC_DEBUG("gcry_cipher_open error: %s\n", gcry_strerror(error));
-        return FALSE;
+        return error;
     }
 
     error = gcry_cipher_setkey(c, newkey, 8);
@@ -4038,11 +4037,9 @@ vncEncryptBytes2(unsigned char *where, const int length, unsigned char *key)
         }
     }
 
-    ret = TRUE;
-
  cleanup:
     gcry_cipher_close(c);
-    return ret;
+    return error;
 }
 
 static gboolean vnc_connection_perform_auth_mslogon(VncConnection *conn)
@@ -4051,6 +4048,7 @@ static gboolean vnc_connection_perform_auth_mslogon(VncConnection *conn)
     struct vnc_dh *dh;
     guchar gen[8], mod[8], resp[8], pub[8], key[8];
     gcry_mpi_t genmpi, modmpi, respmpi, pubmpi, keympi;
+    gcry_error_t error;
     guchar username[256], password[64];
     guint passwordLen, usernameLen;
     gboolean allzeroes = TRUE;
@@ -4108,20 +4106,28 @@ static gboolean vnc_connection_perform_auth_mslogon(VncConnection *conn)
     memcpy(password, priv->cred_password, passwordLen);
     memcpy(username, priv->cred_username, usernameLen);
 
-    if (!vncEncryptBytes2(username, sizeof(username), key))
-        return FALSE;
-    if (!vncEncryptBytes2(password, sizeof(password), key))
-        return FALSE;
+    error = vncEncryptBytes2(username, sizeof(username), key);
+    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
+        goto cleanup;
+    error = vncEncryptBytes2(password, sizeof(password), key);
+    if (gcry_err_code(error) != GPG_ERR_NO_ERROR)
+        goto cleanup;
 
     vnc_connection_write(conn, username, sizeof(username));
     vnc_connection_write(conn, password, sizeof(password));
     vnc_connection_flush(conn);
 
+cleanup:
     gcry_mpi_release(genmpi);
     gcry_mpi_release(modmpi);
     gcry_mpi_release(respmpi);
     vnc_dh_free (dh);
 
+    if (gcry_err_code(error) != GPG_ERR_NO_ERROR) {
+        vnc_connection_set_error(conn, "Unknown authentication failure: %s",
+                                 gcry_strerror(error));
+        return FALSE;
+    }
     return vnc_connection_check_auth_result(conn);
 }