[gtk-vnc] src: rewrite challenge code to use gnutls

[Daniel P. Berrange <dberrange src gnome org>]

commit 7af0e9a9cd32fde35c06aa2a1126211549f9bdcd
Author: Daniel P. Berrangé <dan berrange com>
Date:   Fri Nov 12 11:25:29 2021 +0000

    src: rewrite challenge code to use gnutls
    
    gcrypt refuses to encrypt DES data using an all-zeros key. While this
    might seem reasonable, this is the key that you get when someone sets
    a zero length password in a VNC server.
    
    IOW, the gcrypt behaviour prevents gtk-vnc connecting to a server with
    zero length password.
    
    GNUTLS doesn't have this problem but also doesn't support DES-ECB,
    so we have to fake it by calling DES-CBC for each block individually.
    Conceptually this is horribly inefficient, but since the VNC challenge
    is only 16 bytes long it isn't a problem in the real world.
    
    Signed-off-by: Daniel P. Berrangé <berrange redhat com>

 src/vncconnection.c | 49 +++++++++++++++++++++----------------------------
 1 file changed, 21 insertions(+), 28 deletions(-)
---
diff --git a/src/vncconnection.c b/src/vncconnection.c
index dd32349..8ec6437 100644
--- a/src/vncconnection.c
+++ b/src/vncconnection.c
@@ -3903,45 +3903,38 @@ static gboolean vnc_connection_encrypt_challenge(VncConnection *conn,
                                                  guint8 *challenge,
                                                  guint8 *key)
 {
-    gcry_cipher_hd_t c;
-    gcry_error_t error;
+    int error;
+    size_t i;
+    unsigned char iv[8] = {};
 
     vnc_munge_des_key(key, key);
 
-    error = gcry_cipher_open(&c, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB, 0);
-    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
-        VNC_DEBUG("gcry_cipher_open error: %s\n", gcry_strerror(error));
-        goto error;
-    }
+    for (i = 0; i < 16; i += 8) {
+        gnutls_cipher_hd_t handle;
+        gnutls_datum_t gkey = { (unsigned char *)key, 8 };
+        error = gnutls_cipher_init(&handle,
+                                   GNUTLS_CIPHER_DES_CBC,
+                                   &gkey, NULL);
+        if (error != 0)
+            goto error;
 
-    error = gcry_cipher_setkey(c, key, 8);
-    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
-        VNC_DEBUG("gcry_cipher_setkey error: %s\n", gcry_strerror(error));
-        if (gcry_err_code(error) == GPG_ERR_WEAK_KEY) {
-            vnc_connection_auth_failure(conn, gcry_strerror(error));
+        gnutls_cipher_set_iv(handle, iv, sizeof(iv));
+
+        error = gnutls_cipher_encrypt2(handle,
+                                       challenge + i, 8,
+                                       challenge + i, 8);
+        if (error != 0) {
+            gnutls_cipher_deinit(handle);
+            goto error;
         }
-        gcry_cipher_close(c);
-        goto error;
+        gnutls_cipher_deinit(handle);
     }
 
-    error = gcry_cipher_encrypt(c, challenge, 8, challenge, 8);
-    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
-        VNC_DEBUG("gcry_cipher_encrypt error: %s\n", gcry_strerror(error));
-        gcry_cipher_close(c);
-        goto error;
-    }
-    error = gcry_cipher_encrypt(c, challenge + 8, 8, challenge + 8, 8);
-    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
-        VNC_DEBUG("gcry_cipher_encrypt error: %s\n", gcry_strerror(error));
-        gcry_cipher_close(c);
-        goto error;
-    }
-    gcry_cipher_close(c);
     return TRUE;
 
  error:
     vnc_connection_set_error(conn, "Unknown authentication failure: %s",
-                             gcry_strerror(error));
+                             gnutls_strerror(error));
     return FALSE;
 }