[gtk-vnc] Convert to use gcrypt for d3des routines

[Daniel P. Berrange <dberrange src gnome org>]

commit cbcfdf73e8b5ef5cc6f62b238a7d68736144b14b
Author: Daniel P. Berrangé <berrange redhat com>
Date:   Thu Aug 16 18:40:11 2018 +0100

    Convert to use gcrypt for d3des routines
    
    By munging the DES key we can use the standard APIs for DES
    
    Signed-off-by: Daniel P. Berrangé <berrange redhat com>

 src/Makefile.am     |   1 -
 src/d3des.c         | 674 ----------------------------------------------------
 src/d3des.h         | 162 -------------
 src/vncconnection.c |  97 +++++++-
 4 files changed, 87 insertions(+), 847 deletions(-)
---
diff --git a/src/Makefile.am b/src/Makefile.am
index fceeb3a..5e323a2 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -80,7 +80,6 @@ nodist_libgvnc_1_0_la_HEADERS = \
 
 libgvnc_1_0_la_SOURCES = \
                        coroutine.h \
-                       d3des.h d3des.c \
                        dh.h dh.c \
                        vncpixelformat.h vncpixelformat.c \
                        vncaudioformat.h vncaudioformat.c \
diff --git a/src/vncconnection.c b/src/vncconnection.c
index 52c8365..1119d58 100644
--- a/src/vncconnection.c
+++ b/src/vncconnection.c
@@ -38,9 +38,9 @@
 #include <sys/stat.h>
 
 #include "coroutine.h"
-#include "d3des.h"
 
 #include <gnutls/gnutls.h>
+#include <gnutls/crypto.h>
 #include <gnutls/x509.h>
 #include <gdk-pixbuf/gdk-pixbuf.h>
 
@@ -3631,12 +3631,27 @@ static gboolean vnc_connection_check_auth_result(VncConnection *conn)
     return FALSE;
 }
 
+static void
+vnc_munge_des_key(unsigned char *key, unsigned char *newkey)
+{
+    size_t i;
+    for (i = 0; i < 8; i++) {
+        uint8_t r = key[i];
+        r = (r & 0xf0) >> 4 | (r & 0x0f) << 4;
+        r = (r & 0xcc) >> 2 | (r & 0x33) << 2;
+        r = (r & 0xaa) >> 1 | (r & 0x55) << 1;
+        newkey[i] = r;
+    }
+}
+
 static gboolean vnc_connection_perform_auth_vnc(VncConnection *conn)
 {
     VncConnectionPrivate *priv = conn->priv;
     guint8 challenge[16];
     guint8 key[8];
     gsize keylen;
+    gcry_cipher_hd_t c;
+    gcry_error_t error;
 
     VNC_DEBUG("Do Challenge");
     priv->want_cred_password = TRUE;
@@ -3656,9 +3671,34 @@ static gboolean vnc_connection_perform_auth_vnc(VncConnection *conn)
         keylen = sizeof(key);
     memcpy(key, priv->cred_password, keylen);
 
-    deskey(key, EN0);
-    des(challenge, challenge);
-    des(challenge + 8, challenge + 8);
+    vnc_munge_des_key(key, key);
+
+    error = gcry_cipher_open(&c, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB, 0);
+    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
+        VNC_DEBUG("gcry_cipher_open error: %s\n", gcry_strerror(error));
+        return FALSE;
+    }
+
+    error = gcry_cipher_setkey(c, key, 8);
+    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
+        VNC_DEBUG("gcry_cipher_setkey error: %s\n", gcry_strerror(error));
+        gcry_cipher_close(c);
+        return FALSE;
+    }
+
+    error = gcry_cipher_encrypt(c, challenge, 8, challenge, 8);
+    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
+        VNC_DEBUG("gcry_cipher_encrypt error: %s\n", gcry_strerror(error));
+        gcry_cipher_close(c);
+        return FALSE;
+    }
+    error = gcry_cipher_encrypt(c, challenge + 8, 8, challenge + 8, 8);
+    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
+        VNC_DEBUG("gcry_cipher_encrypt error: %s\n", gcry_strerror(error));
+        gcry_cipher_close(c);
+        return FALSE;
+    }
+    gcry_cipher_close(c);
 
     vnc_connection_write(conn, challenge, 16);
     vnc_connection_flush(conn);
@@ -3670,19 +3710,54 @@ static gboolean vnc_connection_perform_auth_vnc(VncConnection *conn)
  *   Encrypt bytes[length] in memory using key.
  *   Key has to be 8 bytes, length a multiple of 8 bytes.
  */
-static void
+static gboolean
 vncEncryptBytes2(unsigned char *where, const int length, unsigned char *key)
 {
+    gcry_cipher_hd_t c;
     int i, j;
-    deskey(key, EN0);
+    gboolean ret = FALSE;
+    gcry_error_t error;
+    unsigned char newkey[8];
+
+    vnc_munge_des_key(key, newkey);
+
+    error = gcry_cipher_open(&c, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB, 0);
+    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
+        VNC_DEBUG("gcry_cipher_open error: %s\n", gcry_strerror(error));
+        return FALSE;
+    }
+
+    error = gcry_cipher_setkey(c, newkey, 8);
+    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
+        VNC_DEBUG("gcry_cipher_setkey error: %s\n", gcry_strerror(error));
+        goto cleanup;
+    }
+
     for (i = 0; i< 8; i++)
         where[i] ^= key[i];
-    des(where, where);
+
+    error = gcry_cipher_encrypt(c, where, 8, where, 8);
+    if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
+        VNC_DEBUG("gcry_cipher_encrypt error: %s\n", gcry_strerror(error));
+        goto cleanup;
+    }
+
     for (i = 8; i < length; i += 8) {
         for (j = 0; j < 8; j++)
             where[i + j] ^= where[i + j - 8];
-        des(where + i, where + i);
+
+        error = gcry_cipher_encrypt(c, where + i, 8, where + i, 8);
+        if (gcry_err_code (error) != GPG_ERR_NO_ERROR) {
+            VNC_DEBUG("gcry_cipher_encrypt error: %s\n", gcry_strerror(error));
+            goto cleanup;
+        }
     }
+
+    ret = TRUE;
+
+ cleanup:
+    gcry_cipher_close(c);
+    return ret;
 }
 
 static gboolean vnc_connection_perform_auth_mslogon(VncConnection *conn)
@@ -3731,8 +3806,10 @@ static gboolean vnc_connection_perform_auth_mslogon(VncConnection *conn)
     memcpy(password, priv->cred_password, passwordLen);
     memcpy(username, priv->cred_username, usernameLen);
 
-    vncEncryptBytes2(username, sizeof(username), key);
-    vncEncryptBytes2(password, sizeof(password), key);
+    if (!vncEncryptBytes2(username, sizeof(username), key))
+        return FALSE;
+    if (!vncEncryptBytes2(password, sizeof(password), key))
+        return FALSE;
 
     vnc_connection_write(conn, username, sizeof(username));
     vnc_connection_write(conn, password, sizeof(password));