[gtk-vnc] Fix auth subtype choosing logic in vncdisplay

[Daniel P. Berrange <dberrange src gnome org>]

commit 6800dd5d7f665a3587f5f6eb4b44efee361d142c
Author: Daniel P. Berrange <berrange redhat com>
Date:   Fri Sep 17 11:41:29 2010 +0100

    Fix auth subtype choosing logic in vncdisplay
    
    The method for choosing auth subtypes was rather flawed. Rewrite
    it to work correctly. Introduce an explicit list of vencrypt
    subtypes, since these are distinct from main subtypes. Improve
    debug logging in auth choice code.
    
    * src/vncdisplay.c: Fix auth type choice
    * src/vncconnection.c: Improve auth debugging & invoke correct
      signal for auth subtype choice

 src/vncconnection.c |    5 ++-
 src/vncdisplay.c    |   82 +++++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 68 insertions(+), 19 deletions(-)
---
diff --git a/src/vncconnection.c b/src/vncconnection.c
index e3835c9..9ad472a 100644
--- a/src/vncconnection.c
+++ b/src/vncconnection.c
@@ -3832,7 +3832,7 @@ static gboolean vnc_connection_perform_auth_tls(VncConnection *conn)
 
 	if (priv->has_error)
 		return FALSE;
-	vnc_connection_choose_auth(conn, VNC_AUTH_CHOOSE_TYPE, nauth, auth);
+	vnc_connection_choose_auth(conn, VNC_AUTH_CHOOSE_SUBTYPE, nauth, auth);
 	if (priv->has_error)
 		return FALSE;
 
@@ -4804,6 +4804,7 @@ static int vnc_connection_best_path(char **buf,
 		}
 		g_free(tmp);
 	}
+	VNC_DEBUG("Failed to find certificate %s/%s", basedir, basefile);
 	return -1;
 }
 
@@ -4825,6 +4826,8 @@ static gboolean vnc_connection_set_credential_x509(VncConnection *conn,
 #else
 	char *dirs[] = { sysdir };
 #endif
+	for (int i = 0 ; i < sizeof(dirs)/sizeof(dirs[0]) ; i++)
+		VNC_DEBUG("Searching for certs in %s", dirs[i]);
 
 	if (vnc_connection_best_path(&priv->cred_x509_cacert, "CA", "cacert.pem",
 				     dirs, sizeof(dirs)/sizeof(dirs[0])) < 0)
diff --git a/src/vncdisplay.c b/src/vncdisplay.c
index 65b8c3b..e2ef418 100644
--- a/src/vncdisplay.c
+++ b/src/vncdisplay.c
@@ -76,6 +76,7 @@ struct _VncDisplayPrivate
 	gboolean force_size;
 
 	GSList *preferable_auths;
+	GSList *preferable_vencrypt_subauths;
 	size_t keycode_maplen;
 	const guint16 const *keycode_map;
 
@@ -1063,7 +1064,7 @@ static void on_auth_cred(VncConnection *conn G_GNUC_UNUSED,
 	g_signal_emit(G_OBJECT(obj), signals[VNC_AUTH_CREDENTIAL], 0, creds);
 }
 
-static void on_auth_choose_type(VncConnection *conn G_GNUC_UNUSED,
+static void on_auth_choose_type(VncConnection *conn,
 				GValueArray *types,
 				gpointer opaque)
 {
@@ -1072,8 +1073,11 @@ static void on_auth_choose_type(VncConnection *conn G_GNUC_UNUSED,
 	GSList *l;
 	guint i;
 
-	if (!types->n_values)
+	if (!types->n_values) {
+		VNC_DEBUG("No auth types available to choose from");
+		vnc_connection_shutdown(conn);
 		return;
+	}
 
 	for (l = priv->preferable_auths; l; l=l->next) {
 		int pref = GPOINTER_TO_UINT (l->data);
@@ -1081,17 +1085,18 @@ static void on_auth_choose_type(VncConnection *conn G_GNUC_UNUSED,
 		for (i=0; i< types->n_values; i++) {
 			GValue *type = g_value_array_get_nth(types, i);
 			if (pref == g_value_get_enum(type)) {
-				vnc_connection_set_auth_type(priv->conn, pref);
+				vnc_connection_set_auth_type(conn, pref);
 				return;
 			}
 		}
 	}
 
-	GValue *type = g_value_array_get_nth(types, 0);
-	vnc_connection_set_auth_type(priv->conn, g_value_get_enum(type));
+	/* No sub-auth matching our supported auth so have to give up */
+	VNC_DEBUG("No preferred auth type found");
+	vnc_connection_shutdown(conn);
 }
 
-static void on_auth_choose_subtype(VncConnection *conn G_GNUC_UNUSED,
+static void on_auth_choose_subtype(VncConnection *conn,
 				   unsigned int type,
 				   GValueArray *subtypes,
 				   gpointer opaque)
@@ -1101,25 +1106,41 @@ static void on_auth_choose_subtype(VncConnection *conn G_GNUC_UNUSED,
 	GSList *l;
 	guint i;
 
-	if (!subtypes->n_values)
+	if (!subtypes->n_values) {
+		VNC_DEBUG("No subtypes available to choose from");
+		vnc_connection_shutdown(conn);
 		return;
+	}
 
 	if (type == VNC_CONNECTION_AUTH_TLS) {
-		for (l = priv->preferable_auths; l; l=l->next) {
-			int pref = GPOINTER_TO_UINT (l->data);
-
-			for (i=0; i< subtypes->n_values; i++) {
-				GValue *subtype = g_value_array_get_nth(subtypes, i);
-				if (pref == g_value_get_enum(subtype)) {
-					vnc_connection_set_auth_type(priv->conn, pref);
-					return;
-				}
+		l = priv->preferable_auths;
+	} else if (type == VNC_CONNECTION_AUTH_VENCRYPT) {
+		l = priv->preferable_vencrypt_subauths;
+	} else {
+		VNC_DEBUG("Unexpected stackable auth type %d", type);
+		vnc_connection_shutdown(conn);
+		return;
+	}
+
+	for (; l; l=l->next) {
+		int pref = GPOINTER_TO_UINT (l->data);
+
+		/* Don't want to recursively do the same major auth */
+		if (pref == type)
+			continue;
+
+		for (i=0; i< subtypes->n_values; i++) {
+			GValue *subtype = g_value_array_get_nth(subtypes, i);
+			if (pref == g_value_get_enum(subtype)) {
+				vnc_connection_set_auth_subtype(conn, pref);
+				return;
 			}
 		}
 	}
 
-	GValue *subtype = g_value_array_get_nth(subtypes, 0);
-	vnc_connection_set_auth_subtype(priv->conn, g_value_get_enum(subtype));
+	/* No sub-auth matching our supported auth so have to give up */
+	VNC_DEBUG("No preferred auth subtype found");
+	vnc_connection_shutdown(conn);
 }
 
 static void on_auth_failure(VncConnection *conn G_GNUC_UNUSED,
@@ -1482,6 +1503,7 @@ static void vnc_display_finalize (GObject *obj)
 	}
 
 	g_slist_free (priv->preferable_auths);
+	g_slist_free (priv->preferable_vencrypt_subauths);
 
 	vnc_display_keyval_free_entries();
 
@@ -1864,6 +1886,30 @@ static void vnc_display_init(VncDisplay *display)
 	 */
 	priv->preferable_auths = g_slist_append (priv->preferable_auths, GUINT_TO_POINTER (VNC_CONNECTION_AUTH_NONE));
 
+
+	/* Prefered order for VeNCrypt subtypes */
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_X509SASL));
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_X509PLAIN));
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_X509VNC));
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_X509NONE));
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_TLSSASL));
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_TLSPLAIN));
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_TLSVNC));
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_TLSNONE));
+	/*
+	 * Refuse fully cleartext passwords
+	priv->preferable_vencrypt_subauths = g_slist_append(priv->preferable_vencrypt_subauths,
+							    GUINT_TO_POINTER(VNC_CONNECTION_AUTH_VENCRYPT_PLAIN));
+	*/
+
 	priv->conn = vnc_connection_new();
 
 	g_signal_connect(G_OBJECT(priv->conn), "vnc-cursor-changed",