[evolution-ews] Avoid processing OAB records which already exist

[David Woodhouse <dwmw2 src gnome org>]

commit c67e3615d4ee37409b4b39a5f573b1069fb52988
Author: David Woodhouse <David Woodhouse intel com>
Date:   Thu Sep 4 16:24:24 2014 +0100

    Avoid processing OAB records which already exist
    
    By checking the SHA1 before generating the EContact we can halve the time
    taken to process the Intel GAL from about 50s to about 25s.
    
    The extra data is now the key in one GHashTable, and freed with that
    hash table. While the uids are the key in the other and freed with that.

 src/addressbook/e-book-backend-ews.c |   67 +++++++++++++++++++++++-----------
 src/addressbook/ews-oab-decoder.c    |    7 +++-
 src/addressbook/ews-oab-decoder.h    |    5 +++
 src/addressbook/oab-decode-test.c    |    2 +-
 4 files changed, 56 insertions(+), 25 deletions(-)
---
diff --git a/src/addressbook/e-book-backend-ews.c b/src/addressbook/e-book-backend-ews.c
index 7b2e00c..a354ba0 100644
--- a/src/addressbook/e-book-backend-ews.c
+++ b/src/addressbook/e-book-backend-ews.c
@@ -2343,6 +2343,7 @@ ews_remove_old_gal_file (EBookBackendEws *cbews,
 
 struct _db_data {
        GHashTable *uids;
+       GHashTable *sha1s;
        GSList *contact_collector;
        GSList *sha1_collector;
        guint collected_length;
@@ -2354,6 +2355,27 @@ struct _db_data {
        gint percent;
 };
 
+static gboolean
+ews_gal_filter_contact (goffset offset, const gchar *sha1,
+                       gpointer user_data, GError **error)
+{
+       struct _db_data *data = (struct _db_data *) user_data;
+       gchar *uid;
+
+       /* Is there an existing identical record, with the same SHA1? */
+       uid = g_hash_table_lookup (data->sha1s, sha1);
+       if (!uid)
+               return TRUE;
+
+       /* Remove it from the hash tables so it doesn't get deleted at the end. */
+       g_hash_table_remove (data->sha1s, sha1);
+       g_hash_table_remove (data->uids, uid);
+       data->unchanged++;
+
+       /* Don't bother to parse and process this record. */
+       return FALSE;
+}
+
 static void
 ews_gal_store_contact (EContact *contact,
                        goffset offset,
@@ -2364,25 +2386,20 @@ ews_gal_store_contact (EContact *contact,
 {
        struct _db_data *data = (struct _db_data *) user_data;
        EBookBackendEwsPrivate *priv = data->cbews->priv;
-       const gchar *uid = e_contact_get_const (contact, E_CONTACT_UID);
-       gchar *db_sha1 = NULL;
 
        g_return_if_fail (priv->summary != NULL);
 
-       /* Hm, can we not do these two at once? */
-       db_sha1 = g_hash_table_lookup (data->uids, uid);
-       if (g_hash_table_remove (data->uids, uid)) {
-               if (!g_strcmp0 (db_sha1, sha1)) {
-                       data->unchanged++;
-                       goto out;
-               }
-               data->changed++;
-       } else
-               data->added++;
+       if (contact) {
+               const gchar *uid = e_contact_get_const (contact, E_CONTACT_UID);
+               if (g_hash_table_remove (data->uids, uid))
+                       data->changed++;
+               else
+                       data->added++;
 
-       data->contact_collector = g_slist_prepend (data->contact_collector, g_object_ref (contact));
-       data->sha1_collector = g_slist_prepend (data->sha1_collector, g_strdup (sha1));
-       data->collected_length += 1;
+               data->contact_collector = g_slist_prepend (data->contact_collector, g_object_ref (contact));
+               data->sha1_collector = g_slist_prepend (data->sha1_collector, g_strdup (sha1));
+               data->collected_length += 1;
+       }
 
        if (data->collected_length == 1000 || percent >= 100) {
                GSList *l;
@@ -2401,8 +2418,7 @@ ews_gal_store_contact (EContact *contact,
                data->sha1_collector = NULL;
                data->collected_length = 0;
        }
- out:
-       g_free (db_sha1);
+
        if (data->percent != percent) {
                gchar *status_message = NULL;
                GList *list, *link;
@@ -2434,7 +2450,6 @@ static void append_to_list (gpointer key, gpointer val, gpointer user_data)
        GSList **list = user_data;
 
        *list = g_slist_prepend (*list, key);
-       g_free (val);
 }
 
 static gboolean
@@ -2449,13 +2464,16 @@ ews_replace_gal_in_db (EBookBackendEws *cbews,
        gint populated = 0;
        GSList *stale_uids = NULL;
        struct _db_data data;
+       gint64 t1, t2;
 
        g_return_val_if_fail (priv->summary != NULL, FALSE);
 
        data.unchanged = data.changed = data.added = 0;
        data.percent = 0;
        data.uids = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+       data.sha1s = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
 
+       t1 = g_get_monotonic_time ();
        /* remove the old address-book and create a new one in db */
        e_book_sqlite_get_key_value_int (priv->summary, E_BOOK_SQL_IS_POPULATED_KEY, &populated, NULL);
        if (populated) {
@@ -2471,6 +2489,7 @@ ews_replace_gal_in_db (EBookBackendEws *cbews,
                        g_slist_free_1 (l);
 
                        g_hash_table_insert (data.uids, search_data->uid, search_data->extra);
+                       g_hash_table_insert (data.sha1s, search_data->extra, search_data->uid);
 
                        /* We steal these */
                        search_data->extra = search_data->uid = NULL;
@@ -2488,7 +2507,9 @@ ews_replace_gal_in_db (EBookBackendEws *cbews,
        data.cbews = cbews;
        data.cancellable = cancellable;
 
-       ret = ews_oab_decoder_decode (eod, ews_gal_store_contact, &data, cancellable, error);
+       ret = ews_oab_decoder_decode (eod, ews_gal_filter_contact, ews_gal_store_contact, &data, cancellable, 
error);
+       /* Flush final entries */
+       ews_gal_store_contact (NULL, 0, NULL, 100, &data, error);
 
        /* Remove any items which were not present in the new OAB */
        g_hash_table_foreach (data.uids, append_to_list, &stale_uids);
@@ -2498,11 +2519,13 @@ ews_replace_gal_in_db (EBookBackendEws *cbews,
        if (stale_uids && !e_book_sqlite_remove_contacts (priv->summary, stale_uids, cancellable, error))
                ret = FALSE;
 
-       d (g_print("GAL update completed %ssuccessfully. Changed: %d, Unchanged: %d, Added %d, Removed: %d\n",
-                  ret ? "" : "un",
-                  data.changed, data.unchanged, data.added, g_slist_length(stale_uids)));
+       t2 = g_get_monotonic_time ();
+       d (g_print("GAL update completed %ssuccessfully in %ld µs. Added: %d, Changed: %d, Unchanged %d, 
Removed: %d\n",
+                  ret ? "" : "un", t2 - t1,
+                  data.added, data.changed, data.unchanged, g_slist_length(stale_uids)));
 
        g_slist_free (stale_uids);
+       g_hash_table_destroy (data.sha1s);
        g_hash_table_destroy (data.uids);
        /* always notify views as complete, to not left anything behind,
           if the decode was cancelled before full completion */
diff --git a/src/addressbook/ews-oab-decoder.c b/src/addressbook/ews-oab-decoder.c
index c2eb2c2..d819ed2 100644
--- a/src/addressbook/ews-oab-decoder.c
+++ b/src/addressbook/ews-oab-decoder.c
@@ -977,6 +977,7 @@ exit:
 /* Decodes the hdr and address-book records and stores the address-book records inside the db */
 static gboolean
 ews_decode_and_store_oab_records (EwsOabDecoder *eod,
+                                 EwsOabContactFilterCb filter_cb,
                                   EwsOabContactAddedCb cb,
                                   gpointer user_data,
                                   GCancellable *cancellable,
@@ -1038,7 +1039,8 @@ ews_decode_and_store_oab_records (EwsOabDecoder *eod,
 
                memstream = g_memory_input_stream_new_from_data (record_buf, rec_size, NULL);
 
-               if (ews_decode_addressbook_record (eod, memstream,
+               if ((!filter_cb || filter_cb (offset, sum_str, user_data, error)) &&
+                   ews_decode_addressbook_record (eod, memstream,
                                                   contact, priv->oab_props,
                                                   cancellable, error))
                        cb (contact, offset, sum_str,
@@ -1142,6 +1144,7 @@ ews_oab_decoder_set_oab_prop_string (EwsOabDecoder *eod,
  **/
 gboolean
 ews_oab_decoder_decode (EwsOabDecoder *eod,
+                        EwsOabContactFilterCb filter_cb,
                         EwsOabContactAddedCb cb,
                         gpointer user_data,
                         GCancellable *cancellable,
@@ -1166,7 +1169,7 @@ ews_oab_decoder_decode (EwsOabDecoder *eod,
                goto exit;
 
        ret = ews_decode_and_store_oab_records (
-               eod, cb, user_data, cancellable, &err);
+               eod, filter_cb, cb, user_data, cancellable, &err);
 exit:
        if (o_hdr)
                g_free (o_hdr);
diff --git a/src/addressbook/ews-oab-decoder.h b/src/addressbook/ews-oab-decoder.h
index 5e92c36..7017e81 100644
--- a/src/addressbook/ews-oab-decoder.h
+++ b/src/addressbook/ews-oab-decoder.h
@@ -62,12 +62,17 @@ typedef void        (*EwsOabContactAddedCb)         (EContact *contact,
                                                 guint percent_complete,
                                                 gpointer user_data,
                                                 GError **error);
+typedef gboolean (*EwsOabContactFilterCb)      (goffset offset,
+                                                const gchar *sha1,
+                                                gpointer user_data,
+                                                GError **error);
 
 GType          ews_oab_decoder_get_type        (void);
 EwsOabDecoder *        ews_oab_decoder_new             (const gchar *oab_filename,
                                                 const gchar *cache_dir,
                                                 GError **error);
 gboolean       ews_oab_decoder_decode          (EwsOabDecoder *eod,
+                                                EwsOabContactFilterCb filter_cb,
                                                 EwsOabContactAddedCb cb,
                                                 gpointer user_data,
                                                 GCancellable *cancellable,
diff --git a/src/addressbook/oab-decode-test.c b/src/addressbook/oab-decode-test.c
index 8854e21..6a6c7bf 100644
--- a/src/addressbook/oab-decode-test.c
+++ b/src/addressbook/oab-decode-test.c
@@ -83,7 +83,7 @@ main (gint argc,
 
        timer = g_timer_new ();
        g_timer_start (timer);
-       if (!ews_oab_decoder_decode (eod, ews_test_store_contact, &data, NULL, &err)) {
+       if (!ews_oab_decoder_decode (eod, NULL, ews_test_store_contact, &data, NULL, &err)) {
                g_print ("Unable to decode %s \n", err->message);
        }
        g_timer_stop (timer);