[geary/wip/only-incomplete: 1/2] Background prefetcher much smoother and doesn't hang intermittently

[Jim Nelson <jnelson src gnome org>]

commit d1abee7f17a44535740d9f96f4d35d9c1ad78004
Author: Jim Nelson <jim yorba org>
Date:   Tue Apr 15 17:39:14 2014 -0700

    Background prefetcher much smoother and doesn't hang intermittently

 src/engine/imap-db/imap-db-folder.vala             |  103 ++++++++++++++------
 .../imap-engine-account-synchronizer.vala          |    2 +-
 .../imap-engine/imap-engine-email-prefetcher.vala  |   37 +++----
 src/engine/util/util-collection.vala               |   17 +++
 4 files changed, 105 insertions(+), 54 deletions(-)
---
diff --git a/src/engine/imap-db/imap-db-folder.vala b/src/engine/imap-db/imap-db-folder.vala
index 7fa2d04..5d3bd2c 100644
--- a/src/engine/imap-db/imap-db-folder.vala
+++ b/src/engine/imap-db/imap-db-folder.vala
@@ -269,12 +269,6 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
                 SELECT MessageLocationTable.message_id, ordering, remove_marker
                 FROM MessageLocationTable
             """);
-            if (only_incomplete) {
-                sql.append("""
-                    INNER JOIN MessageTable
-                    ON MessageTable.id = MessageLocationTable.message_id
-                """);
-            }
             
             sql.append("WHERE folder_id = ? ");
             
@@ -283,23 +277,23 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
             else
                 sql.append("AND ordering <= ? ");
             
-            if (only_incomplete)
-                sql.append_printf("AND fields != %d ", Geary.Email.Field.ALL);
-            
             if (oldest_to_newest)
                 sql.append("ORDER BY ordering ASC ");
             else
                 sql.append("ORDER BY ordering DESC ");
             
-            sql.append("LIMIT ?");
-            
             Db.Statement stmt = cx.prepare(sql.str);
             stmt.bind_rowid(0, folder_id);
             stmt.bind_int64(1, start_uid.value);
-            stmt.bind_int(2, count);
             
             locations = do_results_to_locations(stmt.exec(cancellable), flags, cancellable);
             
+            if (only_incomplete)
+                do_remove_complete_locations(cx, locations, cancellable);
+            
+            if (locations.size > count)
+                locations = locations.slice(0, count);
+            
             return Db.TransactionOutcome.SUCCESS;
         }, cancellable);
         
@@ -389,16 +383,7 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
                 FROM MessageLocationTable
             """);
             
-            if (only_incomplete) {
-                sql.append("""
-                    INNER JOIN MessageTable
-                    ON MessageTable.id = MessageLocationTable.message_id
-                """);
-            }
-            
             sql.append("WHERE folder_id = ? AND ordering >= ? AND ordering <= ? ");
-            if (only_incomplete)
-                sql.append_printf(" AND fields != %d ", Geary.Email.Field.ALL);
             
             Db.Statement stmt = cx.prepare(sql.str);
             stmt.bind_rowid(0, folder_id);
@@ -407,6 +392,9 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
             
             locations = do_results_to_locations(stmt.exec(cancellable), flags, cancellable);
             
+            if (only_incomplete)
+                do_remove_complete_locations(cx, locations, cancellable);
+            
             return Db.TransactionOutcome.SUCCESS;
         }, cancellable);
         
@@ -436,13 +424,6 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
                 FROM MessageLocationTable
             """);
             
-            if (only_incomplete) {
-                sql.append("""
-                    INNER JOIN MessageTable
-                    ON MessageTable.id = MessageLocationTable.message_id
-                """);
-            }
-            
             if (locs.size != 1) {
                 sql.append("WHERE ordering IN (");
                 bool first = true;
@@ -458,9 +439,6 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
                 sql.append_printf("WHERE ordering = '%s' ", locs[0].uid.to_string());
             }
             
-            if (only_incomplete)
-                sql.append_printf(" AND fields != %d ", Geary.Email.Field.ALL);
-            
             sql.append("AND folder_id = ? ");
             
             Db.Statement stmt = cx.prepare(sql.str);
@@ -468,6 +446,9 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
             
             locations = do_results_to_locations(stmt.exec(cancellable), flags, cancellable);
             
+            if (only_incomplete)
+                do_remove_complete_locations(cx, locations, cancellable);
+            
             return Db.TransactionOutcome.SUCCESS;
         }, cancellable);
         
@@ -2042,6 +2023,66 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
         return locations;
     }
     
+    // Use a separate step to strip out complete emails because original implementation (using an
+    // INNER JOIN) was horribly slow under load
+    private void do_remove_complete_locations(Db.Connection cx, Gee.List<LocationIdentifier>? locations,
+        Cancellable? cancellable) throws Error {
+        if (locations == null || locations.size == 0)
+            return;
+        
+        StringBuilder sql = new StringBuilder("""
+            SELECT id, fields FROM MessageTable WHERE id IN (
+        """);
+        bool first = true;
+        for (int ctr = 0; ctr < locations.size; ctr++) {
+            if (!first)
+                sql.append(",");
+            
+            sql.append(locations[ctr].message_id.to_string());
+            first = false;
+        }
+        sql.append(")");
+        
+        Db.Statement stmt = cx.prepare(sql.str);
+        Db.Result results = stmt.exec(cancellable);
+        
+        Gee.HashSet<int64?> complete_locations = new Gee.HashSet<int64?>(Collection.int64_hash_func,
+            Collection.int64_equal_func);
+        while (!results.finished) {
+            if (results.int_at(1) == Geary.Email.Field.ALL)
+                complete_locations.add(results.int64_at(0));
+            
+            results.next(cancellable);
+        }
+        
+        if (complete_locations.size == 0) {
+            debug("No complete locations found (out of %d)", locations.size);
+            
+            return;
+        }
+        
+        // simple optimization, but worth it
+        if (complete_locations.size == locations.size) {
+            debug("All %d locations, complete, clearing return list", locations.size);
+            locations.clear();
+            
+            return;
+        }
+        
+        debug("%d complete locations out of %d found, removing...", complete_locations.size,
+            locations.size);
+        
+        int original_size = locations.size;
+        Gee.Iterator<LocationIdentifier> iter = locations.iterator();
+        while (iter.next()) {
+            if (complete_locations.contains(iter.get().message_id))
+                iter.remove();
+        }
+        
+        debug("Removed, %d -> %d locations remaining", original_size, locations.size);
+        assert(locations.size == (original_size - complete_locations.size));
+    }
+    
     private LocationIdentifier? do_get_location_for_id(Db.Connection cx, ImapDB.EmailIdentifier id,
         ListFlags flags, Cancellable? cancellable) throws Error {
         Db.Statement stmt = cx.prepare("""
diff --git a/src/engine/imap-engine/imap-engine-account-synchronizer.vala 
b/src/engine/imap-engine/imap-engine-account-synchronizer.vala
index 8d53d50..63a96aa 100644
--- a/src/engine/imap-engine/imap-engine-account-synchronizer.vala
+++ b/src/engine/imap-engine/imap-engine-account-synchronizer.vala
@@ -6,7 +6,7 @@
 
 private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
     private const int FETCH_DATE_RECEIVED_CHUNK_COUNT = 25;
-    private const int SYNC_DELAY_SEC = 15;
+    private const int SYNC_DELAY_SEC = 5;
     
     public GenericAccount account { get; private set; }
     
diff --git a/src/engine/imap-engine/imap-engine-email-prefetcher.vala 
b/src/engine/imap-engine/imap-engine-email-prefetcher.vala
index 4fd3a47..1c25ed4 100644
--- a/src/engine/imap-engine/imap-engine-email-prefetcher.vala
+++ b/src/engine/imap-engine/imap-engine-email-prefetcher.vala
@@ -15,7 +15,6 @@ private class Geary.ImapEngine.EmailPrefetcher : Object {
     public const int PREFETCH_DELAY_SEC = 1;
     
     private const Geary.Email.Field PREFETCH_FIELDS = Geary.Email.Field.ALL;
-    private const int PREFETCH_IDS_CHUNKS = 500;
     private const int PREFETCH_CHUNK_BYTES = 32 * 1024;
     
     public Nonblocking.CountingSemaphore active_sem { get; private set;
@@ -80,7 +79,10 @@ private class Geary.ImapEngine.EmailPrefetcher : Object {
     }
     
     // emails should include PROPERTIES
-    private void schedule_prefetch(Gee.Collection<Geary.Email> emails) {
+    private void schedule_prefetch(Gee.Collection<Geary.Email>? emails) {
+        if (emails == null || emails.size == 0)
+            return;
+        
         debug("%s: scheduling %d emails for prefetching", folder.to_string(), emails.size);
         
         prefetch_emails.add_all(emails);
@@ -103,26 +105,18 @@ private class Geary.ImapEngine.EmailPrefetcher : Object {
     }
     
     private async void do_prepare_all_local_async() {
-        Geary.EmailIdentifier? lowest = null;
-        for (;;) {
-            Gee.List<Geary.Email>? list = null;
-            try {
-                list = yield folder.local_folder.list_email_by_id_async((ImapDB.EmailIdentifier) lowest,
-                    PREFETCH_IDS_CHUNKS, Geary.Email.Field.PROPERTIES, 
ImapDB.Folder.ListFlags.ONLY_INCOMPLETE,
-                    cancellable);
-            } catch (Error err) {
-                debug("Error while list local emails for %s: %s", folder.to_string(), err.message);
-            }
-            
-            if (list == null || list.size == 0)
-                break;
-            
-            // find lowest for next iteration
-            lowest = Geary.EmailIdentifier.sort_emails(list).first().id;
-            
-            schedule_prefetch(list);
+        Gee.List<Geary.Email>? list = null;
+        try {
+            debug("Listing all emails needing prefetching in %s...", folder.to_string());
+            list = yield folder.local_folder.list_email_by_id_async(null, int.MAX,
+                Geary.Email.Field.PROPERTIES, ImapDB.Folder.ListFlags.ONLY_INCOMPLETE, cancellable);
+            debug("Listed all emails needing prefetching in %s", folder.to_string());
+        } catch (Error err) {
+            debug("Error while list local emails for %s: %s", folder.to_string(), err.message);
         }
         
+        schedule_prefetch(list);
+        
         active_sem.blind_notify();
     }
     
@@ -136,8 +130,7 @@ private class Geary.ImapEngine.EmailPrefetcher : Object {
             debug("Error while list local emails for %s: %s", folder.to_string(), err.message);
         }
         
-        if (list != null && list.size > 0)
-            schedule_prefetch(list);
+        schedule_prefetch(list);
         
         active_sem.blind_notify();
     }
diff --git a/src/engine/util/util-collection.vala b/src/engine/util/util-collection.vala
index 7c86c2e..7ec7a65 100644
--- a/src/engine/util/util-collection.vala
+++ b/src/engine/util/util-collection.vala
@@ -126,6 +126,23 @@ public inline static uint int64_hash(int64 value) {
 }
 
 /**
+ * To be used as hash_func for Gee collections.
+ */
+public uint int64_hash_func(int64? n) {
+    return hash_memory((uint8 *) n, sizeof(int64));
+}
+
+/**
+ * To be used as equal_func for Gee collections.
+ */
+public bool int64_equal_func(int64? a, int64? b) {
+    int64 *bia = (int64 *) a;
+    int64 *bib = (int64 *) b;
+    
+    return (*bia) == (*bib);
+}
+
+/**
  * A rotating-XOR hash that can be used to hash memory buffers of any size.
  */
 public static uint hash_memory(void *ptr, size_t bytes) {