[geary/wip/713150-conversations: 7/7] Fix load problem, can now load messages all the way to end of folder

[Jim Nelson <jnelson src gnome org>]

commit 434d298e396b8eb17d961d7181505691020a754b
Author: Jim Nelson <jim yorba org>
Date:   Thu Feb 26 16:07:10 2015 -0800

    Fix load problem, can now load messages all the way to end of folder

 src/engine/app/app-conversation-monitor.vala |   63 +++++++-------------------
 src/engine/imap-db/imap-db-account.vala      |   35 +++------------
 src/engine/imap-db/imap-db-conversation.vala |   31 +++++++++++++
 3 files changed, 53 insertions(+), 76 deletions(-)
---
diff --git a/src/engine/app/app-conversation-monitor.vala b/src/engine/app/app-conversation-monitor.vala
index 8bc92fe..b186e9b 100644
--- a/src/engine/app/app-conversation-monitor.vala
+++ b/src/engine/app/app-conversation-monitor.vala
@@ -382,40 +382,32 @@ public class Geary.App.ConversationMonitor : BaseObject {
             throw close_err;
     }
     
-    private async int load_by_id_async(Geary.EmailIdentifier? initial_id, int count,
+    private async void load_by_id_async(Geary.EmailIdentifier? initial_id, int count,
         Geary.Folder.ListFlags flags, Cancellable? cancellable) {
-        int loaded = 0;
-        
         notify_scan_started();
         try {
             // list by required_flags to ensure all are present in local store
-            loaded = yield process_email_async(folder.path,
+            yield process_email_async(folder.path,
                 yield folder.list_email_by_id_async(initial_id, count, required_fields, flags, cancellable));
         } catch (Error err) {
             notify_scan_error(err);
         } finally {
             notify_scan_completed();
         }
-        
-        return loaded;
     }
     
-    private async int load_by_sparse_id(Gee.Collection<Geary.EmailIdentifier> ids,
+    private async void load_by_sparse_id(Gee.Collection<Geary.EmailIdentifier> ids,
         Geary.Folder.ListFlags flags, Cancellable? cancellable) {
-        int loaded = 0;
-        
         notify_scan_started();
         try {
             // list by required_flags to ensure all are present in local store
-            loaded = yield process_email_async(folder.path,
+            yield process_email_async(folder.path,
                 yield folder.list_email_by_sparse_id_async(ids, required_fields, flags, cancellable));
         } catch (Error err) {
             notify_scan_error(err);
         } finally {
             notify_scan_completed();
         }
-        
-        return loaded;
     }
     
     // NOTE: This is called from a background thread.
@@ -443,35 +435,24 @@ public class Geary.App.ConversationMonitor : BaseObject {
         return true;
     }
     
-    private async int process_email_async(FolderPath path, Gee.Collection<Geary.Email>? emails) {
+    private async void process_email_async(FolderPath path, Gee.Collection<Geary.Email>? emails) {
         if (emails == null || emails.size == 0)
-            return 0;
+            return;
         
         Gee.Set<EmailIdentifier> ids = traverse<Email>(emails)
             .map<EmailIdentifier>(email => email.id)
             .to_hash_set();
         
-        return yield process_email_ids_async(path, ids);
+        yield process_email_ids_async(path, ids);
     }
     
-    private async int process_email_ids_async(FolderPath path, Gee.Collection<Geary.EmailIdentifier>? 
email_ids) {
+    private async void process_email_ids_async(FolderPath path, Gee.Collection<Geary.EmailIdentifier>? 
email_ids) {
         if (email_ids == null || email_ids.size == 0)
-            return 0;
+            return;
         
         Logging.debug(Logging.Flag.CONVERSATIONS, "[%s] ConversationMonitor::process_email: %d emails",
             folder.to_string(), email_ids.size);
         
-        // don't re-process existing emails
-        Gee.Collection<EmailIdentifier> trimmed_ids = traverse<EmailIdentifier>(email_ids)
-            .filter(id => !all_email_id_to_conversation.has_key(id))
-            .to_hash_set();
-        
-        debug("Filtering existing identifiers: %d => %d", email_ids.size, trimmed_ids.size);
-        
-        email_ids = trimmed_ids;
-        if (email_ids.size == 0)
-            return 0;
-        
         Gee.Collection<AssociatedEmails>? associations = null;
         try {
             associations = yield folder.account.local_search_associated_emails_async(
@@ -481,14 +462,13 @@ public class Geary.App.ConversationMonitor : BaseObject {
         }
         
         if (associations == null || associations.size == 0)
-            return 0;
+            return;
         
         debug("%d email ids, %d associations", email_ids.size, associations.size);
         
         Gee.HashSet<Conversation> added = new Gee.HashSet<Conversation>();
         Gee.HashMultiMap<Conversation, Email> appended = new Gee.HashMultiMap<Conversation, Email>();
         
-        int loaded = 0;
         foreach (AssociatedEmails association in associations) {
             // Get all EmailIdentifiers in association
             Gee.HashSet<EmailIdentifier> associated_ids = traverse<Email>(association.emails)
@@ -530,8 +510,6 @@ public class Geary.App.ConversationMonitor : BaseObject {
                 // and they're from this folder and not another one
                 if (email_ids.contains(email.id) && path.equal_to(folder.path))
                     primary_email_id_to_conversation[email.id] = conversation;
-                
-                loaded++;
             }
             
             // if new, added, otherwise appended (if not already added)
@@ -556,8 +534,6 @@ public class Geary.App.ConversationMonitor : BaseObject {
         
         Logging.debug(Logging.Flag.CONVERSATIONS, "[%s] ConversationMonitor::process_email completed: %d 
emails",
             folder.to_string(), email_ids.size);
-        
-        return loaded;
     }
     
     // TODO
@@ -740,35 +716,28 @@ public class Geary.App.ConversationMonitor : BaseObject {
         }
         
         Geary.EmailIdentifier? low_id = yield get_lowest_email_id_async(null);
-        
-        int loaded;
         if (low_id != null && !is_insert) {
             // Load at least as many messages as remianing conversations.
             int num_to_load = min_window_count - conversations.size;
             if (num_to_load < WINDOW_FILL_MESSAGE_COUNT)
                 num_to_load = WINDOW_FILL_MESSAGE_COUNT;
             
-            for (;;) {
-                debug("FILLWINDOW: low_id=%s num_to_load=%d", low_id.to_string(), num_to_load);
-                loaded = yield load_by_id_async(low_id, num_to_load, flags, cancellable_monitor);
-                if (loaded != 0)
-                    break;
-                
-                num_to_load += WINDOW_FILL_MESSAGE_COUNT;
-            }
+            debug("FILLWINDOW: low_id=%s num_to_load=%d", low_id.to_string(), num_to_load);
+            yield load_by_id_async(low_id, num_to_load, flags, cancellable_monitor);
         } else {
             // No existing messages or an insert invalidated our existing list,
             // need to start from scratch.
             debug("FILLWINDOW: low_id=%s is_insert=%s min_window_count=%d",
                 (low_id != null) ? low_id.to_string() : "(null)", is_insert.to_string(),
                 min_window_count);
-            loaded = yield load_by_id_async(null, min_window_count, flags, cancellable_monitor);
+            yield load_by_id_async(null, min_window_count, flags, cancellable_monitor);
         }
         
         // Run again to make sure we're full unless we ran out of messages.
         int final_message_count = get_email_count();
-        if (loaded == 0 || final_message_count != initial_message_count) {
-            debug("FILLWINDOW: Schedule refill: loaded=%d %d vs. %d (%d in folder, %d found)", loaded, 
final_message_count, initial_message_count,
+        if (final_message_count != initial_message_count) {
+            debug("FILLWINDOW: Schedule refill: final=%d initial=%d (%d in folder, %d found)",
+                final_message_count, initial_message_count,
                 folder.properties.email_total, primary_email_id_to_conversation.size);
             operation_queue.add(new FillWindowOperation(this, false));
         } else {
diff --git a/src/engine/imap-db/imap-db-account.vala b/src/engine/imap-db/imap-db-account.vala
index ee2516a..738510e 100644
--- a/src/engine/imap-db/imap-db-account.vala
+++ b/src/engine/imap-db/imap-db-account.vala
@@ -707,46 +707,23 @@ private class Geary.ImapDB.Account : BaseObject {
                 if (found_ids.contains(db_id))
                     continue;
                 
-                Db.Statement stmt = cx.prepare("""
-                    SELECT conversation_id
-                    FROM MessageConversationTable
-                    WHERE message_id = ?
-                """);
-                stmt.bind_rowid(0, db_id.message_id);
-                
-                Db.Result result = stmt.exec(cancellable);
-                if (result.finished || result.is_null_at(0))
+                Gee.HashSet<ImapDB.EmailIdentifier>? associated_ids =
+                    Conversation.do_fetch_associated_email_ids(cx, db_id.message_id, cancellable);
+                if (associated_ids == null || associated_ids.size == 0)
                     continue;
                 
-                int64 conversation_id = result.rowid_at(0);
-                
-                stmt = cx.prepare("""
-                    SELECT message_id
-                    FROM MessageConversationTable
-                    WHERE conversation_id = ?
-                """);
-                stmt.bind_rowid(0, conversation_id);
-                
-                result = stmt.exec(cancellable);
+                found_ids.add_all(associated_ids);
                 
                 AssociatedEmails association = new AssociatedEmails();
-                while (!result.finished) {
-                    if (result.is_null_at(0)) {
-                        result.next(cancellable);
-                        
-                        continue;
-                    }
-                    
+                foreach (ImapDB.EmailIdentifier associated_id in associated_ids) {
                     Email? email;
                     Gee.Collection<FolderPath?>? known_paths;
-                    do_fetch_message(cx, result.rowid_at(0), requested_fields, search_predicate,
+                    do_fetch_message(cx, associated_id.message_id, requested_fields, search_predicate,
                         out email, out known_paths, cancellable);
                     if (email != null) {
                         association.add(email, known_paths);
                         found_ids.add((ImapDB.EmailIdentifier) email.id);
                     }
-                    
-                    result.next(cancellable);
                 }
                 
                 if (association.emails.size > 0)
diff --git a/src/engine/imap-db/imap-db-conversation.vala b/src/engine/imap-db/imap-db-conversation.vala
index 4917b24..35319d0 100644
--- a/src/engine/imap-db/imap-db-conversation.vala
+++ b/src/engine/imap-db/imap-db-conversation.vala
@@ -234,5 +234,36 @@ private int64 do_merge_conversations(Db.Connection cx, Gee.Set<int64?> conversat
     return conversation_id;
 }
 
+private Gee.HashSet<ImapDB.EmailIdentifier>? do_fetch_associated_email_ids(Db.Connection cx,
+    int64 message_id, Cancellable? cancellable) throws Error {
+    Db.Statement stmt = cx.prepare("""
+        SELECT conversation_id
+        FROM MessageConversationTable
+        WHERE message_id = ?
+    """);
+    stmt.bind_rowid(0, message_id);
+    
+    Db.Result result = stmt.exec(cancellable);
+    if (result.finished || result.is_null_at(0))
+        return null;
+    
+    int64 conversation_id = result.rowid_at(0);
+    
+    stmt = cx.prepare("""
+        SELECT message_id
+        FROM MessageConversationTable
+        WHERE conversation_id = ?
+    """);
+    stmt.bind_rowid(0, conversation_id);
+    
+    Gee.HashSet<ImapDB.EmailIdentifier> associated_message_ids = new Gee.HashSet<ImapDB.EmailIdentifier>();
+    for (result = stmt.exec(cancellable); !result.finished; result.next(cancellable)) {
+        if (!result.is_null_at(0))
+            associated_message_ids.add(new ImapDB.EmailIdentifier(result.rowid_at(0), null));
+    }
+    
+    return associated_message_ids.size > 0 ? associated_message_ids : null;
+}
+
 }