[geary/wip/only-incomplete] Break up create/merge emails into chunks

[Jim Nelson <jnelson src gnome org>]

commit 2cdc6827c72ba3656b113d3bcb41ff1b71b00937
Author: Jim Nelson <jim yorba org>
Date:   Tue Apr 29 17:41:05 2014 -0700

    Break up create/merge emails into chunks
    
    Prevent holding the database for too long with any transaction from
    the server, especially when large vector expansions occur.

 src/engine/imap-db/imap-db-folder.vala             |   89 ++++++++++++--------
 .../imap-engine-account-synchronizer.vala          |    2 +-
 .../imap-engine/imap-engine-minimal-folder.vala    |    4 +-
 3 files changed, 55 insertions(+), 40 deletions(-)
---
diff --git a/src/engine/imap-db/imap-db-folder.vala b/src/engine/imap-db/imap-db-folder.vala
index bbdb262..813aeef 100644
--- a/src/engine/imap-db/imap-db-folder.vala
+++ b/src/engine/imap-db/imap-db-folder.vala
@@ -21,6 +21,7 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
     private const int LIST_EMAIL_WITH_MESSAGE_CHUNK_COUNT = 10;
     private const int LIST_EMAIL_METADATA_COUNT = 100;
     private const int LIST_EMAIL_FIELDS_CHUNK_COUNT = 500;
+    private const int CREATE_MERGE_EMAIL_CHUNK_COUNT = 100;
     
     [Flags]
     public enum ListFlags {
@@ -180,45 +181,59 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
     public async Gee.Map<Geary.Email, bool> create_or_merge_email_async(Gee.Collection<Geary.Email> emails,
         Cancellable? cancellable) throws Error {
         Gee.HashMap<Geary.Email, bool> results = new Gee.HashMap<Geary.Email, bool>();
-        Gee.ArrayList<Geary.EmailIdentifier> complete_ids = new Gee.ArrayList<Geary.EmailIdentifier>();
-        Gee.Collection<Contact> updated_contacts = new Gee.ArrayList<Contact>();
-        int total_unread_change = 0;
-        yield db.exec_transaction_async(Db.TransactionType.RW, (cx) => {
-            foreach (Geary.Email email in emails) {
-                Gee.Collection<Contact>? contacts_this_email = null;
-                Geary.Email.Field pre_fields;
-                Geary.Email.Field post_fields;
-                int unread_change = 0;
-                bool created = do_create_or_merge_email(cx, email, out pre_fields,
-                    out post_fields, out contacts_this_email, ref unread_change, cancellable);
-                
-                if (contacts_this_email != null)
-                    updated_contacts.add_all(contacts_this_email);
-                
-                results.set(email, created);
-                
-                // in essence, only fire the "email-completed" signal if the local version didn't
-                // have all the fields but after the create/merge now does
-                if (post_fields.is_all_set(Geary.Email.Field.ALL) && 
!pre_fields.is_all_set(Geary.Email.Field.ALL))
-                    complete_ids.add(email.id);
-                
-                // Update unread count in DB.
-                do_add_to_unread_count(cx, unread_change, cancellable);
+        
+        Gee.ArrayList<Geary.Email> list = traverse<Geary.Email>(emails).to_array_list();
+        int index = 0;
+        while (index < list.size) {
+            int stop = Numeric.int_ceiling(index + CREATE_MERGE_EMAIL_CHUNK_COUNT, list.size);
+            Gee.List<Geary.Email> slice = list.slice(index, stop);
+            debug("%s: creating/merging %d emails (%d:%d/%d)", to_string(), slice.size, index, stop,
+                list.size);
+            
+            Gee.ArrayList<Geary.EmailIdentifier> complete_ids = new Gee.ArrayList<Geary.EmailIdentifier>();
+            Gee.Collection<Contact> updated_contacts = new Gee.ArrayList<Contact>();
+            int total_unread_change = 0;
+            yield db.exec_transaction_async(Db.TransactionType.RW, (cx) => {
+                foreach (Geary.Email email in slice) {
+                    Gee.Collection<Contact>? contacts_this_email = null;
+                    Geary.Email.Field pre_fields;
+                    Geary.Email.Field post_fields;
+                    int unread_change = 0;
+                    bool created = do_create_or_merge_email(cx, email, out pre_fields,
+                        out post_fields, out contacts_this_email, ref unread_change, cancellable);
+                    
+                    if (contacts_this_email != null)
+                        updated_contacts.add_all(contacts_this_email);
+                    
+                    results.set(email, created);
+                    
+                    // in essence, only fire the "email-completed" signal if the local version didn't
+                    // have all the fields but after the create/merge now does
+                    if (post_fields.is_all_set(Geary.Email.Field.ALL) && 
!pre_fields.is_all_set(Geary.Email.Field.ALL))
+                        complete_ids.add(email.id);
+                    
+                    // Update unread count in DB.
+                    do_add_to_unread_count(cx, unread_change, cancellable);
+                    
+                    total_unread_change += unread_change;
+                }
                 
-                total_unread_change += unread_change;
-            }
+                return Db.TransactionOutcome.COMMIT;
+            }, cancellable);
             
-            return Db.TransactionOutcome.COMMIT;
-        }, cancellable);
-        
-        if (updated_contacts.size > 0)
-            contact_store.update_contacts(updated_contacts);
-        
-        // Update the email_unread properties.
-        properties.set_status_unseen((properties.email_unread + total_unread_change).clamp(0, int.MAX));
-        
-        if (complete_ids.size > 0)
-            email_complete(complete_ids);
+            if (updated_contacts.size > 0)
+                contact_store.update_contacts(updated_contacts);
+            
+            // Update the email_unread properties.
+            properties.set_status_unseen((properties.email_unread + total_unread_change).clamp(0, int.MAX));
+            
+            if (complete_ids.size > 0)
+                email_complete(complete_ids);
+            
+            index = stop;
+            if (index < list.size)
+                yield Scheduler.sleep_ms_async(100);
+        }
         
         return results;
     }
diff --git a/src/engine/imap-engine/imap-engine-account-synchronizer.vala 
b/src/engine/imap-engine/imap-engine-account-synchronizer.vala
index 63a96aa..0b7ea84 100644
--- a/src/engine/imap-engine/imap-engine-account-synchronizer.vala
+++ b/src/engine/imap-engine/imap-engine-account-synchronizer.vala
@@ -359,7 +359,7 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
                     break;
                 }
                 
-                current_epoch = current_epoch.add_months(-3);
+                current_epoch = current_epoch.add_months(-1);
                 
                 // if past max_epoch, then just pull in everything and be done with it
                 if (current_epoch.compare(max_epoch) < 0) {
diff --git a/src/engine/imap-engine/imap-engine-minimal-folder.vala 
b/src/engine/imap-engine/imap-engine-minimal-folder.vala
index 1e337bd..40e4f92 100644
--- a/src/engine/imap-engine/imap-engine-minimal-folder.vala
+++ b/src/engine/imap-engine/imap-engine-minimal-folder.vala
@@ -1315,7 +1315,7 @@ private class Geary.ImapEngine.MinimalFolder : Geary.AbstractFolder, Geary.Folde
                 new Imap.MessageSet.uid_range(new Imap.UID(Imap.UID.MIN), before_uid.previous(true))));
         }
         
-        debug("find_earliest_email_async: %s", criteria.to_string());
+        debug("%s: find_earliest_email_async: %s", to_string(), criteria.to_string());
         
         Gee.List<Geary.Email> accumulator = new Gee.ArrayList<Geary.Email>();
         ServerSearchEmail op = new ServerSearchEmail(this, criteria, Geary.Email.Field.NONE,
@@ -1338,7 +1338,7 @@ private class Geary.ImapEngine.MinimalFolder : Geary.AbstractFolder, Geary.Folde
                 earliest_id = email_id;
         }
         
-        debug("find_earliest_email_async: found %s",
+        debug("%s: find_earliest_email_async: found %s", to_string(),
             earliest_id != null ? earliest_id.to_string() : "(null)");
         
         return earliest_id;