[geary/wip/713530-sync] Make background sync less onerous

[Jim Nelson <jnelson src gnome org>]

commit 9e268821bedc0fc00aa2b8924318e2940bc643ee
Author: Jim Nelson <jim yorba org>
Date:   Fri Mar 14 11:36:18 2014 -0700

    Make background sync less onerous
    
    This commit makes three modifications:
    
    * Introduces some sleep() calls in AccountSynchronizer and
      EmailPrefetcher to avoid thrashing hard when under load
    * Modifies the AccountSynchronizer not to go back so far before
      giving up and to traverse time by three months instead of one
      to locate a folder's epoch email quicker
    * If the max epoch time is reached, just pull in all emails, as
      the server search is apparently not working

 .../imap-engine-account-synchronizer.vala          |   76 ++++++++++++--------
 .../imap-engine/imap-engine-email-prefetcher.vala  |    2 +
 .../imap-engine/imap-engine-minimal-folder.vala    |    5 ++
 3 files changed, 52 insertions(+), 31 deletions(-)
---
diff --git a/src/engine/imap-engine/imap-engine-account-synchronizer.vala 
b/src/engine/imap-engine/imap-engine-account-synchronizer.vala
index d979754..8d53d50 100644
--- a/src/engine/imap-engine/imap-engine-account-synchronizer.vala
+++ b/src/engine/imap-engine/imap-engine-account-synchronizer.vala
@@ -16,6 +16,7 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
     private Cancellable? bg_cancellable = null;
     private Nonblocking.Semaphore stopped = new Nonblocking.Semaphore();
     private Gee.HashSet<FolderPath> unavailable_paths = new Gee.HashSet<FolderPath>();
+    private DateTime max_epoch = new DateTime(new TimeZone.local(), 2000, 1, 1, 0, 0, 0.0);
     
     public AccountSynchronizer(GenericAccount account) {
         this.account = account;
@@ -230,7 +231,7 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
                 epoch = new DateTime.now_local();
                 epoch = epoch.add_days(0 - account.information.prefetch_period_days);
             } else {
-                epoch = new DateTime(new TimeZone.local(), 1971, 1, 1, 0, 0, 0.0);
+                epoch = max_epoch;
             }
             
             bool ok = yield process_folder_async(folder, made_available.remove(folder), epoch);
@@ -337,38 +338,16 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
         Geary.EmailIdentifier? oldest_local_id) throws Error {
         debug("Background sync'ing %s", folder.to_string());
         
+        // wait for the folder to be fully opened to be sure we have all the most current
+        // information
+        yield folder.wait_for_open_async(bg_cancellable);
+        
         // only perform vector expansion if oldest isn't old enough
         if (oldest_local == null || oldest_local.compare(epoch) > 0) {
-            // go back one month at a time to the epoch, performing a little vector expansion at a
+            // go back three months at a time to the epoch, performing a little vector expansion at a
             // time rather than all at once (which will stall the replay queue)
             DateTime current_epoch = (oldest_local != null) ? oldest_local : new DateTime.now_local();
             do {
-                current_epoch = current_epoch.add_months(-1);
-                
-                // don't go past epoch
-                if (current_epoch.compare(epoch) < 0)
-                    current_epoch = epoch;
-                
-                debug("Background sync'ing %s to %s", folder.to_string(), current_epoch.to_string());
-                Geary.EmailIdentifier? epoch_id = yield folder.find_earliest_email_async(current_epoch,
-                    oldest_local_id, bg_cancellable);
-                if (epoch_id == null && current_epoch.compare(epoch) <= 0) {
-                    debug("Unable to locate epoch messages on remote folder %s%s, fetching one past 
oldest...",
-                        folder.to_string(),
-                        (oldest_local_id != null) ? " earlier than oldest local" : "");
-                    
-                    // if there's nothing between the oldest local and the epoch, that means the
-                    // mail just prior to our local oldest is oldest than the epoch; rather than
-                    // continually thrashing looking for something that's just out of reach, add it
-                    // to the folder and be done with it ... note that this even works if oldest_local_id
-                    // is null, as that means the local folder is empty and so we should at least
-                    // pull the first one to get a marker of age
-                    yield folder.list_email_by_id_async(oldest_local_id, 1, Geary.Email.Field.NONE,
-                        Geary.Folder.ListFlags.NONE, bg_cancellable);
-                } else if (epoch_id != null) {
-                    oldest_local_id = epoch_id;
-                }
-                
                 // look for complete synchronization of UIDs (i.e. complete vector normalization)
                 // no need to keep searching once this happens
                 int local_count = yield 
folder.local_folder.get_email_count_async(ImapDB.Folder.ListFlags.NONE,
@@ -379,15 +358,50 @@ private class Geary.ImapEngine.AccountSynchronizer : Geary.BaseObject {
                     
                     break;
                 }
+                
+                current_epoch = current_epoch.add_months(-3);
+                
+                // if past max_epoch, then just pull in everything and be done with it
+                if (current_epoch.compare(max_epoch) < 0) {
+                    debug("Background sync reached max epoch of %s, fetching all mail from %s",
+                        max_epoch.to_string(), folder.to_string());
+                    
+                    yield folder.list_email_by_id_async(null, 1, Geary.Email.Field.NONE,
+                        Geary.Folder.ListFlags.OLDEST_TO_NEWEST, bg_cancellable);
+                } else {
+                    // don't go past proscribed epoch
+                    if (current_epoch.compare(epoch) < 0)
+                        current_epoch = epoch;
+                    
+                    debug("Background sync'ing %s to %s", folder.to_string(), current_epoch.to_string());
+                    Geary.EmailIdentifier? earliest_span_id = yield 
folder.find_earliest_email_async(current_epoch,
+                        oldest_local_id, bg_cancellable);
+                    if (earliest_span_id == null && current_epoch.compare(epoch) <= 0) {
+                        debug("Unable to locate epoch messages on remote folder %s%s, fetching one past 
oldest...",
+                            folder.to_string(),
+                            (oldest_local_id != null) ? " earlier than oldest local" : "");
+                        
+                        // if there's nothing between the oldest local and the epoch, that means the
+                        // mail just prior to our local oldest is oldest than the epoch; rather than
+                        // continually thrashing looking for something that's just out of reach, add it
+                        // to the folder and be done with it ... note that this even works if oldest_local_id
+                        // is null, as that means the local folder is empty and so we should at least
+                        // pull the first one to get a marker of age
+                        yield folder.list_email_by_id_async(oldest_local_id, 1, Geary.Email.Field.NONE,
+                            Geary.Folder.ListFlags.NONE, bg_cancellable);
+                    } else if (earliest_span_id != null) {
+                        // use earliest email from that span for the next round
+                        oldest_local_id = earliest_span_id;
+                    }
+                }
+                
+                yield Scheduler.sleep_ms_async(200);
             } while (current_epoch.compare(epoch) > 0);
         } else {
             debug("No expansion necessary for %s, oldest local (%s) is before epoch (%s)",
                 folder.to_string(), oldest_local.to_string(), epoch.to_string());
         }
         
-        // wait for the folder to be fully opened to give the email prefetcher a chance to start up
-        yield folder.wait_for_open_async(bg_cancellable);
-        
         // always give email prefetcher time to finish its work
         debug("Waiting for email prefetcher to complete %s...", folder.to_string());
         try {
diff --git a/src/engine/imap-engine/imap-engine-email-prefetcher.vala 
b/src/engine/imap-engine/imap-engine-email-prefetcher.vala
index 619e102..13a2c44 100644
--- a/src/engine/imap-engine/imap-engine-email-prefetcher.vala
+++ b/src/engine/imap-engine/imap-engine-email-prefetcher.vala
@@ -216,6 +216,8 @@ private class Geary.ImapEngine.EmailPrefetcher : Object {
             
             if (!keep_going)
                 break;
+            
+            yield Scheduler.sleep_ms_async(200);
         }
         
         // get any remaining
diff --git a/src/engine/imap-engine/imap-engine-minimal-folder.vala 
b/src/engine/imap-engine/imap-engine-minimal-folder.vala
index 45c1aa1..096225d 100644
--- a/src/engine/imap-engine/imap-engine-minimal-folder.vala
+++ b/src/engine/imap-engine/imap-engine-minimal-folder.vala
@@ -1283,6 +1283,8 @@ private class Geary.ImapEngine.MinimalFolder : Geary.AbstractFolder, Geary.Folde
                 new Imap.MessageSet.uid_range(new Imap.UID(Imap.UID.MIN), before_uid.previous(true))));
         }
         
+        debug("find_earliest_email_async: %s", criteria.to_string());
+        
         Gee.List<Geary.Email> accumulator = new Gee.ArrayList<Geary.Email>();
         ServerSearchEmail op = new ServerSearchEmail(this, criteria, Geary.Email.Field.NONE,
             accumulator, cancellable);
@@ -1304,6 +1306,9 @@ private class Geary.ImapEngine.MinimalFolder : Geary.AbstractFolder, Geary.Folde
                 earliest_id = email_id;
         }
         
+        debug("find_earliest_email_async: found %s",
+            earliest_id != null ? earliest_id.to_string() : "(null)");
+        
         return earliest_id;
     }