[geary/wip/714134-gc] More work -- this doesn't build

[Jim Nelson <jnelson src gnome org>]

commit 7de044c3b5d52a9a9a1775c5715b0a633468444a
Author: Jim Nelson <jim yorba org>
Date:   Wed Dec 17 19:44:42 2014 -0800

    More work -- this doesn't build

 sql/version-024.sql                      |   15 +++-
 src/engine/imap-db/imap-db-database.vala |   30 +++++-
 src/engine/imap-db/imap-db-gc.vala       |  170 ++++++++++++++++++++++++++++-
 3 files changed, 203 insertions(+), 12 deletions(-)
---
diff --git a/sql/version-024.sql b/sql/version-024.sql
index 4925784..4da75c5 100644
--- a/sql/version-024.sql
+++ b/sql/version-024.sql
@@ -3,9 +3,22 @@
 -- collected) after all references to them have been removed from the database without worrying
 -- about deleting them first and the database transaction failing.
 --
+-- Also add GarbageCollectionTable, a single-row table holding various information about when
+-- GC has occurred and when it should occur next.
+--
 
 CREATE TABLE DeleteAttachmentFileTable (
     id INTEGER PRIMARY KEY,
     filename TEXT NOT NULL
-)
+);
+
+CREATE TABLE GarbageCollectionTable (
+    id INTEGER PRIMARY KEY,
+    last_gc_time_t INTEGER DEFAULT NULL,
+    last_vacuum_time_t INTEGER DEFAULT NULL,
+    reaped_messages_since_last_vacuum INTEGER DEFAULT 0
+);
+
+-- Insert a single row with a well-known rowid with default values
+INSERT INTO GarbageCollectionTable (id) VALUES (0);
 
diff --git a/src/engine/imap-db/imap-db-database.vala b/src/engine/imap-db/imap-db-database.vala
index 2459313..223bf5a 100644
--- a/src/engine/imap-db/imap-db-database.vala
+++ b/src/engine/imap-db/imap-db-database.vala
@@ -11,16 +11,17 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase {
     private const int OPEN_PUMP_EVENT_LOOP_MSEC = 100;
     
     private ProgressMonitor upgrade_monitor;
+    private ProgressMonitor vacuum_monitor;
     private string account_owner_email;
     private bool new_db = false;
-    private GC? gc = null;
     private Cancellable gc_cancellable = new Cancellable();
     
     public Database(File db_dir, File schema_dir, ProgressMonitor upgrade_monitor,
-        string account_owner_email) {
+        ProgressMonitor vacuum_monitor, string account_owner_email) {
         base (get_db_file(db_dir), schema_dir);
         
         this.upgrade_monitor = upgrade_monitor;
+        this.vacuum_monitor = vacuum_monitor;
         this.account_owner_email = account_owner_email;
     }
     
@@ -34,12 +35,31 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase {
      * This should only be done from the main thread, as it is designed to pump the event loop
      * while the database is being opened and updated.
      */
-    public new void open(Db.DatabaseFlags flags, Cancellable? cancellable) throws Error {
+    public async void open_async(Db.DatabaseFlags flags, Cancellable? cancellable) throws Error {
         open_background(flags, on_prepare_database_connection, pump_event_loop,
             OPEN_PUMP_EVENT_LOOP_MSEC, cancellable);
         
-        gc = new GC(this, Priority.LOW);
-        gc.run_async.begin(gc_cancellable, on_gc_run_async_completed);
+        if (cancellable != null)
+            cancellable.cancelled.connect(on_open_cancelled);
+        
+        GC gc = new GC(this, Priority.LOW);
+        
+        GC.RecommendedOperation recommended = yield gc.should_run_async(gc_cancellable);
+        if (recommended == GC.RecommendedOperation.NONE)
+            return;
+        
+        if ((op & GC.RecommendedOperation.VACUUM) != 0) {
+            if (!vacuum_monitor.is_in_progress)
+                vacuum_monitor.notify_start();
+            
+            yield gc.vacuum_async(gc_cancellable);
+            
+            if (vacuum_monitor.is_in_progress)
+                vacuum_monitor.notify_finish();
+        }
+        
+        if ((op & GC.RecommendedOperation.GC) != 0) {
+        }
     }
     
     private void on_gc_run_async_completed(Object? object, AsyncResult result) {
diff --git a/src/engine/imap-db/imap-db-gc.vala b/src/engine/imap-db/imap-db-gc.vala
index f2245ce..8a05cf5 100644
--- a/src/engine/imap-db/imap-db-gc.vala
+++ b/src/engine/imap-db/imap-db-gc.vala
@@ -17,11 +17,18 @@
  */
 
 private class Geary.ImapDB.GC {
+    // Maximum number of days between GC runs.
+    private const int GC_DAYS_SPAN = 10;
+    
+    // Number of reaped messages since last vacuum indicating another vacuum should occur
+    private const int VACUUM_WHEN_REAPED_REACHED = 1000;
+    
     // Days old from today an unlinked email message must be to be reaped by the garbage collector
     private const int UNLINKED_DAYS = 31;
     
-    // Amount of time to sleep between various GC iterations to give other operations a chance
-    private const uint SLEEP_MSEC = 50;
+    // Amount of time to sleep between various database-bound GC iterations to give other
+    // transactions a chance
+    private const uint SLEEP_MSEC = 15;
     
     // Number of files to delete from the DeleteAttachmentFileTable per iteration
     private const int DELETE_ATTACHMENT_PER = 5;
@@ -30,6 +37,16 @@ private class Geary.ImapDB.GC {
     private const int ENUM_DIR_PER = 10;
     
     /**
+     * See { link should_run_async}.
+     */
+    [Flag]
+    public enum RecommendedOperation {
+        NONE = 0,
+        GC,
+        VACUUM
+    }
+    
+    /**
      * Indicates the garbage collector is running.
      *
      * { link run_async} will return immediately if called while running.
@@ -47,24 +64,109 @@ private class Geary.ImapDB.GC {
     }
     
     /**
+     * Returns if the GC should be executed (via { link run_async}).
+     */
+    public async RecommendedOperation should_run_async(Cancellable? cancellable) throws Error {
+        DateTime? last_gc_time, last_vacuum_time;
+        int reaped_messages_since_last_vacuum;
+        yield fetch_gc_info_async(out last_gc_time, out last_vacuum_time, out 
reaped_messages_since_last_vacuum,
+            cancellable);
+        
+        RecommendedOperation op = RecommendedOperation.NONE;
+        DateTime now = new DateTime.now_local();
+        
+        // GC every GC_DAYS_SPAN unless never executed, in which case run now
+        int64 days;
+        if (last_gc_time == null) {
+            // null means GC has never executed
+            debug("[%s] Recommending garbage collection: never run", to_string());
+            op |= RecommendedOperation.GC;
+        } else if (elapsed_days(now, last, out days) >= GC_DAYS_SPAN) {
+            debug("[%s] Recommending garbage collection: %d days since last run", to_string(), days);
+            op |= RecommendedOperation.GC;
+        }
+        
+        // VACUUM is not something we do regularly, but rather look for a lot of reaped messages
+        // as indicator it's time; the last_vacuum_time is stored in case want to add that to the
+        // heuristic later
+        if (reaped_messages_since_last_vacuum >= VACUUM_WHEN_REAPED_REACHED) {
+            debug("[%s] Recommending database vacuum: %d messages reaped since last vacuum",
+                to_string(), reaped_messages_since_last_vacuum);
+            op |= RecommendedOperation.VACUUM;
+        }
+        
+        return op;
+    }
+    
+    private static int64 elapsed_days(DateTime end, DateTime start, out int64 days) {
+        days = end.difference(start) / TimeSpan.DAY;
+        
+        return days;
+    }
+    
+    /**
+     * Should only be called from the foreground thread.
+     *
+     * Since this will lock the database for an extended period of time, the user should be
+     * presented a busy indicator.  Operations against the database should not be expected to run
+     * until this completes.  Cancellation will not work once vacuuming has started.
+     */
+    public async void vacuum_async(Cancellable? cancellable) throws Error {
+        if (is_running)
+            return;
+        
+        is_running = true;
+        try {
+            debug("[%s] Starting vacuum of IMAP database", to_string());
+            yield internal_vacuum_async(cancellable);
+            debug("[%s] Completed vacuum of IMAP database", to_string());
+        } finally {
+            is_running = false;
+        }
+    }
+    
+    private async void internal_vacuum_async(Cancellable? cancellable) throws Error {
+        db.exec_transaction_async(Db.TransactionType.WO, (cx) => {
+            cx.exec("VACUUM", cancellable);
+            
+            DateTime now = new DateTime.now_local();
+            
+            // update last vacuum time and messages reaped since last vacuum
+            Db.Statement stmt = cx.prepare("""
+                UPDATE GarbageCollectionTable
+                SET last_vacuum_time_t = ?, reaped_messages_since_last_vacuum = ?
+                WHERE id = 0
+            """);
+            stmt.bind_int64(0, now.to_unix());
+            stmt.bind_int(1, 0);
+            
+            stmt.exec(cancellable);
+            
+            return Db.TransactionOutcome.COMMIT;
+        }, cancellable);
+    }
+    
+    /**
      * Should only be called from the foreground thread.
+     *
+     * Returns immediately if { link is_running} is true.
      */
-    public async void run_async(Cancellable? cancellable) throws Error {
+    public async void gc_async(Cancellable? cancellable) throws Error {
         if (is_running)
             return;
         
         is_running = true;
         try {
             debug("[%s] Starting garbage collection of IMAP database", to_string());
-            yield internal_run_async(cancellable);
+            yield internal_gc_async(cancellable);
             debug("[%s] Completed garbage collection of IMAP database", to_string());
         } finally {
             is_running = false;
         }
     }
     
-    private async void internal_run_async(Cancellable? cancellable) throws Error {
-        DateTime now = new DateTime.now(new TimeZone.local());
+    private async void internal_gc_async(Cancellable? cancellable) throws Error {
+        DateTime now = new DateTime.now_local();
         DateTime reap_date = now.add_days(0 - UNLINKED_DAYS);
         
         debug("[%s] Garbage collector reaping date: %s (%s)", to_string(), reap_date.to_string(),
@@ -167,6 +269,24 @@ private class Geary.ImapDB.GC {
         count = yield delete_empty_attachment_directories_async(null, null, cancellable);
         
         message("[%s] Deleted %d empty attachment directories", to_string(), count);
+        
+        //
+        // A full GC cycle completed -- store date for next time.
+        //
+        
+        yield db.exec_transaction_async(Db.TransactionType.WO, (cx) => {
+            DateTime now = new DateTime.now_local();
+            Db.Statement stmt = cx.prepare("""
+                UPDATE GarbageCollectionTable
+                SET last_gc_time_t = ?
+                WHERE id = 0
+            """);
+            stmt.bind_int64(now.to_unix());
+            
+            stmt.exec(cancellable);
+            
+            return Db.TransactionOutcome.COMMIT;
+        }, cancellable);
     }
     
     private async void reap_message_async(int64 message_id, Cancellable? cancellable) throws Error {
@@ -262,6 +382,16 @@ private class Geary.ImapDB.GC {
             }
             
             //
+            // Add to the garbage collection count since last vacuum
+            //
+            
+            cx.exec("""
+                UPDATE GarbageCollectionTable
+                SET reaped_messages_since_last_vacuum = reaped_messages_since_last_vacuum + 1
+                WHERE id = 0
+            """);
+            
+            //
             // Done; other than on-disk attachment files, message is now garbage collected.
             //
             
@@ -391,6 +521,34 @@ private class Geary.ImapDB.GC {
         return deleted;
     }
     
+    private async void fetch_gc_info_async(out DateTime? last_gc_time, out DateTime? last_vacuum_time,
+        out int reaped_messages_since_last_vacuum, Cancellable? cancellable) throws Error {
+        // dealing with out arguments for an async method inside a closure is asking for trouble
+        int64 last_gc_time_t = -1, last_vacuum_time_t = -1;
+        int reaped_count = -1;
+        yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => {
+            Db.Result result = cx.query("""
+                SELECT last_gc_time_t, last_vacuum_time_t, reaped_messages_since_last_vacuum
+                FROM GarbageCollectionTable
+                WHERE id = 0
+            """);
+            
+            if (result.finished)
+                Db.TransactionOutcome.FAILURE;
+            
+            // NULL indicates GC/vacuum has not run
+            last_gc_time_t = !result.is_null_at(0) ? result.int64_at(0) : -1;
+            last_vacuum_time_t = !result.is_null_at(1) ? result.int64_at(1) : -1;
+            reaped_count = result.int64_at(2);
+            
+            return Db.TransactionOutcome.SUCCESS;
+        }, cancellable);
+        
+        last_gc_time = (last_gc_time_t >= 0) ? new Date.from_unix_local(last_gc_time_t) : null;
+        last_vacuum_time = (last_vacuum_time_t >= 0) ? new Date.from_unix_local(last_vacuum_time_t) : null;
+        reaped_messages_since_last_vacuum = reaped_count;
+    }
+    
     public string to_string() {
         return "GC:%s".printf(db.db_file.get_path());
     }