[geary/wip/714802-chron: 3/3] Properly load chronologically newest email first in conversation

[Jim Nelson <jnelson src gnome org>]

commit 9b51e6916b1b47c041d5ca2a50a1556911622f39
Author: Jim Nelson <jim yorba org>
Date:   Fri Jan 23 19:44:54 2015 -0800

    Properly load chronologically newest email first in conversation
    
    This also starts to fix a bug where folders with too few emails to
    trigger the scrollbar weren't loading the additional emails

 src/client/application/geary-controller.vala       |    8 ++-
 .../conversation-list/conversation-list-view.vala  |   10 +++
 src/engine/api/geary-folder.vala                   |   24 +++++++
 src/engine/api/geary-search-folder.vala            |   18 ++++++
 src/engine/app/app-conversation-monitor.vala       |   31 +++++++++-
 src/engine/imap-db/imap-db-folder.vala             |   64 ++++++++++++++++++++
 src/engine/imap-db/outbox/smtp-outbox-folder.vala  |   28 +++++++++
 .../imap-engine/imap-engine-minimal-folder.vala    |    8 +++
 8 files changed, 187 insertions(+), 4 deletions(-)
---
diff --git a/src/client/application/geary-controller.vala b/src/client/application/geary-controller.vala
index 2bae336..654857d 100644
--- a/src/client/application/geary-controller.vala
+++ b/src/client/application/geary-controller.vala
@@ -1394,8 +1394,12 @@ public class GearyController : Geary.BaseObject {
     }
     
     private void on_load_more() {
-        debug("on_load_more");
-        current_conversations.min_window_count += MIN_CONVERSATION_COUNT;
+        if (!current_conversations.all_messages_loaded)
+            current_conversations.min_window_count += MIN_CONVERSATION_COUNT;
+        
+        debug("on_load_more: all_messages_loaded=%s min_window_count=%d",
+            current_conversations.all_messages_loaded.to_string(),
+            current_conversations.min_window_count);
     }
     
     private void on_select_folder_completed(Object? source, AsyncResult result) {
diff --git a/src/client/conversation-list/conversation-list-view.vala 
b/src/client/conversation-list/conversation-list-view.vala
index 08cf5f1..d7c5b64 100644
--- a/src/client/conversation-list/conversation-list-view.vala
+++ b/src/client/conversation-list/conversation-list-view.vala
@@ -103,11 +103,21 @@ public class ConversationListView : Gtk.TreeView {
     }
     
     private void on_scan_completed() {
+        debug("ON SCAN COMPLETED");
+        
         enable_load_more = true;
         
         // Select first conversation.
         if (GearyApplication.instance.config.autoselect)
             select_first_conversation();
+        
+        // if no scrollbars, load more ... need to do this until scrollbars show in order to allow
+        // the user a way to manually load more when they're ready
+        Gtk.Adjustment vadj = ((Gtk.Scrollable) this).get_vadjustment();
+        if (vadj.upper <= vadj.page_size) {
+            debug("NO SCROLLBAR, LOADING MORE");
+            load_more();
+        }
     }
     
     private void on_conversation_removed(Geary.App.Conversation conversation) {
diff --git a/src/engine/api/geary-folder.vala b/src/engine/api/geary-folder.vala
index bf5b1cf..e3820c9 100644
--- a/src/engine/api/geary-folder.vala
+++ b/src/engine/api/geary-folder.vala
@@ -474,6 +474,10 @@ public abstract class Geary.Folder : BaseObject {
      * from newest to oldest, with null being the newest email.  If set, the direction is reversed
      * and null indicates the oldest email.
      *
+     * Note that "oldest" and "newest" refer to their ''positional ordering'' in the Folder, which
+     * does not necessarily correspond to the Date: header field or their
+     * { link EmailProperties.date_received}.
+     *
      * If not null, the EmailIdentifier ''must'' have originated from this Folder.
      *
      * To fetch all available messages in one call, use a count of int.MAX.
@@ -539,6 +543,26 @@ public abstract class Geary.Folder : BaseObject {
         Geary.Email.Field required_fields, ListFlags flags, Cancellable? cancellable = null) throws Error;
     
     /**
+     * Find the chronologically newest { link EmailIdentifier} in the { link Folder} according to
+     * its { link EmailProperties.date_received}.
+     *
+     * Because the positional ordering of Email in a Folder is distinct from its Date: and
+     * date received, this offers a simple search mechanism for finding the chronological "start"
+     * of the email as sorted from newest to oldest..  Note that only the ''local'' store is
+     * searched; this does not query the remote store.
+     *
+     * offset_from_top is the zero-based email count from the top ("newest" in the terminology of
+     * the list methods).  Thus, list_email_by_id_async(null, offset_from_top) will return this
+     * email.
+     *
+     * Returns false if the local store is empty.
+     *
+     * The Folder must be opened prior to attempting this operation.
+     */
+    public abstract async bool fetch_local_newest_async(out Geary.EmailIdentifier? newest_id,
+        out DateTime? newest_date, out int offset_from_top, Cancellable? cancellable = null) throws Error;
+    
+    /**
      * Used for debugging.  Should not be used for user-visible labels.
      */
     public virtual string to_string() {
diff --git a/src/engine/api/geary-search-folder.vala b/src/engine/api/geary-search-folder.vala
index c064a62..cdc31ca 100644
--- a/src/engine/api/geary-search-folder.vala
+++ b/src/engine/api/geary-search-folder.vala
@@ -380,6 +380,24 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
         return yield account.local_fetch_email_async(id, required_fields, cancellable);
     }
     
+    public override async bool fetch_local_newest_async(out Geary.EmailIdentifier? newest_id,
+        out DateTime? newest_date, out int offset_from_top, Cancellable? cancellable = null) throws Error {
+        if (search_results == null || search_results.size == 0) {
+            newest_id = null;
+            newest_date = null;
+            offset_from_top = 0;
+            
+            return false;
+        }
+        
+        ImapDB.SearchEmailIdentifier search_id = search_results.first();
+        newest_id = search_id;
+        newest_date = search_id.date_received;
+        offset_from_top = 0;
+        
+        return true;
+    }
+    
     public virtual async void remove_email_async(Gee.List<Geary.EmailIdentifier> email_ids,
         Cancellable? cancellable = null) throws Error {
         Gee.MultiMap<Geary.EmailIdentifier, Geary.FolderPath>? ids_to_folders
diff --git a/src/engine/app/app-conversation-monitor.vala b/src/engine/app/app-conversation-monitor.vala
index ecad09f..d4f8eed 100644
--- a/src/engine/app/app-conversation-monitor.vala
+++ b/src/engine/app/app-conversation-monitor.vala
@@ -31,6 +31,10 @@ public class Geary.App.ConversationMonitor : BaseObject {
     public Geary.Folder folder { get; private set; }
     public bool reestablish_connections { get; set; default = true; }
     public bool is_monitoring { get; private set; default = false; }
+    
+    /**
+     * TODO: Only allow increasing window count if FillWindowOperation is not outstanding.
+     */
     public int min_window_count { get { return _min_window_count; }
         set {
             _min_window_count = value;
@@ -38,6 +42,11 @@ public class Geary.App.ConversationMonitor : BaseObject {
         }
     }
     
+    /**
+     * Indicates no more messages can be loaded by increasing the min_window_count.
+     */
+    public bool all_messages_loaded { get; private set; default = false; }
+    
     public Geary.ProgressMonitor progress_monitor { get { return operation_queue.progress_monitor; } }
     
     private ConversationSet conversations = new ConversationSet();
@@ -314,8 +323,22 @@ public class Geary.App.ConversationMonitor : BaseObject {
     
     internal async void local_load_async() {
         debug("ConversationMonitor seeding with local email for %s", folder.to_string());
+        
+        int count = min_window_count;
         try {
-            yield load_by_id_async(null, min_window_count, Folder.ListFlags.LOCAL_ONLY, cancellable_monitor);
+            int offset_from_top;
+            bool found = yield folder.fetch_local_newest_async(null, null, out offset_from_top,
+                cancellable_monitor);
+            if (found && (offset_from_top + 1) > count) {
+                count = offset_from_top + 1;
+                debug("SEEDING %d TO GET NEWEST EMAIL", count);
+            }
+        } catch (Error err) {
+            debug("Error fetching local newest email: %s", err.message);
+        }
+        
+        try {
+            yield load_by_id_async(null, count, Folder.ListFlags.LOCAL_ONLY, cancellable_monitor);
         } catch (Error e) {
             debug("Error loading local messages: %s", e.message);
         }
@@ -728,8 +751,10 @@ public class Geary.App.ConversationMonitor : BaseObject {
     
     private void on_folder_opened(Geary.Folder.OpenState state, int count) {
         // once remote is open, reseed with messages from the earliest ID to the latest
-        if (state == Geary.Folder.OpenState.BOTH || state == Geary.Folder.OpenState.REMOTE)
+        if (state == Geary.Folder.OpenState.BOTH || state == Geary.Folder.OpenState.REMOTE) {
             operation_queue.add(new ReseedOperation(this, state.to_string()));
+            operation_queue.add(new FillWindowOperation(this, false));
+        }
     }
     
     /**
@@ -783,6 +808,8 @@ public class Geary.App.ConversationMonitor : BaseObject {
             }
         }
         
+        all_messages_loaded = conversations.get_email_count() == initial_message_count;
+        
         // Run again to make sure we're full unless we ran out of messages.
         if (conversations.get_email_count() != initial_message_count)
             operation_queue.add(new FillWindowOperation(this, is_insert));
diff --git a/src/engine/imap-db/imap-db-folder.vala b/src/engine/imap-db/imap-db-folder.vala
index 1919515..f1fc3eb 100644
--- a/src/engine/imap-db/imap-db-folder.vala
+++ b/src/engine/imap-db/imap-db-folder.vala
@@ -134,6 +134,70 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
         return count;
     }
     
+    // Returns newest email in database by its internaldate, not position or uid
+    public async bool get_newest_id_async(ListFlags flags, out ImapDB.EmailIdentifier? newest_id,
+        out DateTime? newest_date, out int offset_from_top, Cancellable? cancellable) throws Error {
+        int64 message_id = Db.INVALID_ROWID;
+        int64 internaldate_time_t = -1;
+        int offset = -1;
+        yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => {
+            Db.Statement stmt = cx.prepare("""
+                SELECT id, internaldate_time_t
+                FROM MessageTable
+                WHERE id IN (
+                    SELECT message_id
+                    FROM MessageLocationTable
+                    WHERE folder_id = ?
+                )
+                ORDER BY internaldate_time_t DESC
+                LIMIT 1
+            """);
+            stmt.bind_rowid(0, folder_id);
+            
+            Db.Result result = stmt.exec(cancellable);
+            if (result.finished)
+                return Db.TransactionOutcome.DONE;
+            
+            message_id = result.rowid_at(0);
+            internaldate_time_t = result.int64_at(1);
+            
+            stmt = cx.prepare("""
+                SELECT COUNT(*)
+                FROM MessageLocationTable
+                WHERE folder_id = ? AND ordering >= (
+                    SELECT ordering
+                    FROM MessageLocationTable
+                    WHERE message_id = ? AND folder_id = ?
+                )
+            """);
+            stmt.bind_rowid(0, folder_id);
+            stmt.bind_rowid(1, message_id);
+            stmt.bind_rowid(2, folder_id);
+            
+            result = stmt.exec(cancellable);
+            if (!result.finished)
+                offset = Numeric.int_floor(result.int_at(0) - 1, 0);
+            
+            return Db.TransactionOutcome.DONE;
+        }, cancellable);
+        
+        if (message_id == Db.INVALID_ROWID) {
+            newest_id = null;
+            newest_date = null;
+            offset_from_top = 0;
+            
+            return false;
+        }
+        
+        newest_id = new ImapDB.EmailIdentifier(message_id, null);
+        newest_date = (internaldate_time_t > -1) ? new DateTime.from_unix_utc(internaldate_time_t) : null;
+        offset_from_top = offset;
+        
+        debug("NEWEST: %s %s offset=%d", newest_id.to_string(), newest_date.to_string(), offset_from_top);
+        
+        return true;
+    }
+    
     // Updates both the FolderProperties and the value in the local store.
     public async void update_remote_status_message_count(int count, Cancellable? cancellable) throws Error {
         if (count < 0)
diff --git a/src/engine/imap-db/outbox/smtp-outbox-folder.vala 
b/src/engine/imap-db/outbox/smtp-outbox-folder.vala
index 745967b..2e7cf9a 100644
--- a/src/engine/imap-db/outbox/smtp-outbox-folder.vala
+++ b/src/engine/imap-db/outbox/smtp-outbox-folder.vala
@@ -535,6 +535,34 @@ private class Geary.SmtpOutboxFolder : Geary.AbstractLocalFolder, Geary.FolderSu
         return row_to_email(row);
     }
     
+    public override async bool fetch_local_newest_async(out Geary.EmailIdentifier? newest_id,
+        out DateTime? newest_date, out int offset_from_top, Cancellable? cancellable = null) throws Error {
+        check_open();
+        
+        SmtpOutboxEmailIdentifier? id = null;
+        yield db.exec_transaction_async(Db.TransactionType.RO, (cx) => {
+            Db.Statement stmt = cx.prepare("""
+                SELECT id, ordering
+                FROM SmtpOutboxTable
+                ORDER BY ordering DESC
+                LIMIT 1
+            """);
+            
+            Db.Result result = stmt.exec(cancellable);
+            if (!result.finished)
+                id = new SmtpOutboxEmailIdentifier(result.rowid_at(0), result.int64_at(1));
+            
+            return Db.TransactionOutcome.DONE;
+        }, cancellable);
+        
+        // TODO: Store date in table
+        newest_id = id;
+        newest_date = null;
+        offset_from_top = 0;
+        
+        return id != null;
+    }
+    
     private async void mark_email_as_sent_async(SmtpOutboxEmailIdentifier outbox_id,
         Cancellable? cancellable = null) throws Error {
         yield db.exec_transaction_async(Db.TransactionType.WR, (cx) => {
diff --git a/src/engine/imap-engine/imap-engine-minimal-folder.vala 
b/src/engine/imap-engine/imap-engine-minimal-folder.vala
index 39d1e26..7be93da 100644
--- a/src/engine/imap-engine/imap-engine-minimal-folder.vala
+++ b/src/engine/imap-engine/imap-engine-minimal-folder.vala
@@ -1290,6 +1290,14 @@ private class Geary.ImapEngine.MinimalFolder : Geary.Folder, Geary.FolderSupport
         return op.email;
     }
     
+    public override async bool fetch_local_newest_async(out Geary.EmailIdentifier? newest_id,
+        out DateTime? newest_date, out int offset_from_top, Cancellable? cancellable = null) throws Error {
+        check_open("fetch_local_newest_async");
+        
+        return yield local_folder.get_newest_id_async(ImapDB.Folder.ListFlags.NONE, out newest_id,
+            out newest_date, out offset_from_top, cancellable);
+    }
+    
     // Helper function for child classes dealing with the delete/archive question.  This method will
     // mark the message as deleted and expunge it.
     protected async void expunge_email_async(Gee.List<Geary.EmailIdentifier> email_ids,