[geary/wip/713830-disposition: 4/4] Upgrade database to re-populate affected attachments

[Jim Nelson <jnelson src gnome org>]

commit cc9507e28445d171c058ed3570998cc0854e5afa
Author: Jim Nelson <jim yorba org>
Date:   Wed Jul 30 19:21:33 2014 -0700

    Upgrade database to re-populate affected attachments
    
    Prior commit only worked for new messages.  This will go back and
    rebuild attachments that were skipped before.

 sql/version-022.sql                      |    4 +
 src/engine/imap-db/imap-db-database.vala |  104 ++++++++++++++++++++++++++++++
 src/engine/imap-db/imap-db-folder.vala   |   31 ++++++++-
 3 files changed, 136 insertions(+), 3 deletions(-)
---
diff --git a/sql/version-022.sql b/sql/version-022.sql
new file mode 100644
index 0000000..df69e30
--- /dev/null
+++ b/sql/version-022.sql
@@ -0,0 +1,4 @@
+--
+-- Dummy database upgrade to repopulate attachments.  Bug #713830 revealed that
+-- non-text and non-image files with no Content-Disposition were being dropped.
+--
diff --git a/src/engine/imap-db/imap-db-database.vala b/src/engine/imap-db/imap-db-database.vala
index b0292d0..c775895 100644
--- a/src/engine/imap-db/imap-db-database.vala
+++ b/src/engine/imap-db/imap-db-database.vala
@@ -105,6 +105,10 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase {
             case 19:
                 post_upgrade_validate_contacts();
             break;
+            
+            case 22:
+                post_rebuild_attachments();
+            break;
         }
     }
     
@@ -402,6 +406,106 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase {
         }
     }
     
+    // Version 22
+    private void post_rebuild_attachments() {
+        try {
+            exec_transaction(Db.TransactionType.RW, (cx) => {
+                Db.Statement stmt = cx.prepare("""
+                    SELECT id, header, body
+                    FROM MessageTable
+                    WHERE (fields & ?) = ?
+                    """);
+                stmt.bind_int(0, Geary.Email.REQUIRED_FOR_MESSAGE);
+                stmt.bind_int(1, Geary.Email.REQUIRED_FOR_MESSAGE);
+                
+                Db.Result results = stmt.exec();
+                if (results.finished)
+                    return Db.TransactionOutcome.ROLLBACK;
+                
+                Gee.HashSet<int64?> deleted_ids = new Gee.HashSet<int64?>(
+                    Collection.int64_hash_func, Collection.int64_equal_func);
+                do {
+                    int64 message_id = results.rowid_at(0);
+                    Geary.Memory.Buffer header = results.string_buffer_at(1);
+                    Geary.Memory.Buffer body = results.string_buffer_at(2);
+                    
+                    Geary.RFC822.Message message;
+                    try {
+                        message = new Geary.RFC822.Message.from_parts(
+                            new RFC822.Header(header), new RFC822.Text(body));
+                    } catch (Error err) {
+                        debug("Error decoding message: %s", err.message);
+                        
+                        continue;
+                    }
+                    
+                    // build a list of attachments in the message itself
+                    Gee.List<GMime.Part> msg_attachments = message.get_attachments();
+                    
+                    // get known attachments stored in database and on disk
+                    Gee.List<Geary.Attachment>? known_attachments = ImapDB.Folder.do_list_attachments(
+                        cx, message_id, null);
+                    int known_attachments_count = (known_attachments != null)
+                        ? known_attachments.size : 0;
+                    
+                    // if the same count, consider all present and accounted for
+                    if (msg_attachments.size == known_attachments_count)
+                        continue;
+                    
+                    // delete all attachments for this message
+                    try {
+                        Geary.ImapDB.Folder.do_delete_attachments(cx, message_id);
+                    } catch (Error err) {
+                        debug("Error deleting existing attachments: %s", err.message);
+                        
+                        continue;
+                    }
+                    
+                    // rebuild all
+                    try {
+                        Geary.ImapDB.Folder.do_save_attachments_db(cx, message_id, msg_attachments,
+                            this, null);
+                    } catch (Error err) {
+                        debug("Error saving attachments: %s", err.message);
+                        
+                        // fallthrough
+                    }
+                    
+                    deleted_ids.add(message_id);
+                } while (results.next());
+                
+                // rebuild rows with potentially new attachments
+                if (deleted_ids.size > 0) {
+                    StringBuilder builder = new StringBuilder("""
+                        DELETE FROM MessageSearchTable WHERE docid IN (
+                    """);
+                    bool first = true;
+                    foreach (int64 message_id in deleted_ids) {
+                        if (!first)
+                            builder.append(", ");
+                        
+                        builder.append(message_id.to_string());
+                        first = false;
+                    }
+                    builder.append(")");
+                    
+                    try {
+                        cx.exec(builder.str);
+                    } catch (Error err) {
+                        debug("Unable to do partial delete of search table: %s", err.message);
+                        
+                        throw err;
+                    }
+                }
+                
+                return Db.TransactionOutcome.COMMIT;
+            });
+        } catch (Error e) {
+            debug("Error populating old inline attachments during upgrade to database schema 13: %s",
+                e.message);
+        }
+    }
+    
     private void on_prepare_database_connection(Db.Connection cx) throws Error {
         cx.set_busy_timeout_msec(Db.Connection.RECOMMENDED_BUSY_TIMEOUT_MSEC);
         cx.set_foreign_keys(true);
diff --git a/src/engine/imap-db/imap-db-folder.vala b/src/engine/imap-db/imap-db-folder.vala
index 4dbfbb2..9ce333a 100644
--- a/src/engine/imap-db/imap-db-folder.vala
+++ b/src/engine/imap-db/imap-db-folder.vala
@@ -1871,7 +1871,7 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
         unread_count_change += new_unread_count;
     }
     
-    private static Gee.List<Geary.Attachment>? do_list_attachments(Db.Connection cx, int64 message_id,
+    public static Gee.List<Geary.Attachment>? do_list_attachments(Db.Connection cx, int64 message_id,
         Cancellable? cancellable) throws Error {
         Db.Statement stmt = cx.prepare("""
             SELECT id, filename, mime_type, filesize, disposition
@@ -1909,8 +1909,9 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
             return;
         
         foreach (GMime.Part attachment in attachments) {
-            string mime_type = attachment.get_content_type().to_string();
-            string disposition = attachment.get_disposition();
+            unowned GMime.ContentType? content_type = attachment.get_content_type();
+            string mime_type = (content_type != null) ? content_type.to_string() : 
"application/octet-stream";
+            string? disposition = attachment.get_disposition();
             string filename = RFC822.Utils.get_clean_attachment_filename(attachment);
             
             // Convert the attachment content into a usable ByteArray.
@@ -2001,6 +2002,30 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
         }
     }
     
+    public static void do_delete_attachments(Db.Connection cx, int64 message_id)
+        throws Error {
+        Gee.List<Geary.Attachment>? attachments = do_list_attachments(cx, message_id, null);
+        if (attachments == null || attachments.size == 0)
+            return;
+        
+        // delete all files
+        foreach (Geary.Attachment attachment in attachments) {
+            try {
+                attachment.file.delete(null);
+            } catch (Error err) {
+                debug("Unable to delete file %s: %s", attachment.file.get_path(), err.message);
+            }
+        }
+        
+        // remove all from attachment table
+        Db.Statement stmt = new Db.Statement(cx, """
+            DELETE FROM MessageAttachmentTable WHERE message_id = ?
+        """);
+        stmt.bind_rowid(0, message_id);
+        
+        stmt.exec();
+    }
+    
     /**
      * Adds a value to the unread count.  If this makes the unread count negative, it will be
      * set to zero.