[geary/wip/714317-hide-html-in-preview] Synthesise PREVIEW field if HEADER and BODY were also listed.

[Michael Gratton <mjog src gnome org>]

commit 9757e6e358734b0b45c3662cc6fbe448b44113b5
Author: Michael James Gratton <mike vee net>
Date:   Tue Dec 20 23:05:43 2016 +1100

    Synthesise PREVIEW field if HEADER and BODY were also listed.
    
    Since we can now generate a preview for a Geary.Email if it can construct
    its RFC822 message, if a PREVIEW is requested and both both HEADER and
    BODY are also requested, don't issue a seperate command for the PREVIEW.
    
    As a result, messages downloaded via the email prefetcher will get
    previews based on the complete message body.
    
    * src/engine/imap-engine/imap-engine-email-prefetcher.vala: Update the
      comment about synthesise ENVELOPE and PREVIEW, since we now do just
      that from PREVIEW.
    
    * src/engine/imap/api/imap-folder.vala (Folder::assemble_list_commands):
      If the requested flags contains PREVIEW, but BODY and HEADER are also
      requested, don't bother adding in additional commands for the PREVIEW.
      (Folder::fetched_data_to_email): When setting the body, see if we can
      also set the preview and if so, do so.

 .../imap-engine/imap-engine-email-prefetcher.vala  |   18 +++--
 src/engine/imap/api/imap-folder.vala               |   81 ++++++++++++++------
 2 files changed, 67 insertions(+), 32 deletions(-)
---
diff --git a/src/engine/imap-engine/imap-engine-email-prefetcher.vala 
b/src/engine/imap-engine/imap-engine-email-prefetcher.vala
index 367f1d9..2d1eacc 100644
--- a/src/engine/imap-engine/imap-engine-email-prefetcher.vala
+++ b/src/engine/imap-engine/imap-engine-email-prefetcher.vala
@@ -173,14 +173,16 @@ private class Geary.ImapEngine.EmailPrefetcher : Object {
             return;
         
         debug("do_prefetch_batch_async %s start_total=%d", folder.to_string(), emails.size);
-        
-        // Big TODO: The engine needs to be able to synthesize ENVELOPE (and any of the fields
-        // constituting it) and PREVIEW from HEADER and BODY if available.  When it can do that
-        // won't need to prefetch ENVELOPE or PREVIEW; prefetching HEADER and BODY will be enough.
-        
-        // Another big TODO: The engine needs to be able to chunk BODY requests so a large email
-        // doesn't monopolize the pipe and prevent other requests from going through
-        
+
+        // Big TODO: The engine needs to be able to synthesize
+        // ENVELOPE (and any of the fields constituting it) from
+        // HEADER if available.  When it can do that won't need to
+        // prefetch ENVELOPE; prefetching HEADER will be enough.
+
+        // Another big TODO: The engine needs to be able to chunk BODY
+        // requests so a large email doesn't monopolize the pipe and
+        // prevent other requests from going through
+
         Gee.HashSet<Geary.EmailIdentifier> ids = new Gee.HashSet<Geary.EmailIdentifier>();
         int64 chunk_bytes = 0;
         int count = 0;
diff --git a/src/engine/imap/api/imap-folder.vala b/src/engine/imap/api/imap-folder.vala
index f563ba8..f860024 100644
--- a/src/engine/imap/api/imap-folder.vala
+++ b/src/engine/imap/api/imap-folder.vala
@@ -483,14 +483,23 @@ private class Geary.Imap.Folder : BaseObject {
         } else {
             body_specifier = null;
         }
-        
-        // PREVIEW requires two separate commands
-        if (fields.require(Email.Field.PREVIEW)) {
-            // Get the preview text (the initial MAX_PREVIEW_BYTES of the first MIME section
+
+        // PREVIEW obtains the content type and a truncated version of
+        // the first part of the message, which often leads to poor
+        // results. It can also be also be synthesised from the
+        // email's RFC822 message in fetched_data_to_email, if the
+        // fields needed for reconstructing the RFC822 message are
+        // present. If so, rely on that and don't also request any
+        // additional data for the preview here.
+        if (fields.require(Email.Field.PREVIEW) &&
+            !fields.require(Email.REQUIRED_FOR_MESSAGE)) {
+            // Get the preview text (the initial MAX_PREVIEW_BYTES of
+            // the first MIME section
+
             preview_specifier = new FetchBodyDataSpecifier.peek(FetchBodyDataSpecifier.SectionPart.NONE,
                 { 1 }, 0, Geary.Email.MAX_PREVIEW_BYTES, null);
             cmds.add(new FetchCommand.body_data_type(msg_set, preview_specifier));
-            
+
             // Also get the character set to properly decode it
             preview_charset_specifier = new FetchBodyDataSpecifier.peek(
                 FetchBodyDataSpecifier.SectionPart.MIME, { 1 }, -1, -1, null);
@@ -499,7 +508,7 @@ private class Geary.Imap.Folder : BaseObject {
             preview_specifier = null;
             preview_charset_specifier = null;
         }
-        
+
         // PROPERTIES and FLAGS are a separate command
         if (fields.requires_any(Email.Field.PROPERTIES | Email.Field.FLAGS)) {
             Gee.List<FetchDataSpecifier> data_types = new Gee.ArrayList<FetchDataSpecifier>();
@@ -990,24 +999,12 @@ private class Geary.Imap.Folder : BaseObject {
         // the server, so use requested fields for determination
         if (required_but_not_set(Geary.Email.Field.REFERENCES, required_fields, email))
             email.set_full_references(message_id, in_reply_to, references);
-        
-        // if body was requested, get it now
-        if (body_specifier != null) {
-            if (fetched_data.body_data_map.has_key(body_specifier)) {
-                email.set_message_body(new Geary.RFC822.Text(
-                    fetched_data.body_data_map.get(body_specifier)));
-            } else {
-                message("[%s] No body specifier \"%s\" found", folder_name,
-                    body_specifier.to_string());
-                foreach (FetchBodyDataSpecifier specifier in fetched_data.body_data_map.keys)
-                    message("[%s] has %s", folder_name, specifier.to_string());
-            }
-        }
-        
-        // if preview was requested, get it now ... both identifiers must be supplied if one is
+
+        // if preview was requested, get it now ... both identifiers
+        // must be supplied if one is
         if (preview_specifier != null || preview_charset_specifier != null) {
             assert(preview_specifier != null && preview_charset_specifier != null);
-            
+
             if (fetched_data.body_data_map.has_key(preview_specifier)
                 && fetched_data.body_data_map.has_key(preview_charset_specifier)) {
                 email.set_message_preview(new RFC822.PreviewText.with_header(
@@ -1020,10 +1017,46 @@ private class Geary.Imap.Folder : BaseObject {
                     message("[%s] has %s", folder_name, specifier.to_string());
             }
         }
-        
+
+        // If body was requested, get it now. We also set the preview
+        // here from the body if possible since for HTML messages at
+        // least there's a lot of boilerplate HTML to wade through to
+        // get some actual preview text, which usually requires more
+        // than Geary.Email.MAX_PREVIEW_BYTES will allow for
+        if (body_specifier != null) {
+            if (fetched_data.body_data_map.has_key(body_specifier)) {
+                email.set_message_body(new Geary.RFC822.Text(
+                    fetched_data.body_data_map.get(body_specifier)));
+
+                // Try to set the preview
+                Geary.RFC822.Message? message = null;
+                try {
+                    message = email.get_message();
+                } catch (Error e) {
+                    // Not enough fields to construct the message
+                }
+                if (message != null) {
+                    string preview = message.get_preview();
+                    if (preview.length > Geary.Email.MAX_PREVIEW_BYTES) {
+                        preview = Geary.String.safe_byte_substring(
+                            preview, Geary.Email.MAX_PREVIEW_BYTES
+                        );
+                    }
+                    email.set_message_preview(
+                        new RFC822.PreviewText.from_string(preview)
+                    );
+                }
+            } else {
+                message("[%s] No body specifier \"%s\" found", folder_name,
+                    body_specifier.to_string());
+                foreach (FetchBodyDataSpecifier specifier in fetched_data.body_data_map.keys)
+                    message("[%s] has %s", folder_name, specifier.to_string());
+            }
+        }
+
         return email;
     }
-    
+
     // Returns a no-message-id ImapDB.EmailIdentifier with the UID stored in it.
     // This method does not take a cancellable; there is currently no way to tell if an email was
     // created or not if exec_commands_async() is cancelled during the append.  For atomicity's sake,