[evolution-data-server] [Camel] Speed up message filtering on headers

[Milan Crha <mcrha src gnome org>]

commit 98705252198922b191324cbfed51065dcd4c3839
Author: Milan Crha <mcrha redhat com>
Date:   Fri Oct 6 11:25:39 2017 +0200

    [Camel] Speed up message filtering on headers
    
    When a filter had been set on some headers it always caused message
    download, even when the CamelMessageInfo contained the headers. This
    checks whether headers are available in the CamelMessageInfo and if
    so, then it is using them, rather than downloading the whole message.
    To have this for every provider, the CamelFolderSummary's
    message_info_new_from_headers() automatically stores passed-in
    headers into the created CamelMessageInfo.

 src/camel/camel-filter-search.c  |  129 +++++++++++++++++++++++++++----------
 src/camel/camel-folder-summary.c |    2 +
 src/camel/camel-search-private.c |   71 +++++++++++++++++----
 src/camel/camel-search-private.h |    4 +
 4 files changed, 158 insertions(+), 48 deletions(-)
---
diff --git a/src/camel/camel-filter-search.c b/src/camel/camel-filter-search.c
index cf8d943..964fb7e 100644
--- a/src/camel/camel-filter-search.c
+++ b/src/camel/camel-filter-search.c
@@ -173,6 +173,57 @@ camel_filter_search_get_message (FilterMessageSearch *fms,
        return fms->message;
 }
 
+static const gchar *
+camel_filter_search_get_header (FilterMessageSearch *fms,
+                               struct _CamelSExp *sexp,
+                               const gchar *name)
+{
+       const CamelNameValueArray *headers;
+       CamelMimeMessage *message;
+       const gchar *value;
+
+       g_return_val_if_fail (fms != NULL, NULL);
+       g_return_val_if_fail (name != NULL, NULL);
+
+       headers = fms->info ? camel_message_info_get_headers (fms->info) : NULL;
+
+       if (headers) {
+               value = camel_name_value_array_get_named (headers, CAMEL_COMPARE_CASE_INSENSITIVE, name);
+
+               while (value && g_ascii_isspace (*value))
+                       value++;
+
+               return value;
+       }
+
+       message = camel_filter_search_get_message (fms, sexp);
+       if (!message)
+               return NULL;
+
+       return camel_medium_get_header (CAMEL_MEDIUM (message), name);
+}
+
+static const CamelNameValueArray *
+camel_filter_search_get_headers (FilterMessageSearch *fms,
+                                struct _CamelSExp *sexp)
+{
+       const CamelNameValueArray *headers;
+
+       g_return_val_if_fail (fms != NULL, NULL);
+
+       headers = fms->info ? camel_message_info_get_headers (fms->info) : NULL;
+
+       if (!headers) {
+               CamelMimeMessage *message;
+
+               message = camel_filter_search_get_message (fms, sexp);
+               if (message)
+                       headers = camel_medium_get_headers (CAMEL_MEDIUM (message));
+       }
+
+       return headers;
+}
+
 static gboolean
 check_header_in_message_info (FilterMessageSearch *fms,
                               gint argc,
@@ -204,7 +255,9 @@ check_header_in_message_info (FilterMessageSearch *fms,
 
        info = fms->info;
 
-       if (!info)
+       /* If there are headers available, then prefer them, thus the addresses are used
+          as received, not as they are stored in the folder summary. */
+       if (!info || camel_message_info_get_headers (info))
                return FALSE;
 
        name = argv[0]->value.string;
@@ -298,30 +351,21 @@ check_header (struct _CamelSExp *f,
                                }
                        }
                } else if (fms->message || !check_header_in_message_info (fms, argc, argv, how, &matched)) {
-                       CamelMimeMessage *message;
-                       CamelMimePart *mime_part;
                        const CamelNameValueArray *headers;
                        const gchar *charset = NULL;
                        camel_search_t type = CAMEL_SEARCH_TYPE_ENCODED;
-                       CamelContentType *ct;
                        guint ii;
                        const gchar *header_name = NULL, *header_value = NULL;
 
-                       message = camel_filter_search_get_message (fms, f);
-                       mime_part = CAMEL_MIME_PART (message);
+                       headers = camel_filter_search_get_headers (fms, f);
 
-                       if (camel_search_header_is_address (name))
+                       if (camel_search_header_is_address (name)) {
                                type = CAMEL_SEARCH_TYPE_ADDRESS_ENCODED;
-                       else if (message) {
-                               ct = camel_mime_part_get_content_type (mime_part);
-                               if (ct) {
-                                       charset = camel_content_type_param (ct, "charset");
-                                       charset = camel_iconv_charset_name (charset);
-                               }
+                       } else {
+                               charset = camel_search_get_default_charset_from_headers (headers);
                        }
 
-                       headers = camel_medium_get_headers (CAMEL_MEDIUM (mime_part));
-                       for (ii = 0; camel_name_value_array_get (headers, ii, &header_name, &header_value); 
ii++) {
+                       for (ii = 0; !matched && camel_name_value_array_get (headers, ii, &header_name, 
&header_value); ii++) {
                                /* empty name means any header */
                                if (!name || !*name || !g_ascii_strcasecmp (header_name, name)) {
                                        for (i = 1; i < argc && !matched; i++) {
@@ -402,17 +446,17 @@ header_exists (struct _CamelSExp *f,
                struct _CamelSExpResult **argv,
                FilterMessageSearch *fms)
 {
-       CamelMimeMessage *message;
        gboolean matched = FALSE;
        CamelSExpResult *r;
        gint i;
 
-       message = camel_filter_search_get_message (fms, f);
-
        for (i = 0; i < argc && !matched; i++) {
                if (argv[i]->type == CAMEL_SEXP_RES_STRING) {
-                       matched = camel_medium_get_header (CAMEL_MEDIUM (message), argv[i]->value.string) != 
NULL;
-                       camel_filter_search_log (fms, "Header '%s' does %sexist", argv[i]->value.string, 
matched ? "" : "not ");
+                       const gchar *name = argv[i]->value.string;
+
+                       matched = camel_filter_search_get_header (fms, f, name) != NULL;
+
+                       camel_filter_search_log (fms, "Header '%s' does %sexist", name, matched ? "" : "not 
");
                }
        }
 
@@ -429,16 +473,13 @@ header_regex (struct _CamelSExp *f,
               FilterMessageSearch *fms)
 {
        CamelSExpResult *r = camel_sexp_result_new (f, CAMEL_SEXP_RES_BOOL);
-       CamelMimeMessage *message;
        regex_t pattern;
        gchar *contents = NULL;
 
-       message = camel_filter_search_get_message (fms, f);
-
        if (argc > 1 && argv[0]->type == CAMEL_SEXP_RES_STRING
            && (contents = camel_search_get_header_decoded (argv[0]->value.string,
-                   camel_medium_get_header (CAMEL_MEDIUM (message), argv[0]->value.string),
-                   camel_search_get_default_charset_from_message (message)))
+                       camel_filter_search_get_header (fms, f, argv[0]->value.string),
+                       camel_search_get_default_charset_from_headers (camel_filter_search_get_headers (fms, 
f))))
            && camel_search_build_match_regex (&pattern, CAMEL_SEARCH_MATCH_REGEX | CAMEL_SEARCH_MATCH_ICASE, 
argc - 1, argv + 1, fms->error) == 0) {
                r->value.boolean = regexec (&pattern, contents, 0, NULL, 0) == 0;
                camel_filter_search_log (fms, "Regex on header '%s' does %smatch value '%s'",
@@ -461,15 +502,17 @@ header_full_regex (struct _CamelSExp *f,
                    FilterMessageSearch *fms)
 {
        CamelSExpResult *r = camel_sexp_result_new (f, CAMEL_SEXP_RES_BOOL);
-       CamelMimeMessage *message;
        regex_t pattern;
-       gchar *contents;
 
        if (camel_search_build_match_regex (&pattern, CAMEL_SEARCH_MATCH_REGEX | CAMEL_SEARCH_MATCH_ICASE | 
CAMEL_SEARCH_MATCH_NEWLINE,
                                           argc, argv, fms->error) == 0) {
-               message = camel_filter_search_get_message (fms, f);
-               contents = camel_search_get_all_headers_decoded (message);
+               const CamelNameValueArray *headers = camel_filter_search_get_headers (fms, f);
+               gchar *contents;
+
+               contents = camel_search_get_headers_decoded (headers, NULL);
+
                r->value.boolean = regexec (&pattern, contents, 0, NULL, 0) == 0;
+
                g_free (contents);
                regfree (&pattern);
        } else
@@ -622,12 +665,20 @@ get_sent_date (struct _CamelSExp *f,
                struct _CamelSExpResult **argv,
                FilterMessageSearch *fms)
 {
-       CamelMimeMessage *message;
+       gint64 sent_date;
        CamelSExpResult *r;
 
-       message = camel_filter_search_get_message (fms, f);
+       if (fms->info) {
+               sent_date = camel_message_info_get_date_sent (fms->info);
+       } else {
+               CamelMimeMessage *message;
+
+               message = camel_filter_search_get_message (fms, f);
+               sent_date = camel_mime_message_get_date (message, NULL);
+       }
+
        r = camel_sexp_result_new (f, CAMEL_SEXP_RES_INT);
-       r->value.number = camel_mime_message_get_date (message, NULL);
+       r->value.number = sent_date;
 
        camel_filter_search_log (fms, "Got sent date '%" G_GINT64_FORMAT "'", (gint64) r->value.number);
 
@@ -640,12 +691,20 @@ get_received_date (struct _CamelSExp *f,
                    struct _CamelSExpResult **argv,
                    FilterMessageSearch *fms)
 {
-       CamelMimeMessage *message;
+       gint64 received_date;
        CamelSExpResult *r;
 
-       message = camel_filter_search_get_message (fms, f);
+       if (fms->info) {
+               received_date = camel_message_info_get_date_received (fms->info);
+       } else {
+               CamelMimeMessage *message;
+
+               message = camel_filter_search_get_message (fms, f);
+               received_date = camel_mime_message_get_date_received (message, NULL);
+       }
+
        r = camel_sexp_result_new (f, CAMEL_SEXP_RES_INT);
-       r->value.number = camel_mime_message_get_date_received (message, NULL);
+       r->value.number = received_date;
 
        camel_filter_search_log (fms, "Got received date '%" G_GINT64_FORMAT "'", (gint64) r->value.number);
 
diff --git a/src/camel/camel-folder-summary.c b/src/camel/camel-folder-summary.c
index 612d8f2..fb54663 100644
--- a/src/camel/camel-folder-summary.c
+++ b/src/camel/camel-folder-summary.c
@@ -3026,6 +3026,8 @@ message_info_new_from_headers (CamelFolderSummary *summary,
                camel_message_info_take_references (mi, references);
        }
 
+       camel_message_info_take_headers (mi, camel_name_value_array_copy (headers));
+
        camel_message_info_set_abort_notifications (mi, FALSE);
 
        return mi;
diff --git a/src/camel/camel-search-private.c b/src/camel/camel-search-private.c
index 29cbabe..41adee7 100644
--- a/src/camel/camel-search-private.c
+++ b/src/camel/camel-search-private.c
@@ -792,6 +792,36 @@ camel_search_get_default_charset_from_message (CamelMimeMessage *message)
 }
 
 /**
+ * camel_search_get_default_charset_from_headers:
+ * @headers: a #CamelNameValueArray
+ *
+ * Returns: Default charset from the Content-Type header of the @headers; if none cannot be determined,
+ *    UTF-8 is returned.
+ *
+ * Since: 3.28
+ **/
+const gchar *
+camel_search_get_default_charset_from_headers (const CamelNameValueArray *headers)
+{
+       CamelContentType *ct = NULL;
+       const gchar *content, *charset = NULL;
+
+       if ((content = headers ? camel_name_value_array_get_named (headers, CAMEL_COMPARE_CASE_INSENSITIVE, 
"Content-Type") : NULL)
+            && (ct = camel_content_type_decode (content)))
+               charset = camel_content_type_param (ct, "charset");
+
+       if (!charset)
+               charset = "utf-8";
+
+       charset = camel_iconv_charset_name (charset);
+
+       if (ct)
+               camel_content_type_unref (ct);
+
+       return charset;
+}
+
+/**
  * camel_search_get_header_decoded:
  * @header_name: the header name
  * @header_value: the header value
@@ -832,31 +862,28 @@ camel_search_get_header_decoded (const gchar *header_name,
 }
 
 /**
- * camel_search_get_all_headers_decoded:
- * @message: a #CamelMessage
+ * camel_search_get_headers_decoded:
+ * @headers: a #CamelNameValueArray
+ * @default_charset: (nullable): default charset to use; or %NULL, to detect from Content-Type of @headers
  *
- * Returns: (transfer full): All headers of the @message, decoded where needed.
+ * Returns: (transfer full): The @headers, decoded where needed.
  *    Free the returned pointer with g_free() when done with it.
  *
- * Since: 3.22
+ * Since: 3.28
  **/
 gchar *
-camel_search_get_all_headers_decoded (CamelMimeMessage *message)
+camel_search_get_headers_decoded (const CamelNameValueArray *headers,
+                                 const gchar *default_charset)
 {
-       CamelMedium *medium;
        GString *str;
-       const CamelNameValueArray *headers;
-       const gchar *default_charset;
        guint ii, length;
 
-       g_return_val_if_fail (CAMEL_IS_MIME_MESSAGE (message), NULL);
-
-       medium = CAMEL_MEDIUM (message);
-       headers = camel_medium_get_headers (medium);
        if (!headers)
                return NULL;
 
-       default_charset = camel_search_get_default_charset_from_message (message);
+       if (!default_charset)
+               default_charset = camel_search_get_default_charset_from_headers (headers);
+
        str = g_string_new ("");
 
        length = camel_name_value_array_get_length (headers);
@@ -888,3 +915,21 @@ camel_search_get_all_headers_decoded (CamelMimeMessage *message)
 
        return g_string_free (str, FALSE);
 }
+
+/**
+ * camel_search_get_all_headers_decoded:
+ * @message: a #CamelMessage
+ *
+ * Returns: (transfer full): All headers of the @message, decoded where needed.
+ *    Free the returned pointer with g_free() when done with it.
+ *
+ * Since: 3.22
+ **/
+gchar *
+camel_search_get_all_headers_decoded (CamelMimeMessage *message)
+{
+       g_return_val_if_fail (CAMEL_IS_MIME_MESSAGE (message), NULL);
+
+       return camel_search_get_headers_decoded (camel_medium_get_headers (CAMEL_MEDIUM (message)),
+               camel_search_get_default_charset_from_message (message));
+}
diff --git a/src/camel/camel-search-private.h b/src/camel/camel-search-private.h
index 5fee6f7..64df42a 100644
--- a/src/camel/camel-search-private.h
+++ b/src/camel/camel-search-private.h
@@ -103,9 +103,13 @@ void               camel_search_words_free         (struct _camel_search_words *words);
 gboolean       camel_search_header_is_address  (const gchar *header_name);
 const gchar *  camel_search_get_default_charset_from_message
                                                (CamelMimeMessage *message);
+const gchar *  camel_search_get_default_charset_from_headers
+                                               (const CamelNameValueArray *headers);
 gchar *                camel_search_get_header_decoded (const gchar *header_name,
                                                 const gchar *header_value,
                                                 const gchar *default_charset);
+gchar *                camel_search_get_headers_decoded(const CamelNameValueArray *headers,
+                                                const gchar *default_charset);
 gchar *                camel_search_get_all_headers_decoded
                                                (CamelMimeMessage *message);