[geary/wip/search-columns-714494: 1/2] WIP ideas for the new field searches
- From: Charles Lindsay <clindsay src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [geary/wip/search-columns-714494: 1/2] WIP ideas for the new field searches
- Date: Sat, 14 Dec 2013 02:13:52 +0000 (UTC)
commit a2dca989dc15e41488349c59820415facd887128
Author: Charles Lindsay <chaz yorba org>
Date: Wed Jul 24 18:28:13 2013 -0700
WIP ideas for the new field searches
Probably doesn't compile. I still need to finish up in ImapDB.Account.
src/CMakeLists.txt | 1 +
src/engine/abstract/geary-abstract-account.vala | 4 +-
src/engine/api/geary-account.vala | 8 +-
src/engine/api/geary-search-folder.vala | 16 ++-
src/engine/api/geary-search-query.vala | 44 +++++++
src/engine/imap-db/imap-db-account.vala | 133 +++++++++++++++-----
.../imap-engine/imap-engine-generic-account.vala | 10 +-
7 files changed, 167 insertions(+), 49 deletions(-)
---
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fab8709..d9e96a4 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -39,6 +39,7 @@ engine/api/geary-named-flag.vala
engine/api/geary-named-flags.vala
engine/api/geary-progress-monitor.vala
engine/api/geary-search-folder.vala
+engine/api/geary-search-query.vala
engine/api/geary-service-provider.vala
engine/api/geary-special-folder-type.vala
diff --git a/src/engine/abstract/geary-abstract-account.vala b/src/engine/abstract/geary-abstract-account.vala
index f9ca814..4ee8115 100644
--- a/src/engine/abstract/geary-abstract-account.vala
+++ b/src/engine/abstract/geary-abstract-account.vala
@@ -116,11 +116,11 @@ public abstract class Geary.AbstractAccount : BaseObject, Geary.Account {
public abstract async Geary.Email local_fetch_email_async(Geary.EmailIdentifier email_id,
Geary.Email.Field required_fields, Cancellable? cancellable = null) throws Error;
- public abstract async Gee.Collection<Geary.EmailIdentifier>? local_search_async(string query,
+ public abstract async Gee.Collection<Geary.EmailIdentifier>? local_search_async(Geary.SearchQuery query,
int limit = 100, int offset = 0, Gee.Collection<Geary.FolderPath?>? folder_blacklist = null,
Gee.Collection<Geary.EmailIdentifier>? search_ids = null, Cancellable? cancellable = null) throws
Error;
- public abstract async Gee.Collection<string>? get_search_matches_async(string query,
+ public abstract async Gee.Collection<string>? get_search_matches_async(Geary.SearchQuery query,
Gee.Collection<Geary.EmailIdentifier> ids, Cancellable? cancellable = null) throws Error;
public abstract async Gee.MultiMap<Geary.EmailIdentifier, Geary.FolderPath>?
get_containing_folders_async(
diff --git a/src/engine/api/geary-account.vala b/src/engine/api/geary-account.vala
index 38b0c39..f2dc933 100644
--- a/src/engine/api/geary-account.vala
+++ b/src/engine/api/geary-account.vala
@@ -305,21 +305,21 @@ public interface Geary.Account : BaseObject {
Geary.Email.Field required_fields, Cancellable? cancellable = null) throws Error;
/**
- * Performs a search with the given query string. Optionally, a list of folders not to search
+ * Performs a search with the given query. Optionally, a list of folders not to search
* can be passed as well as a list of email identifiers to restrict the search to only those messages.
* Returns a list of EmailIdentifiers, or null if there are no results.
* The list is limited to a maximum number of results and starting offset,
* so you can walk the table. limit can be negative to mean "no limit" but
* offset must not be negative.
*/
- public abstract async Gee.Collection<Geary.EmailIdentifier>? local_search_async(string query,
+ public abstract async Gee.Collection<Geary.EmailIdentifier>? local_search_async(Geary.SearchQuery query,
int limit = 100, int offset = 0, Gee.Collection<Geary.FolderPath?>? folder_blacklist = null,
Gee.Collection<Geary.EmailIdentifier>? search_ids = null, Cancellable? cancellable = null) throws
Error;
/**
- * Given a list of mail IDs, returns a list of words that match the given query string.
+ * Given a list of mail IDs, returns a list of words that match for the query.
*/
- public abstract async Gee.Collection<string>? get_search_matches_async(string query,
+ public abstract async Gee.Collection<string>? get_search_matches_async(Geary.SearchQuery query,
Gee.Collection<Geary.EmailIdentifier> ids, Cancellable? cancellable = null) throws Error;
/**
diff --git a/src/engine/api/geary-search-folder.vala b/src/engine/api/geary-search-folder.vala
index 499d0a4..dcd41d0 100644
--- a/src/engine/api/geary-search-folder.vala
+++ b/src/engine/api/geary-search-folder.vala
@@ -55,7 +55,7 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
Geary.SpecialFolderType.TRASH,
// Orphan emails (without a folder) are also excluded; see ctor.
};
- private string? search_query = null;
+ private Geary.SearchQuery? search_query = null;
private Gee.TreeSet<ImapDB.SearchEmailIdentifier> search_results;
private Geary.Nonblocking.Mutex result_mutex = new Geary.Nonblocking.Mutex();
@@ -113,7 +113,7 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
}
}
- private async void append_new_email_async(string query, Geary.Folder folder,
+ private async void append_new_email_async(Geary.SearchQuery query, Geary.Folder folder,
Gee.Collection<Geary.EmailIdentifier> ids, Cancellable? cancellable) throws Error {
int result_mutex_token = yield result_mutex.claim_async();
@@ -144,7 +144,7 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
append_new_email_async.begin(search_query, folder, ids, null, on_append_new_email_complete);
}
- private async void handle_removed_email_async(string query, Geary.Folder folder,
+ private async void handle_removed_email_async(Geary.SearchQuery query, Geary.Folder folder,
Gee.Collection<Geary.EmailIdentifier> ids, Cancellable? cancellable) throws Error {
int result_mutex_token = yield result_mutex.claim_async();
@@ -213,19 +213,21 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
}
private async void set_search_query_async(string query, Cancellable? cancellable = null) throws Error {
+ Geary.SearchQuery search_query = new Geary.SearchQuery(query);
+
int result_mutex_token = yield result_mutex.claim_async();
Error? error = null;
try {
- yield do_search_async(query, null, null, cancellable);
+ yield do_search_async(search_query, null, null, cancellable);
} catch(Error e) {
error = e;
}
result_mutex.release(ref result_mutex_token);
- search_query = query;
- search_query_changed(query);
+ this.search_query = search_query;
+ search_query_changed(search_query.raw);
if (error != null)
throw error;
@@ -237,7 +239,7 @@ public class Geary.SearchFolder : Geary.AbstractLocalFolder, Geary.FolderSupport
// considered to be a delta and are added or subtracted from the full set.
// add_ids are new ids to search for, remove_ids are ids in our result set
// that will be removed if this search doesn't turn them up.
- private async void do_search_async(string query, Gee.Collection<Geary.EmailIdentifier>? add_ids,
+ private async void do_search_async(Geary.SearchQuery query, Gee.Collection<Geary.EmailIdentifier>?
add_ids,
Gee.Collection<ImapDB.SearchEmailIdentifier>? remove_ids, Cancellable? cancellable) throws Error {
// There are three cases here: 1) replace full result set, where the
// *_ids parameters are both null, 2) add to result set, where just
diff --git a/src/engine/api/geary-search-query.vala b/src/engine/api/geary-search-query.vala
new file mode 100644
index 0000000..c03783d
--- /dev/null
+++ b/src/engine/api/geary-search-query.vala
@@ -0,0 +1,44 @@
+/* Copyright 2013 Yorba Foundation
+ *
+ * This software is licensed under the GNU Lesser General Public License
+ * (version 2.1 or later). See the COPYING file in this distribution.
+ */
+
+/**
+ * An object to hold state for various search subsystems that might need to
+ * parse the same text string different ways. The only interaction the API
+ * user should have with this is creating new ones and then passing them off to
+ * the search methods in the engine.
+ *
+ * TODO: support anything other than ImapDB.Account's search methods.
+ */
+public class Geary.SearchQuery : BaseObject {
+ public string raw { get; private set; }
+ public bool parsed { get; internal set; default = false; }
+
+ // Not using a MultiMap because we (might) need a guarantee of order.
+ private Gee.HashMap<string?, Gee.ArrayList<string>> field_map
+ = new Gee.HashMap<string?, Gee.ArrayList<string>>();
+
+ public SearchQuery(string query) {
+ raw = query;
+ }
+
+ internal void add_token(string? field, string token) {
+ if (!field_map.has_key(field))
+ field_map.set(field, new Gee.ArrayList<string>());
+
+ field_map.get(field).add(token);
+ }
+
+ internal Gee.Collection<string?> get_fields() {
+ return field_map.keys;
+ }
+
+ internal Gee.List<string>? get_tokens(string? field) {
+ if (!field_map.has_key(field))
+ return null;
+
+ return field_map.get(field);
+ }
+}
diff --git a/src/engine/imap-db/imap-db-account.vala b/src/engine/imap-db/imap-db-account.vala
index 1cbbd9e..27e72f2 100644
--- a/src/engine/imap-db/imap-db-account.vala
+++ b/src/engine/imap-db/imap-db-account.vala
@@ -624,29 +624,70 @@ private class Geary.ImapDB.Account : BaseObject {
return (messages.size == 0 ? null : messages);
}
- public string prepare_search_query(string raw_query) {
- // Two goals here:
- // 1) append an * after every term so it becomes a prefix search
- // (see <https://www.sqlite.org/fts3.html#section_3>), and
- // 2) strip out common words/operators that might get interpreted as
- // search operators.
+ private string? extract_field_from_token(string[] parts, ref string token) {
+ // Map of user-supplied search field names to column names.
+ Gee.HashMap<string, string> field_names = new Gee.HashMap<string, string>();
+ /// Can be typed in the search box like attachment:file.txt to find
+ /// messages with attachments with a particular name.
+ field_names.set(_("attachment"), "attachment");
+ /// Can be typed in the search box like bcc:johndoe example com to find
+ /// messages bcc'd to a particular person.
+ field_names.set(_("bcc"), "bcc");
+ /// Can be typed in the search box like body:word to find the word only
+ /// if it occurs in the body of a message.
+ field_names.set(_("body"), "body");
+ /// Can be typed in the search box like cc:johndoe example com to find
+ /// messages cc'd to a particular person.
+ field_names.set(_("cc"), "cc");
+ /// Can be typed in the search box like from:johndoe example com to
+ /// find messages from a particular sender.
+ field_names.set(_("from"), "from_field");
+ /// Can be typed in the search box like subject:word to find the word
+ /// only if it occurs in the subject of a message.
+ field_names.set(_("subject"), "subject");
+ /// Can be typed in the search box like to:johndoe example com to find
+ /// messages received by a particular person.
+ field_names.set(_("to"), "receivers");
+
+ string key = parts[0].down();
+ if (key in field_names.keys) {
+ token = parts[1];
+ return field_names.get(key);
+ }
+
+ return null;
+ }
+
+ private void prepare_search_query(Geary.SearchQuery query) {
+ if (query.parsed)
+ return;
+
+ // A few goals here:
+ // 1) Append an * after every term so it becomes a prefix search
+ // (see <https://www.sqlite.org/fts3.html#section_3>)
+ // 2) Strip out common words/operators that might get interpreted as
+ // search operators
+ // 3) Parse each word into a list of which field it applies to, so
+ // you can do "to:johndoe example com thing" (quotes excluded)
+ // to find messages to John containing the word thing
// We ignore everything inside quotes to give the user a way to
// override our algorithm here. The idea is to offer one search query
// syntax for Geary that we can use locally and via IMAP, etc.
- string quote_balanced = raw_query;
- if (Geary.String.count_char(raw_query, '"') % 2 != 0) {
+ string quote_balanced = query.raw;
+ if (Geary.String.count_char(query.raw, '"') % 2 != 0) {
// Remove the last quote if it's not balanced. This has the
// benefit of showing decent results as you type a quoted phrase.
- int last_quote = raw_query.last_index_of_char('"');
+ int last_quote = query.raw.last_index_of_char('"');
assert(last_quote >= 0);
- quote_balanced = raw_query.splice(last_quote, last_quote + 1, " ");
+ quote_balanced = query.raw.splice(last_quote, last_quote + 1, " ");
}
- string[] words = quote_balanced.split_set(" \t\r\n:()%*\\");
+ string[] words = quote_balanced.split_set(" \t\r\n()%*\\");
bool in_quote = false;
- StringBuilder prepared_query = new StringBuilder();
foreach (string s in words) {
+ string? field = null;
+
s = s.strip();
int quotes = Geary.String.count_char(s, '"');
@@ -667,19 +708,39 @@ private class Geary.ImapDB.Account : BaseObject {
if (s == "")
continue;
+ // TODO: support quotes after :
+ string[] parts = s.split(":", 2);
+ if (parts.length > 1)
+ field = extract_field_from_token(parts, ref s);
+
s = "\"" + s + "*\"";
}
if (in_quote && quotes % 2 != 0)
in_quote = false;
- prepared_query.append(s);
- prepared_query.append(" ");
+ query.add_token(field, s);
}
assert(!in_quote);
- return prepared_query.str.strip();
+ query.parsed = true;
+ }
+
+ private Gee.HashMap<string, string> get_query_phrases(Geary.SearchQuery query) {
+ prepare_search_query(query);
+
+ Gee.HashMap<string, string> phrases = new Gee.HashMap<string, string>();
+ foreach (string? field in query.get_fields()) {
+ string? phrase = null;
+ Gee.List<string>? tokens = query.get_tokens(field);
+ if (tokens != null)
+ phrase = string.joinv(" ", tokens.to_array()).strip();
+
+ if (!Geary.String.is_empty(phrase))
+ phrases.set((field == null ? "MessageSearchTable" : field), phrase);
+ }
+ return phrases;
}
// Append each id in the collection to the StringBuilder, in a format
@@ -716,11 +777,16 @@ private class Geary.ImapDB.Account : BaseObject {
return sql.str;
}
- public async Gee.Collection<Geary.EmailIdentifier>? search_async(string prepared_query,
+ public async Gee.Collection<Geary.EmailIdentifier>? search_async(Geary.SearchQuery query,
int limit = 100, int offset = 0, Gee.Collection<Geary.FolderPath?>? folder_blacklist = null,
Gee.Collection<Geary.EmailIdentifier>? search_ids = null, Cancellable? cancellable = null) throws
Error {
check_open();
+ Gee.HashMap<string, string> query_phrases = get_query_phrases(query);
+ if (query_phrases.size == 0)
+ return null;
+ string[] ordered_fields = query_phrases.keys.to_array();
+
Gee.ArrayList<ImapDB.SearchEmailIdentifier> search_results
= new Gee.ArrayList<ImapDB.SearchEmailIdentifier>();
@@ -738,28 +804,35 @@ private class Geary.ImapDB.Account : BaseObject {
// own), it cuts the running time roughly in half of how it was
// before. The short version is: modify with extreme caution. See
// <http://redmine.yorba.org/issues/7372>.
- string sql = """
+ StringBuilder sql = new StringBuilder();
+ sql.append("""
SELECT id, internaldate_time_t
FROM MessageTable
INDEXED BY MessageTableInternalDateTimeTIndex
WHERE id IN (
SELECT id
FROM MessageSearchTable
- WHERE MessageSearchTable MATCH ?
- )
- """;
+ WHERE 1=1
+ """);
+ foreach (string field in ordered_fields)
+ sql.append(" AND %s MATCH ?".printf(field));
+ sql.append(")");
+
if (blacklisted_ids_sql != "")
- sql += " AND id NOT IN (%s)".printf(blacklisted_ids_sql);
+ sql.append(" AND id NOT IN (%s)".printf(blacklisted_ids_sql));
if (!Geary.String.is_empty(search_ids_sql))
- sql += " AND id IN (%s)".printf(search_ids_sql);
- sql += " ORDER BY internaldate_time_t DESC";
+ sql.append(" AND id IN (%s)".printf(search_ids_sql));
+ sql.append(" ORDER BY internaldate_time_t DESC");
if (limit > 0)
- sql += " LIMIT ? OFFSET ?";
- Db.Statement stmt = cx.prepare(sql);
- stmt.bind_string(0, prepared_query);
+ sql.append(" LIMIT ? OFFSET ?");
+
+ Db.Statement stmt = cx.prepare(sql.str);
+ int bind_index = 0;
+ foreach (string field in ordered_fields)
+ stmt.bind_string(bind_index++, query_phrases.get(field));
if (limit > 0) {
- stmt.bind_int(1, limit);
- stmt.bind_int(2, offset);
+ stmt.bind_int(bind_index++, limit);
+ stmt.bind_int(bind_index++, offset);
}
Db.Result result = stmt.exec(cancellable);
@@ -796,7 +869,7 @@ private class Geary.ImapDB.Account : BaseObject {
}
}
- public async Gee.Collection<string>? get_search_matches_async(string raw_query, string prepared_query,
+ public async Gee.Collection<string>? get_search_matches_async(Geary.SearchQuery query,
Gee.Collection<ImapDB.EmailIdentifier> ids, Cancellable? cancellable = null) throws Error {
check_open();
@@ -847,7 +920,7 @@ private class Geary.ImapDB.Account : BaseObject {
return Db.TransactionOutcome.DONE;
}, cancellable);
- add_literal_matches(raw_query, search_matches);
+ add_literal_matches(query.raw, search_matches);
return (search_matches.size == 0 ? null : search_matches);
}
diff --git a/src/engine/imap-engine/imap-engine-generic-account.vala
b/src/engine/imap-engine/imap-engine-generic-account.vala
index d62f639..d26aef2 100644
--- a/src/engine/imap-engine/imap-engine-generic-account.vala
+++ b/src/engine/imap-engine/imap-engine-generic-account.vala
@@ -567,20 +567,18 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount {
return yield local.fetch_email_async(check_id(email_id), required_fields, cancellable);
}
- public override async Gee.Collection<Geary.EmailIdentifier>? local_search_async(string query,
+ public override async Gee.Collection<Geary.EmailIdentifier>? local_search_async(Geary.SearchQuery query,
int limit = 100, int offset = 0, Gee.Collection<Geary.FolderPath?>? folder_blacklist = null,
Gee.Collection<Geary.EmailIdentifier>? search_ids = null, Cancellable? cancellable = null) throws
Error {
if (offset < 0)
throw new EngineError.BAD_PARAMETERS("Offset must not be negative");
- return yield local.search_async(local.prepare_search_query(query),
- limit, offset, folder_blacklist, search_ids, cancellable);
+ return yield local.search_async(query, limit, offset, folder_blacklist, search_ids, cancellable);
}
- public override async Gee.Collection<string>? get_search_matches_async(string query,
+ public override async Gee.Collection<string>? get_search_matches_async(Geary.SearchQuery query,
Gee.Collection<Geary.EmailIdentifier> ids, Cancellable? cancellable = null) throws Error {
- return yield local.get_search_matches_async(query, local.prepare_search_query(query),
- check_ids(ids), cancellable);
+ return yield local.get_search_matches_async(query, check_ids(ids), cancellable);
}
public override async Gee.MultiMap<Geary.EmailIdentifier, Geary.FolderPath>?
get_containing_folders_async(
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
