[tracker] libtracker-miner: Disable monitors during crawling on large indexing trees

[Carlos Garnacho <carlosg src gnome org>]

commit 1ee338bcf8d7303d262384d49a3c0319bb8f2970
Author: Carlos Garnacho <carlosg gnome org>
Date:   Tue Jul 11 01:13:21 2017 +0200

    libtracker-miner: Disable monitors during crawling on large indexing trees
    
    If the number of indexed folders exceed the TrackerMonitor limit,
    acknowledge that the filesystem image shall be possibly out of date soon
    and disable monitors temporarily during crawling.
    
    This results in a major speedup in the crawling phase (both initial indexing
    and the one performed on every startup to update the Tracker concept of the
    filesystem), because an excessive amount of directory monitors make the
    syscalls performed during crawling inherently slower.
    
    Testing with the same sample folders than commit 2e7e70a2ea (and the same
    quarter million files in 17k folders), this results on ~4x faster
    tracker-miner-fs startups. The accumulated improvements took regular startup
    stituations (i.e. just ensuring the filesystem is up-to-date) from roughly
    4m to 22s here. Everything measured on i7/ssd, otherwise idle, ...

 src/libtracker-miner/tracker-file-notifier.c |   59 ++++++++++++++++++++++++++
 1 files changed, 59 insertions(+), 0 deletions(-)
---
diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c
index 495685a..0f847c3 100644
--- a/src/libtracker-miner/tracker-file-notifier.c
+++ b/src/libtracker-miner/tracker-file-notifier.c
@@ -1639,6 +1639,43 @@ tracker_file_notifier_finalize (GObject *object)
 }
 
 static void
+check_disable_monitor (TrackerFileNotifier *notifier)
+{
+       TrackerFileNotifierPrivate *priv;
+       TrackerSparqlCursor *cursor;
+       gint64 folder_count = 0;
+       GError *error = NULL;
+
+       priv = notifier->priv;
+       cursor = tracker_sparql_connection_query (priv->connection,
+                                                 "SELECT COUNT(?f) { ?f a nfo:Folder }",
+                                                 NULL, &error);
+
+       if (!error && tracker_sparql_cursor_next (cursor, NULL, &error)) {
+               folder_count = tracker_sparql_cursor_get_integer (cursor, 0);
+               tracker_sparql_cursor_close (cursor);
+       }
+
+       if (error) {
+               g_warning ("Could not get folder count: %s\n", error->message);
+               g_error_free (error);
+       } else if (folder_count > tracker_monitor_get_limit (priv->monitor)) {
+               /* If the folder count exceeds the monitor limit, there's
+                * nothing we can do anyway to prevent possibly out of date
+                * content. As it is the case no matter what we try, fully
+                * embrace it instead, and disable monitors until after crawling
+                * has been performed. This dramatically improves crawling time
+                * as monitors are inherently expensive.
+                */
+               g_info ("Temporarily disabling monitors until crawling is "
+                       "completed. Too many folders to monitor anyway");
+               tracker_monitor_set_enabled (priv->monitor, FALSE);
+       }
+
+       g_clear_object (&cursor);
+}
+
+static void
 tracker_file_notifier_constructed (GObject *object)
 {
        TrackerFileNotifierPrivate *priv;
@@ -1683,6 +1720,26 @@ tracker_file_notifier_constructed (GObject *object)
        g_signal_connect (priv->crawler, "finished",
                          G_CALLBACK (crawler_finished_cb),
                          object);
+
+       check_disable_monitor (TRACKER_FILE_NOTIFIER (object));
+}
+
+static void
+tracker_file_notifier_real_finished (TrackerFileNotifier *notifier)
+{
+       TrackerFileNotifierPrivate *priv;
+
+       priv = notifier->priv;
+
+       if (!tracker_monitor_get_enabled (priv->monitor)) {
+               /* If the monitor was disabled on ::constructed (see
+                * check_disable_monitor()), enable it back again.
+                * This will lazily create all missing directory
+                * monitors.
+                */
+               g_info ("Re-enabling directory monitors");
+               tracker_monitor_set_enabled (priv->monitor, TRUE);
+       }
 }
 
 static void
@@ -1695,6 +1752,8 @@ tracker_file_notifier_class_init (TrackerFileNotifierClass *klass)
        object_class->get_property = tracker_file_notifier_get_property;
        object_class->constructed = tracker_file_notifier_constructed;
 
+       klass->finished = tracker_file_notifier_real_finished;
+
        signals[FILE_CREATED] =
                g_signal_new ("file-created",
                              G_TYPE_FROM_CLASS (klass),