[tracker] tracker-miner-fs: Implement writeback atop TrackerMinerFS



commit d1b0e46875cbdacfab044fb5efcaed2f4720cd2f
Author: Carlos Garnacho <carlosg gnome org>
Date:   Sun Jul 2 15:53:04 2017 +0200

    tracker-miner-fs: Implement writeback atop TrackerMinerFS
    
    This implementation just pauses the miner while there's writeback
    items to process, and uses the filter_event vmethod to perform
    maintenance of writeback tasks. Functionally, is not that different
    from the TrackerMinerFS implementation.

 src/miners/fs/tracker-miner-files.c          |  196 ++++++++++++++++++++++++++
 src/miners/fs/tracker-miner-files.h          |    8 +
 src/miners/fs/tracker-writeback-dispatcher.c |   18 ++--
 src/miners/fs/tracker-writeback-listener.c   |   10 +-
 4 files changed, 218 insertions(+), 14 deletions(-)
---
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index c27584c..5fa9e5c 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -108,6 +108,9 @@ struct TrackerMinerFilesPrivate {
        GList *extraction_queue;
 
        TrackerThumbnailer *thumbnailer;
+
+       GHashTable *writeback_tasks;
+       gboolean paused_for_writeback;
 };
 
 typedef struct {
@@ -125,6 +128,13 @@ enum {
        PROP_CONFIG
 };
 
+enum {
+       WRITEBACK,
+       N_SIGNALS
+};
+
+static guint signals[N_SIGNALS] = { 0 };
+
 static void        miner_files_set_property             (GObject              *object,
                                                          guint                 param_id,
                                                          const GValue         *value,
@@ -229,6 +239,89 @@ G_DEFINE_TYPE_WITH_CODE (TrackerMinerFiles, tracker_miner_files, TRACKER_TYPE_MI
                                                 miner_files_initable_iface_init));
 
 static void
+sync_writeback_pause_state (TrackerMinerFiles *mf)
+{
+       guint n_writeback_tasks = g_hash_table_size (mf->private->writeback_tasks);
+
+       if (n_writeback_tasks > 0 && !mf->private->paused_for_writeback) {
+               tracker_miner_pause (TRACKER_MINER (mf));
+               mf->private->paused_for_writeback = TRUE;
+       } else if (n_writeback_tasks == 0 && mf->private->paused_for_writeback) {
+               mf->private->paused_for_writeback = FALSE;
+               tracker_miner_resume (TRACKER_MINER (mf));
+       }
+}
+
+static void
+writeback_remove_recursively (TrackerMinerFiles *mf,
+                             GFile             *file)
+{
+       GHashTableIter iter;
+       GFile *writeback_file;
+
+       if (g_hash_table_size (mf->private->writeback_tasks) == 0)
+               return;
+
+       /* Remove and cancel writeback tasks in this directory */
+       g_hash_table_iter_init (&iter, mf->private->writeback_tasks);
+       while (g_hash_table_iter_next (&iter, (gpointer*) &writeback_file, NULL)) {
+               if (g_file_equal (writeback_file, file) ||
+                   g_file_has_prefix (writeback_file, file)) {
+                       g_hash_table_iter_remove (&iter);
+               }
+       }
+
+       sync_writeback_pause_state (mf);
+}
+
+static gboolean
+miner_files_filter_event (TrackerMinerFS          *fs,
+                          TrackerMinerFSEventType  type,
+                          GFile                   *file,
+                          GFile                   *source_file)
+{
+       TrackerMinerFiles *mf = TRACKER_MINER_FILES (fs);
+       GCancellable *cancellable;
+
+       switch (type) {
+       case TRACKER_MINER_FS_EVENT_CREATED:
+               break;
+       case TRACKER_MINER_FS_EVENT_UPDATED:
+               /* If the file is in the writeback task pool, this is the
+                * file update applying it, so the event should be filtered
+                * out.
+                */
+               if (g_hash_table_lookup_extended (mf->private->writeback_tasks, file,
+                                                 NULL, (gpointer*) &cancellable)) {
+                       if (!cancellable) {
+                               /* The task was already notified, we can remove
+                                * it now, so later updates will be processed.
+                                */
+                               g_hash_table_remove (mf->private->writeback_tasks, file);
+                               sync_writeback_pause_state (mf);
+                       }
+
+                       /* There is a writeback task, pending or satisfied.
+                        * Either way, this update should be ignored.
+                        */
+                       return TRUE;
+               }
+               break;
+       case TRACKER_MINER_FS_EVENT_DELETED:
+               writeback_remove_recursively (mf, file);
+               break;
+       case TRACKER_MINER_FS_EVENT_MOVED:
+               /* If the origin file is also being written back,
+                * cancel it as this is an external operation.
+                */
+               writeback_remove_recursively (mf, source_file);
+               break;
+       }
+
+       return FALSE;
+}
+
+static void
 tracker_miner_files_class_init (TrackerMinerFilesClass *klass)
 {
        GObjectClass *object_class = G_OBJECT_CLASS (klass);
@@ -244,6 +337,34 @@ tracker_miner_files_class_init (TrackerMinerFilesClass *klass)
        miner_fs_class->remove_file = miner_files_remove_file;
        miner_fs_class->remove_children = miner_files_remove_children;
        miner_fs_class->move_file = miner_files_move_file;
+       miner_fs_class->filter_event = miner_files_filter_event;
+
+       /**
+        * TrackerMinerFiles::writeback-file:
+        * @miner: the #TrackerMinerFiles
+        * @file: a #GFile
+        * @rdf_types: the set of RDF types
+        * @results: (element-type GStrv): a set of results prepared by the preparation query
+        * @cancellable: a #GCancellable
+        *
+        * The ::writeback-file signal is emitted whenever a file must be written
+        * back
+        *
+        * Returns: %TRUE on success, %FALSE otherwise
+        **/
+       signals[WRITEBACK] =
+               g_signal_new ("writeback",
+                             G_OBJECT_CLASS_TYPE (object_class),
+                             G_SIGNAL_RUN_LAST,
+                             0, NULL,
+                             NULL,
+                             NULL,
+                             G_TYPE_BOOLEAN,
+                             4,
+                             G_TYPE_FILE,
+                             G_TYPE_STRV,
+                             G_TYPE_PTR_ARRAY,
+                             G_TYPE_CANCELLABLE);
 
        g_object_class_install_property (object_class,
                                         PROP_CONFIG,
@@ -259,6 +380,17 @@ tracker_miner_files_class_init (TrackerMinerFilesClass *klass)
 }
 
 static void
+cancel_and_unref (gpointer data)
+{
+       GCancellable *cancellable = data;
+
+       if (cancellable) {
+               g_cancellable_cancel (cancellable);
+               g_object_unref (cancellable);
+       }
+}
+
+static void
 tracker_miner_files_init (TrackerMinerFiles *mf)
 {
        TrackerMinerFilesPrivate *priv;
@@ -297,6 +429,10 @@ tracker_miner_files_init (TrackerMinerFiles *mf)
 
        priv->mtime_check = TRUE;
        priv->quark_mount_point_uuid = g_quark_from_static_string ("tracker-mount-point-uuid");
+
+       priv->writeback_tasks = g_hash_table_new_full (g_file_hash,
+                                                      (GEqualFunc) g_file_equal,
+                                                      NULL, cancel_and_unref);
 }
 
 static void
@@ -330,6 +466,8 @@ miner_files_initable_init (GInitable     *initable,
        fs = TRACKER_MINER_FS (initable);
        indexing_tree = tracker_miner_fs_get_indexing_tree (fs);
        tracker_indexing_tree_set_filter_hidden (indexing_tree, TRUE);
+       g_signal_connect_swapped (indexing_tree, "directory-removed",
+                                 G_CALLBACK (writeback_remove_recursively), mf);
 
        miner_files_update_filters (mf);
 
@@ -682,6 +820,7 @@ miner_files_finalize (GObject *object)
        }
 
        g_list_free (priv->extraction_queue);
+       g_hash_table_destroy (priv->writeback_tasks);
 
        G_OBJECT_CLASS (tracker_miner_files_parent_class)->finalize (object);
 }
@@ -3359,3 +3498,60 @@ tracker_miner_files_set_mtime_checking (TrackerMinerFiles *mf,
 {
        mf->private->mtime_check = mtime_check;
 }
+
+void
+tracker_miner_files_writeback_file (TrackerMinerFiles *mf,
+                                    GFile             *file,
+                                    GStrv              rdf_types,
+                                    GPtrArray         *results)
+{
+       GCancellable *cancellable;
+
+       if (!g_hash_table_contains (mf->private->writeback_tasks, file)) {
+               cancellable = g_cancellable_new ();
+               g_hash_table_insert (mf->private->writeback_tasks, file, cancellable);
+               sync_writeback_pause_state (mf);
+               g_signal_emit (mf, signals[WRITEBACK], 0, file, rdf_types,
+                              results, cancellable);
+       }
+}
+
+/**
+ * tracker_miner_files_writeback_notify:
+ * @fs: a #TrackerMinerFS
+ * @file: a #GFile
+ * @error: a #GError with the error that happened during processing, or %NULL.
+ *
+ * Notifies @fs that all writing back on @file has been finished, if any error
+ * happened during file data processing, it should be passed in @error, else
+ * that parameter will contain %NULL to reflect success.
+ **/
+void
+tracker_miner_files_writeback_notify (TrackerMinerFiles *mf,
+                                      GFile             *file,
+                                      const GError      *error)
+{
+       GCancellable *cancellable;
+
+       g_return_if_fail (TRACKER_IS_MINER_FILES (mf));
+       g_return_if_fail (G_IS_FILE (file));
+
+       cancellable = g_hash_table_lookup (mf->private->writeback_tasks, file);
+
+       if (!cancellable)
+               return;
+
+       if (error) {
+               gchar *uri = g_file_get_uri (file);
+               g_warning ("Writeback on %s got error: %s\n",
+                          uri, error->message);
+               g_free (uri);
+       }
+
+       /* Drop the cancellable, it will be detected on the next file
+        * update in miner_files_filter_event().
+        */
+       g_hash_table_steal (mf->private->writeback_tasks, file);
+       g_hash_table_insert (mf->private->writeback_tasks, file, NULL);
+       g_object_unref (cancellable);
+}
diff --git a/src/miners/fs/tracker-miner-files.h b/src/miners/fs/tracker-miner-files.h
index c9522ec..c671407 100644
--- a/src/miners/fs/tracker-miner-files.h
+++ b/src/miners/fs/tracker-miner-files.h
@@ -82,6 +82,14 @@ void     tracker_miner_files_set_need_mtime_check (gboolean needed);
 void     tracker_miner_files_set_mtime_checking   (TrackerMinerFiles *miner,
                                                    gboolean           mtime_checking);
 
+void     tracker_miner_files_writeback_file       (TrackerMinerFiles *mf,
+                                                   GFile             *file,
+                                                   GStrv              rdf_types,
+                                                   GPtrArray         *results);
+void     tracker_miner_files_writeback_notify     (TrackerMinerFiles *mf,
+                                                   GFile             *file,
+                                                   const GError      *error);
+
 G_END_DECLS
 
 #endif /* __TRACKER_MINER_FS_FILES_H__ */
diff --git a/src/miners/fs/tracker-writeback-dispatcher.c b/src/miners/fs/tracker-writeback-dispatcher.c
index 60765da..fc4e906 100644
--- a/src/miners/fs/tracker-writeback-dispatcher.c
+++ b/src/miners/fs/tracker-writeback-dispatcher.c
@@ -21,9 +21,9 @@
 
 #include <libtracker-common/tracker-dbus.h>
 #include <libtracker-sparql/tracker-sparql.h>
-#include <libtracker-miner/tracker-miner.h>
 
 #include "tracker-writeback-dispatcher.h"
+#include "tracker-miner-files.h"
 
 #define TRACKER_WRITEBACK_SERVICE   "org.freedesktop.Tracker1.Writeback"
 #define TRACKER_WRITEBACK_PATH      "/org/freedesktop/Tracker1/Writeback"
@@ -31,7 +31,7 @@
 
 typedef struct {
        GFile *file;
-       TrackerMinerFS *fs;
+       TrackerMinerFiles *fs;
        GPtrArray *results;
        GStrv rdf_types;
        TrackerWritebackDispatcher *self; /* weak */
@@ -196,7 +196,7 @@ writeback_dispatcher_initable_init (GInitable    *initable,
        }
 
        g_signal_connect_object (priv->files_miner,
-                                "writeback-file",
+                                "writeback",
                                 G_CALLBACK (writeback_dispatcher_writeback_file),
                                 initable,
                                 G_CONNECT_AFTER);
@@ -262,10 +262,10 @@ retry_idle (gpointer user_data)
 {
        WritebackFileData *data = user_data;
 
-       tracker_miner_fs_writeback_file (data->fs,
-                                        data->file,
-                                        data->rdf_types,
-                                        data->results);
+       tracker_miner_files_writeback_file (data->fs,
+                                           data->file,
+                                           data->rdf_types,
+                                           data->results);
 
        writeback_file_data_free (data);
 
@@ -288,13 +288,13 @@ writeback_file_finished  (GObject      *source_object,
                 * happens on unmount of the FS event, for example */
 
                g_debug ("Retrying write-back after unmount (timeout in 5 seconds)");
-               tracker_miner_fs_writeback_notify (data->fs, data->file, NULL);
+               tracker_miner_files_writeback_notify (data->fs, data->file, NULL);
 
                data->retry_timeout = g_timeout_add_seconds (5, retry_idle, data);
                data->retries++;
 
        } else {
-               tracker_miner_fs_writeback_notify (data->fs, data->file, error);
+               tracker_miner_files_writeback_notify (data->fs, data->file, error);
                writeback_file_data_free (data);
        }
 }
diff --git a/src/miners/fs/tracker-writeback-listener.c b/src/miners/fs/tracker-writeback-listener.c
index 36ebec6..c861cae 100644
--- a/src/miners/fs/tracker-writeback-listener.c
+++ b/src/miners/fs/tracker-writeback-listener.c
@@ -21,9 +21,9 @@
 
 #include <libtracker-common/tracker-dbus.h>
 #include <libtracker-sparql/tracker-sparql.h>
-#include <libtracker-miner/tracker-miner.h>
 
 #include "tracker-writeback-listener.h"
+#include "tracker-miner-files.h"
 
 #define TRACKER_SERVICE                 "org.freedesktop.Tracker1"
 #define TRACKER_RESOURCES_OBJECT        "/org/freedesktop/Tracker1/Resources"
@@ -332,10 +332,10 @@ sparql_query_cb (GObject      *object,
                }
 
                if (results != NULL && results->len > 0) {
-                       tracker_miner_fs_writeback_file (TRACKER_MINER_FS (priv->files_miner),
-                                                        file,
-                                                        data->rdf_types,
-                                                        results);
+                       tracker_miner_files_writeback_file (priv->files_miner,
+                                                           file,
+                                                           data->rdf_types,
+                                                           results);
                } else {
                        g_message ("  No files qualify for updates");
                }


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]