[libdazzle] recursive-monitor: add a recursive directory monitor

[Christian Hergert <chergert src gnome org>]

commit 16bc655d52c376c086ea5fdf47219cf73d6fe9ec
Author: Christian Hergert <chergert redhat com>
Date:   Thu Nov 30 00:03:41 2017 -0800

    recursive-monitor: add a recursive directory monitor
    
    Since GFileMonitor uses a single inotify FD, we can use a bunch
    of GFileMonitor while only using a single FD and not risk
    hitting the fd ulimit. We still have a max watch limit with
    inotify, but we're in better shape there as the max value is
    much larger. (We probably race against tracker there, but
    we'll see).

 src/dazzle.h                           |    1 +
 src/files/dzl-recursive-file-monitor.c |  400 ++++++++++++++++++++++++++++++++
 src/files/dzl-recursive-file-monitor.h |   33 +++
 src/files/meson.build                  |    2 +
 tests/meson.build                      |    9 +
 tests/test-recursive-monitor.c         |  142 +++++++++++
 6 files changed, 587 insertions(+), 0 deletions(-)
---
diff --git a/src/dazzle.h b/src/dazzle.h
index c5657bb..bfc8333 100644
--- a/src/dazzle.h
+++ b/src/dazzle.h
@@ -47,6 +47,7 @@ G_BEGIN_DECLS
 #include "cache/dzl-task-cache.h"
 #include "files/dzl-directory-model.h"
 #include "files/dzl-directory-reaper.h"
+#include "files/dzl-recursive-file-monitor.h"
 #include "graphing/dzl-cpu-graph.h"
 #include "graphing/dzl-cpu-model.h"
 #include "graphing/dzl-graph-column.h"
diff --git a/src/files/dzl-recursive-file-monitor.c b/src/files/dzl-recursive-file-monitor.c
new file mode 100644
index 0000000..c7bf08f
--- /dev/null
+++ b/src/files/dzl-recursive-file-monitor.c
@@ -0,0 +1,400 @@
+/* dzl-recursive-file-monitor.c
+ *
+ * Copyright (C) 2017 Christian Hergert <chergert redhat com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define G_LOG_DOMAIN "dzl-recursive-file-monitor"
+
+#include "files/dzl-recursive-file-monitor.h"
+#include "util/dzl-macros.h"
+
+struct _DzlRecursiveFileMonitor
+{
+  GObject       parent_instance;
+
+  GFile        *root;
+  GCancellable *cancellable;
+
+  GMutex        monitor_lock;
+  GHashTable   *monitors_by_file;
+  GHashTable   *files_by_monitor;
+
+  guint         start_handler;
+};
+
+enum {
+  PROP_0,
+  PROP_ROOT,
+  N_PROPS
+};
+
+enum {
+  CHANGED,
+  N_SIGNALS
+};
+
+static gboolean
+dzl_recursive_file_monitor_watch (DzlRecursiveFileMonitor  *self,
+                                  GFile                    *directory,
+                                  GCancellable             *cancellable,
+                                  GError                  **error);
+
+G_DEFINE_TYPE (DzlRecursiveFileMonitor, dzl_recursive_file_monitor, G_TYPE_OBJECT)
+
+static GParamSpec *properties [N_PROPS];
+static guint signals [N_SIGNALS];
+
+static void
+dzl_recursive_file_monitor_unwatch (DzlRecursiveFileMonitor *self,
+                                    GFile                   *file)
+{
+  GFileMonitor *monitor;
+
+  g_assert (DZL_IS_RECURSIVE_FILE_MONITOR (self));
+  g_assert (G_IS_FILE (file));
+
+  monitor = g_hash_table_lookup (self->monitors_by_file, file);
+
+  if (monitor != NULL)
+    {
+      g_object_ref (monitor);
+      g_file_monitor_cancel (monitor);
+      g_hash_table_remove (self->monitors_by_file, file);
+      g_hash_table_remove (self->files_by_monitor, monitor);
+      g_object_unref (monitor);
+    }
+}
+
+static void
+dzl_recursive_file_monitor_changed (DzlRecursiveFileMonitor *self,
+                                    GFile                   *file,
+                                    GFile                   *other_file,
+                                    GFileMonitorEvent        event,
+                                    GFileMonitor            *monitor)
+{
+  g_assert (DZL_IS_RECURSIVE_FILE_MONITOR (self));
+  g_assert (G_IS_FILE (file));
+  g_assert (!other_file || G_IS_FILE (file));
+  g_assert (G_IS_FILE_MONITOR (monitor));
+
+  if (event == G_FILE_MONITOR_EVENT_DELETED)
+    {
+      g_mutex_lock (&self->monitor_lock);
+      if (g_hash_table_contains (self->monitors_by_file, file))
+        dzl_recursive_file_monitor_unwatch (self, file);
+      g_mutex_unlock (&self->monitor_lock);
+    }
+  else if (event == G_FILE_MONITOR_EVENT_CREATED)
+    {
+      if (g_file_query_file_type (file, 0, NULL) == G_FILE_TYPE_DIRECTORY)
+        {
+          g_mutex_lock (&self->monitor_lock);
+          dzl_recursive_file_monitor_watch (self, file, self->cancellable, NULL);
+          g_mutex_unlock (&self->monitor_lock);
+        }
+    }
+
+  g_signal_emit (self, signals [CHANGED], 0, file, other_file, event);
+}
+
+static gboolean
+dzl_recursive_file_monitor_watch (DzlRecursiveFileMonitor  *self,
+                                  GFile                    *directory,
+                                  GCancellable             *cancellable,
+                                  GError                  **error)
+{
+  g_autoptr(GFileMonitor) monitor = NULL;
+
+  g_assert (DZL_IS_RECURSIVE_FILE_MONITOR (self));
+  g_assert (G_IS_FILE (directory));
+  g_assert (G_IS_CANCELLABLE (cancellable));
+
+  monitor = g_file_monitor_directory (directory, 0, cancellable, error);
+
+  if (monitor == NULL)
+    return FALSE;
+
+  g_signal_connect_object (monitor,
+                           "changed",
+                           G_CALLBACK (dzl_recursive_file_monitor_changed),
+                           self,
+                           G_CONNECT_SWAPPED);
+
+  g_hash_table_insert (self->monitors_by_file,
+                       g_object_ref (directory),
+                       g_object_ref (monitor));
+
+  g_hash_table_insert (self->files_by_monitor,
+                       g_object_ref (monitor),
+                       g_object_ref (directory));
+
+  return TRUE;
+}
+
+static void
+dzl_recursive_file_monitor_worker (GTask        *task,
+                                   gpointer      source_object,
+                                   gpointer      task_data,
+                                   GCancellable *cancellable)
+{
+  DzlRecursiveFileMonitor *self = source_object;
+  g_autoptr(GFileEnumerator) enumerator = NULL;
+  g_autoptr(GError) error = NULL;
+  gpointer infoptr;
+  GFile *root = task_data;
+
+  g_assert (DZL_IS_RECURSIVE_FILE_MONITOR (self));
+  g_assert (G_IS_FILE (root));
+
+  /* Short circuit if we were cancelled */
+  if (g_cancellable_is_cancelled (cancellable))
+    return;
+
+  /* Make sure our root is a directory that exists */
+  if (g_file_query_file_type (root, 0, cancellable) != G_FILE_TYPE_DIRECTORY)
+    return;
+
+  g_mutex_lock (&self->monitor_lock);
+
+  if (!dzl_recursive_file_monitor_watch (self, root, cancellable, &error))
+    goto cleanup;
+
+  enumerator = g_file_enumerate_children (root,
+                                          G_FILE_ATTRIBUTE_STANDARD_NAME","
+                                          G_FILE_ATTRIBUTE_STANDARD_TYPE,
+                                          G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                                          cancellable, &error);
+
+  while (NULL != (infoptr = g_file_enumerator_next_file (enumerator, cancellable, &error)))
+    {
+      g_autoptr(GFileInfo) info = infoptr;
+      g_autoptr(GFile) child = NULL;
+
+      if (g_file_info_get_file_type (info) != G_FILE_TYPE_DIRECTORY)
+        continue;
+
+      child = g_file_get_child (root, g_file_info_get_name (info));
+
+      if (!dzl_recursive_file_monitor_watch (self, child, cancellable, &error))
+        break;
+    }
+
+cleanup:
+  g_file_enumerator_close (enumerator, cancellable, NULL);
+
+  g_mutex_unlock (&self->monitor_lock);
+
+  if (error != NULL)
+    g_task_return_error (task, g_steal_pointer (&error));
+  else
+    g_task_return_boolean (task, TRUE);
+}
+
+static gboolean
+dzl_recursive_file_monitor_start (gpointer data)
+{
+  DzlRecursiveFileMonitor *self = data;
+  g_autoptr(GTask) task = NULL;
+
+  g_assert (DZL_IS_RECURSIVE_FILE_MONITOR (self));
+  g_assert (G_IS_FILE (self->root));
+
+  self->start_handler = 0;
+
+  task = g_task_new (self, self->cancellable, NULL, NULL);
+  g_task_set_source_tag (task, dzl_recursive_file_monitor_start);
+  g_task_set_priority (task, G_PRIORITY_LOW + 100);
+  g_task_set_task_data (task, g_object_ref (self->root), g_object_unref);
+  g_task_set_return_on_cancel (task, TRUE);
+  g_task_run_in_thread (task, dzl_recursive_file_monitor_worker);
+
+  return G_SOURCE_REMOVE;
+}
+
+static void
+dzl_recursive_file_monitor_constructed (GObject *object)
+{
+  DzlRecursiveFileMonitor *self = (DzlRecursiveFileMonitor *)object;
+
+  G_OBJECT_CLASS (dzl_recursive_file_monitor_parent_class)->constructed (object);
+
+  if (self->root == NULL)
+    {
+      g_warning ("%s created without a root directory", G_OBJECT_TYPE_NAME (self));
+      return;
+    }
+
+  self->start_handler = g_idle_add (dzl_recursive_file_monitor_start, self);
+}
+
+static void
+dzl_recursive_file_monitor_dispose (GObject *object)
+{
+  DzlRecursiveFileMonitor *self = (DzlRecursiveFileMonitor *)object;
+
+  dzl_clear_source (&self->start_handler);
+  g_cancellable_cancel (self->cancellable);
+
+  G_OBJECT_CLASS (dzl_recursive_file_monitor_parent_class)->dispose (object);
+}
+
+static void
+dzl_recursive_file_monitor_finalize (GObject *object)
+{
+  DzlRecursiveFileMonitor *self = (DzlRecursiveFileMonitor *)object;
+
+  g_clear_object (&self->root);
+  g_clear_object (&self->cancellable);
+
+  g_clear_pointer (&self->files_by_monitor, g_hash_table_unref);
+  g_clear_pointer (&self->monitors_by_file, g_hash_table_unref);
+
+  G_OBJECT_CLASS (dzl_recursive_file_monitor_parent_class)->finalize (object);
+}
+
+static void
+dzl_recursive_file_monitor_get_property (GObject    *object,
+                                         guint       prop_id,
+                                         GValue     *value,
+                                         GParamSpec *pspec)
+{
+  DzlRecursiveFileMonitor *self = DZL_RECURSIVE_FILE_MONITOR (object);
+
+  switch (prop_id)
+    {
+    case PROP_ROOT:
+      g_value_set_object (value, dzl_recursive_file_monitor_get_root (self));
+      break;
+
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+    }
+}
+
+static void
+dzl_recursive_file_monitor_set_property (GObject      *object,
+                                         guint         prop_id,
+                                         const GValue *value,
+                                         GParamSpec   *pspec)
+{
+  DzlRecursiveFileMonitor *self = DZL_RECURSIVE_FILE_MONITOR (object);
+
+  switch (prop_id)
+    {
+    case PROP_ROOT:
+      self->root = g_value_dup_object (value);
+      break;
+
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+    }
+}
+
+static void
+dzl_recursive_file_monitor_class_init (DzlRecursiveFileMonitorClass *klass)
+{
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
+
+  object_class->constructed = dzl_recursive_file_monitor_constructed;
+  object_class->dispose = dzl_recursive_file_monitor_dispose;
+  object_class->finalize = dzl_recursive_file_monitor_finalize;
+  object_class->get_property = dzl_recursive_file_monitor_get_property;
+  object_class->set_property = dzl_recursive_file_monitor_set_property;
+
+  properties [PROP_ROOT] =
+    g_param_spec_object ("root",
+                         "Root",
+                         "The root directory to monitor",
+                         G_TYPE_FILE,
+                         (G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS));
+  
+  g_object_class_install_properties (object_class, N_PROPS, properties);
+
+  /**
+   * DzlRecursiveFileMonitor::changed:
+   * @self: a #DzlRecursiveFileMonitor
+   * @file: a #GFile
+   * @other_file: (nullable): a #GFile for the other file when applicable
+   * @event: the #GFileMonitorEvent event
+   *
+   * This event is similar to #GFileMonitor::changed but can be fired from
+   * any of the monitored directories in the recursive mount.
+   *
+   * Since: 3.28
+   */
+  signals [CHANGED] =
+    g_signal_new ("changed",
+                  G_TYPE_FROM_CLASS (klass),
+                  G_SIGNAL_RUN_LAST,
+                  0, NULL, NULL, NULL,
+                  G_TYPE_NONE, 3, G_TYPE_FILE, G_TYPE_FILE, G_TYPE_FILE_MONITOR_EVENT);
+}
+
+static void
+dzl_recursive_file_monitor_init (DzlRecursiveFileMonitor *self)
+{
+  g_mutex_init (&self->monitor_lock);
+  self->cancellable = g_cancellable_new ();
+  self->files_by_monitor = g_hash_table_new_full (NULL, NULL, g_object_unref, g_object_unref);
+  self->monitors_by_file = g_hash_table_new_full (g_file_hash,
+                                                  (GEqualFunc) g_file_equal,
+                                                  g_object_unref,
+                                                  g_object_unref);
+}
+
+DzlRecursiveFileMonitor *
+dzl_recursive_file_monitor_new (GFile *file)
+{
+  g_return_val_if_fail (G_IS_FILE (file), NULL);
+
+  return g_object_new (DZL_TYPE_RECURSIVE_FILE_MONITOR,
+                       "root", file,
+                       NULL);
+}
+
+/**
+ * dzl_recursive_file_monitor_cancel:
+ * @self: a #DzlRecursiveFileMonitor
+ *
+ * Cancels the recursive file monitor.
+ *
+ * Since: 3.28
+ */
+void
+dzl_recursive_file_monitor_cancel (DzlRecursiveFileMonitor *self)
+{
+  g_return_if_fail (DZL_IS_RECURSIVE_FILE_MONITOR (self));
+
+  g_object_run_dispose (G_OBJECT (self));
+}
+
+/**
+ * dzl_recursive_file_monitor_get_root:
+ * @self: a #DzlRecursiveFileMonitor
+ *
+ * Gets the root directory used forthe file monitor.
+ *
+ * Returns: (transfer none): a #GFile
+ *
+ * Since: 3.28
+ */
+GFile *
+dzl_recursive_file_monitor_get_root (DzlRecursiveFileMonitor *self)
+{
+  g_return_val_if_fail (DZL_IS_RECURSIVE_FILE_MONITOR (self), NULL);
+
+  return self->root;
+}
diff --git a/src/files/dzl-recursive-file-monitor.h b/src/files/dzl-recursive-file-monitor.h
new file mode 100644
index 0000000..2174875
--- /dev/null
+++ b/src/files/dzl-recursive-file-monitor.h
@@ -0,0 +1,33 @@
+/* dzl-recursive-file-monitor.h
+ *
+ * Copyright (C) 2017 Christian Hergert <chergert redhat com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <gio/gio.h>
+
+G_BEGIN_DECLS
+
+#define DZL_TYPE_RECURSIVE_FILE_MONITOR (dzl_recursive_file_monitor_get_type())
+
+G_DECLARE_FINAL_TYPE (DzlRecursiveFileMonitor, dzl_recursive_file_monitor, DZL, RECURSIVE_FILE_MONITOR, 
GObject)
+
+DzlRecursiveFileMonitor *dzl_recursive_file_monitor_new      (GFile                   *root);
+GFile                   *dzl_recursive_file_monitor_get_root (DzlRecursiveFileMonitor *self);
+void                     dzl_recursive_file_monitor_cancel   (DzlRecursiveFileMonitor *self);
+
+G_END_DECLS
diff --git a/src/files/meson.build b/src/files/meson.build
index d777b6a..b574b15 100644
--- a/src/files/meson.build
+++ b/src/files/meson.build
@@ -1,11 +1,13 @@
 files_headers = [
   'dzl-directory-model.h',
   'dzl-directory-reaper.h',
+  'dzl-recursive-file-monitor.h',
 ]
 
 files_sources = [
   'dzl-directory-model.c',
   'dzl-directory-reaper.c',
+  'dzl-recursive-file-monitor.c',
 ]
 
 libdazzle_public_headers += files(files_headers)
diff --git a/tests/meson.build b/tests/meson.build
index 17d32cb..79bcd69 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -343,4 +343,13 @@ test_pattern_spec = executable('test-pattern-spec', 'test-pattern-spec.c',
 )
 test('test-pattern-spec', test_pattern_spec, env: test_env)
 
+# Test is not automated because file system notification is fairly
+# racey due to what else is going on with the system and how big
+# the notification queue depth is.
+test_recursive_monitor = executable('test-recursive-monitor', 'test-recursive-monitor.c',
+        c_args: test_cflags,
+     link_args: test_link_args,
+  dependencies: libdazzle_deps + [libdazzle_dep],
+)
+
 endif
diff --git a/tests/test-recursive-monitor.c b/tests/test-recursive-monitor.c
new file mode 100644
index 0000000..be6edc9
--- /dev/null
+++ b/tests/test-recursive-monitor.c
@@ -0,0 +1,142 @@
+#include <dazzle.h>
+#include <glib/gstdio.h>
+
+static const gchar *layer1[] = { "a", "b", "c", "d", "e", "f", "g", "h", "i", NULL };
+static const gchar *layer2[] = { "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", NULL };
+static GMainLoop *main_loop;
+static GHashTable *created;
+static GHashTable *deleted;
+
+static void
+sync_for_changes (void)
+{
+  GMainContext *context = g_main_loop_get_context (main_loop);
+  gint64 enter = g_get_monotonic_time ();
+
+  /*
+   * we need to spin a little bit while we wait for things
+   * to complete.
+   */
+
+  for (;;)
+    {
+      gint64 now;
+
+      if (g_main_context_pending (context))
+        g_main_context_iteration (context, FALSE);
+
+      now = g_get_monotonic_time ();
+
+      /* Spin at least a milliseconds */
+      if ((now - enter) > (G_USEC_PER_SEC / 1000))
+        break;
+    }
+
+  g_main_context_iteration (context, FALSE);
+}
+
+static void
+monitor_changed_cb (DzlRecursiveFileMonitor *monitor,
+                    GFile                   *file,
+                    GFile                   *other_file,
+                    GFileMonitorEvent        event,
+                    gpointer                 data)
+{
+  sync_for_changes ();
+
+  if (event == G_FILE_MONITOR_EVENT_CREATED)
+    g_hash_table_insert (created, g_object_ref (file), NULL);
+  else if (event == G_FILE_MONITOR_EVENT_DELETED)
+    g_hash_table_insert (deleted, g_object_ref (file), NULL);
+}
+
+static void
+test_basic (void)
+{
+  g_autoptr(DzlRecursiveFileMonitor) monitor = NULL;
+  g_autoptr(GFile) dir = g_file_new_for_path ("recursive-dir");
+  g_autoptr(GPtrArray) dirs = NULL;
+  gint r;
+
+  main_loop = g_main_loop_new (NULL, FALSE);
+
+  created = g_hash_table_new_full (g_file_hash, (GEqualFunc) g_file_equal, g_object_unref, g_object_unref);
+  deleted = g_hash_table_new_full (g_file_hash, (GEqualFunc) g_file_equal, g_object_unref, g_object_unref);
+
+  if (g_file_test ("recursive-dir", G_FILE_TEST_EXISTS))
+    {
+      g_autoptr(DzlDirectoryReaper) reaper = dzl_directory_reaper_new ();
+
+      dzl_directory_reaper_add_directory (reaper, dir, 0);
+      dzl_directory_reaper_execute (reaper, NULL, NULL);
+
+      r = g_rmdir ("recursive-dir");
+      g_assert_cmpint (r, ==, 0);
+    }
+
+  r = g_mkdir ("recursive-dir", 0750);
+  g_assert_cmpint (r, ==, 0);
+
+  monitor = dzl_recursive_file_monitor_new (dir);
+  g_assert (monitor != NULL);
+  sync_for_changes ();
+
+  g_signal_connect (monitor,
+                    "changed",
+                    G_CALLBACK (monitor_changed_cb),
+                    NULL);
+
+  /* Make a bunch of directories while we monitor. We'll add files to
+   * the directories afterwards, and then ensure we got notified. This
+   * allows us to ensure that we track changes as we add dirs.
+   */
+  dirs = g_ptr_array_new_with_free_func (g_object_unref);
+
+  for (guint i = 0; layer1[i]; i++)
+    {
+      g_autofree gchar *first = g_build_filename ("recursive-dir", layer1[i], NULL);
+      g_autoptr(GFile) file1 = g_file_new_for_path (first);
+
+      r = g_mkdir (first, 0750);
+      g_assert_cmpint (r, ==, 0);
+      sync_for_changes ();
+
+      g_ptr_array_add (dirs, g_object_ref (file1));
+
+      g_assert (g_hash_table_contains (created, file1));
+
+      for (guint j = 0; layer2[j]; j++)
+        {
+          g_autofree gchar *second = g_build_filename (first, layer2[j], NULL);
+          g_autoptr(GFile) file2 = g_file_new_for_path (second);
+
+          r = g_mkdir (second, 0750);
+          g_assert_cmpint (r, ==, 0);
+          sync_for_changes ();
+
+          g_assert (g_hash_table_contains (created, file2));
+
+          g_ptr_array_add (dirs, g_object_ref (file2));
+        }
+    }
+
+  for (guint i = dirs->len; i > 0; i--)
+    {
+      GFile *file = g_ptr_array_index (dirs, i - 1);
+
+      r = g_file_delete (file, NULL, NULL);
+      g_assert_cmpint (r, ==, TRUE);
+      sync_for_changes ();
+
+      g_assert (g_hash_table_contains (deleted, file));
+    }
+}
+
+gint
+main (gint argc,
+      gchar *argv[])
+{
+  g_test_init (&argc, &argv, NULL);
+  g_test_add_func ("/Dazzle/RecursiveFileMonitor/basic", test_basic);
+  return g_test_run ();
+}