[tracker-miners/sam/file-processed-signal: 17/18] functional-tests: Add miner-on-demand test

[Sam Thursfield <sthursfield src gnome org>]

commit 5d2c92c9eabce2b3a3a0dab45005a0fb976ddb62
Author: Sam Thursfield <sam afuera me uk>
Date:   Mon Mar 9 01:47:31 2020 +0100

    functional-tests: Add miner-on-demand test
    
    This is for testing IndexFile, IndexFileForProcess and the
    FileProcessed signal.

 tests/functional-tests/fixtures.py        |  50 +++++++-
 tests/functional-tests/helpers.py         | 201 ++++++++++++++++++++++++++++++
 tests/functional-tests/meson.build        |   1 +
 tests/functional-tests/miner-on-demand.py | 130 +++++++++++++++++++
 tests/functional-tests/minerfshelper.py   | 139 ---------------------
 5 files changed, 376 insertions(+), 145 deletions(-)
---
diff --git a/tests/functional-tests/fixtures.py b/tests/functional-tests/fixtures.py
index b48447e33..5d3bca03b 100644
--- a/tests/functional-tests/fixtures.py
+++ b/tests/functional-tests/fixtures.py
@@ -25,7 +25,7 @@ Fixtures used by the tracker-miners functional-tests.
 import gi
 gi.require_version('Gst', '1.0')
 gi.require_version('Tracker', '3.0')
-from gi.repository import GLib
+from gi.repository import Gio, GLib
 from gi.repository import Tracker
 
 import errno
@@ -42,7 +42,7 @@ import unittest as ut
 import trackertestutils.dconf
 import trackertestutils.helpers
 import configuration as cfg
-from minerfshelper import MinerFsHelper
+import helpers
 
 log = logging.getLogger(__name__)
 
@@ -68,6 +68,18 @@ def tracker_test_main():
                             handlers=[handler_stderr, handler_stdout],
                             format='%(message)s')
 
+    # Avoid the test process sending itself SIGTERM via the GDBusConnection
+    # 'exit-on-close' feature.
+    #
+    # This can happen if something calls a g_bus_get_*() function -- I saw
+    # problems when I started using GFile in a test, which causes GIO and GVFS
+    # to connect to the session bus in the background. There may be an
+    # underlying bug at work, but anyway it's important that our tests don't
+    # randomly terminate themselves.
+    #
+    session_bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+    session_bus.set_exit_on_close(False)
+
     ut.main(failfast=True, verbosity=2)
 
 
@@ -113,9 +125,10 @@ class TrackerMinerTest(ut.TestCase):
                 for key, value in contents.items():
                     dconf.write(schema_name, key, value)
 
-            self.miner_fs = MinerFsHelper(self.sandbox.get_connection())
+            self.miner_fs = helpers.MinerFsHelper(self.sandbox.get_connection())
             self.miner_fs.start()
-            self.miner_fs.start_watching_progress()
+
+            self.extractor = helpers.ExtractorHelper(self.sandbox.get_connection())
 
             self.tracker = trackertestutils.helpers.StoreHelper(
                 self.miner_fs.get_sparql_connection())
@@ -141,8 +154,32 @@ class TrackerMinerTest(ut.TestCase):
         if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == True:
             self.fail("Resource <%s> should not exist" % urn)
 
-    def await_document_inserted(self, path, content=None):
+    def assertFileNotIndexed(self, url_or_path):
+        if isinstance(url_or_path, pathlib.Path):
+            url = url_or_path.as_uri()
+        else:
+            assert url_or_path.startswith('file://')
+            url = url_or_path
+
+        if self.tracker.ask("ASK { ?r a rdfs:Resource ; nie:url <%s> }" % url) == True:
+            self.fail("File <%s> should not be indexed" % url)
+
+    def assertFileIndexed(self, url_or_path):
+        if isinstance(url_or_path, pathlib.Path):
+            url = url_or_path.as_uri()
+        else:
+            assert url_or_path.startswith('file://')
+            url = url_or_path
+
+        if self.tracker.ask("ASK { ?r a rdfs:Resource ; nie:url <%s> }" % url) == False:
+            self.fail("File <%s> should be indexed, but is not." % url)
+
+    def await_document_inserted(self, path, content=None, timeout=cfg.DEFAULT_TIMEOUT):
         """Wraps await_insert() context manager."""
+
+        # Safety check, you can get confused if you pass a URI because
+        # pathlib.Path.as_uri() will convert a valid URI into an invalid one.
+        assert isinstance(path, pathlib.Path) or not path.startswith('file://')
         url = self.uri(path)
 
         expected = [
@@ -154,7 +191,8 @@ class TrackerMinerTest(ut.TestCase):
             content_escaped = Tracker.sparql_escape_string(content)
             expected += [f'nie:plainTextContent "{content_escaped}"']
 
-        return self.tracker.await_insert(DOCUMENTS_GRAPH, '; '.join(expected))
+        return self.tracker.await_insert(
+            DOCUMENTS_GRAPH, '; '.join(expected), timeout=timeout)
 
     def await_document_uri_change(self, resource_id, from_path, to_path):
         """Wraps await_update() context manager."""
diff --git a/tests/functional-tests/helpers.py b/tests/functional-tests/helpers.py
new file mode 100644
index 000000000..8e47b7e48
--- /dev/null
+++ b/tests/functional-tests/helpers.py
@@ -0,0 +1,201 @@
+#
+# Copyright (C) 2010, Nokia <ivan frade nokia com>
+# Copyright (C) 2018,2020 Sam Thursfield <sam afuera me uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+import gi
+gi.require_version('Tracker', '3.0')
+from gi.repository import Gio, GLib, GObject
+from gi.repository import Tracker
+
+import dataclasses
+import logging
+import pathlib
+
+import trackertestutils.mainloop
+
+import configuration
+
+
+log = logging.getLogger(__name__)
+
+
+class WakeupCycleTimeoutException(RuntimeError):
+    pass
+
+
+@dataclasses.dataclass
+class FileProcessedResult():
+    path : pathlib.Path
+    status : bool
+
+
+class FileProcessedError(Exception):
+    pass
+
+
+class await_files_processed():
+    """Context manager to await file processing by Tracker miners & extractors.
+
+    """
+    def __init__(self, miner, expected, timeout=None):
+        self.miner = miner
+        self.expected = expected
+        self.timeout = timeout or configuration.DEFAULT_TIMEOUT
+
+        self.loop = trackertestutils.mainloop.MainLoop()
+
+        self.gfiles = []
+        self.status = {}
+        for result in self.expected:
+            gfile = Gio.File.new_for_path(str(result.path))
+            self.gfiles.append(gfile)
+            self.status[gfile.get_path()] = result.status
+
+    def __enter__(self):
+        if len(self.expected) == 1:
+            log.info("Awaiting files-processed signal from %s for file %s",
+                     self.miner, self.expected[0].path)
+        else:
+            log.info("Awaiting %i files-processed signals from %s",
+                     len(self.expected), self.miner)
+
+        def signal_cb(proxy, sender_name, signal_name, parameters):
+            if signal_name == 'FilesProcessed':
+                array = parameters.unpack()[0]
+                for uri, success, message in array:
+                    log.debug("Processing file-processed event for %s", uri)
+
+                    matched = False
+
+                    signal_gfile = Gio.File.new_for_uri(uri)
+
+                    expected_gfile = None
+                    expected_status = None
+                    for expected_gfile in self.gfiles:
+                        if expected_gfile.get_path() == signal_gfile.get_path():
+                            expected_status = self.status[expected_gfile.get_path()]
+                            if success == expected_status:
+                                log.debug("Matched %s", uri)
+                                matched = True
+                                break
+                            else:
+                                raise FileProcessedError(f"{uri}: Expected status {expected_status}, got 
{success}")
+
+                    if matched:
+                        self.gfiles.remove(expected_gfile)
+
+                    if len(self.gfiles) == 0:
+                        log.info("All files were processed, exiting loop.")
+                        self.loop.quit()
+
+        def timeout_cb():
+            log.info("Timeout fired after %s seconds", self.timeout)
+            raise trackertestutils.helpers.AwaitTimeoutException(
+                f"Timeout awaiting files-processed signal on {self.miner}")
+
+        self.signal_id = self.miner.connect('g-signal', signal_cb)
+        self.timeout_id = GLib.timeout_add_seconds(self.timeout, timeout_cb)
+
+        return
+
+    def __exit__(self, etype, evalue, etraceback):
+        if etype is not None:
+            return False
+
+        self.loop.run_checked()
+        log.debug("Main loop finished")
+
+        GLib.source_remove(self.timeout_id)
+        GObject.signal_handler_disconnect(self.miner, self.signal_id)
+
+        return True
+
+
+class MinerFsHelper ():
+
+    MINERFS_BUSNAME = "org.freedesktop.Tracker3.Miner.Files"
+    MINERFS_OBJ_PATH = "/org/freedesktop/Tracker3/Miner/Files"
+    MINER_IFACE = "org.freedesktop.Tracker3.Miner"
+    MINERFS_INDEX_OBJ_PATH = "/org/freedesktop/Tracker3/Miner/Files/Index"
+    MINER_INDEX_IFACE = "org.freedesktop.Tracker3.Miner.Files.Index"
+
+    def __init__(self, dbus_connection):
+        self.log = logging.getLogger(__name__)
+
+        self.bus = dbus_connection
+
+        self.loop = trackertestutils.mainloop.MainLoop()
+
+        self.miner_iface = Gio.DBusProxy.new_sync(
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
+            self.MINERFS_BUSNAME, self.MINERFS_OBJ_PATH, self.MINER_IFACE)
+
+        self.index = Gio.DBusProxy.new_sync(
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
+            self.MINERFS_BUSNAME, self.MINERFS_INDEX_OBJ_PATH, self.MINER_INDEX_IFACE)
+
+    def start(self):
+        self.miner_iface.Start()
+
+    def stop(self):
+        self.miner_iface.Stop()
+
+    def get_sparql_connection(self):
+        return Tracker.SparqlConnection.bus_new(
+            'org.freedesktop.Tracker3.Miner.Files', None, self.bus)
+
+    def index_file(self, uri):
+        log.debug("IndexFile(%s)", uri)
+        return self.index.IndexFile('(s)', uri)
+
+    def index_file_for_process(self, uri):
+        log.debug("IndexFileForProcess(%s)", uri)
+        return self.index.IndexFileForProcess('(s)', uri)
+
+    def await_file_processed(self, path, status=True):
+        expected = [FileProcessedResult(path, status)]
+        return await_files_processed(self.miner_iface, expected)
+
+    def await_files_processed(self, expected):
+        return await_files_processed(self.miner_iface, expected)
+
+
+class ExtractorHelper ():
+
+    EXTRACTOR_BUSNAME = "org.freedesktop.Tracker3.Miner.Extract"
+    EXTRACTOR_OBJ_PATH = "/org/freedesktop/Tracker3/Miner/Extract"
+    MINER_IFACE = "org.freedesktop.Tracker3.Miner"
+
+    def __init__(self, dbus_connection):
+        self.log = logging.getLogger(__name__)
+
+        self.bus = dbus_connection
+
+        self.loop = trackertestutils.mainloop.MainLoop()
+
+        self.miner_iface = Gio.DBusProxy.new_sync(
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
+            self.EXTRACTOR_BUSNAME, self.EXTRACTOR_OBJ_PATH, self.MINER_IFACE)
+
+    def await_file_processed(self, path, status=True, timeout=None):
+        expected = [FileProcessedResult(path, status)]
+        return await_files_processed(self.miner_iface, expected, timeout=timeout)
+
+    def await_files_processed(self, expected, timeout=None):
+        return await_files_processed(self.miner_iface, expected, timeout=timeout)
diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build
index 402423eda..b33466b1e 100644
--- a/tests/functional-tests/meson.build
+++ b/tests/functional-tests/meson.build
@@ -105,6 +105,7 @@ endif
 
 functional_tests = [
   'miner-basic',
+  'miner-on-demand',
   'miner-resource-removal',
   'fts-basic',
   'fts-file-operations',
diff --git a/tests/functional-tests/miner-on-demand.py b/tests/functional-tests/miner-on-demand.py
new file mode 100755
index 000000000..b8b92f9cb
--- /dev/null
+++ b/tests/functional-tests/miner-on-demand.py
@@ -0,0 +1,130 @@
+# Copyright (C) 2020, Sam Thursfield (sam afuera me uk)
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA  02110-1301, USA.
+
+from gi.repository import Gio, GLib, GObject
+
+import logging
+import pathlib
+
+import trackertestutils
+
+import configuration
+import helpers
+import fixtures
+
+log = logging.getLogger(__file__)
+
+
+class MinerOnDemandTest(fixtures.TrackerMinerTest):
+    """
+    Tests on-demand indexing and signals that report indexing status.
+
+    This covers the IndexFile and IndexFileForProcess D-Bus methods, and the
+    FileProcessed signal.
+    """
+
+    def setUp(self):
+        fixtures.TrackerMinerTest.setUp(self)
+
+    def create_test_file(self, path):
+        testfile = pathlib.Path(self.workdir).joinpath(path)
+        testfile.parent.mkdir(parents=True, exist_ok=True)
+        testfile.write_text("Hello, I'm a test file.")
+        return testfile
+
+    def create_test_directory_tree(self):
+        testdir = pathlib.Path(self.workdir).joinpath('test-not-monitored')
+        for dirname in ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']:
+            subdir = testdir.joinpath(dirname)
+            subdir.mkdir(parents=True)
+            subdir_file = subdir.joinpath('content.txt')
+            subdir_file.write_text("Hello, I'm a test file in a subdirectory")
+        return testdir
+
+    def test_index_file_basic(self):
+        """
+        Test on-demand indexing of a file.
+        """
+
+        testfile = self.create_test_file('test-not-monitored/on-demand.txt')
+        self.assertFileNotIndexed(testfile.as_uri())
+        with self.await_document_inserted(testfile):
+            with self.extractor.await_file_processed(testfile):
+                with self.miner_fs.await_file_processed(testfile):
+                    self.miner_fs.index_file(testfile.as_uri())
+        self.assertFileIndexed(testfile.as_uri())
+
+    def test_index_file_not_found(self):
+        """
+        On-demand indexing of a file, but it's missing.
+        """
+
+        self.assertFileNotIndexed('file:///test-missing')
+        with self.assertRaises(GLib.GError) as e:
+            self.miner_fs.index_file('file:///test-missing')
+        assert 
e.exception.message.startswith('GDBus.Error:org.freedesktop.Tracker.Miner.Files.Index.Error.FileNotFound:')
+
+    def await_failsafe_marker_inserted(self, graph, path, timeout=configuration.DEFAULT_TIMEOUT):
+        url = path.as_uri()
+        expected = [
+            f'a rdfs:Resource. <{url}> nie:dataSource tracker:extractor-failure-data-source'
+        ]
+
+        return self.tracker.await_insert(graph, '; '.join(expected), timeout=timeout)
+
+    def test_index_extractor_error(self):
+        """
+        On-demand indexing of a file, but the extractor can't extract it.
+        """
+
+        # This file will be processed by the mp3 or gstreamer extractor due to
+        # its extension, but it's not a valid MP3.
+        testfile = self.create_test_file('test-not-monitored/invalid.mp3')
+
+        with self.extractor.await_file_processed(testfile, False, timeout=5000):
+            self.miner_fs.index_file(testfile.as_uri())
+
+    def test_index_directory_basic(self):
+        """
+        Test on-demand indexing of a directory with different types of file.
+
+        One file is eligible for indexing, the others are not for various
+        reasons.
+        """
+
+        testdir = pathlib.Path(self.workdir).joinpath('test-not-monitored')
+
+        test_eligible = self.create_test_file('test-not-monitored/eligible.txt')
+        test_not_eligible_tmp = self.create_test_file('test-not-monitored/not-eligible.tmp')
+        test_not_eligible_hidden = self.create_test_file('test-not-monitored/.not-eligible')
+
+        test_not_eligible_dir = testdir.joinpath('.not-eligible-dir')
+        test_not_eligible_dir.mkdir()
+
+        expected = [
+            helpers.FileProcessedResult(test_eligible, True),
+            helpers.FileProcessedResult(test_not_eligible_dir, False),
+            helpers.FileProcessedResult(test_not_eligible_hidden, False),
+            helpers.FileProcessedResult(test_not_eligible_tmp, False),
+        ]
+
+        with self.miner_fs.await_files_processed(expected):
+            self.miner_fs.index_file(testdir.as_uri())
+
+
+if __name__ == "__main__":
+    fixtures.tracker_test_main()