[tracker-miners/sam/file-processed-signal: 17/18] functional-tests: Add miner-on-demand test
- From: Sam Thursfield <sthursfield src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker-miners/sam/file-processed-signal: 17/18] functional-tests: Add miner-on-demand test
- Date: Sat, 2 May 2020 11:43:01 +0000 (UTC)
commit 5d2c92c9eabce2b3a3a0dab45005a0fb976ddb62
Author: Sam Thursfield <sam afuera me uk>
Date: Mon Mar 9 01:47:31 2020 +0100
functional-tests: Add miner-on-demand test
This is for testing IndexFile, IndexFileForProcess and the
FileProcessed signal.
tests/functional-tests/fixtures.py | 50 +++++++-
tests/functional-tests/helpers.py | 201 ++++++++++++++++++++++++++++++
tests/functional-tests/meson.build | 1 +
tests/functional-tests/miner-on-demand.py | 130 +++++++++++++++++++
tests/functional-tests/minerfshelper.py | 139 ---------------------
5 files changed, 376 insertions(+), 145 deletions(-)
---
diff --git a/tests/functional-tests/fixtures.py b/tests/functional-tests/fixtures.py
index b48447e33..5d3bca03b 100644
--- a/tests/functional-tests/fixtures.py
+++ b/tests/functional-tests/fixtures.py
@@ -25,7 +25,7 @@ Fixtures used by the tracker-miners functional-tests.
import gi
gi.require_version('Gst', '1.0')
gi.require_version('Tracker', '3.0')
-from gi.repository import GLib
+from gi.repository import Gio, GLib
from gi.repository import Tracker
import errno
@@ -42,7 +42,7 @@ import unittest as ut
import trackertestutils.dconf
import trackertestutils.helpers
import configuration as cfg
-from minerfshelper import MinerFsHelper
+import helpers
log = logging.getLogger(__name__)
@@ -68,6 +68,18 @@ def tracker_test_main():
handlers=[handler_stderr, handler_stdout],
format='%(message)s')
+ # Avoid the test process sending itself SIGTERM via the GDBusConnection
+ # 'exit-on-close' feature.
+ #
+ # This can happen if something calls a g_bus_get_*() function -- I saw
+ # problems when I started using GFile in a test, which causes GIO and GVFS
+ # to connect to the session bus in the background. There may be an
+ # underlying bug at work, but anyway it's important that our tests don't
+ # randomly terminate themselves.
+ #
+ session_bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+ session_bus.set_exit_on_close(False)
+
ut.main(failfast=True, verbosity=2)
@@ -113,9 +125,10 @@ class TrackerMinerTest(ut.TestCase):
for key, value in contents.items():
dconf.write(schema_name, key, value)
- self.miner_fs = MinerFsHelper(self.sandbox.get_connection())
+ self.miner_fs = helpers.MinerFsHelper(self.sandbox.get_connection())
self.miner_fs.start()
- self.miner_fs.start_watching_progress()
+
+ self.extractor = helpers.ExtractorHelper(self.sandbox.get_connection())
self.tracker = trackertestutils.helpers.StoreHelper(
self.miner_fs.get_sparql_connection())
@@ -141,8 +154,32 @@ class TrackerMinerTest(ut.TestCase):
if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == True:
self.fail("Resource <%s> should not exist" % urn)
- def await_document_inserted(self, path, content=None):
+ def assertFileNotIndexed(self, url_or_path):
+ if isinstance(url_or_path, pathlib.Path):
+ url = url_or_path.as_uri()
+ else:
+ assert url_or_path.startswith('file://')
+ url = url_or_path
+
+ if self.tracker.ask("ASK { ?r a rdfs:Resource ; nie:url <%s> }" % url) == True:
+ self.fail("File <%s> should not be indexed" % url)
+
+ def assertFileIndexed(self, url_or_path):
+ if isinstance(url_or_path, pathlib.Path):
+ url = url_or_path.as_uri()
+ else:
+ assert url_or_path.startswith('file://')
+ url = url_or_path
+
+ if self.tracker.ask("ASK { ?r a rdfs:Resource ; nie:url <%s> }" % url) == False:
+ self.fail("File <%s> should be indexed, but is not." % url)
+
+ def await_document_inserted(self, path, content=None, timeout=cfg.DEFAULT_TIMEOUT):
"""Wraps await_insert() context manager."""
+
+ # Safety check, you can get confused if you pass a URI because
+ # pathlib.Path.as_uri() will convert a valid URI into an invalid one.
+ assert isinstance(path, pathlib.Path) or not path.startswith('file://')
url = self.uri(path)
expected = [
@@ -154,7 +191,8 @@ class TrackerMinerTest(ut.TestCase):
content_escaped = Tracker.sparql_escape_string(content)
expected += [f'nie:plainTextContent "{content_escaped}"']
- return self.tracker.await_insert(DOCUMENTS_GRAPH, '; '.join(expected))
+ return self.tracker.await_insert(
+ DOCUMENTS_GRAPH, '; '.join(expected), timeout=timeout)
def await_document_uri_change(self, resource_id, from_path, to_path):
"""Wraps await_update() context manager."""
diff --git a/tests/functional-tests/helpers.py b/tests/functional-tests/helpers.py
new file mode 100644
index 000000000..8e47b7e48
--- /dev/null
+++ b/tests/functional-tests/helpers.py
@@ -0,0 +1,201 @@
+#
+# Copyright (C) 2010, Nokia <ivan frade nokia com>
+# Copyright (C) 2018,2020 Sam Thursfield <sam afuera me uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+import gi
+gi.require_version('Tracker', '3.0')
+from gi.repository import Gio, GLib, GObject
+from gi.repository import Tracker
+
+import dataclasses
+import logging
+import pathlib
+
+import trackertestutils.mainloop
+
+import configuration
+
+
+log = logging.getLogger(__name__)
+
+
+class WakeupCycleTimeoutException(RuntimeError):
+ pass
+
+
+@dataclasses.dataclass
+class FileProcessedResult():
+ path : pathlib.Path
+ status : bool
+
+
+class FileProcessedError(Exception):
+ pass
+
+
+class await_files_processed():
+ """Context manager to await file processing by Tracker miners & extractors.
+
+ """
+ def __init__(self, miner, expected, timeout=None):
+ self.miner = miner
+ self.expected = expected
+ self.timeout = timeout or configuration.DEFAULT_TIMEOUT
+
+ self.loop = trackertestutils.mainloop.MainLoop()
+
+ self.gfiles = []
+ self.status = {}
+ for result in self.expected:
+ gfile = Gio.File.new_for_path(str(result.path))
+ self.gfiles.append(gfile)
+ self.status[gfile.get_path()] = result.status
+
+ def __enter__(self):
+ if len(self.expected) == 1:
+ log.info("Awaiting files-processed signal from %s for file %s",
+ self.miner, self.expected[0].path)
+ else:
+ log.info("Awaiting %i files-processed signals from %s",
+ len(self.expected), self.miner)
+
+ def signal_cb(proxy, sender_name, signal_name, parameters):
+ if signal_name == 'FilesProcessed':
+ array = parameters.unpack()[0]
+ for uri, success, message in array:
+ log.debug("Processing file-processed event for %s", uri)
+
+ matched = False
+
+ signal_gfile = Gio.File.new_for_uri(uri)
+
+ expected_gfile = None
+ expected_status = None
+ for expected_gfile in self.gfiles:
+ if expected_gfile.get_path() == signal_gfile.get_path():
+ expected_status = self.status[expected_gfile.get_path()]
+ if success == expected_status:
+ log.debug("Matched %s", uri)
+ matched = True
+ break
+ else:
+ raise FileProcessedError(f"{uri}: Expected status {expected_status}, got
{success}")
+
+ if matched:
+ self.gfiles.remove(expected_gfile)
+
+ if len(self.gfiles) == 0:
+ log.info("All files were processed, exiting loop.")
+ self.loop.quit()
+
+ def timeout_cb():
+ log.info("Timeout fired after %s seconds", self.timeout)
+ raise trackertestutils.helpers.AwaitTimeoutException(
+ f"Timeout awaiting files-processed signal on {self.miner}")
+
+ self.signal_id = self.miner.connect('g-signal', signal_cb)
+ self.timeout_id = GLib.timeout_add_seconds(self.timeout, timeout_cb)
+
+ return
+
+ def __exit__(self, etype, evalue, etraceback):
+ if etype is not None:
+ return False
+
+ self.loop.run_checked()
+ log.debug("Main loop finished")
+
+ GLib.source_remove(self.timeout_id)
+ GObject.signal_handler_disconnect(self.miner, self.signal_id)
+
+ return True
+
+
+class MinerFsHelper ():
+
+ MINERFS_BUSNAME = "org.freedesktop.Tracker3.Miner.Files"
+ MINERFS_OBJ_PATH = "/org/freedesktop/Tracker3/Miner/Files"
+ MINER_IFACE = "org.freedesktop.Tracker3.Miner"
+ MINERFS_INDEX_OBJ_PATH = "/org/freedesktop/Tracker3/Miner/Files/Index"
+ MINER_INDEX_IFACE = "org.freedesktop.Tracker3.Miner.Files.Index"
+
+ def __init__(self, dbus_connection):
+ self.log = logging.getLogger(__name__)
+
+ self.bus = dbus_connection
+
+ self.loop = trackertestutils.mainloop.MainLoop()
+
+ self.miner_iface = Gio.DBusProxy.new_sync(
+ self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
+ self.MINERFS_BUSNAME, self.MINERFS_OBJ_PATH, self.MINER_IFACE)
+
+ self.index = Gio.DBusProxy.new_sync(
+ self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
+ self.MINERFS_BUSNAME, self.MINERFS_INDEX_OBJ_PATH, self.MINER_INDEX_IFACE)
+
+ def start(self):
+ self.miner_iface.Start()
+
+ def stop(self):
+ self.miner_iface.Stop()
+
+ def get_sparql_connection(self):
+ return Tracker.SparqlConnection.bus_new(
+ 'org.freedesktop.Tracker3.Miner.Files', None, self.bus)
+
+ def index_file(self, uri):
+ log.debug("IndexFile(%s)", uri)
+ return self.index.IndexFile('(s)', uri)
+
+ def index_file_for_process(self, uri):
+ log.debug("IndexFileForProcess(%s)", uri)
+ return self.index.IndexFileForProcess('(s)', uri)
+
+ def await_file_processed(self, path, status=True):
+ expected = [FileProcessedResult(path, status)]
+ return await_files_processed(self.miner_iface, expected)
+
+ def await_files_processed(self, expected):
+ return await_files_processed(self.miner_iface, expected)
+
+
+class ExtractorHelper ():
+
+ EXTRACTOR_BUSNAME = "org.freedesktop.Tracker3.Miner.Extract"
+ EXTRACTOR_OBJ_PATH = "/org/freedesktop/Tracker3/Miner/Extract"
+ MINER_IFACE = "org.freedesktop.Tracker3.Miner"
+
+ def __init__(self, dbus_connection):
+ self.log = logging.getLogger(__name__)
+
+ self.bus = dbus_connection
+
+ self.loop = trackertestutils.mainloop.MainLoop()
+
+ self.miner_iface = Gio.DBusProxy.new_sync(
+ self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
+ self.EXTRACTOR_BUSNAME, self.EXTRACTOR_OBJ_PATH, self.MINER_IFACE)
+
+ def await_file_processed(self, path, status=True, timeout=None):
+ expected = [FileProcessedResult(path, status)]
+ return await_files_processed(self.miner_iface, expected, timeout=timeout)
+
+ def await_files_processed(self, expected, timeout=None):
+ return await_files_processed(self.miner_iface, expected, timeout=timeout)
diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build
index 402423eda..b33466b1e 100644
--- a/tests/functional-tests/meson.build
+++ b/tests/functional-tests/meson.build
@@ -105,6 +105,7 @@ endif
functional_tests = [
'miner-basic',
+ 'miner-on-demand',
'miner-resource-removal',
'fts-basic',
'fts-file-operations',
diff --git a/tests/functional-tests/miner-on-demand.py b/tests/functional-tests/miner-on-demand.py
new file mode 100755
index 000000000..b8b92f9cb
--- /dev/null
+++ b/tests/functional-tests/miner-on-demand.py
@@ -0,0 +1,130 @@
+# Copyright (C) 2020, Sam Thursfield (sam afuera me uk)
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+
+from gi.repository import Gio, GLib, GObject
+
+import logging
+import pathlib
+
+import trackertestutils
+
+import configuration
+import helpers
+import fixtures
+
+log = logging.getLogger(__file__)
+
+
+class MinerOnDemandTest(fixtures.TrackerMinerTest):
+ """
+ Tests on-demand indexing and signals that report indexing status.
+
+ This covers the IndexFile and IndexFileForProcess D-Bus methods, and the
+ FileProcessed signal.
+ """
+
+ def setUp(self):
+ fixtures.TrackerMinerTest.setUp(self)
+
+ def create_test_file(self, path):
+ testfile = pathlib.Path(self.workdir).joinpath(path)
+ testfile.parent.mkdir(parents=True, exist_ok=True)
+ testfile.write_text("Hello, I'm a test file.")
+ return testfile
+
+ def create_test_directory_tree(self):
+ testdir = pathlib.Path(self.workdir).joinpath('test-not-monitored')
+ for dirname in ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']:
+ subdir = testdir.joinpath(dirname)
+ subdir.mkdir(parents=True)
+ subdir_file = subdir.joinpath('content.txt')
+ subdir_file.write_text("Hello, I'm a test file in a subdirectory")
+ return testdir
+
+ def test_index_file_basic(self):
+ """
+ Test on-demand indexing of a file.
+ """
+
+ testfile = self.create_test_file('test-not-monitored/on-demand.txt')
+ self.assertFileNotIndexed(testfile.as_uri())
+ with self.await_document_inserted(testfile):
+ with self.extractor.await_file_processed(testfile):
+ with self.miner_fs.await_file_processed(testfile):
+ self.miner_fs.index_file(testfile.as_uri())
+ self.assertFileIndexed(testfile.as_uri())
+
+ def test_index_file_not_found(self):
+ """
+ On-demand indexing of a file, but it's missing.
+ """
+
+ self.assertFileNotIndexed('file:///test-missing')
+ with self.assertRaises(GLib.GError) as e:
+ self.miner_fs.index_file('file:///test-missing')
+ assert
e.exception.message.startswith('GDBus.Error:org.freedesktop.Tracker.Miner.Files.Index.Error.FileNotFound:')
+
+ def await_failsafe_marker_inserted(self, graph, path, timeout=configuration.DEFAULT_TIMEOUT):
+ url = path.as_uri()
+ expected = [
+ f'a rdfs:Resource. <{url}> nie:dataSource tracker:extractor-failure-data-source'
+ ]
+
+ return self.tracker.await_insert(graph, '; '.join(expected), timeout=timeout)
+
+ def test_index_extractor_error(self):
+ """
+ On-demand indexing of a file, but the extractor can't extract it.
+ """
+
+ # This file will be processed by the mp3 or gstreamer extractor due to
+ # its extension, but it's not a valid MP3.
+ testfile = self.create_test_file('test-not-monitored/invalid.mp3')
+
+ with self.extractor.await_file_processed(testfile, False, timeout=5000):
+ self.miner_fs.index_file(testfile.as_uri())
+
+ def test_index_directory_basic(self):
+ """
+ Test on-demand indexing of a directory with different types of file.
+
+ One file is eligible for indexing, the others are not for various
+ reasons.
+ """
+
+ testdir = pathlib.Path(self.workdir).joinpath('test-not-monitored')
+
+ test_eligible = self.create_test_file('test-not-monitored/eligible.txt')
+ test_not_eligible_tmp = self.create_test_file('test-not-monitored/not-eligible.tmp')
+ test_not_eligible_hidden = self.create_test_file('test-not-monitored/.not-eligible')
+
+ test_not_eligible_dir = testdir.joinpath('.not-eligible-dir')
+ test_not_eligible_dir.mkdir()
+
+ expected = [
+ helpers.FileProcessedResult(test_eligible, True),
+ helpers.FileProcessedResult(test_not_eligible_dir, False),
+ helpers.FileProcessedResult(test_not_eligible_hidden, False),
+ helpers.FileProcessedResult(test_not_eligible_tmp, False),
+ ]
+
+ with self.miner_fs.await_files_processed(expected):
+ self.miner_fs.index_file(testdir.as_uri())
+
+
+if __name__ == "__main__":
+ fixtures.tracker_test_main()
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]