[tracker/sam/tracker-2.3-developer-experience: 4/45] functional-tests: Start daemons through D-Bus autolaunch



commit b69c5ee2040c5ce15be89066558d2c8de88f911d
Author: Sam Thursfield <sam afuera me uk>
Date:   Tue Aug 27 00:04:11 2019 +0300

    functional-tests: Start daemons through D-Bus autolaunch
    
    Instead of manually running and managing Tracker daemon processes
    manually in the test, we now rely on our private D-Bus daemon to
    do so.
    
    This makes the test environment more like a real Tracker deployment.
    
    Log output from the D-Bus daemon is now captured and output through
    the Python logging system. This allows for finer-grained filtering
    of output from the tests themselves and from the Tracker daemons.
    
    Some test code is changed to support the new model.

 meson.build                                    |   9 +-
 tests/functional-tests/01-insertion.py         |   2 -
 tests/functional-tests/02-sparql-bugs.py       |   5 -
 tests/functional-tests/03-fts-functions.py     |   2 -
 tests/functional-tests/04-group-concat.py      |   2 -
 tests/functional-tests/05-coalesce.py          |   2 -
 tests/functional-tests/06-distance.py          |   2 -
 tests/functional-tests/07-graph.py             |   2 -
 tests/functional-tests/08-unique-insertions.py |   2 -
 tests/functional-tests/09-concurrent-query.py  |   2 -
 tests/functional-tests/14-signals.py           |   5 +-
 tests/functional-tests/15-statistics.py        |   2 -
 tests/functional-tests/16-collation.py         |   1 -
 tests/functional-tests/17-ontology-changes.py  | 172 +++++------------
 tests/functional-tests/configuration.json.in   |  10 +-
 tests/functional-tests/configuration.py        |  43 ++++-
 tests/functional-tests/meson.build             |  38 ++--
 tests/functional-tests/storetest.py            |  27 ++-
 tests/test-bus.conf.in                         |   1 +
 utils/sandbox/tracker-sandbox.py               | 117 +-----------
 utils/trackertestutils/dbusdaemon.py           | 198 ++++++++++++++++++++
 utils/trackertestutils/dconf.py                |  69 ++-----
 utils/trackertestutils/helpers.py              | 245 +++++++------------------
 utils/trackertestutils/meson.build             |   1 +
 24 files changed, 419 insertions(+), 540 deletions(-)
---
diff --git a/meson.build b/meson.build
index 82c75ae4e..631b2aee3 100644
--- a/meson.build
+++ b/meson.build
@@ -289,14 +289,7 @@ conf.set('domain_ontologies_dir', join_paths('${datadir}', 'tracker', 'domain-on
 conf.set('FUNCTIONAL_TESTS_ONTOLOGIES_DIR', join_paths(meson.current_source_dir(), 'tests', 
'functional-tests', 'test-ontologies'))
 conf.set('FUNCTIONAL_TESTS_TRACKER_STORE_PATH', join_paths(meson.current_build_dir(), 'src', 
'tracker-store', 'tracker-store'))
 
-# This is set in an awkward way for compatibility with Autoconf. Switch it
-# to a normal boolean once we get rid of the Autotools build system. It's
-# only used in tests/functional-tests/common/utils/configuration.py.in.
-if get_option('journal')
-    conf.set('DISABLE_JOURNAL_TRUE', 'true')
-else
-    conf.set('DISABLE_JOURNAL_TRUE', '')
-endif
+conf.set10('DISABLE_JOURNAL', get_option('journal'))
 
 configure_file(input: 'config.h.meson.in',
                output: 'config.h',
diff --git a/tests/functional-tests/01-insertion.py b/tests/functional-tests/01-insertion.py
index 0bed7aeb3..1611b4464 100755
--- a/tests/functional-tests/01-insertion.py
+++ b/tests/functional-tests/01-insertion.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/02-sparql-bugs.py b/tests/functional-tests/02-sparql-bugs.py
index 4305ea0a9..bf0d8ecbb 100755
--- a/tests/functional-tests/02-sparql-bugs.py
+++ b/tests/functional-tests/02-sparql-bugs.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
@@ -111,21 +109,18 @@ class TrackerStoreSparqlBugsTests (CommonTrackerStoreTest):
                 """
 
         results1 = self.tracker.query(query1)
-        print("1", results1)
         self.assertEqual(len(results1), 1)
         self.assertEqual(len(results1[0]), 2)
         self.assertEqual(results1[0][0], "contact:test")
         self.assertEqual(results1[0][1], "98653")
 
         results2 = self.tracker.query(query2)
-        print("2", results2)
         self.assertEqual(len(results2), 1)
         self.assertEqual(len(results2[0]), 2)
         self.assertEqual(results2[0][0], "contact:test")
         self.assertEqual(results2[0][1], "98653")
 
         results3 = self.tracker.query(query3)
-        print("3", results3)
         self.assertEqual(len(results3), 1)
         self.assertEqual(len(results3[0]), 2)
         self.assertEqual(results3[0][0], "contact:test")
diff --git a/tests/functional-tests/03-fts-functions.py b/tests/functional-tests/03-fts-functions.py
index 46c43f368..0938a7e52 100755
--- a/tests/functional-tests/03-fts-functions.py
+++ b/tests/functional-tests/03-fts-functions.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/04-group-concat.py b/tests/functional-tests/04-group-concat.py
index a8064a828..06e06265b 100755
--- a/tests/functional-tests/04-group-concat.py
+++ b/tests/functional-tests/04-group-concat.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/05-coalesce.py b/tests/functional-tests/05-coalesce.py
index 48d8e6eb6..eddf3148e 100755
--- a/tests/functional-tests/05-coalesce.py
+++ b/tests/functional-tests/05-coalesce.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/06-distance.py b/tests/functional-tests/06-distance.py
index 80d35dfb9..b2a22d255 100755
--- a/tests/functional-tests/06-distance.py
+++ b/tests/functional-tests/06-distance.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/07-graph.py b/tests/functional-tests/07-graph.py
index aad935f77..07517f8dc 100755
--- a/tests/functional-tests/07-graph.py
+++ b/tests/functional-tests/07-graph.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/08-unique-insertions.py b/tests/functional-tests/08-unique-insertions.py
index 9c9578a5c..94b1f3625 100755
--- a/tests/functional-tests/08-unique-insertions.py
+++ b/tests/functional-tests/08-unique-insertions.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/09-concurrent-query.py b/tests/functional-tests/09-concurrent-query.py
index b24fdcc40..dacd396aa 100755
--- a/tests/functional-tests/09-concurrent-query.py
+++ b/tests/functional-tests/09-concurrent-query.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/14-signals.py b/tests/functional-tests/14-signals.py
index 242ae8480..cae97d46d 100755
--- a/tests/functional-tests/14-signals.py
+++ b/tests/functional-tests/14-signals.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
@@ -52,7 +50,7 @@ class TrackerStoreSignalsTests (CommonTrackerStoreTest):
         self.loop = GLib.MainLoop()
         self.timeout_id = 0
 
-        self.bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+        self.bus = self.sandbox.get_connection()
 
         self.results_classname = None
         self.results_deletes = None
@@ -125,7 +123,6 @@ class TrackerStoreSignalsTests (CommonTrackerStoreTest):
         """
         self.__connect_signal()
         self.tracker.update(CONTACT)
-        time.sleep(1)
         self.__wait_for_signal()
 
         # validate results
diff --git a/tests/functional-tests/15-statistics.py b/tests/functional-tests/15-statistics.py
index 6f6ca3014..56fa38eb7 100755
--- a/tests/functional-tests/15-statistics.py
+++ b/tests/functional-tests/15-statistics.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
diff --git a/tests/functional-tests/16-collation.py b/tests/functional-tests/16-collation.py
index 40a993d82..62765a77a 100755
--- a/tests/functional-tests/16-collation.py
+++ b/tests/functional-tests/16-collation.py
@@ -1,4 +1,3 @@
-#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
diff --git a/tests/functional-tests/17-ontology-changes.py b/tests/functional-tests/17-ontology-changes.py
index 8cf7db220..7b7e591f4 100755
--- a/tests/functional-tests/17-ontology-changes.py
+++ b/tests/functional-tests/17-ontology-changes.py
@@ -1,5 +1,3 @@
-#!/usr/bin/python3
-#
 # Copyright (C) 2010, Nokia <ivan frade nokia com>
 # Copyright (C) 2019, Sam Thursfield <sam afuera me uk>
 #
@@ -26,8 +24,8 @@ changes and checking if the data is still there.
 
 from gi.repository import GLib
 
-import logging
 import os
+import pathlib
 import shutil
 import re
 import tempfile
@@ -48,101 +46,6 @@ XSD_INTEGER = "http://www.w3.org/2001/XMLSchema#integer";
 
 TEST_PREFIX = "http://example.org/ns#";
 
-TEST_ENV_VARS = {"LC_COLLATE": "en_GB.utf8"}
-
-REASONABLE_TIMEOUT = 5
-
-log = logging.getLogger()
-
-
-class UnableToBootException (Exception):
-    pass
-
-
-class TrackerSystemAbstraction (object):
-
-    def __init__(self, settings=None):
-        self.store = None
-        self._dirs = {}
-
-    def xdg_data_home(self):
-        return os.path.join(self._basedir, 'data')
-
-    def xdg_cache_home(self):
-        return os.path.join(self._basedir, 'cache')
-
-    def set_up_environment(self, settings=None, ontodir=None):
-        """
-        Sets up the XDG_*_HOME variables and make sure the directories exist
-
-        Settings should be a dict mapping schema names to dicts that hold the
-        settings that should be changed in those schemas. The contents dicts
-        should map key->value, where key is a key name and value is a suitable
-        GLib.Variant instance.
-        """
-        self._basedir = tempfile.mkdtemp()
-
-        self._dirs = {
-            "XDG_DATA_HOME": self.xdg_data_home(),
-            "XDG_CACHE_HOME": self.xdg_cache_home()
-        }
-
-        for var, directory in list(self._dirs.items()):
-            os.makedirs(directory)
-            os.makedirs(os.path.join(directory, 'tracker'))
-            os.environ[var] = directory
-
-        if ontodir:
-            log.debug("export %s=%s", "TRACKER_DB_ONTOLOGIES_DIR", ontodir)
-            os.environ["TRACKER_DB_ONTOLOGIES_DIR"] = ontodir
-
-        for var, value in TEST_ENV_VARS.items():
-            log.debug("export %s=%s", var, value)
-            os.environ[var] = value
-
-        # Previous loop should have set DCONF_PROFILE to the test location
-        if settings is not None:
-            self._apply_settings(settings)
-
-    def _apply_settings(self, settings):
-        for schema_name, contents in settings.items():
-            dconf = trackertestutils.dconf.DConfClient(schema_name)
-            dconf.reset()
-            for key, value in contents.items():
-                dconf.write(key, value)
-
-    def tracker_store_testing_start(self, confdir=None, ontodir=None):
-        """
-        Stops any previous instance of the store, calls set_up_environment,
-        and starts a new instances of the store
-        """
-        self.set_up_environment(confdir, ontodir)
-
-        self.store = trackertestutils.helpers.StoreHelper(cfg.TRACKER_STORE_PATH)
-        self.store.start()
-
-    def tracker_store_restart_with_new_ontologies(self, ontodir):
-        self.store.stop()
-        if ontodir:
-            os.environ["TRACKER_DB_ONTOLOGIES_DIR"] = ontodir
-        try:
-            self.store.start()
-        except GLib.Error:
-            raise UnableToBootException(
-                "Unable to boot the store \n(" + str(e) + ")")
-
-    def finish(self):
-        """
-        Stop all running processes and remove all test data.
-        """
-
-        if self.store:
-            self.store.stop()
-
-        for path in list(self._dirs.values()):
-            shutil.rmtree(path)
-        os.rmdir(self._basedir)
-
 
 class OntologyChangeTestTemplate (ut.TestCase):
     """
@@ -158,35 +61,43 @@ class OntologyChangeTestTemplate (ut.TestCase):
     Check doc in those methods for the specific details.
     """
 
-    def get_ontology_dir(self, param):
-        return os.path.join(cfg.TEST_ONTOLOGIES_DIR, param)
-
     def setUp(self):
-        self.system = TrackerSystemAbstraction()
+        self.tmpdir = tempfile.mkdtemp(prefix='tracker-test-')
 
     def tearDown(self):
-        self.system.finish()
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
 
-    def template_test_ontology_change(self):
+    def get_ontology_dir(self, param):
+        return str(pathlib.Path(__file__).parent.joinpath('test-ontologies', param))
 
+    def template_test_ontology_change(self):
         self.set_ontology_dirs()
 
-        basic_ontologies = self.get_ontology_dir(self.FIRST_ONTOLOGY_DIR)
-        modified_ontologies = self.get_ontology_dir(self.SECOND_ONTOLOGY_DIR)
+        self.__assert_ontology_dates(self.FIRST_ONTOLOGY_DIR, self.SECOND_ONTOLOGY_DIR)
 
-        self.__assert_ontology_dates(basic_ontologies, modified_ontologies)
+        extra_env = cfg.test_environment(self.tmpdir)
+        extra_env['LC_COLLATE'] = 'en_GB.utf8'
+        extra_env['TRACKER_DB_ONTOLOGIES_DIR'] = self.get_ontology_dir(self.FIRST_ONTOLOGY_DIR)
 
-        self.system.tracker_store_testing_start(ontodir=basic_ontologies)
-        self.tracker = self.system.store
+        sandbox1 = trackertestutils.helpers.TrackerDBusSandbox(
+            cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env)
+        sandbox1.start()
+
+        self.tracker = trackertestutils.helpers.StoreHelper(sandbox1.get_connection())
+        self.tracker.start_and_wait_for_ready()
 
         self.insert_data()
 
-        try:
-            # Boot the second set of ontologies
-            self.system.tracker_store_restart_with_new_ontologies(
-                modified_ontologies)
-        except UnableToBootException as e:
-            self.fail(str(self.__class__) + " " + str(e))
+        sandbox1.stop()
+
+        # Boot the second set of ontologies
+        extra_env['TRACKER_DB_ONTOLOGIES_DIR'] = self.get_ontology_dir(self.SECOND_ONTOLOGY_DIR)
+        sandbox2 = trackertestutils.helpers.TrackerDBusSandbox(
+            cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env)
+        sandbox2.start()
+
+        self.tracker = trackertestutils.helpers.StoreHelper(sandbox2.get_connection())
+        self.tracker.start_and_wait_for_ready()
 
         self.validate_status()
 
@@ -233,7 +144,7 @@ class OntologyChangeTestTemplate (ut.TestCase):
                           (member, dbus_result))
         return
 
-    def __assert_ontology_dates(self, first_dir, second_dir):
+    def __assert_ontology_dates(self, first, second):
         """
         Asserts that 91-test.ontology in second_dir has a more recent
         modification time than in first_dir
@@ -241,23 +152,24 @@ class OntologyChangeTestTemplate (ut.TestCase):
         ISO9601_REGEX = "(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)"
 
         def get_ontology_date(ontology):
-            for line in open(ontology, 'r'):
-                if "nao:lastModified" in line:
-                    getmodtime = re.compile(
-                        'nao:lastModified\ \"' + ISO9601_REGEX + '\"')
-                    modtime_match = getmodtime.search(line)
-
-                    if (modtime_match):
-                        nao_date = modtime_match.group(1)
-                        return time.strptime(nao_date, "%Y-%m-%dT%H:%M:%SZ")
-                    else:
-                        print("something funky in", line)
-                    break
+            with open(ontology, 'r') as f:
+                for line in f:
+                    if "nao:lastModified" in line:
+                        getmodtime = re.compile(
+                            'nao:lastModified\ \"' + ISO9601_REGEX + '\"')
+                        modtime_match = getmodtime.search(line)
+
+                        if (modtime_match):
+                            nao_date = modtime_match.group(1)
+                            return time.strptime(nao_date, "%Y-%m-%dT%H:%M:%SZ")
+                        else:
+                            print("something funky in", line)
+                        break
 
         first_date = get_ontology_date(
-            os.path.join(first_dir, "91-test.ontology"))
+            os.path.join(self.get_ontology_dir(first), "91-test.ontology"))
         second_date = get_ontology_date(
-            os.path.join(second_dir, "91-test.ontology"))
+            os.path.join(self.get_ontology_dir(second), "91-test.ontology"))
         if first_date >= second_date:
             self.fail("nao:modifiedTime in '%s' is not more recent in the second ontology" % (
                 "91-test.ontology"))
diff --git a/tests/functional-tests/configuration.json.in b/tests/functional-tests/configuration.json.in
index 46c5126f7..ee23be36f 100644
--- a/tests/functional-tests/configuration.json.in
+++ b/tests/functional-tests/configuration.json.in
@@ -1,5 +1,9 @@
 {
-    "TEST_ONTOLOGIES_DIR": "@FUNCTIONAL_TESTS_ONTOLOGIES_DIR@",
-    "TRACKER_STORE_PATH": "@FUNCTIONAL_TESTS_TRACKER_STORE_PATH@",
-    "disableJournal": "@DISABLE_JOURNAL_TRUE@"
+    "TEST_DBUS_DAEMON_CONFIG_FILE": "@TEST_DBUS_DAEMON_CONFIG_FILE@",
+    "TEST_DCONF_PROFILE": "@TEST_DCONF_PROFILE@",
+    "TEST_GSETTINGS_SCHEMA_DIR": "@TEST_GSETTINGS_SCHEMA_DIR@",
+    "TEST_LANGUAGE_STOP_WORDS_DIR": "@TEST_LANGUAGE_STOP_WORDS_DIR@",
+    "TEST_ONTOLOGIES_DIR": "@TEST_ONTOLOGIES_DIR@",
+    "TEST_DOMAIN_ONTOLOGY_RULE": "@TEST_DOMAIN_ONTOLOGY_RULE@",
+    "disableJournal": "@DISABLE_JOURNAL@"
 }
diff --git a/tests/functional-tests/configuration.py b/tests/functional-tests/configuration.py
index 938ad0f19..249f00e6b 100644
--- a/tests/functional-tests/configuration.py
+++ b/tests/functional-tests/configuration.py
@@ -18,10 +18,10 @@
 # 02110-1301, USA.
 #
 
-
 import json
 import logging
 import os
+import pathlib
 import sys
 
 
@@ -34,13 +34,23 @@ with open(os.environ['TRACKER_FUNCTIONAL_TEST_CONFIG']) as f:
     config = json.load(f)
 
 
-TOP_SRCDIR = os.path.dirname(os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-TOP_BUILDDIR = os.environ['TRACKER_FUNCTIONAL_TEST_BUILD_DIR']
+TEST_DBUS_DAEMON_CONFIG_FILE = config['TEST_DBUS_DAEMON_CONFIG_FILE']
+
+
+disableJournal = bool(config['disableJournal']))
 
-TEST_ONTOLOGIES_DIR = config['TEST_ONTOLOGIES_DIR']
-TRACKER_STORE_PATH = config['TRACKER_STORE_PATH']
-disableJournal = (len(config['disableJournal']) == 0)
+def test_environment(tmpdir):
+    return {
+        'DCONF_PROFILE': config['TEST_DCONF_PROFILE'],
+        'GSETTINGS_SCHEMA_DIR': config['TEST_GSETTINGS_SCHEMA_DIR'],
+        'TRACKER_DB_ONTOLOGIES_DIR': config['TEST_ONTOLOGIES_DIR'],
+        'TRACKER_LANGUAGE_STOP_WORDS_DIR': config['TEST_LANGUAGE_STOP_WORDS_DIR'],
+        'TRACKER_TEST_DOMAIN_ONTOLOGY_RULE': config['TEST_DOMAIN_ONTOLOGY_RULE'],
+        'XDG_CACHE_HOME': os.path.join(tmpdir, 'cache'),
+        'XDG_CONFIG_HOME': os.path.join(tmpdir, 'config'),
+        'XDG_DATA_HOME': os.path.join(tmpdir, 'data'),
+        'XDG_RUNTIME_DIR': os.path.join(tmpdir, 'run'),
+    }
 
 
 def get_environment_boolean(variable):
@@ -56,5 +66,24 @@ def get_environment_boolean(variable):
                            (variable, value))
 
 
+def get_environment_int(variable, default=0):
+    try:
+        return int(os.environ.get(variable))
+    except (TypeError, ValueError):
+        return default
+
+
 if get_environment_boolean('TRACKER_TESTS_VERBOSE'):
+    # Output all logs to stderr
     logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+else:
+    # Output some messages from D-Bus daemon to stderr by default. In practice,
+    # only errors and warnings should be output here unless the environment
+    # contains G_MESSAGES_DEBUG= and/or TRACKER_VERBOSITY=1 or more.
+    handler_stderr = logging.StreamHandler(stream=sys.stderr)
+    handler_stderr.addFilter(logging.Filter('trackertestutils.dbusdaemon.stderr'))
+    handler_stdout = logging.StreamHandler(stream=sys.stderr)
+    handler_stdout.addFilter(logging.Filter('trackertestutils.dbusdaemon.stdout'))
+    logging.basicConfig(level=logging.INFO,
+                        handlers=[handler_stderr, handler_stdout],
+                        format='%(message)s')
diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build
index be3fc2a4a..21813e410 100644
--- a/tests/functional-tests/meson.build
+++ b/tests/functional-tests/meson.build
@@ -1,13 +1,22 @@
-test_runner = configure_file(
-    input: 'test-runner.sh.in',
-    output: 'test-runner.sh',
-    configuration: conf)
-test_runner = find_program(test_runner)
+python = find_program('python3')
+
+# Configure functional tests to run completely from source tree.
+testconf = configuration_data()
+
+config_json_full_path = join_paths(meson.current_build_dir(), 'configuration.json')
+dconf_profile_full_path = join_paths(meson.current_source_dir(), 'trackertest')
+
+testconf.set('TEST_DBUS_DAEMON_CONFIG_FILE', join_paths(build_root, 'tests', 'test-bus.conf'))
+testconf.set('TEST_DCONF_PROFILE', dconf_profile_full_path)
+testconf.set('TEST_DOMAIN_ONTOLOGY_RULE', tracker_uninstalled_domain_rule)
+testconf.set('TEST_GSETTINGS_SCHEMA_DIR', tracker_uninstalled_gsettings_schema_dir)
+testconf.set('TEST_ONTOLOGIES_DIR', tracker_uninstalled_nepomuk_ontologies_dir)
+testconf.set('TEST_LANGUAGE_STOP_WORDS_DIR', tracker_uninstalled_stop_words_dir)
 
 config_json = configure_file(
   input: 'configuration.json.in',
   output: 'configuration.json',
-  configuration: conf
+  configuration: testconf
 )
 
 functional_tests = [
@@ -26,28 +35,21 @@ functional_tests = [
   '17-ontology-changes',
 ]
 
-config_json_full_path = join_paths(meson.current_build_dir(), 'configuration.json')
-dconf_profile_full_path = join_paths(meson.current_source_dir(), 'trackertest')
-
 test_env = environment()
-test_env.set('DCONF_PROFILE', dconf_profile_full_path)
-test_env.set('GSETTINGS_SCHEMA_DIR', tracker_uninstalled_gsettings_schema_dir)
 
 tracker_uninstalled_testutils_dir = join_paths(meson.current_source_dir(), '..', '..', 'utils')
 test_env.prepend('PYTHONPATH', tracker_uninstalled_testutils_dir)
 
-test_env.set('TRACKER_DB_ONTOLOGIES_DIR', tracker_uninstalled_nepomuk_ontologies_dir)
-test_env.set('TRACKER_FUNCTIONAL_TEST_BUILD_DIR', build_root)
 test_env.set('TRACKER_FUNCTIONAL_TEST_CONFIG', config_json_full_path)
-test_env.set('TRACKER_LANGUAGE_STOP_WORDS_DIR', tracker_uninstalled_stop_words_dir)
-test_env.set('TRACKER_TEST_DOMAIN_ONTOLOGY_RULE', tracker_uninstalled_domain_rule)
 
 foreach t: functional_tests
-  test('functional-' + t, test_runner,
-    args: './' + t + '.py',
+  file = '@0@.py'.format(t)
+  test('functional-' + t, python,
+    args: [file],
     env: test_env,
     workdir: meson.current_source_dir(),
     timeout: 60)
 endforeach
 
-subdir('ipc')
+# FIXME
+#subdir('ipc')
diff --git a/tests/functional-tests/storetest.py b/tests/functional-tests/storetest.py
index ed7aa82c5..d96294cc4 100644
--- a/tests/functional-tests/storetest.py
+++ b/tests/functional-tests/storetest.py
@@ -19,6 +19,8 @@
 #
 
 import os
+import shutil
+import tempfile
 import time
 import unittest as ut
 
@@ -35,11 +37,28 @@ class CommonTrackerStoreTest (ut.TestCase):
 
     @classmethod
     def setUpClass(self):
-        extra_env = {'LC_COLLATE': 'en_GB.utf8'}
+        self.tmpdir = tempfile.mkdtemp(prefix='tracker-test-')
 
-        self.tracker = trackertestutils.helpers.StoreHelper(cfg.TRACKER_STORE_PATH)
-        self.tracker.start(extra_env=extra_env)
+        try:
+            extra_env = cfg.test_environment(self.tmpdir)
+            extra_env['LANG'] = 'en_GB.utf8'
+            extra_env['LC_COLLATE'] = 'en_GB.utf8'
+
+            self.sandbox = trackertestutils.helpers.TrackerDBusSandbox(
+                dbus_daemon_config_file=cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env)
+            self.sandbox.start()
+
+            self.tracker = trackertestutils.helpers.StoreHelper(
+                self.sandbox.get_connection())
+            self.tracker.start_and_wait_for_ready()
+            self.tracker.start_watching_updates()
+        except Exception as e:
+            shutil.rmtree(self.tmpdir, ignore_errors=True)
+            raise
 
     @classmethod
     def tearDownClass(self):
-        self.tracker.stop()
+        self.tracker.stop_watching_updates()
+        self.sandbox.stop()
+
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
diff --git a/tests/test-bus.conf.in b/tests/test-bus.conf.in
index 5b4f51ff9..2f4b2ef1b 100644
--- a/tests/test-bus.conf.in
+++ b/tests/test-bus.conf.in
@@ -7,6 +7,7 @@
   <listen>unix:tmpdir=./</listen>
 
   <servicedir>@abs_top_builddir@/tests/services/</servicedir>
+  <standard_session_servicedirs/>
 
   <policy context="default">
     <!-- Allow everything to be sent -->
diff --git a/utils/sandbox/tracker-sandbox.py b/utils/sandbox/tracker-sandbox.py
index cc8ebd786..34ad09ed5 100755
--- a/utils/sandbox/tracker-sandbox.py
+++ b/utils/sandbox/tracker-sandbox.py
@@ -24,6 +24,7 @@
 #
 
 import argparse
+import configparser
 import locale
 import logging
 import os
@@ -33,10 +34,10 @@ import subprocess
 import sys
 import threading
 
-import configparser
-
 from gi.repository import GLib
 
+import trackertestutils.dbusdaemon
+
 # Script
 script_name = 'tracker-sandbox'
 script_version = '1.0'
@@ -84,116 +85,6 @@ log = logging.getLogger('sandbox')
 dbuslog = logging.getLogger('dbus')
 
 
-# Private DBus daemon
-
-class DBusDaemon:
-    """The private D-Bus instance that provides the sandbox's session bus.
-
-    We support reading and writing the session information to a file. This
-    means that if the user runs two sandbox instances on the same data
-    directory at the same time, they will share the same message bus.
-    """
-
-    def __init__(self, session_file=None):
-        self.session_file = session_file
-        self.existing_session = False
-        self.process = None
-
-        try:
-            self.address, self.pid = self.read_session_file(session_file)
-            self.existing_session = True
-        except FileNotFoundError:
-            log.debug("No existing D-Bus session file was found.")
-
-            self.address = None
-            self.pid = None
-
-    def get_session_file(self):
-        """Returns the path to the session file if we created it, or None."""
-        if self.existing_session:
-            return None
-        return self.session_file
-
-    def get_address(self):
-        return self.address
-
-    @staticmethod
-    def read_session_file(session_file):
-        with open(session_file, 'r') as f:
-            content = f.read()
-
-        try:
-            address = content.splitlines()[0]
-            pid = int(content.splitlines()[1])
-        except ValueError:
-            raise RuntimeError(f"D-Bus session file {session_file} is not valid. "
-                                "Remove this file to start a new session.")
-
-        return address, pid
-
-    @staticmethod
-    def write_session_file(session_file, address, pid):
-        os.makedirs(os.path.dirname(session_file), exist_ok=True)
-
-        content = '%s\n%s' % (address, pid)
-        with open(session_file, 'w') as f:
-            f.write(content)
-
-    def start_if_needed(self):
-        if self.existing_session:
-            log.debug('Using existing D-Bus session from file "%s" with address "%s"'
-                      ' with PID %d' % (self.session_file, self.address, self.pid))
-        else:
-            dbus_command = ['dbus-daemon', '--session', '--print-address=1', '--print-pid=1']
-            self.process = subprocess.Popen(dbus_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-            try:
-                self.address = self.process.stdout.readline().strip().decode('ascii')
-                self.pid = int(self.process.stdout.readline().strip().decode('ascii'))
-            except ValueError:
-                error = self.process.stderr.read().strip().decode('unicode-escape')
-                raise RuntimeError(f"Failed to start D-Bus daemon.\n{error}")
-
-            log.debug("Using new D-Bus session with address '%s' with PID %d",
-                      self.address, self.pid)
-
-            self.write_session_file(self.session_file, self.address, self.pid)
-            log.debug("Wrote D-Bus session file at %s", self.session_file)
-
-            # We must read from the pipes continuously, otherwise the daemon
-            # process will block.
-            self._threads=[threading.Thread(target=self.pipe_to_log, args=(self.process.stdout, 'stdout'), 
daemon=True),
-                           threading.Thread(target=self.pipe_to_log, args=(self.process.stderr, 'stderr'), 
daemon=True)]
-            self._threads[0].start()
-            self._threads[1].start()
-
-    def stop(self):
-        if self.process:
-            log.debug("  Stopping DBus daemon")
-            self.process.terminate()
-            self.process.wait()
-
-    def pipe_to_log(self, pipe, source):
-        """This function processes the output from our dbus-daemon instance."""
-        while True:
-            line_raw = pipe.readline()
-
-            if len(line_raw) == 0:
-                break
-
-            line = line_raw.decode('utf-8').rstrip()
-
-            if line.startswith('(tracker-'):
-                # We set G_MESSAGES_PREFIXED=all, meaning that all log messages
-                # output by Tracker processes have a prefix. Note that
-                # g_print() will NOT be captured here.
-                dbuslog.info(line)
-            else:
-                # Log messages from other daemons, including the dbus-daemon
-                # itself, go here. Any g_print() messages also end up here.
-                dbuslog.debug(line)
-
-
 # Environment / Clean up
 
 def environment_unset(dbus):
@@ -260,7 +151,7 @@ def environment_set(index_location, prefix, verbosity=0):
     dbus_session_file = os.path.join(
         os.environ['XDG_RUNTIME_DIR'], 'dbus-session')
 
-    dbus = DBusDaemon(dbus_session_file)
+    dbus = trackertestutils.dbusdaemon.DBusDaemon(dbus_session_file)
     dbus.start_if_needed()
 
     # Important, other subprocesses must use our new bus
diff --git a/utils/trackertestutils/dbusdaemon.py b/utils/trackertestutils/dbusdaemon.py
new file mode 100644
index 000000000..43fe8f146
--- /dev/null
+++ b/utils/trackertestutils/dbusdaemon.py
@@ -0,0 +1,198 @@
+# Copyright (C) 2018,2019, Sam Thursfield <sam afuera me uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+from gi.repository import Gio
+
+import logging
+import os
+import signal
+import subprocess
+import threading
+
+log = logging.getLogger(__name__)
+dbus_stderr_log = logging.getLogger(__name__ + '.stderr')
+dbus_stdout_log = logging.getLogger(__name__ + '.stdout')
+
+
+class DaemonNotStartedError(Exception):
+    pass
+
+
+class DBusDaemon:
+    """The private D-Bus instance that provides the sandbox's session bus.
+
+    We support reading and writing the session information to a file. This
+    means that if the user runs two sandbox instances on the same data
+    directory at the same time, they will share the same message bus.
+
+    """
+
+    def __init__(self, session_file=None):
+        self.session_file = session_file
+        self.existing_session = False
+        self.process = None
+
+        self.address = None
+        self.pid = None
+
+        self._gdbus_connection = None
+        self._previous_sigterm_handler = None
+
+        self._threads = []
+
+        if session_file:
+            try:
+                self.address, self.pid = self.read_session_file(session_file)
+                self.existing_session = True
+            except FileNotFoundError:
+                log.debug("No existing D-Bus session file was found.")
+
+    def get_session_file(self):
+        """Returns the path to the session file if we created it, or None."""
+        if self.existing_session:
+            return None
+        return self.session_file
+
+    def get_address(self):
+        if self.address is None:
+            raise DaemonNotStartedError()
+        return self.address
+
+    def get_connection(self):
+        if self._gdbus_connection is None:
+            raise DaemonNotStartedError()
+        return self._gdbus_connection
+
+    @staticmethod
+    def read_session_file(session_file):
+        with open(session_file, 'r') as f:
+            content = f.read()
+
+        try:
+            address = content.splitlines()[0]
+            pid = int(content.splitlines()[1])
+        except ValueError:
+            raise RuntimeError(f"D-Bus session file {session_file} is not valid. "
+                                "Remove this file to start a new session.")
+
+        return address, pid
+
+    @staticmethod
+    def write_session_file(session_file, address, pid):
+        os.makedirs(os.path.dirname(session_file), exist_ok=True)
+
+        content = '%s\n%s' % (address, pid)
+        with open(session_file, 'w') as f:
+            f.write(content)
+
+    def start_if_needed(self, config_file=None, env=None):
+        if self.existing_session:
+            log.debug('Using existing D-Bus session from file "%s" with address "%s"'
+                      ' with PID %d' % (self.session_file, self.address, self.pid))
+        else:
+            dbus_command = ['dbus-daemon', '--print-address=1', '--print-pid=1']
+            if config_file:
+                dbus_command += ['--config-file=' + config_file]
+            else:
+                dbus_command += ['--session']
+            log.debug("Running: %s", dbus_command)
+            self.process = subprocess.Popen(
+                dbus_command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+            self._previous_sigterm_handler = signal.signal(
+                signal.SIGTERM, self._sigterm_handler)
+
+            try:
+                self.address = self.process.stdout.readline().strip().decode('ascii')
+                self.pid = int(self.process.stdout.readline().strip().decode('ascii'))
+            except ValueError:
+                error = self.process.stderr.read().strip().decode('unicode-escape')
+                raise RuntimeError(f"Failed to start D-Bus daemon.\n{error}")
+
+            log.debug("Using new D-Bus session with address '%s' with PID %d",
+                      self.address, self.pid)
+
+            if self.session_file:
+                self.write_session_file(self.session_file, self.address, self.pid)
+                log.debug("Wrote D-Bus session file at %s", self.session_file)
+
+            # We must read from the pipes continuously, otherwise the daemon
+            # process will block.
+            self._threads=[threading.Thread(target=self.pipe_to_log, args=(self.process.stdout, 
dbus_stdout_log), daemon=True),
+                           threading.Thread(target=self.pipe_to_log, args=(self.process.stderr, 
dbus_stdout_log), daemon=True)]
+            self._threads[0].start()
+            self._threads[1].start()
+
+        self._gdbus_connection = Gio.DBusConnection.new_for_address_sync(
+            self.address,
+            Gio.DBusConnectionFlags.AUTHENTICATION_CLIENT |
+            Gio.DBusConnectionFlags.MESSAGE_BUS_CONNECTION, None, None)
+
+        log.debug("Pinging the new D-Bus daemon...")
+        self.ping_sync()
+
+    def stop(self):
+        if self.process:
+            log.debug("  Stopping DBus daemon")
+            self.process.terminate()
+            self.process.wait()
+            self.process = None
+        if len(self._threads) > 0:
+            log.debug("  Stopping %i pipe reader threads", len(self._threads))
+            for thread in self._threads:
+                thread.join()
+            self.threads = []
+        if self._previous_sigterm_handler:
+            signal.signal(signal.SIGTERM, self._previous_sigterm_handler)
+            self._previous_sigterm_handler = None
+
+    def pipe_to_log(self, pipe, dbuslog):
+        """This function processes the output from our dbus-daemon instance."""
+        while True:
+            line_raw = pipe.readline()
+
+            if len(line_raw) == 0:
+                break
+
+            line = line_raw.decode('utf-8').rstrip()
+
+            if line.startswith('(tracker-'):
+                # We set G_MESSAGES_PREFIXED=all, meaning that all log messages
+                # output by Tracker processes have a prefix. Note that
+                # g_print() will NOT be captured here.
+                dbuslog.info(line)
+            else:
+                # Log messages from other daemons, including the dbus-daemon
+                # itself, go here. Any g_print() messages also end up here.
+                dbuslog.debug(line)
+        log.debug("Thread stopped")
+
+        # I'm not sure why this is needed, or if it's correct, but without it
+        # we see warnings like this:
+        #
+        #    ResourceWarning: unclosed file <_io.BufferedReader name=3>
+        pipe.close()
+
+    def _sigterm_handler(self, signal, frame):
+        log.info("Received signal %s", signal)
+        self.stop()
+
+    def ping_sync(self):
+        self._gdbus_connection.call_sync(
+            'org.freedesktop.DBus', '/', 'org.freedesktop.DBus', 'GetId',
+            None, None, Gio.DBusCallFlags.NONE, 10000, None)
diff --git a/utils/trackertestutils/dconf.py b/utils/trackertestutils/dconf.py
index 4ad0e88e9..fe6d981fb 100644
--- a/utils/trackertestutils/dconf.py
+++ b/utils/trackertestutils/dconf.py
@@ -18,11 +18,9 @@
 # 02110-1301, USA.
 #
 
-from gi.repository import GLib
-from gi.repository import Gio
-
 import logging
 import os
+import subprocess
 
 log = logging.getLogger(__name__)
 
@@ -36,28 +34,23 @@ class DConfClient(object):
     this reason, and the constructor will fail if this isn't the profile in
     use, to avoid any risk of modifying or removing your real configuration.
 
-    The constructor will fail if DConf is not the default backend, because this
-    probably indicates that the memory backend is in use. Without DConf the
-    required configuration changes will not take effect, causing many tests to
-    break.
+    We use the `gsettings` binary rather than using the Gio.Settings API.
+    This is to avoid the need to set DCONF_PROFILE in our own process
+    environment.
     """
 
-    def __init__(self, schema):
-        self._settings = Gio.Settings.new(schema)
-
-        backend = self._settings.get_property('backend')
-        self._check_settings_backend_is_dconf(backend)
-        self._check_using_correct_dconf_profile()
-
-    def _check_settings_backend_is_dconf(self, backend):
-        typename = type(backend).__name__.split('.')[-1]
-        if typename != 'DConfSettingsBackend':
-            raise Exception(
-                "The functional tests require DConf to be the default "
-                "GSettings backend. Got %s instead." % typename)
+    def __init__(self, sandbox):
+        self.env = os.environ
+        self.env.update(sandbox.extra_env)
+        self.env['DBUS_SESSION_BUS_ADDRESS'] = sandbox.daemon.get_address()
 
     def _check_using_correct_dconf_profile(self):
-        profile = os.environ["DCONF_PROFILE"]
+        profile = self.env.get("DCONF_PROFILE")
+        if not profile:
+            raise Exception(
+                "DCONF_PROFILE is not set in the environment. This class must "
+                "be created inside a TrackerDBussandbox to avoid risk of "
+                "interfering with real settings.")
         if not os.path.exists(profile):
             raise Exception(
                 "Unable to find DConf profile '%s'. Check that Tracker and "
@@ -66,35 +59,11 @@ class DConfClient(object):
 
         assert os.path.basename(profile) == "trackertest"
 
-    def write(self, key, value):
+    def write(self, schema, key, value):
         """
         Write a settings value.
         """
-        self._settings.set_value(key, value)
-
-    def read(self, schema, key):
-        """
-        Read a settings value.
-        """
-        return self._settings.get_value(key)
-
-    def reset(self):
-        """
-        Remove all stored values, resetting configuration to the default.
-
-        This can be done by removing the entire 'trackertest' configuration
-        database.
-        """
-
-        self._check_using_correct_dconf_profile()
-
-        # XDG_CONFIG_HOME is useless, so we use HOME. This code should not be
-        # needed unless for some reason the test is not being run via the
-        # 'test-runner.sh' script.
-        dconf_db = os.path.join(os.environ["HOME"],
-                                ".config",
-                                "dconf",
-                                "trackertest")
-        if os.path.exists(dconf_db):
-            log.debug("[Conf] Removing dconf database: %s", dconf_db)
-            os.remove(dconf_db)
+        subprocess.run(['gsettings', 'set', schema, key, value.print_(False)],
+                       env=self.env,
+                       stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE)
diff --git a/utils/trackertestutils/helpers.py b/utils/trackertestutils/helpers.py
index 2b218e5d0..45fa67242 100644
--- a/utils/trackertestutils/helpers.py
+++ b/utils/trackertestutils/helpers.py
@@ -24,8 +24,8 @@ from gi.repository import GLib
 import atexit
 import logging
 import os
-import subprocess
 
+from . import dbusdaemon
 from . import mainloop
 
 log = logging.getLogger(__name__)
@@ -54,171 +54,9 @@ def _cleanup_processes():
 atexit.register(_cleanup_processes)
 
 
-class Helper:
+class StoreHelper():
     """
-    Abstract helper for Tracker processes. Launches the process
-    and waits for it to appear on the session bus.
-
-    The helper will fail if the process is already running. Use
-    test-runner.sh to ensure the processes run inside a separate DBus
-    session bus.
-
-    The process is watched using a timed GLib main loop source. If the process
-    exits with an error code, the test will abort the next time the main loop
-    is entered (or straight away if currently running the main loop).
-    """
-
-    STARTUP_TIMEOUT = 200   # milliseconds
-    SHUTDOWN_TIMEOUT = 200  #
-
-    def __init__(self, helper_name, bus_name, process_path):
-        self.name = helper_name
-        self.bus_name = bus_name
-        self.process_path = process_path
-
-        self.log = logging.getLogger(f'{__name__}.{self.name}')
-
-        self.process = None
-        self.available = False
-
-        self.loop = mainloop.MainLoop()
-
-        self.bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
-
-    def _start_process(self, command_args=None, extra_env=None):
-        global _process_list
-        _process_list.append(self)
-
-        command = [self.process_path] + (command_args or [])
-        self.log.debug("Starting %s.", ' '.join(command))
-
-        env = os.environ
-        if extra_env:
-            self.log.debug("  starting with extra environment: %s", extra_env)
-            env.update(extra_env)
-
-        try:
-            return subprocess.Popen(command, env=env)
-        except OSError as e:
-            raise RuntimeError("Error starting %s: %s" % (self.process_path, e))
-
-    def _bus_name_appeared(self, connection, name, owner):
-        self.log.debug("%s appeared on the message bus, owned by %s", name, owner)
-        self.available = True
-        self.loop.quit()
-
-    def _bus_name_vanished(self, connection, name):
-        self.log.debug("%s vanished from the message bus", name)
-        self.available = False
-        self.loop.quit()
-
-    def _process_watch_cb(self):
-        if self.process_watch_timeout == 0:
-            # GLib seems to call the timeout after we've removed it
-            # sometimes, which causes errors unless we detect it.
-            return False
-
-        status = self.process.poll()
-
-        if status is None:
-            return True    # continue
-        elif status == 0 and not self.abort_if_process_exits_with_status_0:
-            return True    # continue
-        else:
-            self.process_watch_timeout = 0
-            raise RuntimeError(f"{self.name} exited with status: {self.status}")
-
-    def _process_startup_timeout_cb(self):
-        self.log.debug(f"Process timeout of {self.STARTUP_TIMEOUT}ms was called")
-        self.loop.quit()
-        self.timeout_id = None
-        return False
-
-    def start(self, command_args=None, extra_env=None):
-        """
-        Start an instance of process and wait for it to appear on the bus.
-        """
-        if self.process is not None:
-            raise RuntimeError("%s: already started" % self.name)
-
-        self._bus_name_watch_id = Gio.bus_watch_name_on_connection(
-            self.bus, self.bus_name, Gio.BusNameWatcherFlags.NONE,
-            self._bus_name_appeared, self._bus_name_vanished)
-
-        # We expect the _bus_name_vanished callback to be called here,
-        # causing the loop to exit again.
-        self.loop.run_checked()
-
-        if self.available:
-            # It's running, but we didn't start it...
-            raise RuntimeError("Unable to start test instance of %s: "
-                               "already running" % self.name)
-
-        self.process = self._start_process(command_args=command_args,
-                                           extra_env=extra_env)
-        self.log.debug('Started with PID %i', self.process.pid)
-
-        self.process_startup_timeout = GLib.timeout_add(
-            self.STARTUP_TIMEOUT, self._process_startup_timeout_cb)
-
-        self.abort_if_process_exits_with_status_0 = True
-
-        # Run the loop until the bus name appears, or the process dies.
-        self.loop.run_checked()
-
-        self.abort_if_process_exits_with_status_0 = False
-
-    def stop(self):
-        global _process_list
-
-        if self.process is None:
-            # Seems that it didn't even start...
-            return
-
-        if self.process.poll() == None:
-            GLib.source_remove(self.process_startup_timeout)
-            self.process_startup_timeout = 0
-
-            self.process.terminate()
-            returncode = self.process.wait(timeout=self.SHUTDOWN_TIMEOUT * 1000)
-            if returncode is None:
-                self.log.debug("Process failed to terminate in time, sending kill!")
-                self.process.kill()
-                self.process.wait()
-            elif returncode > 0:
-                self.log.warn("Process returned error code %s", returncode)
-
-        self.log.debug("Process stopped.")
-
-        # Run the loop to handle the expected name_vanished signal.
-        self.loop.run_checked()
-        Gio.bus_unwatch_name(self._bus_name_watch_id)
-
-        self.process = None
-        _process_list.remove(self)
-
-    def kill(self):
-        global _process_list
-
-        if self.process_watch_timeout != 0:
-            GLib.source_remove(self.process_watch_timeout)
-            self.process_watch_timeout = 0
-
-        self.process.kill()
-
-        # Name owner changed callback should take us out from this loop
-        self.loop.run_checked()
-        Gio.bus_unwatch_name(self._bus_name_watch_id)
-
-        self.process = None
-        _process_list.remove(self)
-
-        self.log.debug("Process killed.")
-
-
-class StoreHelper (Helper):
-    """
-    Helper for starting and testing the tracker-store daemon.
+    Helper for testing the tracker-store daemon.
     """
 
     TRACKER_BUSNAME = 'org.freedesktop.Tracker1'
@@ -234,32 +72,41 @@ class StoreHelper (Helper):
     TRACKER_STATUS_OBJ_PATH = "/org/freedesktop/Tracker1/Status"
     STATUS_IFACE = "org.freedesktop.Tracker1.Status"
 
-    def __init__(self, process_path):
-        Helper.__init__(self, "tracker-store", self.TRACKER_BUSNAME, process_path)
+    def __init__(self, dbus_connection):
+        self.log = logging.getLogger(__name__)
+        self.loop = mainloop.MainLoop()
 
-    def start(self, command_args=None, extra_env=None):
-        Helper.start(self, command_args, extra_env)
+        self.bus = dbus_connection
+        self.graph_updated_handler_id = 0
 
         self.resources = Gio.DBusProxy.new_sync(
-            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
             self.TRACKER_BUSNAME, self.TRACKER_OBJ_PATH, self.RESOURCES_IFACE)
 
         self.backup_iface = Gio.DBusProxy.new_sync(
-            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
             self.TRACKER_BUSNAME, self.TRACKER_BACKUP_OBJ_PATH, self.BACKUP_IFACE)
 
         self.stats_iface = Gio.DBusProxy.new_sync(
-            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
             self.TRACKER_BUSNAME, self.TRACKER_STATS_OBJ_PATH, self.STATS_IFACE)
 
         self.status_iface = Gio.DBusProxy.new_sync(
-            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None,
             self.TRACKER_BUSNAME, self.TRACKER_STATUS_OBJ_PATH, self.STATUS_IFACE)
 
+    def start_and_wait_for_ready(self):
+        # The daemon is autostarted as soon as a method is called.
+        #
+        # We set a big timeout to avoid interfering when a daemon is being
+        # interactively debugged.
         self.log.debug("Calling %s.Wait() method", self.STATUS_IFACE)
-        self.status_iface.Wait()
+        self.status_iface.call_sync('Wait', None, Gio.DBusCallFlags.NONE, 1000000, None)
         self.log.debug("Ready")
 
+    def start_watching_updates(self):
+        assert self.graph_updated_handler_id == 0
+
         self.reset_graph_updates_tracking()
 
         def signal_handler(proxy, sender_name, signal_name, parameters):
@@ -268,12 +115,13 @@ class StoreHelper (Helper):
 
         self.graph_updated_handler_id = self.resources.connect(
             'g-signal', signal_handler)
+        self.log.debug("Watching for updates from Resources interface")
 
-    def stop(self):
-        Helper.stop(self)
-
+    def stop_watching_updates(self):
         if self.graph_updated_handler_id != 0:
+            self.log.debug("No longer watching for updates from Resources interface")
             self.resources.disconnect(self.graph_updated_handler_id)
+            self.graph_updated_handler_id = 0
 
     # A system to follow GraphUpdated and make sure all changes are tracked.
     # This code saves every change notification received, and exposes methods
@@ -328,6 +176,7 @@ class StoreHelper (Helper):
         """
         assert (self.inserts_match_function == None)
         assert (self.class_to_track == None), "Already waiting for resource of type %s" % self.class_to_track
+        assert (self.graph_updated_handler_id != 0), "You must call start_watching_updates() first."
 
         self.class_to_track = rdf_class
 
@@ -412,6 +261,7 @@ class StoreHelper (Helper):
         """
         assert (self.deletes_match_function == None)
         assert (self.class_to_track == None)
+        assert (self.graph_updated_handler_id != 0), "You must call start_watching_updates() first."
 
         def find_resource_deletion(deletes_list):
             self.log.debug("find_resource_deletion: looking for %i in %s", id, deletes_list)
@@ -443,8 +293,8 @@ class StoreHelper (Helper):
             # Run the event loop until the correct notification arrives
             try:
                 self.loop.run_checked()
-            except GraphUpdateTimeoutException:
-                raise GraphUpdateTimeoutException("Resource %i has not been deleted." % id)
+            except GraphUpdateTimeoutException as e:
+                raise GraphUpdateTimeoutException("Resource %i has not been deleted." % id) from e
             self.deletes_match_function = None
             self.class_to_track = None
 
@@ -457,6 +307,7 @@ class StoreHelper (Helper):
         assert (self.inserts_match_function == None)
         assert (self.deletes_match_function == None)
         assert (self.class_to_track == None)
+        assert (self.graph_updated_handler_id != 0), "You must call start_watching_updates() first."
 
         self.log.debug("Await change to %i %s (%i, %i existing)", subject_id, property_uri, 
len(self.inserts_list), len(self.deletes_list))
 
@@ -581,3 +432,39 @@ class StoreHelper (Helper):
             return False
         else:
             raise Exception("Something fishy is going on")
+
+
+class TrackerDBusSandbox:
+    """
+    Private D-Bus session bus which executes a sandboxed Tracker instance.
+
+    """
+    def __init__(self, dbus_daemon_config_file, extra_env=None):
+        self.dbus_daemon_config_file = dbus_daemon_config_file
+        self.extra_env = extra_env or {}
+
+        self.daemon = dbusdaemon.DBusDaemon()
+
+    def start(self):
+        env = os.environ
+        env.update(self.extra_env)
+        env['G_MESSAGES_PREFIXED'] = 'all'
+
+        # Precreate runtime dir, to avoid this warning from dbus-daemon:
+        #
+        #    Unable to set up transient service directory: XDG_RUNTIME_DIR 
"/home/sam/tracker-tests/tmp_59i3ev1/run" not available: No such file or directory
+        #
+        xdg_runtime_dir = env.get('XDG_RUNTIME_DIR')
+        if xdg_runtime_dir:
+            os.makedirs(xdg_runtime_dir, exist_ok=True)
+
+        log.info("Starting D-Bus daemon for sandbox.")
+        log.debug("Added environment variables: %s", self.extra_env)
+        self.daemon.start_if_needed(self.dbus_daemon_config_file, env=env)
+
+    def stop(self):
+        log.info("Stopping D-Bus daemon for sandbox.")
+        self.daemon.stop()
+
+    def get_connection(self):
+        return self.daemon.get_connection()
diff --git a/utils/trackertestutils/meson.build b/utils/trackertestutils/meson.build
index 99573e323..a144794d5 100644
--- a/utils/trackertestutils/meson.build
+++ b/utils/trackertestutils/meson.build
@@ -1,5 +1,6 @@
 sources = [
   '__init__.py',
+  'dbusdaemon.py',
   'dconf.py',
   'helpers.py',
   'mainloop.py'


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]