[tracker-miners/sam/index-file-sync: 11/18] functional-tests: Rework TrackerMinerTest fixtures

[Sam Thursfield <sthursfield src gnome org>]

commit 795ce8495436d9a586d80e55913a721c3ab4155e
Author: Sam Thursfield <sam afuera me uk>
Date:   Mon Mar 9 01:37:11 2020 +0100

    functional-tests: Rework TrackerMinerTest fixtures
    
    This will allow adding tests for on-demand indexing.

 tests/functional-tests/fixtures.py    | 114 ++++++++--------------------------
 tests/functional-tests/miner-basic.py |  47 +++++++++++++-
 2 files changed, 72 insertions(+), 89 deletions(-)
---
diff --git a/tests/functional-tests/fixtures.py b/tests/functional-tests/fixtures.py
index 4941bb3e0..9cd0001a7 100644
--- a/tests/functional-tests/fixtures.py
+++ b/tests/functional-tests/fixtures.py
@@ -37,7 +37,6 @@ import shutil
 import subprocess
 import time
 import unittest as ut
-from itertools import chain
 
 import trackertestutils.dconf
 import trackertestutils.helpers
@@ -47,15 +46,13 @@ from minerfshelper import MinerFsHelper
 log = logging.getLogger(__name__)
 
 
-DEFAULT_TEXT = "Some stupid content, to have a test file"
-
-
-def ensure_dir_exists(dirname):
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+class TrackerMinerTest(ut.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(TrackerMinerTest, self).__init__(*args, **kwargs)
 
+        self.workdir = cfg.create_monitored_test_dir()
+        self.indexed_dir = os.path.join(self.workdir, 'test-monitored')
 
-class TrackerMinerTest(ut.TestCase):
     def config(self):
         settings = {
             'org.freedesktop.Tracker.Miner.Files': {
@@ -71,57 +68,32 @@ class TrackerMinerTest(ut.TestCase):
         return settings
 
     def setUp(self):
-        self.workdir = cfg.create_monitored_test_dir()
+        extra_env = cfg.test_environment(self.workdir)
+        extra_env['LANG'] = 'en_GB.utf8'
 
-        self.indexed_dir = os.path.join(self.workdir, 'test-monitored')
+        self.sandbox = trackertestutils.helpers.TrackerDBusSandbox(
+            dbus_daemon_config_file=cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env)
 
-        # It's important that this directory exists BEFORE we start Tracker:
-        # it won't monitor an indexing root for changes if it doesn't exist,
-        # it'll silently ignore it instead. See the tracker_crawler_start()
-        # function.
-        ensure_dir_exists(self.indexed_dir)
+        self.sandbox.start()
 
         try:
-            extra_env = cfg.test_environment(self.workdir)
-            extra_env['LANG'] = 'en_GB.utf8'
-
-            self.sandbox = trackertestutils.helpers.TrackerDBusSandbox(
-                dbus_daemon_config_file=cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env)
-
-            self.sandbox.start()
-
-            try:
-                for schema_name, contents in self.config().items():
-                    dconf = trackertestutils.dconf.DConfClient(self.sandbox)
-                    for key, value in contents.items():
-                        dconf.write(schema_name, key, value)
-
-                # We must create the test data before the miner does its
-                # initial crawl, or it may miss some files due
-                # https://gitlab.gnome.org/GNOME/tracker-miners/issues/79.
-                monitored_files = self.create_test_data()
-
-                self.miner_fs = MinerFsHelper(self.sandbox.get_connection())
-                self.miner_fs.start()
-                self.miner_fs.start_watching_progress()
-
-                self.tracker = trackertestutils.helpers.StoreHelper(
-                    self.miner_fs.get_sparql_connection())
-
-                for tf in monitored_files:
-                    url = self.uri(tf)
-                    self.tracker.ensure_resource(f"a nfo:Document ; nie:url <{url}>")
-            except Exception:
-                self.sandbox.stop()
-                raise
+            for schema_name, contents in self.config().items():
+                dconf = trackertestutils.dconf.DConfClient(self.sandbox)
+                for key, value in contents.items():
+                    dconf.write(schema_name, key, value)
+
+            self.miner_fs = MinerFsHelper(self.sandbox.get_connection())
+            self.miner_fs.start()
+            self.miner_fs.start_watching_progress()
+
+            self.tracker = trackertestutils.helpers.StoreHelper(
+                self.miner_fs.get_sparql_connection())
         except Exception:
-            self.remove_test_data()
-            cfg.remove_monitored_test_dir(self.workdir)
+            self.sandbox.stop()
             raise
 
     def tearDown(self):
         self.sandbox.stop()
-        self.remove_test_data()
         cfg.remove_monitored_test_dir(self.workdir)
 
     def path(self, filename):
@@ -130,32 +102,6 @@ class TrackerMinerTest(ut.TestCase):
     def uri(self, filename):
         return "file://" + os.path.join(self.workdir, filename)
 
-    def create_test_data(self):
-        monitored_files = [
-            'test-monitored/file1.txt',
-            'test-monitored/dir1/file2.txt',
-            'test-monitored/dir1/dir2/file3.txt'
-        ]
-
-        unmonitored_files = [
-            'test-no-monitored/file0.txt'
-        ]
-
-        for tf in chain(monitored_files, unmonitored_files):
-            testfile = self.path(tf)
-            ensure_dir_exists(os.path.dirname(testfile))
-            with open(testfile, 'w') as f:
-                f.write(DEFAULT_TEXT)
-
-        return monitored_files
-
-    def remove_test_data(self):
-        try:
-            shutil.rmtree(os.path.join(self.workdir, 'test-monitored'))
-            shutil.rmtree(os.path.join(self.workdir, 'test-no-monitored'))
-        except Exception as e:
-            log.warning("Failed to remove temporary data dir: %s", e)
-
     def assertResourceExists(self, urn):
         if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == False:
             self.fail("Resource <%s> does not exist" % urn)
@@ -203,11 +149,12 @@ class TrackerMinerFTSTest (TrackerMinerTest):
     Superclass to share methods. Shouldn't be run by itself.
     """
 
-    def prepare_directories(self):
-        # Override content from the base class
-        pass
-
     def setUp(self):
+        # It's very important to make this directory BEFORE the miner starts.
+        # If a configured root doesn't exist when the miner starts up, it will
+        # be ignored even after it's created.
+        os.makedirs(self.indexed_dir, exist_ok=True)
+
         super(TrackerMinerFTSTest, self).setUp()
 
         self.testfile = "test-monitored/miner-fts-test.txt"
@@ -427,13 +374,6 @@ class TrackerWritebackTest (TrackerMinerTest):
         values['org.freedesktop.Tracker.Miner.Files']['enable-writeback'] = GLib.Variant.new_boolean(True)
         return values
 
-    def create_test_data(self):
-        return []
-
-    def remove_test_data(self):
-        for test_file in pathlib.Path(self.indexed_dir).iterdir():
-            test_file.unlink()
-
     def datadir_path(self, filename):
         """Returns the full path to a writeback test file."""
         datadir = os.path.join(os.path.dirname(__file__), 'test-writeback-data')
diff --git a/tests/functional-tests/miner-basic.py b/tests/functional-tests/miner-basic.py
index 658a4f34b..5a4fb9559 100755
--- a/tests/functional-tests/miner-basic.py
+++ b/tests/functional-tests/miner-basic.py
@@ -27,6 +27,7 @@ Check the basic data of the files is updated accordingly in tracker.
 """
 
 
+import itertools
 import logging
 import os
 import shutil
@@ -38,14 +39,56 @@ import fixtures
 
 log = logging.getLogger(__name__)
 
-NFO_DOCUMENT = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Document'
+DEFAULT_TEXT = "Some stupid content, to have a test file"
 
 
 class MinerCrawlTest(fixtures.TrackerMinerTest):
     """
-    Test cases to check if miner is able to monitor files that are created, deleted or moved
+    Tests crawling and monitoring of configured content locations.
     """
 
+    def setUp(self):
+        # We must create the test data before the miner does its
+        # initial crawl, or it may miss some files due
+        # https://gitlab.gnome.org/GNOME/tracker-miners/issues/79.
+        monitored_files = self.create_test_data()
+
+        try:
+            # Start the miner.
+            fixtures.TrackerMinerTest.setUp(self)
+
+            for tf in monitored_files:
+                url = self.uri(tf)
+                self.tracker.ensure_resource(f"a nfo:Document ; nie:url <{url}>")
+        except Exception:
+            cfg.remove_monitored_test_dir(self.workdir)
+            raise
+
+    def create_test_data(self):
+        # It's important that this directory exists BEFORE we start Tracker:
+        # it won't monitor an indexing root for changes if it doesn't exist,
+        # it'll silently ignore it instead. See the tracker_crawler_start()
+        # function.
+        os.makedirs(self.indexed_dir, exist_ok=True)
+
+        monitored_files = [
+            'test-monitored/file1.txt',
+            'test-monitored/dir1/file2.txt',
+            'test-monitored/dir1/dir2/file3.txt'
+        ]
+
+        unmonitored_files = [
+            'test-no-monitored/file0.txt'
+        ]
+
+        for tf in itertools.chain(monitored_files, unmonitored_files):
+            testfile = self.path(tf)
+            os.makedirs(os.path.dirname(testfile), exist_ok=True)
+            with open(testfile, 'w') as f:
+                f.write(DEFAULT_TEXT)
+
+        return monitored_files
+
     def __get_text_documents(self):
         return self.tracker.query("""
           SELECT ?url WHERE {