[tracker-miners/sam/tracker-3.0-functional-tests: 1/3] functional-tests: Update for Tracker 3.0 changes
- From: Sam Thursfield <sthursfield src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker-miners/sam/tracker-3.0-functional-tests: 1/3] functional-tests: Update for Tracker 3.0 changes
- Date: Tue, 11 Feb 2020 00:16:51 +0000 (UTC)
commit af4c9686b0d4f86d71c747a1ecc69d7cb33d6a2b
Author: Sam Thursfield <sam afuera me uk>
Date: Sat Feb 8 19:55:27 2020 +0100
functional-tests: Update for Tracker 3.0 changes
tests/functional-tests/300-miner-basic-ops.py | 91 ++--
.../functional-tests/301-miner-resource-removal.py | 57 +--
tests/functional-tests/310-fts-basic.py | 17 +-
tests/functional-tests/311-fts-file-operations.py | 23 +-
tests/functional-tests/312-fts-stopwords.py | 16 +-
tests/functional-tests/400-extractor-metadata.py | 10 +-
.../401-extractor-flac-cuesheet.py | 9 +-
tests/functional-tests/410-extractor-decorator.py | 22 +-
tests/functional-tests/500-writeback-images.py | 31 +-
.../501-writeback-image-details.py | 11 +-
tests/functional-tests/502-writeback-audio.py | 9 +-
tests/functional-tests/600-applications-camera.py | 122 ++---
tests/functional-tests/601-applications-sync.py | 14 +-
tests/functional-tests/applicationstest.py | 93 ----
tests/functional-tests/configuration.py | 3 +
tests/functional-tests/datagenerator.py | 75 +++
tests/functional-tests/extractor.py | 228 ---------
tests/functional-tests/fixtures.py | 545 +++++++++++++++++++++
tests/functional-tests/meson.build | 7 +
tests/functional-tests/minerfshelper.py | 17 +-
tests/functional-tests/minertest.py | 230 ---------
tests/functional-tests/system.py | 259 ----------
tests/functional-tests/writebacktest.py | 164 -------
23 files changed, 823 insertions(+), 1230 deletions(-)
---
diff --git a/tests/functional-tests/300-miner-basic-ops.py b/tests/functional-tests/300-miner-basic-ops.py
index 290145f8e..658a4f34b 100755
--- a/tests/functional-tests/300-miner-basic-ops.py
+++ b/tests/functional-tests/300-miner-basic-ops.py
@@ -1,5 +1,5 @@
# Copyright (C) 2010, Nokia (ivan frade nokia com)
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -20,24 +20,28 @@
# TODO:
# These tests are for files... we need to write them for folders!
#
+
"""
Monitor a test directory and copy/move/remove/update files and folders there.
Check the basic data of the files is updated accordingly in tracker.
"""
+
+
import logging
import os
import shutil
import time
-
import unittest as ut
-from minertest import CommonTrackerMinerTest
+
+import fixtures
+
log = logging.getLogger(__name__)
NFO_DOCUMENT = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Document'
-class MinerCrawlTest (CommonTrackerMinerTest):
+class MinerCrawlTest(fixtures.TrackerMinerTest):
"""
Test cases to check if miner is able to monitor files that are created, deleted or moved
"""
@@ -101,11 +105,12 @@ class MinerCrawlTest (CommonTrackerMinerTest):
"""
source = os.path.join(self.workdir, "test-no-monitored", "file0.txt")
dest = os.path.join(self.workdir, "test-monitored", "file0.txt")
- shutil.copyfile(source, dest)
- dest_id, dest_urn = self.tracker.await_resource_inserted(NFO_DOCUMENT, self.uri(dest))
+ with self.await_document_inserted(dest) as resource:
+ shutil.copyfile(source, dest)
+ dest_id = resource.id
- # verify if miner indexed this file.
+ # Verify if miner indexed this file.
result = self.__get_text_documents()
self.assertEqual(len(result), 4)
unpacked_result = [r[0] for r in result]
@@ -115,9 +120,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
self.assertIn(self.uri("test-monitored/file0.txt"), unpacked_result)
# Clean the new file so the test directory is as before
- log.debug("Remove and wait")
- os.remove(dest)
- self.tracker.await_resource_deleted(NFO_DOCUMENT, dest_id)
+ with self.tracker.await_delete(dest_id):
+ os.remove(dest)
def test_03_copy_from_monitored_to_unmonitored(self):
"""
@@ -148,9 +152,10 @@ class MinerCrawlTest (CommonTrackerMinerTest):
"""
source = os.path.join(self.workdir, "test-monitored", "file1.txt")
dest = os.path.join(self.workdir, "test-monitored", "dir1", "dir2", "file-test04.txt")
- shutil.copyfile(source, dest)
- dest_id, dest_urn = self.tracker.await_resource_inserted(NFO_DOCUMENT, self.uri(dest))
+ with self.await_document_inserted(dest) as resource:
+ shutil.copyfile(source, dest)
+ dest_id = resource.id
result = self.__get_text_documents()
self.assertEqual(len(result), 4)
@@ -160,9 +165,9 @@ class MinerCrawlTest (CommonTrackerMinerTest):
self.assertIn(self.uri("test-monitored/dir1/dir2/file3.txt"), unpacked_result)
self.assertIn(self.uri("test-monitored/dir1/dir2/file-test04.txt"), unpacked_result)
- # Clean the file
- os.remove(dest)
- self.tracker.await_resource_deleted(NFO_DOCUMENT, dest_id)
+ with self.tracker.await_delete(dest_id):
+ os.remove(dest)
+
self.assertEqual(3, self.tracker.count_instances("nfo:TextDocument"))
@ut.skip("https://gitlab.gnome.org/GNOME/tracker-miners/issues/56";)
@@ -172,8 +177,10 @@ class MinerCrawlTest (CommonTrackerMinerTest):
"""
source = os.path.join(self.workdir, "test-no-monitored", "file0.txt")
dest = os.path.join(self.workdir, "test-monitored", "dir1", "file-test05.txt")
- shutil.move(source, dest)
- dest_id, dest_urn = self.tracker.await_resource_inserted(NFO_DOCUMENT, self.uri(dest))
+
+ with self.await_document_inserted(dest) as resource:
+ shutil.move(source, dest)
+ dest_id = resource.id
result = self.__get_text_documents()
self.assertEqual(len(result), 4)
@@ -183,9 +190,9 @@ class MinerCrawlTest (CommonTrackerMinerTest):
self.assertIn(self.uri("test-monitored/dir1/dir2/file3.txt"), unpacked_result)
self.assertIn(self.uri("test-monitored/dir1/file-test05.txt"), unpacked_result)
- # Clean the file
- os.remove(dest)
- self.tracker.await_resource_deleted(NFO_DOCUMENT, dest_id)
+ with self.tracker.await_delete(dest_id):
+ os.remove(dest)
+
self.assertEqual(3, self.tracker.count_instances("nfo:TextDocument"))
## """ move operation and tracker-miner response test cases """
@@ -198,8 +205,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
source = self.path("test-monitored/dir1/file2.txt")
dest = self.path("test-no-monitored/file2.txt")
source_id = self.tracker.get_resource_id(self.uri(source))
- shutil.move(source, dest)
- self.tracker.await_resource_deleted(NFO_DOCUMENT, source_id)
+ with self.tracker.await_delete(source_id):
+ shutil.move(source, dest)
result = self.__get_text_documents()
self.assertEqual(len(result), 2)
@@ -207,9 +214,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
self.assertIn(self.uri("test-monitored/file1.txt"), unpacked_result)
self.assertIn(self.uri("test-monitored/dir1/dir2/file3.txt"), unpacked_result)
- # Restore the file
- shutil.move(dest, source)
- self.tracker.await_resource_inserted(NFO_DOCUMENT, self.uri(source))
+ with self.await_document_inserted(source):
+ shutil.move(dest, source)
self.assertEqual(3, self.tracker.count_instances("nfo:TextDocument"))
def test_07_move_from_monitored_to_monitored(self):
@@ -220,14 +226,13 @@ class MinerCrawlTest (CommonTrackerMinerTest):
source = self.path("test-monitored/dir1/file2.txt")
dest = self.path("test-monitored/file2.txt")
- resource_id = self.tracker.get_resource_id(url=self.uri(source))
-
source_dir_urn = self.__get_file_urn(os.path.dirname(source))
parent_before = self.__get_parent_urn(source)
self.assertEqual(source_dir_urn, parent_before)
- shutil.move(source, dest)
- self.tracker.await_property_changed(NFO_DOCUMENT, resource_id, 'nie:url')
+ resource_id = self.tracker.get_resource_id(url=self.uri(source))
+ with self.await_document_uri_change(resource_id, source, dest):
+ shutil.move(source, dest)
# Checking fix for NB#214413: After a move operation, nfo:belongsToContainer
# should be changed to the new one
@@ -244,8 +249,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
self.assertIn(self.uri("test-monitored/dir1/dir2/file3.txt"), unpacked_result)
# Restore the file
- shutil.move(dest, source)
- self.tracker.await_property_changed(NFO_DOCUMENT, resource_id, 'nie:url')
+ with self.await_document_uri_change(resource_id, dest, source):
+ shutil.move(dest, source)
result = self.__get_text_documents()
self.assertEqual(len(result), 3)
@@ -258,8 +263,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
"""
victim = self.path("test-monitored/dir1/file2.txt")
victim_id = self.tracker.get_resource_id(self.uri(victim))
- os.remove(victim)
- self.tracker.await_resource_deleted(NFO_DOCUMENT, victim_id)
+ with self.tracker.await_delete(victim_id):
+ os.remove(victim)
result = self.__get_text_documents()
self.assertEqual(len(result), 2)
@@ -268,22 +273,21 @@ class MinerCrawlTest (CommonTrackerMinerTest):
self.assertIn(self.uri("test-monitored/dir1/dir2/file3.txt"), unpacked_result)
# Restore the file
- f = open(victim, "w")
- f.write("Don't panic, everything is fine")
- f.close()
- self.tracker.await_resource_inserted(NFO_DOCUMENT, self.uri(victim))
+ with self.await_document_inserted(victim):
+ with open(victim, "w") as f:
+ f.write("Don't panic, everything is fine")
def test_09_deletion_directory(self):
"""
Delete a directory
"""
victim = self.path("test-monitored/dir1")
- victim_id = self.tracker.get_resource_id(self.uri(victim))
- shutil.rmtree(victim)
file_inside_victim_url = self.uri(os.path.join(victim, "file2.txt"))
file_inside_victim_id = self.tracker.get_resource_id(file_inside_victim_url)
- self.tracker.await_resource_deleted(NFO_DOCUMENT, file_inside_victim_id)
+
+ with self.tracker.await_delete(file_inside_victim_id):
+ shutil.rmtree(victim)
result = self.__get_text_documents()
self.assertEqual(len(result), 1)
@@ -296,10 +300,9 @@ class MinerCrawlTest (CommonTrackerMinerTest):
for f in ["test-monitored/dir1/file2.txt",
"test-monitored/dir1/dir2/file3.txt"]:
filename = self.path(f)
- writer = open(filename, "w")
- writer.write("Don't panic, everything is fine")
- writer.close()
- self.tracker.await_resource_inserted(NFO_DOCUMENT, self.uri(f))
+ with self.await_document_inserted(filename):
+ with open(filename, "w") as f:
+ f.write("Don't panic, everything is fine")
# Check everything is fine
result = self.__get_text_documents()
diff --git a/tests/functional-tests/301-miner-resource-removal.py
b/tests/functional-tests/301-miner-resource-removal.py
index 5296ccae6..4d4d56082 100755
--- a/tests/functional-tests/301-miner-resource-removal.py
+++ b/tests/functional-tests/301-miner-resource-removal.py
@@ -1,5 +1,5 @@
# Copyright (C) 2010, Nokia (ivan frade nokia com)
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -21,20 +21,16 @@ Test that resource removal does not leave debris or clobber too much,
especially in the case where nie:InformationElement != nie:DataObject
"""
-import configuration as cfg
-from minertest import CommonTrackerMinerTest
-
-from gi.repository import GLib
-
import os
+import pathlib
import unittest as ut
-
-NFO_DOCUMENT = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Document'
-NMM_MUSIC_PIECE = 'http://www.tracker-project.org/temp/nmm#MusicPiece'
+# We must import this to set up logging.
+import configuration
+from fixtures import TrackerMinerTest
-class MinerResourceRemovalTest (CommonTrackerMinerTest):
+class MinerResourceRemovalTest(TrackerMinerTest):
def prepare_directories(self):
# Override content from the base class
@@ -47,44 +43,39 @@ class MinerResourceRemovalTest (CommonTrackerMinerTest):
nie:isStoredAs <%s> \
} " % (title, file_urn)
- self.tracker.update(sparql)
-
- return self.tracker.await_resource_inserted(rdf_class=NMM_MUSIC_PIECE,
- title=title)
+ with self.tracker.await_insert(f'a nmm:MusicPiece; nie:title "{title}"') as resource:
+ self.tracker.update(sparql)
+ return resource
def create_test_file(self, file_name):
- file_path = self.path(file_name)
-
- file = open(file_path, 'w')
- file.write("Test")
- file.close()
+ path = pathlib.Path(self.path(file_name))
+ text = "Test"
- return self.tracker.await_resource_inserted(rdf_class=NFO_DOCUMENT,
- url=self.uri(file_name))
+ with self.await_document_inserted(file_name, content=text) as resource:
+ path.write_text(text)
+ return resource
- @ut.skip("https://gitlab.gnome.org/GNOME/tracker-miners/issues/57";)
def test_01_file_deletion(self):
"""
Ensure every logical resource (nie:InformationElement) contained with
in a file is deleted when the file is deleted.
"""
- (file_1_id, file_1_urn) = self.create_test_file("test-monitored/test_1.txt")
- (file_2_id, file_2_urn) = self.create_test_file("test-monitored/test_2.txt")
- (ie_1_id, ie_1_urn) = self.create_test_content(file_1_urn, "Test resource 1")
- (ie_2_id, ie_2_urn) = self.create_test_content(file_2_urn, "Test resource 2")
-
- os.unlink(self.path("test-monitored/test_1.txt"))
+ file_1 = self.create_test_file("test-monitored/test_1.txt")
+ file_2 = self.create_test_file("test-monitored/test_2.txt")
+ ie_1 = self.create_test_content(file_1.urn, "Test resource 1")
+ ie_2 = self.create_test_content(file_2.urn, "Test resource 2")
- self.tracker.await_resource_deleted(NFO_DOCUMENT, file_1_id)
+ with self.tracker.await_delete(file_1.id):
+ os.unlink(self.path("test-monitored/test_1.txt"))
- self.assertResourceMissing(file_1_urn)
+ self.assertResourceMissing(file_1.urn)
# Ensure the logical resource is deleted when the relevant file is
# removed.
- self.assertResourceMissing(ie_1_urn)
+ self.assertResourceMissing(ie_1.urn)
- self.assertResourceExists(file_2_urn)
- self.assertResourceExists(ie_2_urn)
+ self.assertResourceExists(file_2.urn)
+ self.assertResourceExists(ie_2.urn)
# def test_02_removable_device_data (self):
# """
diff --git a/tests/functional-tests/310-fts-basic.py b/tests/functional-tests/310-fts-basic.py
index 8678656ed..4358b69dc 100755
--- a/tests/functional-tests/310-fts-basic.py
+++ b/tests/functional-tests/310-fts-basic.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010, Nokia (ivan frade nokia com)
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -26,17 +26,15 @@
Monitor a directory, copy/move/remove/update text files and check that
the text contents are updated accordingly in the indexes.
"""
-import os
-import shutil
-import locale
-import time
import unittest as ut
-from minertest import CommonTrackerMinerFTSTest, DEFAULT_TEXT
+
+# Must import this for logging.
import configuration as cfg
+import fixtures
-class MinerFTSBasicTest (CommonTrackerMinerFTSTest):
+class MinerFTSBasicTest(fixtures.TrackerMinerFTSTest):
"""
Tests different contents in a single file
"""
@@ -58,8 +56,8 @@ class MinerFTSBasicTest (CommonTrackerMinerFTSTest):
self.assertIn(self.uri(self.testfile), results)
def test_03_long_word(self):
- # TEXT is longer than the 20 characters specified in the fts configuration
- TEXT = "fsfsfsdfskfweeqrewqkmnbbvkdasdjefjewriqjfnc"
+ # TEXT is longer than the 200 characters specified in the fts configuration
+ TEXT = "ai" * 200
self.set_text(TEXT)
results = self.search_word(TEXT)
@@ -95,6 +93,7 @@ class MinerFTSBasicTest (CommonTrackerMinerFTSTest):
TEXT = "abc123"
self.basic_test(TEXT, "abc123")
+ @ut.skip("We don't ignore numbers by default since
https://gitlab.gnome.org/GNOME/tracker/merge_requests/172.";)
def test_10_ignore_numbers(self):
TEXT = "palabra 123123"
self.set_text(TEXT)
diff --git a/tests/functional-tests/311-fts-file-operations.py
b/tests/functional-tests/311-fts-file-operations.py
index ee82b57c4..2a4d2b14b 100755
--- a/tests/functional-tests/311-fts-file-operations.py
+++ b/tests/functional-tests/311-fts-file-operations.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010, Nokia (ivan frade nokia com)
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -28,18 +28,15 @@ the text contents are updated accordingly in the indexes.
"""
import os
import shutil
-import locale
import time
-
import unittest as ut
-from minertest import CommonTrackerMinerFTSTest, DEFAULT_TEXT
-import configuration as cfg
-
-NFO_DOCUMENT = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Document'
+# Needed for log config.
+import configuration as cfg
+import fixtures
-class MinerFTSFileOperationsTest (CommonTrackerMinerFTSTest):
+class MinerFTSFileOperationsTest(fixtures.TrackerMinerFTSTest):
"""
Move, update, delete the files and check the text indexes are updated accordingly.
"""
@@ -52,8 +49,8 @@ class MinerFTSFileOperationsTest (CommonTrackerMinerFTSTest):
self.basic_test(TEXT, "automobile")
id = self._query_id(self.uri(self.testfile))
- os.remove(self.path(self.testfile))
- self.tracker.await_resource_deleted(NFO_DOCUMENT, id)
+ with self.tracker.await_delete(id):
+ os.remove(self.path(self.testfile))
results = self.search_word("automobile")
self.assertEqual(len(results), 0)
@@ -125,10 +122,8 @@ class MinerFTSFileOperationsTest (CommonTrackerMinerFTSTest):
results = self.search_word("airplane")
self.assertEqual(len(results), 0)
- shutil.copyfile(self.path(TEST_16_SOURCE), self.path(TEST_16_DEST))
- self.tracker.await_resource_inserted(rdf_class=NFO_DOCUMENT,
- url=self.uri(TEST_16_DEST),
- required_property='nie:plainTextContent')
+ with self.await_document_inserted(TEST_16_DEST, content=TEXT):
+ shutil.copyfile(self.path(TEST_16_SOURCE), self.path(TEST_16_DEST))
results = self.search_word("airplane")
self.assertEqual(len(results), 1)
diff --git a/tests/functional-tests/312-fts-stopwords.py b/tests/functional-tests/312-fts-stopwords.py
index 7ae37361a..95fa29397 100755
--- a/tests/functional-tests/312-fts-stopwords.py
+++ b/tests/functional-tests/312-fts-stopwords.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010, Nokia (ivan frade nokia com)
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -22,27 +22,26 @@
# TODO:
# These tests are for files... we need to write them for folders!
#
+
"""
Monitor a directory, copy/move/remove/update text files and check that
the text contents are updated accordingly in the indexes.
"""
-import os
-import shutil
-import locale
-import time
+import locale
+import os
import unittest as ut
-from minertest import CommonTrackerMinerFTSTest, DEFAULT_TEXT
+
import configuration as cfg
+import fixtures
-class MinerFTSStopwordsTest (CommonTrackerMinerFTSTest):
+class MinerFTSStopwordsTest(fixtures.TrackerMinerFTSTest):
"""
Search for stopwords in a file
"""
def __get_some_stopwords(self):
-
langcode, encoding = locale.getdefaultlocale()
if "_" in langcode:
langcode = langcode.split("_")[0]
@@ -67,6 +66,7 @@ class MinerFTSStopwordsTest (CommonTrackerMinerFTSTest):
return stopwords
+ @ut.skip("Stopwords are disabled by default since
https://gitlab.gnome.org/GNOME/tracker/merge_requests/172";)
def test_01_stopwords(self):
stopwords = self.__get_some_stopwords()
TEXT = " ".join(["this a completely normal text automobile"] + stopwords)
diff --git a/tests/functional-tests/400-extractor-metadata.py
b/tests/functional-tests/400-extractor-metadata.py
index 5de4035b3..4b4798a9f 100755
--- a/tests/functional-tests/400-extractor-metadata.py
+++ b/tests/functional-tests/400-extractor-metadata.py
@@ -23,17 +23,17 @@ metadata is extracted. Load dynamically the test information from a data
directory (containing xxx.expected files)
"""
-import configuration as cfg
-from extractor import get_tracker_extract_jsonld_output, TrackerExtractTestCase
-import unittest as ut
import json
import os
import shutil
import sys
import tempfile
+import unittest as ut
+import configuration as cfg
+import fixtures
-class GenericExtractionTestCase(TrackerExtractTestCase):
+class GenericExtractionTestCase(fixtures.TrackerExtractTestCase):
"""
Test checks if the tracker extractor is able to retrieve metadata
"""
@@ -86,7 +86,7 @@ class GenericExtractionTestCase(TrackerExtractTestCase):
tmpdir = tempfile.mkdtemp(prefix='tracker-extract-test-')
try:
extra_env = cfg.test_environment(tmpdir)
- result = get_tracker_extract_jsonld_output(extra_env, self.file_to_extract)
+ result = fixtures.get_tracker_extract_jsonld_output(extra_env, self.file_to_extract)
self.__assert_extraction_ok(result)
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
diff --git a/tests/functional-tests/401-extractor-flac-cuesheet.py
b/tests/functional-tests/401-extractor-flac-cuesheet.py
index 42705bc85..46a9fb356 100755
--- a/tests/functional-tests/401-extractor-flac-cuesheet.py
+++ b/tests/functional-tests/401-extractor-flac-cuesheet.py
@@ -26,10 +26,11 @@ import tempfile
import unittest as ut
import configuration as cfg
-from extractor import get_tracker_extract_jsonld_output, create_test_flac, TrackerExtractTestCase
+import datagenerator
+import fixtures
-class FlacCuesheetTest(TrackerExtractTestCase):
+class FlacCuesheetTest(fixtures.TrackerExtractTestCase):
def spec(self, audio_path):
audio_uri = 'file://' + audio_path
return {
@@ -86,9 +87,9 @@ class FlacCuesheetTest(TrackerExtractTestCase):
shutil.copy(os.path.join(datadir, 'audio', 'cuesheet-test.cue'), tmpdir)
audio_path = os.path.join(tmpdir, 'cuesheet-test.flac')
- create_test_flac(audio_path, duration=6*60)
+ datagenerator.create_test_flac(audio_path, duration=6*60)
- result = get_tracker_extract_jsonld_output(
+ result = fixtures.get_tracker_extract_jsonld_output(
cfg.test_environment(tmpdir), audio_path)
self.assert_extract_result_matches_spec(
diff --git a/tests/functional-tests/410-extractor-decorator.py
b/tests/functional-tests/410-extractor-decorator.py
index f6f496d67..d77c035c9 100755
--- a/tests/functional-tests/410-extractor-decorator.py
+++ b/tests/functional-tests/410-extractor-decorator.py
@@ -1,5 +1,5 @@
# Copyright (C) 2016, Sam Thursfield (sam afuera me uk)
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -24,19 +24,18 @@ import os
import shutil
import unittest as ut
-import minertest
+import fixtures
VALID_FILE = os.path.join(
os.path.dirname(__file__), 'test-extraction-data', 'audio',
'mp3-id3v2.4-1.mp3')
-VALID_FILE_CLASS = 'http://www.tracker-project.org/temp/nmm#MusicPiece'
VALID_FILE_TITLE = 'Simply Juvenile'
TRACKER_EXTRACT_FAILURE_DATA_SOURCE = 'tracker:extractor-failure-data-source'
-class ExtractorDecoratorTest(minertest.CommonTrackerMinerTest):
+class ExtractorDecoratorTest(fixtures.TrackerMinerTest):
def test_reextraction(self):
"""Tests whether known files are still re-extracted on user request."""
miner_fs = self.miner_fs
@@ -44,26 +43,25 @@ class ExtractorDecoratorTest(minertest.CommonTrackerMinerTest):
# Insert a valid file and wait extraction of its metadata.
file_path = os.path.join(self.indexed_dir, os.path.basename(VALID_FILE))
- shutil.copy(VALID_FILE, file_path)
- try:
- file_id, file_urn = store.await_resource_inserted(
- VALID_FILE_CLASS, title=VALID_FILE_TITLE)
+ expected = f'a nmm:MusicPiece ; nie:title "{VALID_FILE_TITLE}"'
+ with self.tracker.await_insert(expected) as resource:
+ shutil.copy(VALID_FILE, file_path)
+ file_urn = resource.urn
+ try:
# Remove a key piece of metadata.
# (Writeback must be disabled in the config so that the file
# itself is not changed).
store.update(
'DELETE { GRAPH ?g { <%s> nie:title ?title } }'
' WHERE { GRAPH ?g { <%s> nie:title ?title } }' % (file_urn, file_urn))
- store.await_property_changed(VALID_FILE_CLASS, file_id, 'nie:title')
assert not store.ask('ASK { <%s> nie:title ?title }' % file_urn)
# Request re-indexing (same as `tracker index --file ...`)
- miner_fs.index_file('file://' + os.path.join(self.indexed_dir, file_path))
-
# The extractor should reindex the file and re-add the metadata that we
# deleted, so we should see the nie:title property change.
- store.await_property_changed(VALID_FILE_CLASS, file_id, 'nie:title')
+ with self.tracker.await_insert(f'nie:title "{VALID_FILE_TITLE}"'):
+ miner_fs.index_file('file://' + os.path.join(self.indexed_dir, file_path))
title_result = store.query('SELECT ?title { <%s> nie:title ?title }' % file_urn)
assert len(title_result) == 1
diff --git a/tests/functional-tests/500-writeback-images.py b/tests/functional-tests/500-writeback-images.py
index 46462f55a..6d8c0d6ef 100755
--- a/tests/functional-tests/500-writeback-images.py
+++ b/tests/functional-tests/500-writeback-images.py
@@ -1,5 +1,5 @@
# Copyright (C) 2010, Nokia (ivan frade nokia com)
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -21,20 +21,15 @@
import logging
-import os
-import sys
-import time
-
-from extractor import get_tracker_extract_jsonld_output
-from writebacktest import CommonTrackerWritebackTest
import unittest as ut
-log = logging.getLogger(__name__)
+import configuration
+import fixtures
-REASONABLE_TIMEOUT = 5 # Seconds we wait for tracker-writeback to do the work
+log = logging.getLogger(__name__)
-class WritebackImagesTest (CommonTrackerWritebackTest):
+class WritebackImagesTest(fixtures.TrackerWritebackTest):
"""
Write in tracker store the properties witih writeback support and check
that the new values are actually in the file
@@ -66,7 +61,7 @@ class WritebackImagesTest (CommonTrackerWritebackTest):
self.wait_for_file_change(path, initial_mtime)
log.debug("Got the change")
- results = get_tracker_extract_jsonld_output(self.extra_env, path, mimetype)
+ results = fixtures.get_tracker_extract_jsonld_output(self.extra_env, path, mimetype)
keyDict = expectedKey or prop
self.assertIn(TEST_VALUE, results[keyDict])
@@ -83,20 +78,14 @@ class WritebackImagesTest (CommonTrackerWritebackTest):
}
"""
- CLEAN_VALUE = """
- DELETE {
- <test://writeback-hasTag-test/1> a rdfs:Resource.
- ?u nao:hasTag <test://writeback-hasTag-test/1> .
- } WHERE {
- ?u nao:hasTag <test://writeback-hasTag-test/1> .
- }
- """
+ path = self.prepare_test_image(self.datadir_path(filename))
+ initial_mtime = path.stat().st_mtime
self.tracker.update(SPARQL_TMPL % (filename))
- time.sleep(REASONABLE_TIMEOUT)
+ self.wait_for_file_change(path, initial_mtime)
- results = get_tracker_extract_jsonld_output(self.extra_env, filename, mimetype)
+ results = fixtures.get_tracker_extract_jsonld_output(self.extra_env, filename, mimetype)
self.assertIn("testTag", results["nao:hasTag"])
# JPEG test
diff --git a/tests/functional-tests/501-writeback-image-details.py
b/tests/functional-tests/501-writeback-image-details.py
index f3b53c432..96c16b20e 100755
--- a/tests/functional-tests/501-writeback-image-details.py
+++ b/tests/functional-tests/501-writeback-image-details.py
@@ -1,5 +1,5 @@
# Copyright (C) 2011, Nokia (ivan frade nokia com)
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -23,16 +23,13 @@ import sys
import time
import unittest as ut
-from writebacktest import CommonTrackerWritebackTest as CommonTrackerWritebackTest
-from extractor import get_tracker_extract_jsonld_output
+import fixtures
log = logging.getLogger(__name__)
-REASONABLE_TIMEOUT = 5 # Seconds we wait for tracker-writeback to do the work
-
-class WritebackKeepDateTest (CommonTrackerWritebackTest):
+class WritebackKeepDateTest (fixtures.TrackerWritebackTest):
def setUp(self):
super(WritebackKeepDateTest, self).setUp()
@@ -88,7 +85,7 @@ class WritebackKeepDateTest (CommonTrackerWritebackTest):
self.wait_for_file_change(jpeg_path, initial_mtime)
# Check the value is written in the file
- metadata = get_tracker_extract_jsonld_output(self.extra_env, jpeg_path, "")
+ metadata = fixtures.get_tracker_extract_jsonld_output(self.extra_env, jpeg_path, "")
tags = metadata.get('nao:hasTag', [])
tag_names = [tag['nao:prefLabel'] for tag in tags]
diff --git a/tests/functional-tests/502-writeback-audio.py b/tests/functional-tests/502-writeback-audio.py
index 11e75cbf4..0993b76e5 100755
--- a/tests/functional-tests/502-writeback-audio.py
+++ b/tests/functional-tests/502-writeback-audio.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -18,11 +18,10 @@
import unittest
-from extractor import get_tracker_extract_jsonld_output
-from writebacktest import CommonTrackerWritebackTest
+import fixtures
-class WritebackAudioTest(CommonTrackerWritebackTest):
+class WritebackAudioTest(fixtures.TrackerWritebackTest):
def _writeback_test(self, path):
prop = 'nie:title'
@@ -39,7 +38,7 @@ class WritebackAudioTest(CommonTrackerWritebackTest):
self.wait_for_file_change(path, initial_mtime)
- results = get_tracker_extract_jsonld_output(self.extra_env, path)
+ results = fixtures.get_tracker_extract_jsonld_output(self.extra_env, path)
self.assertIn(TEST_VALUE, results[prop])
def test_writeback_mp3(self):
diff --git a/tests/functional-tests/600-applications-camera.py
b/tests/functional-tests/600-applications-camera.py
index 9163c7315..0c702f139 100755
--- a/tests/functional-tests/600-applications-camera.py
+++ b/tests/functional-tests/600-applications-camera.py
@@ -1,5 +1,5 @@
# Copyright (C) 2011, Nokia Corporation <ivan frade nokia com>
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
@@ -24,18 +24,15 @@ Tests trying to simulate the behaviour of applications working with tracker
import logging
import os
import random
-
import unittest as ut
-from applicationstest import CommonTrackerApplicationTest as CommonTrackerApplicationTest
+import fixtures
-log = logging.getLogger(__name__)
-NMM_PHOTO = 'http://www.tracker-project.org/temp/nmm#Photo'
-NMM_VIDEO = 'http://www.tracker-project.org/temp/nmm#Video'
+log = logging.getLogger(__name__)
-class TrackerCameraTestSuite (CommonTrackerApplicationTest):
+class TrackerCameraTestSuite (fixtures.TrackerApplicationTest):
"""
Common functionality for camera tests.
"""
@@ -46,31 +43,14 @@ class TrackerCameraTestSuite (CommonTrackerApplicationTest):
"""
insert = """
INSERT { <%(urn)s>
- a nie:InformationElement,
- nie:DataObject,
- nfo:Image,
- nfo:Media,
- nfo:Visual,
- nmm:Photo
- }
-
- DELETE { <%(urn)s> nie:mimeType ?_1 }
- WHERE { <%(urn)s> nie:mimeType ?_1 }
-
- INSERT { <%(urn)s>
- a rdfs:Resource ;
- nie:mimeType \"image/jpeg\"
- }
-
- DELETE { <%(urn)s> nie:url ?_2 }
- WHERE { <%(urn)s> nie:url ?_2 }
-
- INSERT { <%(urn)s>
- a rdfs:Resource ;
+ a nie:InformationElement, nie:DataObject, nfo:Image, nfo:Media,
+ nfo:Visual, nmm:Photo ;
+ nie:mimeType \"image/jpeg\" ;
nie:url \"%(file_url)s\" ;
- nie:isStoredAs <%(urn)s>
+ nie:isStoredAs <%(urn)s> .
}
""" % locals()
+ logging.debug("Running: %s", insert)
self.tracker.update(insert)
self.assertEqual(self.get_urn_count_by_url(file_url), 1)
@@ -80,29 +60,11 @@ class TrackerCameraTestSuite (CommonTrackerApplicationTest):
"""
insert = """
INSERT { <%(urn)s>
- a nie:InformationElement,
- nie:DataObject,
- nfo:Video,
- nfo:Media,
- nfo:Visual,
- nmm:Video
- }
-
- DELETE { <%(urn)s> nie:mimeType ?_1 }
- WHERE { <%(urn)s> nie:mimeType ?_1 }
-
- INSERT { <%(urn)s>
- a rdfs:Resource ;
- nie:mimeType \"video/mp4\"
- }
-
- DELETE { <%(urn)s> nie:url ?_2 }
- WHERE { <%(urn)s> nie:url ?_2 }
-
- INSERT { <%(urn)s>
- a rdfs:Resource ;
+ a nie:InformationElement, nie:DataObject, nfo:Video, nfo:Media,
+ nfo:Visual, nmm:Video ;
+ nie:mimeType \"video/mp4\" ;
nie:url \"%(file_url)s\" ;
- nie:isStoredAs <%(urn)s>
+ nie:isStoredAs <%(urn)s> .
}
""" % locals()
self.tracker.update(insert)
@@ -147,17 +109,18 @@ class TrackerCameraPicturesApplicationTests (TrackerCameraTestSuite):
dest_fileuri = "file://" + dest_filepath
self.insert_photo_resource_info(fileurn, dest_fileuri)
+ fileid = self.tracker.get_resource_id_by_uri(fileurn)
# Copy the image to the dest path
- self.slowcopy_file(origin_filepath, dest_filepath)
- assert os.path.exists(dest_filepath)
- dest_id, dest_urn = self.system.store.await_resource_inserted(NMM_PHOTO, dest_fileuri)
+ with self.tracker.await_update(fileid, "", "nfo:contentCreated ?created"):
+ self.slowcopy_file(origin_filepath, dest_filepath)
+ assert os.path.exists(dest_filepath)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 1)
# Clean the new file so the test directory is as before
log.debug("Remove and wait")
- os.remove(dest_filepath)
- self.system.store.await_resource_deleted(NMM_PHOTO, dest_id)
+ with self.tracker.await_delete(fileid):
+ os.remove(dest_filepath)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 0)
def test_02_camera_picture_geolocation(self):
@@ -179,31 +142,32 @@ class TrackerCameraPicturesApplicationTests (TrackerCameraTestSuite):
postaladdressurn = "tracker://test_camera_picture_02_postaladdress/" + str(random.randint(0, 100))
self.insert_photo_resource_info(fileurn, dest_fileuri)
+ fileid = self.tracker.get_resource_id_by_uri(fileurn)
# FIRST, open the file for writing, and just write some garbage, to simulate that
# we already started recording the video...
fdest = open(dest_filepath, 'wb')
- fdest.write("some garbage written here")
- fdest.write("to simulate we're recording something...")
+ fdest.write(b"some garbage written here")
+ fdest.write(b"to simulate we're recording something...")
fdest.seek(0)
# SECOND, set slo:location
self.insert_dummy_location_info(fileurn, geolocationurn, postaladdressurn)
# THIRD, start copying the image to the dest path
- original_file = os.path.join(self.get_data_dir(), self.get_test_image())
- self.slowcopy_file_fd(original_file, fdest)
- fdest.close()
+ with self.tracker.await_update(fileid, "", "nfo:contentCreated ?created"):
+ original_file = os.path.join(self.get_data_dir(), self.get_test_image())
+ self.slowcopy_file_fd(original_file, fdest)
+ fdest.close()
assert os.path.exists(dest_filepath)
# FOURTH, ensure we have only 1 resource
- dest_id, dest_urn = self.system.store.await_resource_inserted(NMM_PHOTO, dest_fileuri)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 1)
# Clean the new file so the test directory is as before
log.debug("Remove and wait")
- os.remove(dest_filepath)
- self.system.store.await_resource_deleted(NMM_PHOTO, dest_id)
+ with self.tracker.await_delete(resource.id):
+ os.remove(dest_filepath)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 0)
@@ -227,15 +191,15 @@ class TrackerCameraVideosApplicationTests (TrackerCameraTestSuite):
self.insert_video_resource_info(fileurn, dest_fileuri)
# Copy the image to the dest path
- self.slowcopy_file(origin_filepath, dest_filepath)
- assert os.path.exists(dest_filepath)
- dest_id, dest_urn = self.system.store.await_resource_inserted(NMM_PHOTO, dest_fileuri)
+ with self.await_photo_inserted(dest_filepath) as resource:
+ self.slowcopy_file(origin_filepath, dest_filepath)
+ assert os.path.exists(dest_filepath)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 1)
# Clean the new file so the test directory is as before
log.debug("Remove and wait")
- os.remove(dest_filepath)
- self.system.store.await_resource_deleted(NMM_PHOTO, dest_id)
+ with self.await_delete(resource.id):
+ os.remove(dest_filepath)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 0)
def test_02_camera_video_geolocation(self):
@@ -262,32 +226,28 @@ class TrackerCameraVideosApplicationTests (TrackerCameraTestSuite):
# FIRST, open the file for writing, and just write some garbage, to simulate that
# we already started recording the video...
fdest = open(dest_filepath, 'wb')
- fdest.write("some garbage written here")
- fdest.write("to simulate we're recording something...")
+ fdest.write(b"some garbage written here")
+ fdest.write(b"to simulate we're recording something...")
fdest.seek(0)
# SECOND, set slo:location
self.insert_dummy_location_info(fileurn, geolocationurn, postaladdressurn)
# THIRD, start copying the image to the dest path
- self.slowcopy_file_fd(origin_filepath, fdest)
- fdest.close()
- assert os.path.exists(dest_filepath)
+ with self.await_photo_inserted(dest_filepath) as resource:
+ self.slowcopy_file_fd(origin_filepath, fdest)
+ fdest.close()
+ assert os.path.exists(dest_filepath)
# FOURTH, ensure we have only 1 resource
- dest_id, dest_urn = self.system.store.await_resource_inserted(NMM_VIDEO, dest_fileuri)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 1)
# Clean the new file so the test directory is as before
log.debug("Remove and wait")
- os.remove(dest_filepath)
- self.system.store.await_resource_deleted(NMM_VIDEO, dest_id)
+ with self.tracker.await_delete(resource.id):
+ os.remove(dest_filepath)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 0)
if __name__ == "__main__":
- print("FIXME: This test is skipped as it currently fails. See:
https://gitlab.gnome.org/GNOME/tracker-miners/issues/55";)
- import sys
- sys.exit(77)
-
ut.main(verbosity=2)
diff --git a/tests/functional-tests/601-applications-sync.py b/tests/functional-tests/601-applications-sync.py
index 7e715f447..931f6d546 100755
--- a/tests/functional-tests/601-applications-sync.py
+++ b/tests/functional-tests/601-applications-sync.py
@@ -1,5 +1,5 @@
# Copyright (C) 2011, Nokia Corporation <ivan frade nokia com>
-# Copyright (C) 2019, Sam Thursfield (sam afuera me uk)
+# Copyright (C) 2019-2020, Sam Thursfield (sam afuera me uk)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
@@ -25,17 +25,15 @@ import logging
import os
import random
import shutil
-
import unittest as ut
-from applicationstest import CommonTrackerApplicationTest as CommonTrackerApplicationTest
+import fixtures
-log = logging.getLogger(__name__)
-NMM_MUSICPIECE = 'http://www.tracker-project.org/temp/nmm#MusicPiece'
+log = logging.getLogger(__name__)
-class TrackerSyncApplicationTests (CommonTrackerApplicationTest):
+class TrackerSyncApplicationTests(fixtures.TrackerApplicationTest):
def test_01_sync_audio_nb219946(self):
"""
@@ -116,8 +114,8 @@ class TrackerSyncApplicationTests (CommonTrackerApplicationTest):
# Clean the new file so the test directory is as before
log.debug("Remove and wait")
- os.remove(dest_filepath)
- self.tracker.await_resource_deleted(NMM_MUSICPIECE, resource_id)
+ with self.tracker.await_delete(resource_id):
+ os.remove(dest_filepath)
self.assertEqual(self.get_urn_count_by_url(dest_fileuri), 0)
self.miner_fs.stop_watching_progress()
diff --git a/tests/functional-tests/configuration.py b/tests/functional-tests/configuration.py
index 7070f86c2..557b0c477 100644
--- a/tests/functional-tests/configuration.py
+++ b/tests/functional-tests/configuration.py
@@ -28,6 +28,9 @@ import tempfile
import sys
+DEFAULT_TIMEOUT = 10
+
+
if 'TRACKER_FUNCTIONAL_TEST_CONFIG' not in os.environ:
raise RuntimeError("The TRACKER_FUNCTIONAL_TEST_CONFIG environment "
"variable must be set to point to the location of "
diff --git a/tests/functional-tests/datagenerator.py b/tests/functional-tests/datagenerator.py
new file mode 100644
index 000000000..705a22445
--- /dev/null
+++ b/tests/functional-tests/datagenerator.py
@@ -0,0 +1,75 @@
+# Copyright (C) 2018-2019, Sam Thursfield <sam afuera me uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+import logging
+import math
+import shlex
+
+import gi
+gi.require_version('Gst', '1.0')
+from gi.repository import Gst
+
+
+log = logging.getLogger(__name__)
+
+
+def create_test_flac(path, duration, timeout=10, tags=None):
+ """
+ Create a .flac audio file for testing purposes.
+
+ FLAC audio doesn't compress test data particularly efficiently, so
+ committing an audio file more than a few seconds long to Git is not
+ practical. This function creates a .flac file containing a test tone.
+ The 'duration' parameter sets the length in seconds of the time.
+
+ The function is guaranteed to return or raise an exception within the
+ number of seconds given in the 'timeout' parameter.
+ """
+
+ Gst.init([])
+
+ num_buffers = math.ceil(duration * 44100 / 1024.0)
+
+ pipeline_src = ' ! '.join([
+ 'audiotestsrc num-buffers=%s samplesperbuffer=1024' % num_buffers,
+ 'capsfilter caps="audio/x-raw,rate=44100"',
+ 'flacenc name=flacenc',
+ 'filesink location="%s"' % str(path),
+ ])
+
+ log.debug("Running pipeline: %s", pipeline_src)
+ pipeline = Gst.parse_launch(pipeline_src)
+
+ if tags:
+ flacenc = pipeline.get_child_by_name('flacenc')
+ flacenc.merge_tags(tags, Gst.TagMergeMode.REPLACE_ALL)
+
+ pipeline.set_state(Gst.State.PLAYING)
+
+ msg = pipeline.get_bus().poll(Gst.MessageType.ERROR | Gst.MessageType.EOS,
+ timeout * Gst.SECOND)
+ if msg and msg.type == Gst.MessageType.EOS:
+ pass
+ elif msg and msg.type == Gst.MessageType.ERROR:
+ raise RuntimeError(msg.parse_error())
+ elif msg:
+ raise RuntimeError("Got unexpected GStreamer message %s" % msg.type)
+ else:
+ raise RuntimeError("Timeout generating test audio file after %i seconds" % timeout)
+
+ pipeline.set_state(Gst.State.NULL)
diff --git a/tests/functional-tests/fixtures.py b/tests/functional-tests/fixtures.py
new file mode 100644
index 000000000..d0e8a5c3a
--- /dev/null
+++ b/tests/functional-tests/fixtures.py
@@ -0,0 +1,545 @@
+#
+# Copyright (C) 2010, Nokia <ivan frade nokia com>
+# Copyright (C) 2018-2020 Sam Thursfield <sam afuera me uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+
+"""
+Fixtures used by the tracker-miners functional-tests.
+"""
+
+import gi
+gi.require_version('Gst', '1.0')
+gi.require_version('Tracker', '3.0')
+from gi.repository import GLib
+from gi.repository import Tracker
+
+import errno
+import json
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import time
+import unittest as ut
+from itertools import chain
+
+import trackertestutils.dconf
+import trackertestutils.helpers
+import configuration as cfg
+from minerfshelper import MinerFsHelper
+
+log = logging.getLogger(__name__)
+
+
+DEFAULT_TEXT = "Some stupid content, to have a test file"
+
+
+def ensure_dir_exists(dirname):
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+
+
+class TrackerMinerTest(ut.TestCase):
+ def config(self):
+ settings = {
+ 'org.freedesktop.Tracker.Miner.Files': {
+ 'enable-writeback': GLib.Variant.new_boolean(False),
+ 'index-recursive-directories': GLib.Variant.new_strv([self.indexed_dir]),
+ 'index-single-directories': GLib.Variant.new_strv([]),
+ 'index-optical-discs': GLib.Variant.new_boolean(False),
+ 'index-removable-devices': GLib.Variant.new_boolean(False),
+ 'index-applications': GLib.Variant.new_boolean(False),
+ 'throttle': GLib.Variant.new_int32(5),
+ }
+ }
+ return settings
+
+ def setUp(self):
+ self.workdir = cfg.create_monitored_test_dir()
+
+ self.indexed_dir = os.path.join(self.workdir, 'test-monitored')
+
+ # It's important that this directory exists BEFORE we start Tracker:
+ # it won't monitor an indexing root for changes if it doesn't exist,
+ # it'll silently ignore it instead. See the tracker_crawler_start()
+ # function.
+ ensure_dir_exists(self.indexed_dir)
+
+ try:
+ extra_env = cfg.test_environment(self.workdir)
+ extra_env['LANG'] = 'en_GB.utf8'
+
+ self.sandbox = trackertestutils.helpers.TrackerDBusSandbox(
+ dbus_daemon_config_file=cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env)
+
+ self.sandbox.start()
+
+ try:
+ for schema_name, contents in self.config().items():
+ dconf = trackertestutils.dconf.DConfClient(self.sandbox)
+ for key, value in contents.items():
+ dconf.write(schema_name, key, value)
+
+ # We must create the test data before the miner does its
+ # initial crawl, or it may miss some files due
+ # https://gitlab.gnome.org/GNOME/tracker-miners/issues/79.
+ monitored_files = self.create_test_data()
+
+ self.miner_fs = MinerFsHelper(self.sandbox.get_connection())
+ self.miner_fs.start()
+ self.miner_fs.start_watching_progress()
+
+ self.tracker = trackertestutils.helpers.StoreHelper(
+ self.miner_fs.get_sparql_connection())
+
+ for tf in monitored_files:
+ url = self.uri(tf)
+ self.tracker.ensure_resource(f"a nfo:Document ; nie:url <{url}>")
+ except Exception:
+ self.sandbox.stop()
+ raise
+ except Exception:
+ self.remove_test_data()
+ cfg.remove_monitored_test_dir(self.workdir)
+ raise
+
+ def tearDown(self):
+ self.sandbox.stop()
+ self.remove_test_data()
+ cfg.remove_monitored_test_dir(self.workdir)
+
+ def path(self, filename):
+ return os.path.join(self.workdir, filename)
+
+ def uri(self, filename):
+ return "file://" + os.path.join(self.workdir, filename)
+
+ def create_test_data(self):
+ monitored_files = [
+ 'test-monitored/file1.txt',
+ 'test-monitored/dir1/file2.txt',
+ 'test-monitored/dir1/dir2/file3.txt'
+ ]
+
+ unmonitored_files = [
+ 'test-no-monitored/file0.txt'
+ ]
+
+ for tf in chain(monitored_files, unmonitored_files):
+ testfile = self.path(tf)
+ ensure_dir_exists(os.path.dirname(testfile))
+ with open(testfile, 'w') as f:
+ f.write(DEFAULT_TEXT)
+
+ return monitored_files
+
+ def remove_test_data(self):
+ try:
+ shutil.rmtree(os.path.join(self.workdir, 'test-monitored'))
+ shutil.rmtree(os.path.join(self.workdir, 'test-no-monitored'))
+ except Exception as e:
+ log.warning("Failed to remove temporary data dir: %s", e)
+
+ def assertResourceExists(self, urn):
+ if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == False:
+ self.fail("Resource <%s> does not exist" % urn)
+
+ def assertResourceMissing(self, urn):
+ if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == True:
+ self.fail("Resource <%s> should not exist" % urn)
+
+ def await_document_inserted(self, path, content=None):
+ """Wraps await_insert() context manager."""
+ url = self.uri(path)
+
+ expected = [
+ 'a nfo:Document',
+ f'nie:url <{url}>',
+ ]
+
+ if content:
+ content_escaped = Tracker.sparql_escape_string(content)
+ expected += [f'nie:plainTextContent "{content_escaped}"']
+
+ return self.tracker.await_insert('; '.join(expected))
+
+ def await_document_uri_change(self, resource_id, from_path, to_path):
+ """Wraps await_update() context manager."""
+ from_url = self.uri(from_path)
+ to_url = self.uri(to_path)
+ return self.tracker.await_update(resource_id,
+ f'nie:url <{from_url}>',
+ f'nie:url <{to_url}>')
+
+ def await_photo_inserted(self, path):
+ url = self.uri(path)
+
+ expected = [
+ 'a nmm:Photo',
+ f'nie:url <{url}>',
+ ]
+
+ return self.tracker.await_insert('; '.join(expected))
+
+
+class TrackerMinerFTSTest (TrackerMinerTest):
+ """
+ Superclass to share methods. Shouldn't be run by itself.
+ """
+
+ def prepare_directories(self):
+ # Override content from the base class
+ pass
+
+ def setUp(self):
+ super(TrackerMinerFTSTest, self).setUp()
+
+ self.testfile = "test-monitored/miner-fts-test.txt"
+
+ def set_text(self, text):
+ text_escaped = Tracker.sparql_escape_string(text)
+ path = pathlib.Path(self.path(self.testfile))
+
+ if path.exists():
+ old_text_escaped = Tracker.sparql_escape_string(path.read_text())
+ resource_id = self.tracker.get_resource_id(self.uri(self.testfile))
+ with self.tracker.await_update(resource_id,
+ f'nie:plainTextContent "{old_text_escaped}"',
+ f'nie:plainTextContent "{text_escaped}"'):
+ path.write_text(text)
+ else:
+ url = self.uri(self.testfile)
+ expected = f'a nfo:Document; nie:url <{url}>; nie:plainTextContent "{text_escaped}"'
+ with self.tracker.await_insert(expected):
+ path.write_text(text)
+
+ def search_word(self, word):
+ """
+ Return list of URIs with the word in them
+ """
+ log.info("Search for: %s", word)
+ results = self.tracker.query("""
+ SELECT ?url WHERE {
+ ?u a nfo:TextDocument ;
+ nie:url ?url ;
+ fts:match '%s'.
+ }
+ """ % (word))
+ return [r[0] for r in results]
+
+ def basic_test(self, text, word):
+ """
+ Save the text on the testfile, search the word
+ and assert the testfile is only result.
+
+ Be careful with the default contents of the text files
+ ( see minertest.py DEFAULT_TEXT )
+ """
+ self.set_text(text)
+ results = self.search_word(word)
+ self.assertEqual(len(results), 1)
+ self.assertIn(self.uri(self.testfile), results)
+
+ def _query_id(self, uri):
+ query = "SELECT tracker:id(?urn) WHERE { ?urn nie:url \"%s\". }" % uri
+ result = self.tracker.query(query)
+ assert len(result) == 1
+ return int(result[0][0])
+
+
+def get_tracker_extract_jsonld_output(extra_env, filename, mime_type=None):
+ """
+ Runs `tracker-extract --file` to extract metadata from a file.
+ """
+
+ tracker_extract = os.path.join(cfg.TRACKER_EXTRACT_PATH)
+ command = [tracker_extract, '--verbosity=0', '--output-format=json-ld', '--file', str(filename)]
+ if mime_type is not None:
+ command.extend(['--mime', mime_type])
+
+ # We depend on parsing the output, so verbosity MUST be 0.
+ extra_env['TRACKER_VERBOSITY'] = '0'
+ # Tell GStreamer not to fork to create the registry
+ extra_env['GST_REGISTRY_FORK'] = 'no'
+ log.debug('Adding to environment: %s', ' '.join('%s=%s' % (k, v) for k, v in extra_env.items()))
+
+ env = os.environ.copy()
+ env.update(extra_env)
+
+ log.debug('Running: %s', ' '.join(command))
+ try:
+ p = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ raise RuntimeError("Did not find tracker-extract binary. Is the 'extract' option disabled?")
+ else:
+ raise RuntimeError("Error running tracker-extract: %s" % (e))
+ stdout, stderr = p.communicate()
+
+ if p.returncode != 0:
+ raise RuntimeError(
+ "tracker-extract returned non-zero exit code: %s\n"
+ "Error output:\n%s\n" % (p.returncode, stderr.decode('utf-8').strip()))
+
+ if len(stderr) > 0:
+ error_output = stderr.decode('utf-8').strip()
+ log.debug("Error output from tracker-extract:\n%s", error_output)
+
+ try:
+ output = stdout.decode('utf-8')
+
+ if len(output.strip()) == 0:
+ raise RuntimeError("tracker-extract didn't return any data.\n"
+ "Error output was: %s" % error_output)
+
+ data = json.loads(output)
+ except ValueError as e:
+ raise RuntimeError("tracker-extract did not return valid JSON data: %s\n"
+ "Output was: %s" % (e, output))
+
+ return data
+
+
+class TrackerExtractTestCase(ut.TestCase):
+ def assertDictHasKey(self, d, key, msg=None):
+ if not isinstance(d, dict):
+ self.fail("Expected dict, got %s" % d)
+ if key not in d:
+ import pdb
+ pdb.set_trace()
+
+ standardMsg = "Missing: %s" % (key)
+ self.fail(self._formatMessage(msg, standardMsg))
+ else:
+ return
+
+ def assertIsURN(self, supposed_uuid, msg=None):
+ import uuid
+
+ try:
+ if (supposed_uuid.startswith("<") and supposed_uuid.endswith(">")):
+ supposed_uuid = supposed_uuid[1:-1]
+
+ uuid.UUID(supposed_uuid)
+ except ValueError:
+ standardMsg = "'%s' is not a valid UUID" % (supposed_uuid)
+ self.fail(self._formatMessage(msg, standardMsg))
+
+ def assert_extract_result_matches_spec(self, spec, result, filename, spec_filename):
+ """
+ Checks tracker-extract json-ld output against the expected result.
+
+ Use get_tracker_extract_jsonld_output() to get the extractor output.
+
+ Look in test-extraction-data/*/*.expected.json for examples of the spec
+ format.
+ """
+
+ error_missing_prop = "Property '%s' hasn't been extracted from file \n'%s'\n (requested on '%s')"
+ error_wrong_value = "on property '%s' from file %s\n (requested on: '%s')"
+ error_wrong_length = "Length mismatch on property '%s' from file %s\n (requested on: '%s')"
+ error_extra_prop = "Property '%s' was explicitely banned for file \n'%s'\n (requested on '%s')"
+ error_extra_prop_v = "Property '%s' with value '%s' was explicitely banned for file \n'%s'\n
(requested on %s')"
+
+ expected_pairs = [] # List of expected (key, value)
+ unexpected_pairs = [] # List of unexpected (key, value)
+ expected_keys = [] # List of expected keys (the key must be there, value doesnt matter)
+
+ for k, v in list(spec.items()):
+ if k.startswith("!"):
+ unexpected_pairs.append((k[1:], v))
+ elif k == '@type':
+ expected_keys.append('@type')
+ else:
+ expected_pairs.append((k, v))
+
+ for prop, expected_value in expected_pairs:
+ self.assertDictHasKey(result, prop,
+ error_missing_prop % (prop, filename, spec_filename))
+ if expected_value == "@URNUUID@":
+ self.assertIsURN(result[prop][0]['@id'],
+ error_wrong_value % (prop, filename, spec_filename))
+ else:
+ if isinstance(expected_value, list):
+ if not isinstance(result[prop], list):
+ raise AssertionError("Expected a list property for %s, but got a %s: %s" % (
+ prop, type(result[prop]).__name__, result[prop]))
+
+ self.assertEqual(len(expected_value), len(result[prop]),
+ error_wrong_length % (prop, filename, spec_filename))
+
+ for i in range(0, len(expected_value)):
+ if isinstance(expected_value[i], dict):
+ self.assert_extract_result_matches_spec(expected_value[i], result[prop][i],
filename, spec_filename)
+ else:
+ self.assertEqual(str(expected_value[i]), str(result[prop][i]),
+ error_wrong_value % (prop, filename, spec_filename))
+ elif isinstance(expected_value, dict):
+ self.assert_extract_result_matches_spec(expected_value, result[prop], filename,
spec_filename)
+ else:
+ self.assertEqual(str(spec[prop]), str(result[prop]),
+ error_wrong_value % (prop, filename, spec_filename))
+
+ for (prop, value) in unexpected_pairs:
+ # There is no prop, or it is but not with that value
+ if (value == ""):
+ self.assertFalse(prop in result,
+ error_extra_prop % (prop, filename, spec_filename))
+ else:
+ if (value == "@URNUUID@"):
+ self.assertIsURN(result[prop][0],
+ error_extra_prop % (prop, filename, spec_filename))
+ else:
+ self.assertNotIn(value, result[prop],
+ error_extra_prop_v % (prop, value, filename, spec_filename))
+
+ for prop in expected_keys:
+ self.assertDictHasKey(result, prop,
+ error_missing_prop % (prop, filename, spec_filename))
+
+
+TEST_FILE_JPEG = "writeback-test-1.jpeg"
+TEST_FILE_TIFF = "writeback-test-2.tif"
+TEST_FILE_PNG = "writeback-test-4.png"
+
+
+class TrackerWritebackTest (TrackerMinerTest):
+ """
+ Superclass to share methods. Shouldn't be run by itself.
+ Start all processes including writeback, miner pointing to WRITEBACK_TMP_DIR
+ """
+
+ def config(self):
+ values = super(TrackerWritebackTest, self).config()
+ values['org.freedesktop.Tracker.Miner.Files']['enable-writeback'] = GLib.Variant.new_boolean(True)
+ return values
+
+ def create_test_data(self):
+ return []
+
+ def remove_test_data(self):
+ for test_file in pathlib.Path(self.indexed_dir).iterdir():
+ test_file.unlink()
+
+ def datadir_path(self, filename):
+ """Returns the full path to a writeback test file."""
+ datadir = os.path.join(os.path.dirname(__file__), 'test-writeback-data')
+ return pathlib.Path(os.path.join(datadir, filename))
+
+ def prepare_test_audio(self, filename):
+ path = pathlib.Path(os.path.join(self.indexed_dir, os.path.basename(filename)))
+ url = path.as_uri()
+
+ # Copy and wait. The extractor adds the nfo:duration property.
+ expected = f'a nfo:Audio ; nie:url <{url}> ; nfo:duration ?duration'
+ with self.tracker.await_insert(expected):
+ shutil.copy(path, self.indexed_dir)
+ return path
+
+ def prepare_test_image(self, source_path):
+ dest_path = pathlib.Path(os.path.join(self.indexed_dir, os.path.basename(source_path)))
+ url = dest_path.as_uri()
+
+ # Copy and wait. The extractor adds the nfo:width property.
+ expected = f'a nfo:Image ; nie:url <{url}> ; nfo:width ?width'
+ with self.tracker.await_insert(expected):
+ shutil.copy(source_path, self.indexed_dir)
+ return dest_path
+
+ def uri(self, filename):
+ return pathlib.Path(filename).as_uri()
+
+ def get_mtime(self, filename):
+ return os.stat(filename).st_mtime
+
+ def wait_for_file_change(self, filename, initial_mtime):
+ start = time.time()
+ while time.time() < start + 5:
+ mtime = os.stat(filename).st_mtime
+ if mtime > initial_mtime:
+ return
+ time.sleep(0.2)
+
+ raise Exception(
+ "Timeout waiting for %s to be updated (mtime has not changed)" %
+ filename)
+
+
+# Copy rate, 10KBps (1024b/100ms)
+SLOWCOPY_RATE = 1024
+
+
+class TrackerApplicationTest (TrackerWritebackTest):
+ def get_urn_count_by_url(self, url):
+ select = """
+ SELECT ?u WHERE { ?u nie:url \"%s\" }
+ """ % (url)
+ return len(self.tracker.query(select))
+
+ def get_test_image(self):
+ TEST_IMAGE = "test-image-1.jpg"
+ return TEST_IMAGE
+
+ def get_test_video(self):
+ TEST_VIDEO = "test-video-1.mp4"
+ return TEST_VIDEO
+
+ def get_test_music(self):
+ TEST_AUDIO = "test-music-1.mp3"
+ return TEST_AUDIO
+
+ def get_data_dir(self):
+ return self.datadir
+
+ def get_dest_dir(self):
+ return self.indexed_dir
+
+ def slowcopy_file_fd(self, src, fdest, rate=SLOWCOPY_RATE):
+ """
+ @rate: bytes per 100ms
+ """
+ log.debug("Copying slowly\n '%s' to\n '%s'", src, fdest.name)
+ fsrc = open(src, 'rb')
+ buffer_ = fsrc.read(rate)
+ while (buffer_ != b""):
+ fdest.write(buffer_)
+ time.sleep(0.1)
+ buffer_ = fsrc.read(rate)
+ fsrc.close()
+
+ def slowcopy_file(self, src, dst, rate=SLOWCOPY_RATE):
+ """
+ @rate: bytes per 100ms
+ """
+ fdest = open(dst, 'wb')
+ self.slowcopy_file_fd(src, fdest, rate)
+ fdest.close()
+
+ def setUp(self):
+ # Use local directory if available. Installation otherwise.
+ if os.path.exists(os.path.join(os.getcwd(),
+ "test-apps-data")):
+ self.datadir = os.path.join(os.getcwd(),
+ "test-apps-data")
+ else:
+ self.datadir = os.path.join(cfg.DATADIR,
+ "tracker-tests",
+ "test-apps-data")
+
+ super(TrackerApplicationTest, self).setUp()
diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build
index aaae6a312..7bf760e8e 100644
--- a/tests/functional-tests/meson.build
+++ b/tests/functional-tests/meson.build
@@ -134,6 +134,13 @@ else
endif
test_env = environment()
+
+if get_option('tracker_core') == 'subproject'
+ tracker_sparql_uninstalled_dir = tracker_subproject.get_variable('tracker_sparql_uninstalled_dir')
+ test_env.prepend('GI_TYPELIB_PATH', tracker_sparql_uninstalled_dir)
+ test_env.prepend('LD_LIBRARY_PATH', tracker_sparql_uninstalled_dir)
+endif
+
test_env.prepend('PYTHONPATH', tracker_uninstalled_testutils_dir)
test_env.set('TRACKER_FUNCTIONAL_TEST_CONFIG', config_json_full_path)
diff --git a/tests/functional-tests/minerfshelper.py b/tests/functional-tests/minerfshelper.py
index 4dd6cf326..23edf2d2c 100644
--- a/tests/functional-tests/minerfshelper.py
+++ b/tests/functional-tests/minerfshelper.py
@@ -18,17 +18,20 @@
# 02110-1301, USA.
-import logging
+import gi
+gi.require_version('Tracker', '3.0')
+from gi.repository import Gio, GLib
+from gi.repository import Tracker
-from gi.repository import Gio
-from gi.repository import GLib
+import logging
import trackertestutils.mainloop
+import configuration
+
log = logging.getLogger(__name__)
-REASONABLE_TIMEOUT = 5
class WakeupCycleTimeoutException(RuntimeError):
pass
@@ -63,6 +66,10 @@ class MinerFsHelper ():
def stop(self):
self.miner_fs.Stop()
+ def get_sparql_connection(self):
+ return Tracker.SparqlConnection.bus_new(
+ 'org.freedesktop.Tracker1.Miner.Files', None, self.bus)
+
def start_watching_progress(self):
self._previous_status = None
self._target_wakeup_count = None
@@ -91,7 +98,7 @@ class MinerFsHelper ():
"""Return the number of wakeup-to-idle cycles the miner-fs completed."""
return self._wakeup_count
- def await_wakeup_count(self, target_wakeup_count, timeout=REASONABLE_TIMEOUT):
+ def await_wakeup_count(self, target_wakeup_count, timeout=configuration.DEFAULT_TIMEOUT):
"""Block until the miner has completed N wakeup-and-idle cycles.
This function is for use by miner-fs tests that should trigger an
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
