[tracker] tracker-fts: Add FTS test infrastructure and first test
- From: Jürg Billeter <juergbi src gnome org>
- To: svn-commits-list gnome org
- Cc:
- Subject: [tracker] tracker-fts: Add FTS test infrastructure and first test
- Date: Tue, 29 Sep 2009 14:16:20 +0000 (UTC)
commit 0cd3c628ec42b0b4d01b16b4609b8beaf8abd66c
Author: Jürg Billeter <j bitron ch>
Date: Tue Sep 29 15:32:47 2009 +0200
tracker-fts: Add FTS test infrastructure and first test
configure.ac | 1 +
src/tracker-fts/tracker-fts.c | 15 ++-
tests/Makefile.am | 1 +
tests/tracker-fts/.gitignore | 2 +
tests/tracker-fts/Makefile.am | 45 +++++++
tests/tracker-fts/data.ontology | 23 ++++
tests/tracker-fts/fts3aa-1.out | 16 +++
tests/tracker-fts/fts3aa-1.rq | 1 +
tests/tracker-fts/fts3aa-2.out | 8 ++
tests/tracker-fts/fts3aa-2.rq | 1 +
tests/tracker-fts/fts3aa-data.rq | 34 +++++
tests/tracker-fts/tracker-fts-test.c | 224 ++++++++++++++++++++++++++++++++++
12 files changed, 366 insertions(+), 5 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index d32aac9..67ff202 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1441,6 +1441,7 @@ AC_CONFIG_FILES([
tests/libtracker-data/turtle/Makefile
tests/libtracker-db/Makefile
tests/Makefile
+ tests/tracker-fts/Makefile
tests/tracker-miner-fs/Makefile
tests/tracker-extract/Makefile
tests/tracker-store/Makefile
diff --git a/src/tracker-fts/tracker-fts.c b/src/tracker-fts/tracker-fts.c
index d8a2302..b1e4c01 100644
--- a/src/tracker-fts/tracker-fts.c
+++ b/src/tracker-fts/tracker-fts.c
@@ -2409,6 +2409,7 @@ struct fulltext_vtab {
char **azColumn; /* column names. malloced */
char **azContentColumn; /* column names in content table; malloced */
TrackerParser *parser; /* tokenizer for inserts and queries */
+ gboolean stop_words;
int max_words;
/* Precompiled statements which we keep as long as the table is
@@ -3412,6 +3413,9 @@ static int constructVtab(
v->parser = tracker_parser_new (language, max_len, min_len);
+ /* disable stop words if TRACKER_FTS_STOP_WORDS is set to 0 - used by tests */
+ v->stop_words = g_strcmp0 (g_getenv ("TRACKER_FTS_STOP_WORDS"), "0") != 0;
+
g_object_unref (language);
@@ -3677,7 +3681,7 @@ static void snippetOffsetsOfColumn(
pVtab = pQuery->pFts;
nColumn = pVtab->nColumn;
- tracker_parser_reset (pVtab->parser, zDoc, nDoc, FALSE, TRUE, TRUE, FALSE);
+ tracker_parser_reset (pVtab->parser, zDoc, nDoc, FALSE, TRUE, pVtab->stop_words, FALSE);
aTerm = pQuery->pTerms;
nTerm = pQuery->nTerms;
@@ -4367,16 +4371,17 @@ static int checkColumnSpecifier(
** term found is marked with nPhrase=0 and OR and "-" syntax is significant.
*/
static int tokenizeSegment(
- TrackerParser *parser, /* The tokenizer to use */
+ fulltext_vtab *v, /* The tokenizer to use */
const char *pSegment, int nSegment, /* Query expression being parsed */
int inPhrase, /* True if within "..." */
Query *pQuery /* Append results here */
){
+ TrackerParser *parser = v->parser;
int firstIndex = pQuery->nTerms;
int iCol;
int nTerm = 1;
- tracker_parser_reset (parser, pSegment, nSegment, FALSE, TRUE, TRUE, TRUE);
+ tracker_parser_reset (parser, pSegment, nSegment, FALSE, TRUE, v->stop_words, TRUE);
while( 1 ){
const char *pToken;
@@ -4508,7 +4513,7 @@ static int parseQuery(
int i;
for(i=iInput; i<nInput && zInput[i]!='"'; ++i){}
if( i>iInput ){
- tokenizeSegment(v->parser, zInput+iInput, i-iInput, inPhrase,
+ tokenizeSegment(v, zInput+iInput, i-iInput, inPhrase,
pQuery);
}
iInput = i;
@@ -4822,7 +4827,7 @@ int Catid,
if (!zText) return SQLITE_OK;
- tracker_parser_reset (parser, zText, strlen (zText), FALSE, TRUE, TRUE, FALSE);
+ tracker_parser_reset (parser, zText, strlen (zText), FALSE, TRUE, v->stop_words, FALSE);
while( 1 ){
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 28d0a11..d588c55 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -5,6 +5,7 @@ SUBDIRS = \
libtracker-common \
libtracker-data \
libtracker-db \
+ tracker-fts \
tracker-miner-fs \
tracker-extract \
tracker-store
diff --git a/tests/tracker-fts/.gitignore b/tests/tracker-fts/.gitignore
new file mode 100644
index 0000000..243ee35
--- /dev/null
+++ b/tests/tracker-fts/.gitignore
@@ -0,0 +1,2 @@
+tracker
+tracker-fts
diff --git a/tests/tracker-fts/Makefile.am b/tests/tracker-fts/Makefile.am
new file mode 100644
index 0000000..60ccb6c
--- /dev/null
+++ b/tests/tracker-fts/Makefile.am
@@ -0,0 +1,45 @@
+include $(top_srcdir)/Makefile.decl
+
+noinst_PROGRAMS = $(TEST_PROGS)
+
+TEST_PROGS += \
+ tracker-fts
+
+INCLUDES = \
+ -DTRACKER_COMPILATION \
+ -DTOP_SRCDIR=\"$(top_srcdir)\" \
+ -DTOP_BUILDDIR=\"$(top_builddir)\" \
+ -DG_LOG_DOMAIN=\"Tracker\" \
+ -I$(top_srcdir)/src \
+ -I$(top_srcdir)/tests/common \
+ $(WARN_CFLAGS) \
+ $(GCOV_CFLAGS) \
+ $(DBUS_CFLAGS) \
+ $(SQLITE3_CFLAGS) \
+ $(GMODULE_CFLAGS) \
+ $(GTHREAD_CFLAGS) \
+ $(GLIB2_CFLAGS)
+
+common_ldadd = \
+ $(top_builddir)/src/libtracker-common/libtracker-common.la \
+ $(top_builddir)/src/libtracker-db/libtracker-db.la \
+ $(top_builddir)/src/libtracker-data/libtracker-data.la \
+ $(top_builddir)/tests/common/libtracker-testcommon.la \
+ $(GCOV_LIBS) \
+ $(GMODULE_LIBS) \
+ $(GTHREAD_LIBS) \
+ $(GLIB2_LIBS)
+
+tracker_fts_SOURCES = \
+ tracker-fts-test.c
+
+tracker_fts_LDADD = $(common_ldadd)
+
+EXTRA_DIST = \
+ data.ontology \
+ fts3aa-data.rq \
+ fts3aa-1.rq \
+ fts3aa-1.out \
+ fts3aa-2.rq \
+ fts3aa-2.out
+
diff --git a/tests/tracker-fts/data.ontology b/tests/tracker-fts/data.ontology
new file mode 100644
index 0000000..d865fe1
--- /dev/null
+++ b/tests/tracker-fts/data.ontology
@@ -0,0 +1,23 @@
+ prefix fts: <http://www.tracker-project.org/ontologies/fts#> .
+ prefix nrl: <http://www.semanticdesktop.org/ontologies/2007/08/15/nrl#> .
+ prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+ prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+ prefix test: <http://www.example.org/test#> .
+ prefix tracker: <http://www.tracker-project.org/ontologies/tracker#> .
+ prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+fts: a tracker:Namespace ;
+ tracker:prefix "fts" .
+
+test: a tracker:Namespace ;
+ tracker:prefix "test" .
+
+test:A a rdfs:Class ;
+ rdfs:subClassOf rdfs:Resource .
+
+test:p a rdf:Property ;
+ nrl:maxCardinality 1 ;
+ rdfs:domain test:A ;
+ rdfs:range xsd:string ;
+ tracker:fulltextIndexed true .
+
diff --git a/tests/tracker-fts/fts3aa-1.out b/tests/tracker-fts/fts3aa-1.out
new file mode 100644
index 0000000..59c0ce4
--- /dev/null
+++ b/tests/tracker-fts/fts3aa-1.out
@@ -0,0 +1,16 @@
+"http://www.example.org/test#1";
+"http://www.example.org/test#3";
+"http://www.example.org/test#5";
+"http://www.example.org/test#7";
+"http://www.example.org/test#9";
+"http://www.example.org/test#11";
+"http://www.example.org/test#13";
+"http://www.example.org/test#15";
+"http://www.example.org/test#17";
+"http://www.example.org/test#19";
+"http://www.example.org/test#21";
+"http://www.example.org/test#23";
+"http://www.example.org/test#25";
+"http://www.example.org/test#27";
+"http://www.example.org/test#29";
+"http://www.example.org/test#31";
diff --git a/tests/tracker-fts/fts3aa-1.rq b/tests/tracker-fts/fts3aa-1.rq
new file mode 100644
index 0000000..2b7f448
--- /dev/null
+++ b/tests/tracker-fts/fts3aa-1.rq
@@ -0,0 +1 @@
+SELECT ?o WHERE { ?o fts:match "one" }
diff --git a/tests/tracker-fts/fts3aa-2.out b/tests/tracker-fts/fts3aa-2.out
new file mode 100644
index 0000000..c31ecf3
--- /dev/null
+++ b/tests/tracker-fts/fts3aa-2.out
@@ -0,0 +1,8 @@
+"http://www.example.org/test#3";
+"http://www.example.org/test#7";
+"http://www.example.org/test#11";
+"http://www.example.org/test#15";
+"http://www.example.org/test#19";
+"http://www.example.org/test#23";
+"http://www.example.org/test#27";
+"http://www.example.org/test#31";
diff --git a/tests/tracker-fts/fts3aa-2.rq b/tests/tracker-fts/fts3aa-2.rq
new file mode 100644
index 0000000..8dd156c
--- /dev/null
+++ b/tests/tracker-fts/fts3aa-2.rq
@@ -0,0 +1 @@
+SELECT ?o WHERE { ?o fts:match "one two" }
diff --git a/tests/tracker-fts/fts3aa-data.rq b/tests/tracker-fts/fts3aa-data.rq
new file mode 100644
index 0000000..cf81a6b
--- /dev/null
+++ b/tests/tracker-fts/fts3aa-data.rq
@@ -0,0 +1,34 @@
+INSERT {
+ test:1 a test:A ; test:p "one" .
+ test:2 a test:A ; test:p "two" .
+ test:3 a test:A ; test:p "one two" .
+ test:4 a test:A ; test:p "three" .
+ test:5 a test:A ; test:p "one three" .
+ test:6 a test:A ; test:p "two three" .
+ test:7 a test:A ; test:p "one two three" .
+ test:8 a test:A ; test:p "four" .
+ test:9 a test:A ; test:p "one four" .
+ test:10 a test:A ; test:p "two four" .
+ test:11 a test:A ; test:p "one two four" .
+ test:12 a test:A ; test:p "three four" .
+ test:13 a test:A ; test:p "one three four" .
+ test:14 a test:A ; test:p "two three four" .
+ test:15 a test:A ; test:p "one two three four" .
+ test:16 a test:A ; test:p "five" .
+ test:17 a test:A ; test:p "one five" .
+ test:18 a test:A ; test:p "two five" .
+ test:19 a test:A ; test:p "one two five" .
+ test:20 a test:A ; test:p "three five" .
+ test:21 a test:A ; test:p "one three five" .
+ test:22 a test:A ; test:p "two three five" .
+ test:23 a test:A ; test:p "one two three five" .
+ test:24 a test:A ; test:p "four five" .
+ test:25 a test:A ; test:p "one four five" .
+ test:26 a test:A ; test:p "two four five" .
+ test:27 a test:A ; test:p "one two four five" .
+ test:28 a test:A ; test:p "three four five" .
+ test:29 a test:A ; test:p "one three four five" .
+ test:30 a test:A ; test:p "two three four five" .
+ test:31 a test:A ; test:p "one two three four five" .
+}
+
diff --git a/tests/tracker-fts/tracker-fts-test.c b/tests/tracker-fts/tracker-fts-test.c
new file mode 100644
index 0000000..02bd3c0
--- /dev/null
+++ b/tests/tracker-fts/tracker-fts-test.c
@@ -0,0 +1,224 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/*
+ * Copyright (C) 2009, Nokia (urho konttori nokia com)
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include <string.h>
+
+#include <glib.h>
+#include <gio/gio.h>
+
+#include <libtracker-db/tracker-db-manager.h>
+
+#include <libtracker-data/tracker-data-manager.h>
+#include <libtracker-data/tracker-data-query.h>
+#include <libtracker-data/tracker-data-update.h>
+#include <libtracker-data/tracker-sparql-query.h>
+#include <libtracker-common/tracker-ontology.h>
+
+typedef struct _TestInfo TestInfo;
+
+struct _TestInfo {
+ const gchar *test_name;
+ gint number_of_queries;
+};
+
+const TestInfo tests[] = {
+ { "fts3aa", 2 },
+ { NULL }
+};
+
+static void
+test_sparql_query (gconstpointer test_data)
+{
+ TrackerDBResultSet *result_set;
+ const TestInfo *test_info;
+ GError *error;
+ GString *test_results;
+ gchar *update, *update_filename;
+ gchar *query, *query_filename;
+ gchar *results, *results_filename;
+ gchar *prefix, *data_prefix, *test_prefix;
+ gint i;
+
+ error = NULL;
+ test_info = test_data;
+
+ /* initialization */
+ prefix = g_build_path (G_DIR_SEPARATOR_S, TOP_SRCDIR, "tests", "tracker-fts", NULL);
+ data_prefix = g_build_filename (prefix, "data", NULL);
+ test_prefix = g_build_filename (prefix, test_info->test_name, NULL);
+ g_free (prefix);
+
+ tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+ data_prefix,
+ NULL, NULL);
+
+ /* load data / perform updates */
+
+ update_filename = g_strconcat (test_prefix, "-data.rq", NULL);
+ g_file_get_contents (update_filename, &update, NULL, &error);
+ g_assert (error == NULL);
+
+ tracker_data_update_sparql (update, &error);
+ g_assert (error == NULL);
+
+ g_free (update_filename);
+ g_free (update);
+
+ /* perform queries */
+
+ for (i = 1; i <= test_info->number_of_queries; i++) {
+ query_filename = g_strdup_printf ("%s-%d.rq", test_prefix, i);
+ g_file_get_contents (query_filename, &query, NULL, &error);
+ g_assert (error == NULL);
+
+ results_filename = g_strdup_printf ("%s-%d.out", test_prefix, i);
+ g_file_get_contents (results_filename, &results, NULL, &error);
+ g_assert (error == NULL);
+
+ result_set = tracker_data_query_sparql (query, &error);
+ g_assert (error == NULL);
+
+ /* compare results with reference output */
+
+ test_results = g_string_new ("");
+
+ if (result_set) {
+ gboolean valid = TRUE;
+ guint col_count;
+ gint col;
+
+ col_count = tracker_db_result_set_get_n_columns (result_set);
+
+ while (valid) {
+ for (col = 0; col < col_count; col++) {
+ GValue value = { 0 };
+
+ _tracker_db_result_set_get_value (result_set, col, &value);
+
+ switch (G_VALUE_TYPE (&value)) {
+ case G_TYPE_INT:
+ g_string_append_printf (test_results, "\"%d\"", g_value_get_int (&value));
+ break;
+ case G_TYPE_DOUBLE:
+ g_string_append_printf (test_results, "\"%f\"", g_value_get_double (&value));
+ break;
+ case G_TYPE_STRING:
+ g_string_append_printf (test_results, "\"%s\"", g_value_get_string (&value));
+ break;
+ default:
+ /* unbound variable */
+ break;
+ }
+
+ if (col < col_count - 1) {
+ g_string_append (test_results, "\t");
+ }
+ }
+
+ g_string_append (test_results, "\n");
+
+ valid = tracker_db_result_set_iter_next (result_set);
+ }
+
+ g_object_unref (result_set);
+ }
+
+ if (strcmp (results, test_results->str)) {
+ /* print result difference */
+ gchar *quoted_results;
+ gchar *command_line;
+ gchar *quoted_command_line;
+ gchar *shell;
+ gchar *diff;
+
+ quoted_results = g_shell_quote (test_results->str);
+ command_line = g_strdup_printf ("echo -n %s | diff -u %s -", quoted_results, results_filename);
+ quoted_command_line = g_shell_quote (command_line);
+ shell = g_strdup_printf ("sh -c %s", quoted_command_line);
+ g_spawn_command_line_sync (shell, &diff, NULL, NULL, &error);
+ g_assert (error == NULL);
+
+ g_error ("%s", diff);
+
+ g_free (quoted_results);
+ g_free (command_line);
+ g_free (quoted_command_line);
+ g_free (shell);
+ g_free (diff);
+ }
+
+ /* cleanup */
+
+ g_free (query_filename);
+ g_free (query);
+ g_free (results_filename);
+ g_free (results);
+ g_string_free (test_results, TRUE);
+ }
+
+ g_free (data_prefix);
+ g_free (test_prefix);
+
+ tracker_data_manager_shutdown ();
+}
+
+int
+main (int argc, char **argv)
+{
+ gint result;
+ gint i;
+ gchar *current_dir;
+
+ g_type_init ();
+
+ if (!g_thread_supported ()) {
+ g_thread_init (NULL);
+ }
+
+ g_test_init (&argc, &argv, NULL);
+
+ current_dir = g_get_current_dir ();
+
+ g_setenv ("XDG_DATA_HOME", current_dir, TRUE);
+ g_setenv ("XDG_CACHE_HOME", current_dir, TRUE);
+ g_setenv ("TRACKER_DB_SQL_DIR", TOP_SRCDIR "/data/db/", TRUE);
+ g_setenv ("TRACKER_DB_ONTOLOGIES_DIR", TOP_SRCDIR "/data/ontologies/", TRUE);
+ g_setenv ("TRACKER_FTS_STOP_WORDS", "0", TRUE);
+
+ g_free (current_dir);
+
+ /* add test cases */
+ for (i = 0; tests[i].test_name; i++) {
+ gchar *testpath;
+
+ testpath = g_strconcat ("/tracker-fts/", tests[i].test_name, NULL);
+ g_test_add_data_func (testpath, &tests[i], test_sparql_query);
+ g_free (testpath);
+ }
+
+ /* run tests */
+ result = g_test_run ();
+
+ /* clean up */
+ g_print ("Removing temporary data\n");
+ g_spawn_command_line_async ("rm -R tracker/", NULL);
+
+ return result;
+}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
