GNOME.org
 

[tracker/fts-limits: 1/2] Removed character-limits to FTS searching.

commit bb1c445262462363693a6cafec9091c189e6c621
Author: Mikael Ottela <mikael ottela ixonos com>
Date:   Fri Feb 19 02:51:29 2010 +0200

    Removed character-limits to FTS searching.

 src/libtracker-fts/tracker-fts.c               |    6 +++---
 src/libtracker-fts/tracker-parser.c            |   10 +++++++---
 src/libtracker-fts/tracker-parser.h            |    3 ++-
 tests/libtracker-fts/prefix/Makefile.am        |    7 +++++++
 tests/libtracker-fts/prefix/data.ontology      |   23 +++++++++++++++++++++++
 tests/libtracker-fts/prefix/fts3prefix-1.out   |    3 +++
 tests/libtracker-fts/prefix/fts3prefix-1.rq    |    1 +
 tests/libtracker-fts/prefix/fts3prefix-2.out   |    5 +++++
 tests/libtracker-fts/prefix/fts3prefix-2.rq    |    1 +
 tests/libtracker-fts/prefix/fts3prefix-data.rq |   13 +++++++++++++
 tests/libtracker-fts/tracker-fts-test.c        |    1 +
 11 files changed, 66 insertions(+), 7 deletions(-)
---
diff --git a/src/libtracker-fts/tracker-fts.c b/src/libtracker-fts/tracker-fts.c
index f9d55ec..a58af78 100644
--- a/src/libtracker-fts/tracker-fts.c
+++ b/src/libtracker-fts/tracker-fts.c
@@ -3629,7 +3629,7 @@ static void snippetOffsetsOfColumn(
   pVtab = pQuery->pFts;
   nColumn = pVtab->nColumn;
 
-  tracker_parser_reset (pVtab->parser, zDoc, nDoc, FALSE, TRUE, pVtab->stop_words, FALSE);
+  tracker_parser_reset (pVtab->parser, zDoc, nDoc, FALSE, TRUE, pVtab->stop_words, FALSE, FALSE);
 
   aTerm = pQuery->pTerms;
   nTerm = pQuery->nTerms;
@@ -4324,7 +4324,7 @@ static int tokenizeSegment(
   int iCol;
   int nTerm = 1;
 
-  tracker_parser_reset (parser, pSegment, nSegment, FALSE, TRUE, v->stop_words, TRUE);
+  tracker_parser_reset (parser, pSegment, nSegment, FALSE, TRUE, v->stop_words, TRUE, FALSE);
 
   while( 1 ){
     const char *pToken;
@@ -4768,7 +4768,7 @@ int Catid,
 
   if (!zText) return SQLITE_OK;
 
-  tracker_parser_reset (parser, zText, strlen (zText), FALSE, TRUE, v->stop_words, FALSE);
+  tracker_parser_reset (parser, zText, strlen (zText), FALSE, TRUE, v->stop_words, FALSE, TRUE);
 
   while( 1 ){
 
diff --git a/src/libtracker-fts/tracker-parser.c b/src/libtracker-fts/tracker-parser.c
index cf1d1e3..fb71ed9 100644
--- a/src/libtracker-fts/tracker-parser.c
+++ b/src/libtracker-fts/tracker-parser.c
@@ -82,6 +82,7 @@ struct TrackerParser {
 	guint                  min_word_length;
 	gboolean               delimit_words;
 	gboolean               parse_reserved_words;
+	gboolean               limit_word_length;
 
 	/* Private members */
 	gchar                   *word;
@@ -323,7 +324,8 @@ parser_next (TrackerParser *parser,
 				}
 
 				if (!is_valid ||
-				    length < parser->min_word_length ||
+				    (parser->limit_word_length &&
+				     length < parser->min_word_length) ||
 				    word_type == TRACKER_PARSER_WORD_NUM) {
 					word_type = TRACKER_PARSER_WORD_IGNORE;
 					is_valid = TRUE;
@@ -362,7 +364,7 @@ parser_next (TrackerParser *parser,
 			}
 		}
 
-		if (length >= parser->max_word_length) {
+		if (parser->limit_word_length && length >= parser->max_word_length) {
 			continue;
 		}
 
@@ -504,7 +506,8 @@ tracker_parser_reset (TrackerParser *parser,
                       gboolean       delimit_words,
                       gboolean       enable_stemmer,
                       gboolean       enable_stop_words,
-                      gboolean       parse_reserved_words)
+                      gboolean       parse_reserved_words,
+		      gboolean       limit_word_length)
 {
 	g_return_if_fail (parser != NULL);
 	g_return_if_fail (txt != NULL);
@@ -519,6 +522,7 @@ tracker_parser_reset (TrackerParser *parser,
 	parser->txt_size = txt_size;
 	parser->txt = txt;
 	parser->parse_reserved_words = parse_reserved_words;
+	parser->limit_word_length = limit_word_length;
 
 	g_free (parser->word);
 	parser->word = NULL;
diff --git a/src/libtracker-fts/tracker-parser.h b/src/libtracker-fts/tracker-parser.h
index 6e6b7fa..e16123d 100644
--- a/src/libtracker-fts/tracker-parser.h
+++ b/src/libtracker-fts/tracker-parser.h
@@ -38,7 +38,8 @@ void           tracker_parser_reset           (TrackerParser   *parser,
                                                gboolean                 delimit_words,
                                                gboolean                 enable_stemmer,
                                                gboolean                 enable_stop_words,
-                                               gboolean                 parse_reserved_words);
+                                               gboolean                 parse_reserved_words,
+					       gboolean                 limit_word_length);
 const gchar *  tracker_parser_next            (TrackerParser   *parser,
                                                gint            *position,
                                                gint            *byte_offset_start,
diff --git a/tests/libtracker-fts/prefix/Makefile.am b/tests/libtracker-fts/prefix/Makefile.am
new file mode 100644
index 0000000..7a39f1e
--- /dev/null
+++ b/tests/libtracker-fts/prefix/Makefile.am
@@ -0,0 +1,7 @@
+include $(top_srcdir)/Makefile.decl
+
+EXTRA_DIST = 				\
+	data.ontology			\
+	fts3prefix-data.rq		\
+	fts3prefix-1.rq			\
+	fts3prefix-2.rq
diff --git a/tests/libtracker-fts/prefix/data.ontology b/tests/libtracker-fts/prefix/data.ontology
new file mode 100644
index 0000000..d865fe1
--- /dev/null
+++ b/tests/libtracker-fts/prefix/data.ontology
@@ -0,0 +1,23 @@
+ prefix fts: <http://www.tracker-project.org/ontologies/fts#> .
+ prefix nrl: <http://www.semanticdesktop.org/ontologies/2007/08/15/nrl#> .
+ prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+ prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+ prefix test: <http://www.example.org/test#> .
+ prefix tracker: <http://www.tracker-project.org/ontologies/tracker#> .
+ prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+fts: a tracker:Namespace ;
+	tracker:prefix "fts" .
+
+test: a tracker:Namespace ;
+	tracker:prefix "test" .
+
+test:A a rdfs:Class ;
+	rdfs:subClassOf rdfs:Resource .
+
+test:p a rdf:Property ;
+	nrl:maxCardinality 1 ;
+	rdfs:domain test:A ;
+	rdfs:range xsd:string ;
+	tracker:fulltextIndexed true .
+
diff --git a/tests/libtracker-fts/prefix/fts3prefix-1.out b/tests/libtracker-fts/prefix/fts3prefix-1.out
new file mode 100644
index 0000000..ff6290b
--- /dev/null
+++ b/tests/libtracker-fts/prefix/fts3prefix-1.out
@@ -0,0 +1,3 @@
+"http://www.example.org/test#4";
+"http://www.example.org/test#5";
+"http://www.example.org/test#8";
diff --git a/tests/libtracker-fts/prefix/fts3prefix-1.rq b/tests/libtracker-fts/prefix/fts3prefix-1.rq
new file mode 100644
index 0000000..3ad76d9
--- /dev/null
+++ b/tests/libtracker-fts/prefix/fts3prefix-1.rq
@@ -0,0 +1 @@
+SELECT ?o WHERE { ?o fts:match "trac*" }
diff --git a/tests/libtracker-fts/prefix/fts3prefix-2.out b/tests/libtracker-fts/prefix/fts3prefix-2.out
new file mode 100644
index 0000000..85d8792
--- /dev/null
+++ b/tests/libtracker-fts/prefix/fts3prefix-2.out
@@ -0,0 +1,5 @@
+"http://www.example.org/test#3";
+"http://www.example.org/test#4";
+"http://www.example.org/test#5";
+"http://www.example.org/test#6";
+"http://www.example.org/test#8";
diff --git a/tests/libtracker-fts/prefix/fts3prefix-2.rq b/tests/libtracker-fts/prefix/fts3prefix-2.rq
new file mode 100644
index 0000000..7127fa2
--- /dev/null
+++ b/tests/libtracker-fts/prefix/fts3prefix-2.rq
@@ -0,0 +1 @@
+SELECT ?o WHERE { ?o fts:match "tr*" }
diff --git a/tests/libtracker-fts/prefix/fts3prefix-data.rq b/tests/libtracker-fts/prefix/fts3prefix-data.rq
new file mode 100644
index 0000000..e84145f
--- /dev/null
+++ b/tests/libtracker-fts/prefix/fts3prefix-data.rq
@@ -0,0 +1,13 @@
+INSERT {
+	test:1 a test:A ; test:p "t" .
+	test:2 a test:A ; test:p "tr" .
+	test:3 a test:A ; test:p "tra" .
+	test:4 a test:A ; test:p "tracker test" .
+	test:5 a test:A ; test:p "tracking tester" .
+	test:6 a test:A ; test:p "trash trash more trash" .
+	test:7 a test:A ; test:p "racker ester" .
+	test:8 a test:A ; test:p "TeStiNg TraCkEr" .
+	test:9 a test:A ; test:p "Prefix search with content" .
+	test:10 a test:A ; test:p "...and a one bit more here" .
+}
+
diff --git a/tests/libtracker-fts/tracker-fts-test.c b/tests/libtracker-fts/tracker-fts-test.c
index 777ae6e..9421323 100644
--- a/tests/libtracker-fts/tracker-fts-test.c
+++ b/tests/libtracker-fts/tracker-fts-test.c
@@ -43,6 +43,7 @@ struct _TestInfo {
 const TestInfo tests[] = {
 	{ "fts3aa", 2 },
 	{ "fts3ae", 1 },
+	{ "prefix/fts3prefix", 2 },
 	{ NULL }
 };
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]