[tracker/parser-unicode-libs-review: 81/85] New FTS parser tester

[Aleksander Morgado <aleksm src gnome org>]

commit b2b7828a173687dc7043f2e0c8c63dbad60ed5bb
Author: Aleksander Morgado <aleksander lanedo com>
Date:   Tue May 4 11:07:23 2010 +0200

    New FTS parser tester

 src/libtracker-fts/tracker-parser.h        |    1 +
 tests/libtracker-fts/Makefile.am           |   10 +-
 tests/libtracker-fts/tracker-parser-test.c |  226 ++++++++++++++++++++++++++++
 3 files changed, 234 insertions(+), 3 deletions(-)
---
diff --git a/src/libtracker-fts/tracker-parser.h b/src/libtracker-fts/tracker-parser.h
index 4fe03a8..3175b22 100644
--- a/src/libtracker-fts/tracker-parser.h
+++ b/src/libtracker-fts/tracker-parser.h
@@ -51,6 +51,7 @@ gchar *        tracker_parser_process_word    (TrackerParser   *parser,
                                                const gchar     *word,
                                                gint             length,
                                                gboolean         do_strip);
+
 void           tracker_parser_free            (TrackerParser   *parser);
 
 G_END_DECLS
diff --git a/tests/libtracker-fts/Makefile.am b/tests/libtracker-fts/Makefile.am
index ac8ec9c..5cdfaeb 100644
--- a/tests/libtracker-fts/Makefile.am
+++ b/tests/libtracker-fts/Makefile.am
@@ -7,7 +7,8 @@ SUBDIRS = 			\
 noinst_PROGRAMS = $(TEST_PROGS)
 
 TEST_PROGS += 								\
-	tracker-fts
+	tracker-fts							\
+	tracker-parser
 
 INCLUDES = 								\
 	-DTRACKER_COMPILATION						\
@@ -34,11 +35,14 @@ common_ldadd =								\
 	$(GTHREAD_LIBS)							\
 	$(GLIB2_LIBS)
 
-tracker_fts_SOURCES = 						\
-	tracker-fts-test.c
+tracker_fts_SOURCES = tracker-fts-test.c
 
 tracker_fts_LDADD = $(common_ldadd)
 
+tracker_parser_SOURCES = tracker-parser-test.c
+
+tracker_parser_LDADD = $(common_ldadd)
+
 EXTRA_DIST = 				\
 	data.ontology			\
 	fts3aa-data.rq			\
diff --git a/tests/libtracker-fts/tracker-parser-test.c b/tests/libtracker-fts/tracker-parser-test.c
new file mode 100644
index 0000000..b2ec05d
--- /dev/null
+++ b/tests/libtracker-fts/tracker-parser-test.c
@@ -0,0 +1,226 @@
+/*
+ * Copyright (C) 2010, Nokia <ivan frade nokia com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#include "config.h"
+
+#include <string.h>
+
+#include <glib.h>
+#include <gio/gio.h>
+
+#include <libtracker-fts/tracker-parser.h>
+#include <libtracker-fts/tracker-fts-config.h>
+#include <libtracker-common/tracker-common.h>
+
+
+#define DEFAULT_MAX_WORD_LENGTH 30
+
+static gint   max_word_length = DEFAULT_MAX_WORD_LENGTH;
+static gchar *text;
+static gchar *filename;
+
+/* Command Line options */
+static const GOptionEntry options [] = {
+	{
+		"max-word-length", 'm', 0,
+		G_OPTION_ARG_INT, &max_word_length,
+		"Maximum word length to consider",
+		NULL
+	},
+	{
+		"text", 't', 0,
+		G_OPTION_ARG_STRING, &text,
+		"Specific text to parse",
+		NULL
+	},
+	{
+		"file", 'f', 0,
+		G_OPTION_ARG_STRING, &filename,
+		"Specific file to parse its contents",
+		NULL
+	},
+	{ NULL }
+};
+
+static gboolean
+setup_context (gint argc,
+               gchar **argv)
+{
+	GOptionContext *context = NULL;
+	GError *error = NULL;
+
+	/* Setup command line options */
+	context = g_option_context_new ("- Test the Tracker FTS parser");
+	g_option_context_add_main_entries (context,
+	                                   options,
+	                                   argv[0]);
+
+	/* Parse input arguments */
+	if (!g_option_context_parse (context,
+	                             &argc,
+	                             &argv,
+	                             &error))
+	{
+		g_printerr ("%s\nRun '%s --help' to see a full list of available "
+		            "command line options.\n",
+		            error->message,
+		            argv[0]);
+		g_error_free (error);
+		return FALSE;
+	}
+
+	g_option_context_free (context);
+	return TRUE;
+}
+
+static gboolean
+load_file_contents (void)
+{
+	GError *error = NULL;
+	GFile *file;
+
+	file = g_file_new_for_commandline_arg (filename);
+	if (!g_file_load_contents (file, NULL, &text, NULL, NULL, &error)) {
+		g_printerr ("Error loading file '%s' contents: '%s'\n",
+		            filename,
+		            error->message);
+		g_error_free (error);
+		g_object_unref (file);
+		return FALSE;
+	}
+	g_object_unref (file);
+	return TRUE;
+}
+
+static gboolean
+run_parsing (void)
+{
+	TrackerLanguage *language;
+	TrackerParser   *parser;
+
+	/* Setup language for parser */
+	language = tracker_language_new (NULL);
+	if (!language) {
+		g_printerr ("Language setup failed!\n");
+		return FALSE;
+	}
+
+	/* Create the parser */
+	parser = tracker_parser_new (language,
+	                             max_word_length);
+	g_object_unref (language);
+	if (!parser) {
+		g_printerr ("Parser creation failed!\n");
+		return FALSE;
+	}
+
+	/* Reset the parser with our string */
+	tracker_parser_reset (parser,
+	                      text,
+	                      strlen (text),
+	                      TRUE,
+	                      TRUE,
+	                      TRUE,
+	                      TRUE);
+
+
+	/* Loop through all words! */
+	while (1) {
+		const gchar *word;
+		gchar *word_hex;
+		gint position;
+		gint byte_offset_start;
+		gint byte_offset_end;
+		gboolean stop_word;
+		gint word_length;
+
+		/* Process next word */
+		word = tracker_parser_next (parser,
+		                            &position,
+		                            &byte_offset_start,
+		                            &byte_offset_end,
+		                            &stop_word,
+		                            &word_length);
+
+		/* Stop loop if no more words */
+		if (!word) {
+			break;
+		}
+
+		word_hex = tracker_strhex (word, word_length, ':');
+
+		g_print ("WORD at %d [%d,%d]: '%s' (%s) (stop? %s)\n",
+		         position,
+		         byte_offset_start,
+		         byte_offset_end,
+		         word,
+		         word_hex,
+		         stop_word ? "yes" : "no");
+
+		g_free (word_hex);
+	}
+
+	tracker_parser_free (parser);
+	return TRUE;
+}
+
+
+int
+main (int argc, char **argv)
+{
+	g_type_init ();
+	if (!g_thread_supported ()) {
+		g_thread_init (NULL);
+	}
+
+	/* Setup context */
+	if (!setup_context (argc, argv)) {
+		g_printerr ("Context setup failed... exiting\n");
+		return -1;
+	}
+
+	/* Either text or file must be given */
+	if (filename == NULL &&
+	    text == NULL) {
+		g_printerr ("Either 'file' or 'text' options should be used\n"
+		            "Run '%s --help' to see a full list of available "
+		            "command line options.\n",
+		            argv[0]);
+		return -2;
+	}
+
+	/* If required, load file contents */
+	if (filename != NULL &&
+	    !load_file_contents ()) {
+		g_printerr ("Loading file '%s' contents failed... exiting\n",
+		            filename);
+		return -3;
+	}
+
+	/* Run the parsing! */
+	if (!run_parsing ()) {
+		g_printerr ("Parsing operation failed... exiting\n");
+		return -4;
+	}
+
+	/* Clean exit */
+	if (filename)
+		g_free (text);
+	return 0;
+}