tracker r1406 - in branches/indexer-split: . src/libtracker-common src/tracker-indexer src/tracker-indexer/modules src/trackerd



Author: carlosg
Date: Tue May 13 12:48:19 2008
New Revision: 1406
URL: http://svn.gnome.org/viewvc/tracker?rev=1406&view=rev

Log:
2008-05-13  Carlos Garnacho  <carlos imendio com>

        Initial work on the separate indexer executable, implemented as a
        modular, single-threaded state machine.

        * src/tracker-indexer/main.c:
        * src/tracker-indexer/tracker-indexer.[ch]:
        * src/tracker-indexer/tracker-indexer-module.[ch]: Added, this is the
        indexer core.

        * src/tracker-indexer/modules/applications.c:
        * src/tracker-indexer/modules/dummy.c:
        * src/tracker-indexer/modules/files.c:
        * src/tracker-indexer/modules/gaim-conversations.c: Added, independent
        modules, called out by the indexer core.

        * src/libtracker-common/tracker-config.[ch]
        (tracker_config_class_init) (config_get_property)
        (config_set_property) (config_load_string_list)
        (tracker_config_get_index_modules): Added configuration option, plus
        methods to know which modules to query.
        (config_create_with_defaults): ensure TrackerConfig contains the
        default values if the configuration file didn't exist.

        * src/tracker-indexer/modules/Makefile.am:
        * src/tracker-indexer/Makefile.am:
        * configure.in: autofoo glue to make this build.


Added:
   branches/indexer-split/src/tracker-indexer/main.c
   branches/indexer-split/src/tracker-indexer/modules/
   branches/indexer-split/src/tracker-indexer/modules/Makefile.am
   branches/indexer-split/src/tracker-indexer/modules/applications.c
   branches/indexer-split/src/tracker-indexer/modules/dummy.c
   branches/indexer-split/src/tracker-indexer/modules/files.c
   branches/indexer-split/src/tracker-indexer/modules/gaim-conversations.c
   branches/indexer-split/src/tracker-indexer/tracker-indexer-module.c
   branches/indexer-split/src/tracker-indexer/tracker-indexer-module.h
   branches/indexer-split/src/tracker-indexer/tracker-indexer.h
Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/configure.ac
   branches/indexer-split/src/libtracker-common/tracker-config.c
   branches/indexer-split/src/libtracker-common/tracker-config.h
   branches/indexer-split/src/tracker-indexer/Makefile.am
   branches/indexer-split/src/tracker-indexer/tracker-indexer.c
   branches/indexer-split/src/trackerd/tracker-process-files.c

Modified: branches/indexer-split/configure.ac
==============================================================================
--- branches/indexer-split/configure.ac	(original)
+++ branches/indexer-split/configure.ac	Tue May 13 12:48:19 2008
@@ -827,6 +827,7 @@
 	src/libtracker-db/Makefile
 	src/libtracker-gtk/Makefile
 	src/tracker-indexer/Makefile
+	src/tracker-indexer/modules/Makefile
 	src/tracker-applet/Makefile
 	src/tracker-applet/tracker-applet.desktop.in
 	src/tracker-search-tool/Makefile

Modified: branches/indexer-split/src/libtracker-common/tracker-config.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-config.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-config.c	Tue May 13 12:48:19 2008
@@ -60,6 +60,7 @@
 #define KEY_LOW_DISK_SPACE_LIMIT		 "LowDiskSpaceLimit"
 #define KEY_INDEX_MOUNTED_DIRECTORIES	         "IndexMountedDirectories"
 #define KEY_INDEX_REMOVABLE_DEVICES		 "IndexRemovableMedia"
+#define KEY_INDEX_MODULES                        "IndexModules"
 
 #define GROUP_EMAILS				 "Emails"
 #define KEY_EMAIL_CLIENT                         "IndexEMailClient"
@@ -96,7 +97,8 @@
 #define DEFAULT_DISABLE_INDEXING_ON_BATTERY	 TRUE
 #define DEFAULT_DISABLE_INDEXING_ON_BATTERY_INIT FALSE
 #define DEFAULT_INDEX_MOUNTED_DIRECTORIES  	 TRUE 
-#define DEFAULT_INDEX_REMOVABLE_DEVICES	         TRUE 
+#define DEFAULT_INDEX_REMOVABLE_DEVICES	         TRUE
+#define DEFAULT_INDEX_MODULES                    "applications;files;gaim-conversations;firefox-history"
 #define DEFAULT_INDEX_EMAIL_CLIENT               "evolution"
 #define DEFAULT_LOW_DISK_SPACE_LIMIT		 1	  /* 0->100 / -1 */
 #define DEFAULT_MAX_TEXT_TO_INDEX		 1048576  /* Bytes */
@@ -141,6 +143,7 @@
 	gint	  low_disk_space_limit;
 	gboolean  index_mounted_directories;
 	gboolean  index_removable_devices;
+	GSList   *index_modules;
 
 	/* Emails */
 	gchar    *email_client;
@@ -202,6 +205,7 @@
 	PROP_LOW_DISK_SPACE_LIMIT,
 	PROP_INDEX_MOUNTED_DIRECTORIES,
 	PROP_INDEX_REMOVABLE_DEVICES,
+	PROP_INDEX_MODULES,
 
 	/* Emails */
 	PROP_EMAIL_CLIENT,
@@ -424,7 +428,12 @@
 							       "which are for removable devices",
 							       DEFAULT_INDEX_REMOVABLE_DEVICES,
 							       G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
-
+	g_object_class_install_property (object_class,
+					 PROP_INDEX_MODULES,
+					 g_param_spec_pointer ("index-modules",
+							       "Used index modules",
+							       "Modules used to index data",
+							       G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
 	/* Emails */
         g_object_class_install_property (object_class,
 					 PROP_EMAIL_CLIENT,
@@ -638,6 +647,9 @@
 	case PROP_INDEX_REMOVABLE_DEVICES:
 		g_value_set_boolean (value, priv->index_removable_devices);
 		break;
+	case PROP_INDEX_MODULES:
+		g_value_set_pointer (value, priv->index_modules);
+		break;
 
 		/* Emails */
 	case PROP_EMAIL_CLIENT:
@@ -782,6 +794,9 @@
 		tracker_config_set_index_removable_devices (TRACKER_CONFIG (object),
 								g_value_get_boolean (value));
 		break;
+	case PROP_INDEX_MODULES:
+		/* Not writable */
+		break;
 
 		/* Emails */
 	case PROP_EMAIL_CLIENT:
@@ -973,22 +988,23 @@
 }
 
 static gboolean
-config_create_with_defaults (const gchar *filename)
+config_create_with_defaults (const gchar *filename,
+			     GKeyFile    *key_file)
 {
-	GKeyFile     *key_file;
 	GError	     *error = NULL;
 	gchar	     *content = NULL;
 	gchar	     *language;
 	const gchar  *watch_directory_roots[2] = { NULL, NULL };
 	const gchar  *empty_string_list[] = { NULL };
-
-	key_file = g_key_file_new ();
+	gchar       **index_modules;
 
 	/* Get default values */
 	language = tracker_language_get_default_code ();
 
 	watch_directory_roots[0] = g_get_home_dir ();
 
+	index_modules = g_strsplit (DEFAULT_INDEX_MODULES, ";", -1);
+
 	/* General */
 	g_key_file_set_integer (key_file, GROUP_GENERAL, KEY_VERBOSITY, DEFAULT_VERBOSITY);
 	g_key_file_set_comment (key_file, GROUP_GENERAL, KEY_VERBOSITY,
@@ -1109,6 +1125,11 @@
 	g_key_file_set_comment (key_file, GROUP_INDEXING, KEY_INDEX_REMOVABLE_DEVICES,
 				" Set to true to enable traversing mounted directories for removable devices",
 				NULL);
+	g_key_file_set_string_list (key_file, GROUP_INDEXING, KEY_INDEX_MODULES,
+				    (const gchar **) index_modules, g_strv_length (index_modules));
+	g_key_file_set_comment (key_file, GROUP_INDEXING, KEY_INDEX_MODULES,
+				" Modules used to extract data, they will be queried in the same order than they're written here",
+				NULL);
 
 	/* Emails */
 	g_key_file_set_string  (key_file, GROUP_EMAILS, KEY_EMAIL_CLIENT, DEFAULT_INDEX_EMAIL_CLIENT);
@@ -1159,7 +1180,6 @@
 
 	content = g_key_file_to_data (key_file, NULL, &error);
 	g_free (language);
-	g_key_file_free (key_file);
 
 	if (error) {
 		g_warning ("Couldn't produce default configuration, %s", error->message);
@@ -1175,6 +1195,7 @@
 	}
 
 	g_print ("Writting default configuration to file:'%s'\n", filename);
+	g_strfreev (index_modules);
 	g_free (content);
 
 	return TRUE;
@@ -1315,6 +1336,12 @@
 				config_string_list_to_gslist ((const gchar **) value, FALSE);
 		}
 	}
+	else if (strcmp (property, "index-modules") == 0) {
+		if (value) {
+			priv->index_modules =
+				config_string_list_to_gslist ((const gchar **) value, FALSE);
+		}
+	}
 	else {
 		g_warning ("Property '%s' not recognized to set string list from key '%s'",
 			   property, key);
@@ -1347,7 +1374,7 @@
 	/* Load options */
 	g_key_file_load_from_file (key_file, filename, G_KEY_FILE_NONE, &error);
 	if (error) {
-		config_create_with_defaults (filename);
+		config_create_with_defaults (filename, key_file);
 		g_clear_error (&error);
 	}
 
@@ -1381,6 +1408,7 @@
 	config_load_int (config, "low-disk-space-limit", key_file, GROUP_INDEXING, KEY_LOW_DISK_SPACE_LIMIT);
 	config_load_boolean (config, "index-mounted-directories", key_file, GROUP_INDEXING, KEY_INDEX_MOUNTED_DIRECTORIES);
 	config_load_boolean (config, "index-removable-devices", key_file, GROUP_INDEXING, KEY_INDEX_REMOVABLE_DEVICES);
+	config_load_string_list (config, "index-modules", key_file, GROUP_INDEXING, KEY_INDEX_MODULES);
 
 	/* Emails */
 	config_load_string (config, "email-client", key_file, GROUP_EMAILS, KEY_EMAIL_CLIENT);
@@ -1726,6 +1754,18 @@
 	return priv->index_removable_devices;
 }
 
+GSList *
+tracker_config_get_index_modules (TrackerConfig *config)
+{
+	TrackerConfigPriv *priv;
+
+	g_return_val_if_fail (TRACKER_IS_CONFIG (config), NULL);
+
+	priv = GET_PRIV (config);
+
+	return priv->index_modules;
+}
+
 const gchar *
 tracker_config_get_email_client (TrackerConfig *config)
 {

Modified: branches/indexer-split/src/libtracker-common/tracker-config.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-config.h	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-config.h	Tue May 13 12:48:19 2008
@@ -73,6 +73,7 @@
 gboolean       tracker_config_get_disable_indexing_on_battery_init (TrackerConfig *config);
 gint           tracker_config_get_low_disk_space_limit             (TrackerConfig *config);
 gboolean       tracker_config_get_index_removable_devices          (TrackerConfig *config);
+GSList *       tracker_config_get_index_modules                    (TrackerConfig *config);
 gboolean       tracker_config_get_index_mounted_directories        (TrackerConfig *config);
 const gchar *  tracker_config_get_email_client                     (TrackerConfig *config);
 gint           tracker_config_get_max_text_to_index                (TrackerConfig *config);

Modified: branches/indexer-split/src/tracker-indexer/Makefile.am
==============================================================================
--- branches/indexer-split/src/tracker-indexer/Makefile.am	(original)
+++ branches/indexer-split/src/tracker-indexer/Makefile.am	Tue May 13 12:48:19 2008
@@ -1,8 +1,10 @@
+SUBDIRS = modules
+
 INCLUDES =								\
 	-DSHAREDIR=\""$(datadir)"\"					\
 	-DLIBDIR=\""$(libdir)"\"					\
 	-DLOCALEDIR=\""$(localedir)"\" 					\
-	-DMAIL_MODULES_DIR=\"$(mail_modulesdir)\"			\
+	-DINDEXER_MODULES_DIR=\""$(libdir)"/tracker/indexer-modules\"	\
 	$(GLIB2_CFLAGS)							\
 	$(GMODULE_CFLAGS)						\
 	$(GMIME_CFLAGS)							\
@@ -43,7 +45,11 @@
 bin_PROGRAMS = tracker-indexer
 
 tracker_indexer_SOURCES =						\
-	tracker-indexer.c
+	main.c								\
+	tracker-indexer.c						\
+	tracker-indexer.h						\
+	tracker-indexer-module.c					\
+	tracker-indexer-module.h
 
 tracker_indexer_LDADD =							\
 	$(GLIB2_LIBS)							\
@@ -57,6 +63,7 @@
 	$(GTHREAD_LIBS)							\
 	-lz								\
 	-lm								\
+	$(top_builddir)/src/libtracker-db/libtracker-db.la 		\
 	$(top_builddir)/src/libtracker-common/libtracker-common.la 	\
 	$(top_builddir)/src/libstemmer/libstemmer-private.la 		\
 	$(top_builddir)/src/xdgmime/libxdgmime.la 			\

Added: branches/indexer-split/src/tracker-indexer/main.c
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/main.c	Tue May 13 12:48:19 2008
@@ -0,0 +1,251 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#include "config.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <locale.h>
+#include <signal.h>
+
+#include <glib/gi18n.h>
+#include <glib-object.h>
+
+#include "tracker-indexer.h"
+
+#define COPYRIGHT							  \
+	"Tracker version " PACKAGE_VERSION "\n"				  \
+	"Copyright (c) 2005-2008 by Jamie McCracken (jamiemcc gnome org)"
+
+#define WARRANTY \
+	"This program is free software and comes without any warranty.\n" \
+	"It is licensed under version 2 or later of the General Public\n" \
+	"License which can be viewed at:\n"				  \
+	"\n"								  \
+	"\thttp://www.gnu.org/licenses/gpl.txt";
+
+static GMainLoop  *main_loop;
+
+static gchar	 **no_watch_dirs;
+static gchar	 **watch_dirs;
+static gchar	 **crawl_dirs;
+static gchar	  *language;
+static gboolean	   disable_indexing;
+static gboolean	   reindex;
+static gboolean	   fatal_errors;
+static gboolean	   low_memory;
+static gint	   throttle = -1;
+static gint	   verbosity;
+static gint	   initial_sleep = -1;
+
+static GOptionEntry entries[] = {
+	{ "exclude-dir", 'e', 0, G_OPTION_ARG_STRING_ARRAY, &no_watch_dirs, 
+	  N_("Directory to exclude from indexing"), 
+	  N_("/PATH/DIR")
+	},
+	{ "include-dir", 'i', 0, G_OPTION_ARG_STRING_ARRAY, &watch_dirs, 
+	  N_("Directory to include in indexing"), 
+	  N_("/PATH/DIR")
+	},
+	{ "crawl-dir", 'c', 0, G_OPTION_ARG_STRING_ARRAY, &crawl_dirs, 
+	  N_("Directory to crawl for indexing at start up only"), 
+	  N_("/PATH/DIR")
+	},
+	{ "no-indexing", 'n', 0, G_OPTION_ARG_NONE, &disable_indexing, 
+	  N_("Disable any indexing or watching taking place"),
+	  NULL 
+	},
+	{ "verbosity", 'v', 0, G_OPTION_ARG_INT, &verbosity, 
+	  N_("Value that controls the level of logging. Valid values "
+	     "are 0=errors, 1=minimal, 2=detailed, 3=debug"), 
+	  N_("VALUE")
+	},
+	{ "throttle", 't', 0, G_OPTION_ARG_INT, &throttle, 
+	  N_("Value to use for throttling indexing. Value must be in "
+	     "range 0-99 (default=0) with lower values increasing "
+	     "indexing speed"), 
+	  N_("VALUE") 
+	},
+	{ "low-memory", 'm', 0, G_OPTION_ARG_NONE, &low_memory, 
+	  N_("Minimizes the use of memory but may slow indexing down"), 
+	  NULL 
+	},
+	{ "initial-sleep", 's', 0, G_OPTION_ARG_INT, &initial_sleep, 
+	  N_("Initial sleep time, just before indexing, in seconds"), 
+	  NULL
+	},
+	{ "language", 'l', 0, G_OPTION_ARG_STRING, &language, 
+	  N_("Language to use for stemmer and stop words list "
+	     "(ISO 639-1 2 characters code)"), 
+	  N_("LANG")
+	},
+	{ "reindex", 'R', 0, G_OPTION_ARG_NONE, &reindex, 
+	  N_("Force a re-index of all content"), 
+	  NULL 
+	},
+	{ "fatal-errors", 'f', 0, G_OPTION_ARG_NONE, &fatal_errors, 
+	  N_("Make tracker errors fatal"), 
+	  NULL 
+	},
+	{ NULL }
+};
+
+static void
+signal_handler (gint signo)
+{
+	static gboolean in_loop = FALSE;
+
+	/* die if we get re-entrant signals handler calls */
+	if (in_loop) {
+		exit (EXIT_FAILURE);
+	}
+
+	switch (signo) {
+	case SIGSEGV:
+		/* we are screwed if we get this so exit immediately! */
+		exit (EXIT_FAILURE);
+	case SIGBUS:
+	case SIGILL:
+	case SIGFPE:
+	case SIGPIPE:
+	case SIGABRT:
+	case SIGTERM:
+	case SIGINT:
+		in_loop = TRUE;
+		g_main_loop_quit (main_loop);
+
+		/*
+		tracker->is_running = FALSE;
+		tracker_end_watching ();
+
+		g_timeout_add_full (G_PRIORITY_LOW,
+				    1,
+				    (GSourceFunc) tracker_do_cleanup,
+				    g_strdup (g_strsignal (signo)), NULL);
+		*/
+	default:
+		if (g_strsignal (signo)) {
+			g_warning ("Received signal: %s", g_strsignal (signo));
+		}
+		break;
+	}
+}
+
+static void
+indexer_finished_cb (TrackerIndexer *indexer,
+		     gpointer	     user_data)
+{
+	g_main_loop_quit (main_loop);
+}
+
+gint
+main (gint argc, gchar *argv[])
+{
+	TrackerIndexer *indexer;
+	GOptionContext *context;
+	GError	       *error = NULL;
+	gchar	       *summary = NULL;
+	gchar	       *example;
+#ifndef OS_WIN32
+	struct sigaction   act;
+	sigset_t	   empty_mask;
+#endif
+
+	g_type_init ();
+	
+	if (!g_thread_supported ())
+		g_thread_init (NULL);
+
+	setlocale (LC_ALL, "");
+
+	bindtextdomain (GETTEXT_PACKAGE, LOCALEDIR);
+	bind_textdomain_codeset (GETTEXT_PACKAGE, "UTF-8");
+	textdomain (GETTEXT_PACKAGE);
+
+	/* Set timezone info */
+	tzset ();
+
+	/* Translators: this messagge will apper immediately after the
+	 * usage string - Usage: COMMAND <THIS_MESSAGE>
+	 */
+	context = g_option_context_new (_("- start the tracker daemon"));
+	example = g_strconcat ("-i ", _("DIRECTORY"), 
+			       "-i ", _("DIRECTORY"),
+			       "-e ", _("DIRECTORY"), 
+			       "-e ", _("DIRECTORY"),
+			       NULL);
+
+#ifdef HAVE_RECENT_GLIB
+	/* Translators: this message will appear after the usage
+	 * string and before the list of options, showing an usage
+	 * example.
+	 */
+	summary = g_strdup_printf (_("To include or exclude multiple directories "
+				     "at the same time, join multiple options like:\n"
+				     "\n"
+				     "\t%s"),
+				   example);
+	g_option_context_set_summary (context, summary);
+#endif /* HAVE_RECENT_GLIB */
+
+	g_option_context_add_main_entries (context, entries, NULL);
+	g_option_context_parse (context, &argc, &argv, &error);
+	g_option_context_free (context);
+	g_free (summary);
+	g_free (example);
+
+	g_print ("\n"
+		 COPYRIGHT "\n"
+		 "\n"
+		 WARRANTY "\n"
+		 "\n");
+
+#ifndef OS_WIN32
+	/* trap signals */
+	sigemptyset (&empty_mask);
+	act.sa_handler = signal_handler;
+	act.sa_mask    = empty_mask;
+	act.sa_flags   = 0;
+	sigaction (SIGTERM, &act, NULL);
+	sigaction (SIGILL,  &act, NULL);
+	sigaction (SIGBUS,  &act, NULL);
+	sigaction (SIGFPE,  &act, NULL);
+	sigaction (SIGHUP,  &act, NULL);
+	sigaction (SIGSEGV, &act, NULL);
+	sigaction (SIGABRT, &act, NULL);
+	sigaction (SIGUSR1, &act, NULL);
+	sigaction (SIGINT,  &act, NULL);
+#endif
+
+	g_print ("Initializing...\n");
+
+	indexer = tracker_indexer_new ();
+	main_loop = g_main_loop_new (NULL, FALSE);
+
+	g_signal_connect (indexer, "finished",
+			  G_CALLBACK (indexer_finished_cb), NULL);
+
+	g_main_loop_run (main_loop);
+
+	g_object_unref (indexer);
+
+	g_print ("Shutting down...\n");
+
+	return EXIT_SUCCESS;
+}

Added: branches/indexer-split/src/tracker-indexer/modules/Makefile.am
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/modules/Makefile.am	Tue May 13 12:48:19 2008
@@ -0,0 +1,30 @@
+module_flags = -module -avoid-version -no-undefined
+indexer_modulesdir = $(libdir)/tracker/indexer-modules
+
+INCLUDES =								\
+	-DSHAREDIR=\""$(datadir)"\"					\
+	-DLIBDIR=\""$(libdir)"\"					\
+	-DLOCALEDIR=\""$(localedir)"\" 					\
+	-DINDEXER_MODULES_DIR=\"$(indexer_modulesdir)\"			\
+	$(GLIB2_CFLAGS)							\
+	$(GMODULE_CFLAGS)						\
+	-I$(top_srcdir)/src
+
+indexer_modules_LTLIBRARIES = 						\
+	libtracker-indexer-applications.la				\
+	libtracker-indexer-files.la					\
+	libtracker-indexer-gaim-conversations.la
+
+libtracker_indexer_applications_la_SOURCES = applications.c
+libtracker_indexer_applications_la_LDFLAGS = $(module_flags)
+libtracker_indexer_applications_la_LIBADD = $(GLIB2_LIBS)
+
+libtracker_indexer_files_la_SOURCES = files.c
+libtracker_indexer_files_la_LDFLAGS = $(module_flags)
+libtracker_indexer_files_la_LIBADD = 					\
+	$(GLIB2_LIBS)							\
+	$(top_builddir)/src/libtracker-db/libtracker-db.la
+
+libtracker_indexer_gaim_conversations_la_SOURCES = gaim-conversations.c
+libtracker_indexer_gaim_conversations_la_LDFLAGS = $(module_flags)
+libtracker_indexer_gaim_conversations_la_LIBADD = $(GLIB2_LIBS)

Added: branches/indexer-split/src/tracker-indexer/modules/applications.c
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/modules/applications.c	Tue May 13 12:48:19 2008
@@ -0,0 +1,167 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#include <stdlib.h>
+#include <glib.h>
+
+#define GROUP_DESKTOP_ENTRY "Desktop Entry"
+#define KEY_TYPE            "Type"
+#define KEY_HIDDEN          "Hidden"
+#define KEY_NAME            "Name"
+#define KEY_GENERIC_NAME    "GenericName"
+#define KEY_COMMENT         "Comment"
+#define KEY_EXECUTABLE      "Exec"
+#define KEY_ICON            "Icon"
+#define KEY_MIMETYPE        "MimeType"
+#define KEY_CATEGORIES      "Categories"
+
+#define METADATA_FILE_NAME        "File:Name"
+#define METADATA_APP_NAME         "App:Name"
+#define METADATA_APP_DISPLAY_NAME "App:DisplayName"
+#define METADATA_APP_GENERIC_NAME "App:GenericName"
+#define METADATA_APP_COMMENT      "App:Comment"
+#define METADATA_APP_EXECUTABLE   "App:Exec"
+#define METADATA_APP_ICON         "App:Icon"
+#define METADATA_APP_MIMETYPE     "App:MimeType"
+#define METADATA_APP_CATEGORIES   "App:Categories"
+
+G_CONST_RETURN gchar *
+tracker_module_get_name (void)
+{
+	/* Return module name here */
+	return "Applications";
+}
+
+gchar **
+tracker_module_get_directories (void)
+{
+	GPtrArray *dirs;
+	gchar *value, *dir;
+
+	dirs = g_ptr_array_new ();
+	value = getenv ("XDG_DATA_HOME");
+
+	if (value) {
+		dir = g_build_filename (value, "applications", NULL);
+	} else {
+		dir = g_build_filename (g_get_home_dir (), ".local/share/applications", NULL);
+	}
+
+	/* Add user defined applications path to service directory list */
+	g_ptr_array_add (dirs, dir);
+
+	/* Add system defined applications path to service directory list */
+	value = getenv ("XDG_DATA_DIRS");
+
+	if (value) {
+		gchar **dir_array;
+		gint i;
+
+		dir_array = g_strsplit (value, ":", 0);
+
+		for (i = 0; dir_array[i]; i++) {
+			dir = g_build_filename (dir_array[i], "applications", NULL);
+			g_ptr_array_add (dirs, dir);
+		}
+
+		g_strfreev (dir_array);
+	} else {
+		g_ptr_array_add (dirs, g_strdup ("/usr/share/applications"));
+		g_ptr_array_add (dirs, g_strdup ("/usr/local/share/applications"));
+	}
+
+	g_ptr_array_add (dirs, NULL);
+
+	return (gchar **) g_ptr_array_free (dirs, FALSE);
+}
+
+static void
+insert_data_from_desktop_file (GHashTable  *metadata,
+			       const gchar *metadata_key,
+			       GKeyFile    *desktop_file,
+			       const gchar *key,
+			       gboolean     use_locale)
+{
+	gchar *str;
+
+	if (use_locale) {
+		str = g_key_file_get_locale_string (desktop_file, GROUP_DESKTOP_ENTRY, key, NULL, NULL);
+	} else {
+		str = g_key_file_get_string (desktop_file, GROUP_DESKTOP_ENTRY, key, NULL);
+	}
+
+	if (str) {
+		g_hash_table_insert (metadata, (gpointer) metadata_key, str);
+	}
+}
+
+GHashTable *
+tracker_module_get_file_metadata (const gchar *file)
+{
+	GHashTable *metadata;
+	GKeyFile *key_file;
+	gchar *type, *filename;
+
+	/* Check we're dealing with a desktop file */
+	if (!g_str_has_suffix (file, ".desktop")) {
+		return NULL;
+	}
+
+	key_file = g_key_file_new ();
+
+	if (!g_key_file_load_from_file (key_file, file, G_KEY_FILE_NONE, NULL)) {
+		g_key_file_free (key_file);
+		return NULL;
+	}
+
+	if (g_key_file_get_boolean (key_file, GROUP_DESKTOP_ENTRY, KEY_HIDDEN, NULL)) {
+		g_key_file_free (key_file);
+		return NULL;
+	}
+
+	type = g_key_file_get_string (key_file, GROUP_DESKTOP_ENTRY, KEY_TYPE, NULL);
+
+	if (!type || g_ascii_strcasecmp (type, "Application") != 0) {
+		g_key_file_free (key_file);
+		g_free (type);
+		return NULL;
+	}
+
+	/* Begin collecting data */
+	metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
+					  NULL,
+					  (GDestroyNotify) g_free);
+
+	insert_data_from_desktop_file (metadata, METADATA_APP_NAME, key_file, KEY_NAME, FALSE);
+	insert_data_from_desktop_file (metadata, METADATA_APP_DISPLAY_NAME, key_file, KEY_NAME, TRUE);
+	insert_data_from_desktop_file (metadata, METADATA_APP_GENERIC_NAME, key_file, KEY_GENERIC_NAME, TRUE);
+	insert_data_from_desktop_file (metadata, METADATA_APP_COMMENT, key_file, KEY_COMMENT, TRUE);
+	insert_data_from_desktop_file (metadata, METADATA_APP_EXECUTABLE, key_file, KEY_EXECUTABLE, FALSE);
+	insert_data_from_desktop_file (metadata, METADATA_APP_ICON, key_file, KEY_ICON, FALSE);
+
+	/* FIXME: mimetypes list and categories? */
+
+	filename = g_filename_display_basename (file);
+	g_hash_table_insert (metadata, METADATA_FILE_NAME, filename);
+
+	g_key_file_free (key_file);
+	g_free (type);
+
+	return metadata;
+}

Added: branches/indexer-split/src/tracker-indexer/modules/dummy.c
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/modules/dummy.c	Tue May 13 12:48:19 2008
@@ -0,0 +1,41 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#include <glib.h>
+
+G_CONST_RETURN gchar *
+tracker_module_get_name (void)
+{
+	/* Return module name here */
+	return "Dummy";
+}
+
+gchar **
+tracker_module_get_directories (void)
+{
+	/* Return directories/files to scan */
+	return NULL;
+}
+
+GHashTable *
+tracker_module_get_file_metadata (const gchar *file)
+{
+	/* Return a hashtable filled with metadata for the file */
+	return NULL;
+}

Added: branches/indexer-split/src/tracker-indexer/modules/files.c
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/modules/files.c	Tue May 13 12:48:19 2008
@@ -0,0 +1,112 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#include <glib.h>
+#include <glib/gstdio.h>
+#include <libtracker-common/tracker-config.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string.h>
+
+#define METADATA_FILE_NAME_DELIMITED "File:NameDelimited"
+#define METADATA_FILE_EXT            "File:Ext"
+#define METADATA_FILE_PATH           "File:Path"
+#define METADATA_FILE_NAME           "File:Name"
+#define METADATA_FILE_LINK           "File:Link"
+#define METADATA_FILE_MIME           "File:Mime"
+#define METADATA_FILE_SIZE           "File:Size"
+#define METADATA_FILE_MODIFIED       "File:Modified"
+#define METADATA_FILE_ACCESSED       "File:Accessed"
+
+static TrackerConfig *config = NULL;
+
+G_CONST_RETURN gchar *
+tracker_module_get_name (void)
+{
+	/* Return module name here */
+	return "Files";
+}
+
+gchar **
+tracker_module_get_directories (void)
+{
+	GSList *watch_roots;
+	GPtrArray *dirs;
+
+	if (!config) {
+		config = tracker_config_new ();
+	}
+
+	watch_roots = tracker_config_get_watch_directory_roots (config);
+	dirs = g_ptr_array_new ();
+
+	for (; watch_roots; watch_roots = watch_roots->next) {
+		g_ptr_array_add (dirs, g_strdup (watch_roots->data));
+	}
+
+	g_ptr_array_add (dirs, NULL);
+
+	return (gchar **) g_ptr_array_free (dirs, FALSE);
+}
+
+GHashTable *
+tracker_module_get_file_metadata (const gchar *file)
+{
+	GHashTable *metadata;
+	struct stat st;
+	const gchar *ext;
+
+	/* FIXME: check exclude extensions */
+
+	g_lstat (file, &st);
+	metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
+					  NULL,
+					  (GDestroyNotify) g_free);
+	ext = strrchr (file, '.');
+
+	if (ext) {
+		g_hash_table_insert (metadata, METADATA_FILE_EXT, g_strdup (ext + 1));
+	}
+
+	g_hash_table_insert (metadata, METADATA_FILE_NAME, g_filename_display_basename (file));
+	g_hash_table_insert (metadata, METADATA_FILE_PATH, g_path_get_dirname (file));
+	g_hash_table_insert (metadata, METADATA_FILE_NAME_DELIMITED,
+			     g_filename_to_utf8 (file, -1, NULL, NULL, NULL));
+
+	if (S_ISLNK (st.st_mode)) {
+		gchar *link_path;
+
+		link_path = g_file_read_link (file, NULL);
+		g_hash_table_insert (metadata, METADATA_FILE_LINK,
+				     g_filename_to_utf8 (link_path, -1, NULL, NULL, NULL));
+		g_free (link_path);
+	}
+
+	/* FIXME, Missing:
+	 *
+	 * File:NameDelimited
+	 * File:Mime
+	 * File:Size
+	 * File:Modified
+	 * File:Accessed
+	 * Call external metadata extractor
+	 */
+
+	return metadata;
+}

Added: branches/indexer-split/src/tracker-indexer/modules/gaim-conversations.c
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/modules/gaim-conversations.c	Tue May 13 12:48:19 2008
@@ -0,0 +1,46 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#include <glib.h>
+
+G_CONST_RETURN gchar *
+tracker_module_get_name (void)
+{
+	/* Return module name here */
+	return "GaimConversations";
+}
+
+gchar **
+tracker_module_get_directories (void)
+{
+	gchar **log_directories;
+
+	log_directories = g_new0 (gchar*, 3);
+	log_directories[0] = g_build_filename (g_get_home_dir(), ".gaim", "logs", NULL);
+	log_directories[1] = g_build_filename (g_get_home_dir(), ".purple", "logs", NULL);
+
+	return log_directories;
+}
+
+GHashTable *
+tracker_module_get_file_metadata (const gchar *file)
+{
+	/* Return a hashtable filled with metadata for the file */
+	return NULL;
+}

Added: branches/indexer-split/src/tracker-indexer/tracker-indexer-module.c
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer-module.c	Tue May 13 12:48:19 2008
@@ -0,0 +1,87 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#include <gmodule.h>
+#include "tracker-indexer-module.h"
+
+typedef const gchar * (* Foo) (void);
+typedef gchar ** (* Bar) (void);
+typedef GHashTable * (* Baz) (const gchar *path);
+
+GModule *
+tracker_indexer_module_load (const gchar *module_name)
+{
+	gchar *full_name, *path;
+	GModule *module;
+
+	g_return_val_if_fail (module_name != NULL, NULL);
+
+	full_name = g_strdup_printf ("libtracker-indexer-%s", module_name);
+	path = g_build_filename (INDEXER_MODULES_DIR, full_name, NULL);
+
+	module = g_module_open (path, G_MODULE_BIND_LOCAL);
+
+	if (!module) {
+		g_warning ("Could not load indexer module '%s', %s\n", module_name, g_module_error ());
+	} else {
+		g_module_make_resident (module);
+	}
+
+	g_free (full_name);
+	g_free (path);
+
+	return module;
+}
+
+G_CONST_RETURN gchar *
+tracker_indexer_module_get_name (GModule *module)
+{
+	Foo func;
+
+	if (g_module_symbol (module, "tracker_module_get_name", (gpointer *) &func)) {
+		return (func) ();
+	}
+
+	return NULL;
+}
+
+gchar **
+tracker_indexer_module_get_directories (GModule *module)
+{
+	Bar func;
+
+	if (g_module_symbol (module, "tracker_module_get_directories", (gpointer *) &func)) {
+		return (func) ();
+        }
+
+	return NULL;
+}
+
+GHashTable *
+tracker_indexer_module_get_file_metadata (GModule     *module,
+					  const gchar *file)
+{
+	Baz func;
+
+	if (g_module_symbol (module, "tracker_module_get_file_metadata", (gpointer *) &func)) {
+		return (func) (file);
+        }
+
+	return NULL;
+}

Added: branches/indexer-split/src/tracker-indexer/tracker-indexer-module.h
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer-module.h	Tue May 13 12:48:19 2008
@@ -0,0 +1,34 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __TRACKER_INDEXER_MODULE_H__
+#define __TRACKER_INDEXER_MODULE_H__
+
+G_BEGIN_DECLS
+
+GModule *               tracker_indexer_module_load              (const gchar *module_name);
+
+G_CONST_RETURN gchar *  tracker_indexer_module_get_name          (GModule     *module);
+gchar **                tracker_indexer_module_get_directories   (GModule     *module);
+GHashTable *            tracker_indexer_module_get_file_metadata (GModule     *module,
+								  const gchar *file);
+
+G_END_DECLS
+
+#endif /* __TRACKER_INDEXER_MODULE_H__ */

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.c	Tue May 13 12:48:19 2008
@@ -1,9 +1,6 @@
-/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
-/*
- * Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
- * Copyright (C) 2007, Michal Pryc (Michal Pryc Sun Com)
- * Copyright (C) 2008, Nokia (urho konttori nokia com)
- *
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
@@ -11,7 +8,7 @@
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
@@ -20,154 +17,443 @@
  * Boston, MA  02110-1301, USA.
  */
 
-#include "config.h"
+/* The indexer works as a state machine, there are 3 different queues:
+ *
+ * * The files queue: the highest priority one, individual files are
+ *   stored here, waiting for metadata extraction, etc... files are
+ *   taken one by one in order to be processed, when this queue is
+ *   empty, a single token from the next queue is processed.
+ *
+ * * The directories queue: directories are stored here, waiting for
+ *   being inspected. When a directory is inspected, contained files
+ *   and directories will be prepended in their respective queues.
+ *   When this queue is empty, a single token from the next queue
+ *   is processed.
+ *
+ * * The modules list: indexing modules are stored here, these modules
+ *   can either prepend the files or directories to be inspected in
+ *   their respective queues.
+ *
+ * Once all queues are empty, all elements have been inspected, and the
+ * indexer will emit the ::finished signal, this behavior can be observed
+ * in the indexing_func() function.
+ */
 
-#include <string.h>
 #include <stdlib.h>
-#include <locale.h>
+#include <gmodule.h>
+#include <libtracker-common/tracker-config.h>
+#include <libtracker-db/tracker-db-interface-sqlite.h>
+#include "tracker-indexer.h"
+#include "tracker-indexer-module.h"
+
+#define TRACKER_INDEXER_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_INDEXER, TrackerIndexerPrivate))
+
+typedef struct TrackerIndexerPrivate TrackerIndexerPrivate;
+typedef struct PathInfo PathInfo;
+
+struct TrackerIndexerPrivate {
+	GQueue *dir_queue;
+	GQueue *file_process_queue;
+
+	GSList *module_names;
+	GSList *current_module;
+	GHashTable *indexer_modules;
+
+	TrackerDBInterface *index;
+
+	TrackerConfig *config;
+
+	guint idle_id;
+};
+
+struct PathInfo {
+	GModule *module;
+	gchar *path;
+};
 
-#include <glib/gi18n.h>
-#include <glib-object.h>
+enum {
+	PROP_0,
+	PROP_RUNNING
+};
 
-#define COPYRIGHT                                                         \
-        "Tracker version " PACKAGE_VERSION "\n"                           \
-        "Copyright (c) 2005-2008 by Jamie McCracken (jamiemcc gnome org)"
-
-#define WARRANTY \
-        "This program is free software and comes without any warranty.\n" \
-        "It is licensed under version 2 or later of the General Public\n" \
-        "License which can be viewed at:\n"                               \
-        "\n"                                                              \
-        "\thttp://www.gnu.org/licenses/gpl.txt";
-
-static gchar    **no_watch_dirs;
-static gchar    **watch_dirs;
-static gchar    **crawl_dirs;
-static gchar     *language;
-static gboolean   disable_indexing;
-static gboolean   reindex;
-static gboolean   fatal_errors;
-static gboolean   low_memory;
-static gint       throttle = -1;
-static gint       verbosity;
-static gint       initial_sleep = -1; 
-
-static GOptionEntry entries[] = {
-	{ "exclude-dir", 'e', 0, G_OPTION_ARG_STRING_ARRAY, &no_watch_dirs, 
-          N_("Directory to exclude from indexing"), 
-          N_("/PATH/DIR")
-        },
-	{ "include-dir", 'i', 0, G_OPTION_ARG_STRING_ARRAY, &watch_dirs, 
-          N_("Directory to include in indexing"), 
-          N_("/PATH/DIR")
-        },
-	{ "crawl-dir", 'c', 0, G_OPTION_ARG_STRING_ARRAY, &crawl_dirs, 
-          N_("Directory to crawl for indexing at start up only"), 
-          N_("/PATH/DIR")
-        },
-	{ "no-indexing", 'n', 0, G_OPTION_ARG_NONE, &disable_indexing, 
-          N_("Disable any indexing or watching taking place"),
-          NULL 
-        },
-	{ "verbosity", 'v', 0, G_OPTION_ARG_INT, &verbosity, 
-          N_("Value that controls the level of logging. Valid values "
-             "are 0=errors, 1=minimal, 2=detailed, 3=debug"), 
-          N_("VALUE")
-        },
-	{ "throttle", 't', 0, G_OPTION_ARG_INT, &throttle, 
-          N_("Value to use for throttling indexing. Value must be in "
-             "range 0-99 (default=0) with lower values increasing "
-             "indexing speed"), 
-          N_("VALUE") 
-        },
-	{ "low-memory", 'm', 0, G_OPTION_ARG_NONE, &low_memory, 
-          N_("Minimizes the use of memory but may slow indexing down"), 
-          NULL 
-        },
-	{ "initial-sleep", 's', 0, G_OPTION_ARG_INT, &initial_sleep, 
-          N_("Initial sleep time, just before indexing, in seconds"), 
-          NULL
-        },
-	{ "language", 'l', 0, G_OPTION_ARG_STRING, &language, 
-          N_("Language to use for stemmer and stop words list "
-             "(ISO 639-1 2 characters code)"), 
-          N_("LANG")
-        },
-	{ "reindex", 'R', 0, G_OPTION_ARG_NONE, &reindex, 
-          N_("Force a re-index of all content"), 
-          NULL 
-        },
-	{ "fatal-errors", 'f', 0, G_OPTION_ARG_NONE, &fatal_errors, 
-          N_("Make tracker errors fatal"), 
-          NULL 
-        },
-	{ NULL }
+enum {
+	FINISHED,
+	LAST_SIGNAL
 };
 
-gint
-main (gint argc, gchar *argv[])
+static guint signals[LAST_SIGNAL] = { 0, };
+
+G_DEFINE_TYPE (TrackerIndexer, tracker_indexer, G_TYPE_OBJECT)
+
+static PathInfo *
+path_info_new (GModule     *module,
+	       const gchar *path)
+{
+	PathInfo *info;
+
+	info = g_slice_new (PathInfo);
+	info->module = module;
+	info->path = g_strdup (path);
+
+	return info;
+}
+
+static void
+path_info_free (PathInfo *info)
+{
+	g_free (info->path);
+	g_slice_free (PathInfo, info);
+}
+
+static void
+tracker_indexer_finalize (GObject *object)
+{
+	TrackerIndexerPrivate *priv;
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (object);
+
+	g_queue_foreach (priv->dir_queue, (GFunc) path_info_free, NULL);
+	g_queue_free (priv->dir_queue);
+
+	g_queue_foreach (priv->file_process_queue, (GFunc) path_info_free, NULL);
+	g_queue_free (priv->file_process_queue);
+
+	g_slist_foreach (priv->module_names, (GFunc) g_free, NULL);
+	g_slist_free (priv->module_names);
+
+	g_hash_table_destroy (priv->indexer_modules);
+
+	g_object_unref (priv->config);
+
+	G_OBJECT_CLASS (tracker_indexer_parent_class)->finalize (object);
+}
+
+static void
+tracker_indexer_set_property (GObject      *object,
+			      guint         prop_id,
+			      const GValue *value,
+			      GParamSpec   *pspec)
+{
+	TrackerIndexer *indexer;
+	TrackerIndexerPrivate *priv;
+
+	indexer = TRACKER_INDEXER (object);
+	priv = TRACKER_INDEXER_GET_PRIVATE (object);
+
+	switch (prop_id) {
+	case PROP_RUNNING:
+		tracker_indexer_set_running (indexer, g_value_get_boolean (value));
+		break;
+	default:
+		G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+	}
+}
+
+static void
+tracker_indexer_get_property (GObject    *object,
+			      guint       prop_id,
+			      GValue     *value,
+			      GParamSpec *pspec)
+{
+	TrackerIndexerPrivate *priv;
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (object);
+
+	switch (prop_id) {
+	case PROP_RUNNING:
+		g_value_set_boolean (value, (priv->idle_id != 0));
+		break;
+	default:
+		G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+	}
+}
+
+static void
+tracker_indexer_class_init (TrackerIndexerClass *class)
+{
+	GObjectClass *object_class = G_OBJECT_CLASS (class);
+
+	object_class->finalize = tracker_indexer_finalize;
+	object_class->set_property = tracker_indexer_set_property;
+	object_class->get_property = tracker_indexer_get_property;
+
+	signals [FINISHED] = g_signal_new ("finished",
+					   G_OBJECT_CLASS_TYPE (object_class),
+					   G_SIGNAL_RUN_LAST,
+					   G_STRUCT_OFFSET (TrackerIndexerClass, finished),
+					   NULL, NULL,
+					   g_cclosure_marshal_VOID__VOID,
+					   G_TYPE_NONE, 0);
+
+	g_object_class_install_property (object_class,
+					 PROP_RUNNING,
+					 g_param_spec_boolean ("running",
+							       "Running",
+							       "Whether the indexer is running",
+							       TRUE,
+							       G_PARAM_READWRITE));
+
+	g_type_class_add_private (object_class,
+				  sizeof (TrackerIndexerPrivate));
+}
+
+static gboolean
+init_indexer (TrackerIndexer *indexer)
+{
+	tracker_indexer_set_running (indexer, TRUE);
+	return FALSE;
+}
+
+TrackerDBInterface *
+create_db_interface (const gchar *filename)
+{
+#if 0
+	TrackerDBInterface *interface;
+	gchar *path;
+
+	path = g_build_filename (g_get_user_cache_dir (),
+				 "tracker",
+				 filename,
+				 NULL);
+
+	interface = tracker_db_interface_sqlite_new (path);
+	g_free (path);
+
+	return interface;
+#endif
+	return NULL;
+}
+
+static void
+tracker_indexer_init (TrackerIndexer *indexer)
+{
+	TrackerIndexerPrivate *priv;
+	gint initial_sleep;
+	GSList *m;
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	priv->dir_queue = g_queue_new ();
+	priv->file_process_queue = g_queue_new ();
+	priv->config = tracker_config_new ();
+
+	priv->index = create_db_interface ("file-meta.db");
+
+	priv->module_names = tracker_config_get_index_modules (priv->config);
+
+	priv->indexer_modules = g_hash_table_new_full (g_direct_hash,
+						       g_direct_equal,
+						       NULL,
+						       (GDestroyNotify) g_module_close);
+
+	for (m = priv->module_names; m; m = m->next) {
+		GModule *module;
+
+		module = tracker_indexer_module_load (m->data);
+
+		if (module) {
+			g_hash_table_insert (priv->indexer_modules,
+					     m->data, module);
+		}
+	}
+
+	initial_sleep = tracker_config_get_initial_sleep (priv->config);
+	g_timeout_add (initial_sleep * 1000, (GSourceFunc) init_indexer, indexer);
+}
+
+static void
+tracker_indexer_add_file (TrackerIndexer *indexer,
+			  PathInfo       *info)
+{
+	TrackerIndexerPrivate *priv;
+
+	g_return_if_fail (info != NULL);
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	g_queue_push_tail (priv->file_process_queue, info);
+}
+
+static void
+tracker_indexer_add_directory (TrackerIndexer *indexer,
+			       PathInfo       *info)
+{
+	TrackerIndexerPrivate *priv;
+
+	g_return_if_fail (info != NULL);
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	g_queue_push_tail (priv->dir_queue, info);
+}
+
+static void
+process_file (TrackerIndexer *indexer,
+	      PathInfo       *info)
+{
+	GHashTable *metadata;
+
+	g_message ("Processing file: %s\n", info->path);
+
+	metadata = tracker_indexer_module_get_file_metadata (info->module, info->path);
+
+	if (metadata) {
+		/* FIXME: store metadata in DB */
+		GList *keys, *k;
+
+		keys = g_hash_table_get_keys (metadata);
+
+		for (k = keys; k; k = k->next) {
+			g_print (" %s = %s\n",
+				 (gchar *) k->data,
+				 (gchar *) g_hash_table_lookup (metadata, k->data));
+		}
+
+		g_hash_table_destroy (metadata);
+		g_list_free (keys);
+	}
+}
+
+static void
+process_directory (TrackerIndexer *indexer,
+		   PathInfo       *info,
+		   gboolean        recurse)
 {
-        GMainLoop      *main_loop;
-	GOptionContext *context;
-	GError         *error = NULL;
-        gchar          *summary = NULL;
-        gchar          *example;
-
-        g_type_init ();
-        
-	if (!g_thread_supported ())
-		g_thread_init (NULL);
-
-	setlocale (LC_ALL, "");
-
-	bindtextdomain (GETTEXT_PACKAGE, LOCALEDIR);
-        bind_textdomain_codeset (GETTEXT_PACKAGE, "UTF-8");
-        textdomain (GETTEXT_PACKAGE);
-
-	/* Set timezone info */
-	tzset ();
-
-        /* Translators: this messagge will apper immediately after the
-         * usage string - Usage: COMMAND <THIS_MESSAGE>
-         */
-	context = g_option_context_new (_("- start the tracker daemon"));
-        example = g_strconcat ("-i ", _("DIRECTORY"), 
-                               "-i ", _("DIRECTORY"),
-			       "-e ", _("DIRECTORY"), 
-                               "-e ", _("DIRECTORY"),
-			       NULL);
-
-#ifdef HAVE_RECENT_GLIB
-        /* Translators: this message will appear after the usage
-         * string and before the list of options, showing an usage
-         * example.
-         */
-        summary = g_strdup_printf (_("To include or exclude multiple directories "
-                                     "at the same time, join multiple options like:\n"
-                                     "\n"
-                                     "\t%s"),
-                                   example);
-        g_option_context_set_summary (context, summary);
-#endif /* HAVE_RECENT_GLIB */
-
-	g_option_context_add_main_entries (context, entries, NULL);
-	g_option_context_parse (context, &argc, &argv, &error);
-	g_option_context_free (context);
-        g_free (summary);
-        g_free (example);
-
-	g_print ("\n"
-                 COPYRIGHT "\n"
-                 "\n"
-                 WARRANTY "\n"
-                 "\n");
+	const gchar *name;
+	GDir *dir;
+
+	g_message ("Processing directory: %s\n", info->path);
+
+	dir = g_dir_open (info->path, 0, NULL);
+
+	if (!dir) {
+		return;
+	}
 
-	g_print ("Initialising...\n");
+	while ((name = g_dir_read_name (dir)) != NULL) {
+		PathInfo *new_info;
+		gchar *path;
+
+		path = g_build_filename (info->path, name, NULL);
+
+		new_info = path_info_new (info->module, path);
+		tracker_indexer_add_file (indexer, new_info);
+
+		if (recurse && g_file_test (path, G_FILE_TEST_IS_DIR)) {
+			new_info = path_info_new (info->module, path);
+			tracker_indexer_add_directory (indexer, new_info);
+		}
+
+		g_free (path);
+	}
+
+	g_dir_close (dir);
+}
+
+static void
+process_module (TrackerIndexer *indexer,
+		const gchar    *module_name)
+{
+	TrackerIndexerPrivate *priv;
+	GModule *module;
+	gchar **dirs;
+	gint i;
+
+	g_message ("Starting module: %s\n", module_name);
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+	module = g_hash_table_lookup (priv->indexer_modules, module_name);
+	dirs = tracker_indexer_module_get_directories (module);
+
+	g_return_if_fail (dirs != NULL);
+
+	for (i = 0; dirs[i]; i++) {
+		PathInfo *info;
+
+		info = path_info_new (module, dirs[i]);
+		tracker_indexer_add_directory (indexer, info);
+	}
+
+	g_strfreev (dirs);
+}
+
+static gboolean
+indexing_func (gpointer data)
+{
+	TrackerIndexer *indexer;
+	TrackerIndexerPrivate *priv;
+	PathInfo *path;
+
+	indexer = (TrackerIndexer *) data;
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	if ((path = g_queue_pop_head (priv->file_process_queue)) != NULL) {
+		/* process file */
+		process_file (indexer, path);
+		path_info_free (path);
+	} else if ((path = g_queue_pop_head (priv->dir_queue)) != NULL) {
+		/* process directory contents */
+		process_directory (indexer, path, TRUE);
+		path_info_free (path);
+	} else {
+		/* dirs/files queues are empty, process the next module */
+		if (!priv->current_module) {
+			priv->current_module = priv->module_names;
+		} else {
+			priv->current_module = priv->current_module->next;
+		}
+
+		if (!priv->current_module) {
+			/* no more modules to query, we're done */
+			g_signal_emit (indexer, signals[FINISHED], 0);
+			return FALSE;
+		}
+
+		process_module (indexer, priv->current_module->data);
+	}
+
+	return TRUE;
+}
+
+TrackerIndexer *
+tracker_indexer_new (void)
+{
+	return g_object_new (TRACKER_TYPE_INDEXER, NULL);
+}
+
+void
+tracker_indexer_set_running (TrackerIndexer *indexer,
+			     gboolean        running)
+{
+	TrackerIndexerPrivate *priv;
+	gboolean changed = FALSE;
+
+	g_return_if_fail (TRACKER_IS_INDEXER (indexer));
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	if (running && priv->idle_id == 0) {
+		priv->idle_id = g_idle_add ((GSourceFunc) indexing_func, indexer);
+		changed = TRUE;
+	} else if (!running && priv->idle_id != 0) {
+		g_source_remove (priv->idle_id);
+		priv->idle_id = 0;
+		changed = TRUE;
+	}
+
+	if (changed) {
+		g_object_notify (G_OBJECT (indexer), "running");
+	}
+}
+
+gboolean
+tracker_indexer_get_running (TrackerIndexer *indexer)
+{
+	TrackerIndexerPrivate *priv;
 
-        main_loop = g_main_loop_new (NULL, FALSE);
-        g_main_loop_run (main_loop);
+	g_return_val_if_fail (TRACKER_IS_INDEXER (indexer), FALSE);
 
-	g_print ("Shutting down...\n");
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
 
-        return EXIT_SUCCESS;
+	return (priv->idle_id == 0);
 }

Added: branches/indexer-split/src/tracker-indexer/tracker-indexer.h
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.h	Tue May 13 12:48:19 2008
@@ -0,0 +1,60 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __TRACKER_INDEXER_H__
+#define __TRACKER_INDEXER_H__
+
+#include <glib.h>
+
+G_BEGIN_DECLS
+
+#include <glib-object.h>
+
+#define TRACKER_TYPE_INDEXER         (tracker_indexer_get_type())
+#define TRACKER_INDEXER(o)           (G_TYPE_CHECK_INSTANCE_CAST ((o), TRACKER_TYPE_INDEXER, TrackerIndexer))
+#define TRACKER_INDEXER_CLASS(c)     (G_TYPE_CHECK_CLASS_CAST ((c),    TRACKER_TYPE_INDEXER, TrackerIndexerClass))
+#define TRACKER_IS_INDEXER(o)        (G_TYPE_CHECK_INSTANCE_TYPE ((o), TRACKER_TYPE_INDEXER))
+#define TRACKER_IS_INDEXER_CLASS(c)  (G_TYPE_CHECK_CLASS_TYPE ((c),    TRACKER_TYPE_INDEXER))
+#define TRACKER_INDEXER_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS ((o),  TRACKER_TYPE_INDEXER, TrackerIndexerClass))
+
+typedef struct TrackerIndexer TrackerIndexer;
+typedef struct TrackerIndexerClass TrackerIndexerClass;
+
+struct TrackerIndexer {
+	GObject parent_instance;
+};
+
+struct TrackerIndexerClass {
+	GObjectClass parent_class;
+
+	void (*finished) (TrackerIndexer *indexer);
+};
+
+GType                 tracker_indexer_get_type       (void) G_GNUC_CONST;
+
+TrackerIndexer *      tracker_indexer_new            (void);
+
+void                  tracker_indexer_set_running    (TrackerIndexer      *indexer,
+						      gboolean             running);
+gboolean              tracker_indexer_get_running    (TrackerIndexer      *indexer);
+
+
+G_END_DECLS
+
+#endif /* __TRACKER_INDEXER_H__ */

Modified: branches/indexer-split/src/trackerd/tracker-process-files.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-process-files.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-process-files.c	Tue May 13 12:48:19 2008
@@ -1527,9 +1527,7 @@
                           tracker);
 
         /* Start processing */
-	g_print ("oooye!\n");
 	g_mutex_unlock (tracker->files_signal_mutex);
-	g_print ("oooye2!\n");
 
         /* Signal state change */
         g_signal_emit_by_name (object, 



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]