tracker r1891 - in branches/indexer-split: . src/libtracker-common src/trackerd

[mr svn gnome org]

Author: mr
Date: Fri Jul 18 17:48:24 2008
New Revision: 1891
URL: http://svn.gnome.org/viewvc/tracker?rev=1891&view=rev

Log:
	* src/libtracker-common/tracker-file-utils.[ch]:
	* src/libtracker-common/tracker-module-config.c: Added functions
	to remove duplicates or hierarchical conflicts between monitor
	directories and monitor recurse directories. 

	* src/trackerd/tracker-marshal.list:
	* src/trackerd/tracker-crawler.c: Emit a signal for each directory
	we process so the processor can add monitors. The bug where we
	don't monitor or crawl the toplevel directory/files for each
	module is fixed now. We now only send files to the indexer AFTER
	we have crawled the file system. This time is insignificant, for
	100k files it takes ~10 seconds.

	* src/trackerd/tracker-processor.c: Removed add_recurse_monitors()
	and add_monitors(), they are not needed. Added the code to handle
	adding monitors from the process-directory callback in the crawler.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/libtracker-common/tracker-file-utils.c
   branches/indexer-split/src/libtracker-common/tracker-file-utils.h
   branches/indexer-split/src/libtracker-common/tracker-module-config.c
   branches/indexer-split/src/trackerd/tracker-crawler.c
   branches/indexer-split/src/trackerd/tracker-crawler.h
   branches/indexer-split/src/trackerd/tracker-marshal.list
   branches/indexer-split/src/trackerd/tracker-processor.c

Modified: branches/indexer-split/src/libtracker-common/tracker-file-utils.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-file-utils.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-file-utils.c	Fri Jul 18 17:48:24 2008
@@ -470,6 +470,98 @@
 	g_slist_free (dirs_to_remove);
 }
 
+gboolean
+tracker_path_is_in_path (const gchar *path,
+			 const gchar *in_path)
+{
+	gchar    *new_path;
+	gchar    *new_in_path;
+	gboolean  is_in_path = FALSE;
+
+	g_return_val_if_fail (path != NULL, FALSE);
+	g_return_val_if_fail (in_path != NULL, FALSE);
+	
+	if (!g_str_has_suffix (path, G_DIR_SEPARATOR_S)) {
+		new_path = g_strconcat (path, G_DIR_SEPARATOR_S, NULL);
+	} else {
+		new_path = g_strdup (path);
+	}
+
+	if (!g_str_has_suffix (in_path, G_DIR_SEPARATOR_S)) {
+		new_in_path = g_strconcat (in_path, G_DIR_SEPARATOR_S, NULL);
+	} else {
+		new_in_path = g_strdup (in_path);
+	}
+	
+	if (g_str_has_prefix (new_path, new_in_path)) {
+		is_in_path = TRUE;
+	} 
+
+	g_free (new_in_path);
+	g_free (new_path);
+
+	return is_in_path;
+}
+
+void
+tracker_path_hash_table_filter_duplicates (GHashTable *roots)
+{
+	GHashTableIter iter1, iter2;
+	gpointer       key;
+
+	g_debug ("Filtering duplicates in path hash table:");
+
+	g_hash_table_iter_init (&iter1, roots);
+	while (g_hash_table_iter_next (&iter1, &key, NULL)) {
+		const gchar *path;
+		
+		path = (const gchar*) key;
+
+		g_hash_table_iter_init (&iter2, roots);
+		while (g_hash_table_iter_next (&iter2, &key, NULL)) {
+			const gchar *in_path;
+
+			in_path = (const gchar*) key;
+
+			if (path == in_path) {
+				continue;
+			}
+
+			if (tracker_path_is_in_path (path, in_path)) {
+				g_debug ("Removing path:'%s', it is in path:'%s'", 
+					 path, in_path);
+
+				g_hash_table_iter_remove (&iter1);
+				g_hash_table_iter_init (&iter1, roots);
+				break;
+			} else if (tracker_path_is_in_path (in_path, path)) {
+				g_debug ("Removing path:'%s', it is in path:'%s'", 
+					 in_path, path);
+
+				g_hash_table_iter_remove (&iter2);
+				g_hash_table_iter_init (&iter1, roots);
+				break;
+			}
+		}
+	}
+
+#ifdef TESTING
+	g_debug ("Using the following roots to crawl:");
+
+	if (TRUE) {
+		GList *keys, *l;
+
+		keys = g_hash_table_get_keys (roots);
+		
+		for (l = keys; l; l = l->next) {
+			g_debug ("  %s", (gchar*) l->data);
+		}
+		
+		g_list_free (keys);
+	}
+#endif /* TESTING */
+}
+
 GSList *
 tracker_path_list_filter_duplicates (GSList *roots)
 {

Modified: branches/indexer-split/src/libtracker-common/tracker-file-utils.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-file-utils.h	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-file-utils.h	Fri Jul 18 17:48:24 2008
@@ -24,21 +24,24 @@
 
 #include <glib.h>
 
-gint     tracker_file_open                   (const gchar *uri,
-					      gboolean     readahead);
-void     tracker_file_close                  (gint         fd,
-					      gboolean     no_longer_needed);
-gboolean tracker_file_unlink                 (const gchar *uri);
-gboolean tracker_file_is_valid               (const gchar *uri);
-gboolean tracker_file_is_directory           (const gchar *uri);
-gboolean tracker_file_is_indexable           (const gchar *uri);
-guint32  tracker_file_get_size               (const gchar *uri);
-gint32   tracker_file_get_mtime              (const gchar *uri);
-gchar *  tracker_file_get_mime_type          (const gchar *uri);
-gchar *  tracker_file_get_vfs_path           (const gchar *uri);
-gchar *  tracker_file_get_vfs_name           (const gchar *uri);
-void     tracker_path_remove                 (const gchar *uri);
-GSList * tracker_path_list_filter_duplicates (GSList      *roots);
-gchar *  tracker_path_evaluate_name          (const gchar *uri);
+gint     tracker_file_open                         (const gchar *uri,
+						    gboolean     readahead);
+void     tracker_file_close                        (gint         fd,
+						    gboolean     no_longer_needed);
+gboolean tracker_file_unlink                       (const gchar *uri);
+gboolean tracker_file_is_valid                     (const gchar *uri);
+gboolean tracker_file_is_directory                 (const gchar *uri);
+gboolean tracker_file_is_indexable                 (const gchar *uri);
+guint32  tracker_file_get_size                     (const gchar *uri);
+gint32   tracker_file_get_mtime                    (const gchar *uri);
+gchar *  tracker_file_get_mime_type                (const gchar *uri);
+gchar *  tracker_file_get_vfs_path                 (const gchar *uri);
+gchar *  tracker_file_get_vfs_name                 (const gchar *uri);
+void     tracker_path_remove                       (const gchar *uri);
+gboolean tracker_path_is_in_path                   (const gchar *path,
+						    const gchar *in_path);
+void     tracker_path_hash_table_filter_duplicates (GHashTable  *roots);
+GSList * tracker_path_list_filter_duplicates       (GSList      *roots);
+gchar *  tracker_path_evaluate_name                (const gchar *uri);
 
 #endif /* __LIBTRACKER_COMMON_FILE_UTILS_H__ */

Modified: branches/indexer-split/src/libtracker-common/tracker-module-config.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-module-config.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-module-config.c	Fri Jul 18 17:48:24 2008
@@ -67,8 +67,12 @@
 static GFileMonitor *monitor;
 
 static void
-module_config_free (ModuleConfig *mc)
+module_destroy_notify (gpointer data)
 {
+	ModuleConfig *mc;
+
+	mc = (ModuleConfig*) data;
+
 	g_list_foreach (mc->index_file_patterns,
 			(GFunc) g_pattern_spec_free,
 			NULL);
@@ -99,14 +103,65 @@
 	g_slice_free (ModuleConfig, mc);
 }
 
+static void
+check_for_monitor_directory_conflicts (ModuleConfig *mc)
+{
+	GHashTableIter iter1, iter2;
+	gpointer       key;
+
+	/* Make sure we don't have duplicates from the monitor
+	 * directories in the monitor recurse directories hash table
+	 * and also, make sure there are no recurse directories higher
+	 * as parents of monitor directories, this would duplicate
+	 * monitors unnecesarily.
+	 */
+	g_hash_table_iter_init (&iter1, mc->monitor_directories);
+	while (g_hash_table_iter_next (&iter1, &key, NULL)) {
+		const gchar *path;
+		
+		path = (const gchar*) key;
+
+		if (g_hash_table_lookup (mc->monitor_recurse_directories, path)) {
+			g_debug ("Removing path:'%s' from monitor directories, "
+				 "ALREADY in monitor recurse directories", 
+				 path);
+
+			g_hash_table_iter_remove (&iter1);
+			g_hash_table_iter_init (&iter1, mc->monitor_directories);
+			continue;
+		}
+
+		g_hash_table_iter_init (&iter2, mc->monitor_recurse_directories);
+		while (g_hash_table_iter_next (&iter2, &key, NULL)) {
+			const gchar *in_path;
+
+			in_path = (const gchar*) key;
+
+			if (path == in_path) {
+				continue;
+			}
+
+			if (tracker_path_is_in_path (path, in_path)) {
+				g_debug ("Removing path:'%s' from monitor directories, "
+					 "ALREADY in monitor recurse directories HIERARCHY", 
+					 path);
+
+				g_hash_table_iter_remove (&iter1);
+				g_hash_table_iter_init (&iter1, mc->monitor_directories);
+				break;
+			}
+		}
+	}
+}
+
 static gchar *
-module_config_get_directory (void)
+get_directory (void)
 {
 	return g_build_path (G_DIR_SEPARATOR_S, SHAREDIR, "tracker", "modules", NULL);
 }
 
 static void
-module_config_set_ignored_file_patterns (ModuleConfig *mc)
+set_ignored_file_patterns (ModuleConfig *mc)
 {
 	GPatternSpec *spec;
 	GList        *ignored_files;
@@ -133,7 +188,7 @@
 }
 
 static void
-module_config_set_ignored_directory_patterns (ModuleConfig *mc)
+set_ignored_directory_patterns (ModuleConfig *mc)
 {
 	GPatternSpec *spec;
 	GList        *ignored_directories;
@@ -160,7 +215,7 @@
 }
 
 static void
-module_config_set_index_file_patterns (ModuleConfig *mc)
+set_index_file_patterns (ModuleConfig *mc)
 {
 	GPatternSpec *spec;
 	GList        *index_files;
@@ -187,9 +242,9 @@
 }
 
 static gboolean
-module_config_load_boolean (GKeyFile    *key_file,
-			    const gchar *group,
-			    const gchar *key)
+load_boolean (GKeyFile    *key_file,
+	      const gchar *group,
+	      const gchar *key)
 {
 	GError   *error = NULL;
 	gboolean  boolean;
@@ -213,10 +268,10 @@
 }
 
 static gchar *
-module_config_load_string (GKeyFile    *key_file,
-			   const gchar *group,
-			   const gchar *key,
-			   gboolean     expand_string_as_path)
+load_string (GKeyFile    *key_file,
+	      const gchar *group,
+	      const gchar *key,
+	      gboolean     expand_string_as_path)
 {
 	GError *error = NULL;
 	gchar  *str;
@@ -249,10 +304,11 @@
 }
 
 static GHashTable *
-module_config_load_string_list (GKeyFile    *key_file,
-				const gchar *group,
-				const gchar *key,
-				gboolean     expand_strings_as_paths)
+load_string_list (GKeyFile    *key_file,
+		  const gchar *group,
+		  const gchar *key,
+		  gboolean     expand_strings_as_paths,
+		  gboolean     remove_hierarchy_dups)
 {
 	GError      *error = NULL;
 	GHashTable  *table;
@@ -307,16 +363,23 @@
 					     GINT_TO_POINTER (1));
 			g_debug ("Got real path:'%s' for '%s'", real_path, *p);
 		}
-
 	}
 
 	g_strfreev (str);
-
+	
+	/* Go through again to make sure we don't have situations
+	 * where /foo and / exist, because of course /foo is
+	 * redundant here where 'remove_hierarchy_dups' is TRUE.
+	 */
+	if (remove_hierarchy_dups) {
+		tracker_path_hash_table_filter_duplicates (table);
+	}
+	
 	return table;
 }
 
 static ModuleConfig *
-module_config_load_file (const gchar *filename)
+load_file (const gchar *filename)
 {
 	GKeyFile     *key_file;
 	GError       *error = NULL;
@@ -343,62 +406,61 @@
 	mc = g_slice_new0 (ModuleConfig);
 
 	/* General */
-	mc->description =
-		module_config_load_string (key_file,
-					   GROUP_GENERAL,
-					   "Description",
-					   FALSE);
-	mc->enabled =
-		module_config_load_boolean (key_file,
-					    GROUP_GENERAL,
-					    "Enabled");
+	mc->description = load_string (key_file,
+				       GROUP_GENERAL,
+				       "Description",
+				       FALSE);
+	mc->enabled = load_boolean (key_file,
+				    GROUP_GENERAL,
+				    "Enabled");
 
 	/* Monitors */
-	mc->monitor_directories =
-		module_config_load_string_list (key_file,
-						GROUP_MONITORS,
-						"Directories",
-						TRUE);
-	mc->monitor_recurse_directories =
-		module_config_load_string_list (key_file,
-						GROUP_MONITORS,
-						"RecurseDirectories",
-						TRUE);
+	mc->monitor_directories = load_string_list (key_file,
+						    GROUP_MONITORS,
+						    "Directories",
+						    TRUE,
+						    FALSE);
+	mc->monitor_recurse_directories = load_string_list (key_file,
+							   GROUP_MONITORS,
+							   "RecurseDirectories",
+							   TRUE,
+							   TRUE);
 
 	/* Ignored */
-	mc->ignored_directories =
-		module_config_load_string_list (key_file,
-						GROUP_IGNORED,
-						"Directories",
-						TRUE);
-	mc->ignored_files =
-		module_config_load_string_list (key_file,
-						GROUP_IGNORED,
-						"Files",
-						FALSE);
+	mc->ignored_directories = load_string_list (key_file,
+						    GROUP_IGNORED,
+						    "Directories",
+						    TRUE,
+						    FALSE);
+	mc->ignored_files = load_string_list (key_file,
+					      GROUP_IGNORED,
+					      "Files",
+					      FALSE,
+					      FALSE);
 
 	/* Index */
-	mc->index_service =
-		module_config_load_string (key_file,
-					   GROUP_INDEX,
-					   "Service",
-					   FALSE);
-	mc->index_mime_types =
-		module_config_load_string_list (key_file,
-						GROUP_INDEX,
-						"MimeTypes",
-						FALSE);
-	mc->index_files =
-		module_config_load_string_list (key_file,
-						GROUP_INDEX,
-						"Files",
-						FALSE);
+	mc->index_service = load_string (key_file,
+					 GROUP_INDEX,
+					 "Service",
+					 FALSE);
+	mc->index_mime_types = load_string_list (key_file,
+						 GROUP_INDEX,
+						 "MimeTypes",
+						 FALSE,
+						 FALSE);
+	mc->index_files = load_string_list (key_file,
+					    GROUP_INDEX,
+					    "Files",
+					    FALSE,
+					    FALSE);
+
+	check_for_monitor_directory_conflicts (mc);
 
 	/* FIXME: Specific options */
 
-	module_config_set_ignored_file_patterns (mc);
-	module_config_set_ignored_directory_patterns (mc);
-	module_config_set_index_file_patterns (mc);
+	set_ignored_file_patterns (mc);
+	set_ignored_directory_patterns (mc);
+	set_index_file_patterns (mc);
 
 	g_key_file_free (key_file);
 
@@ -406,7 +468,7 @@
 }
 
 static gboolean
-module_config_load (void)
+load_directory (void)
 {
 	GFile           *file;
 	GFileEnumerator *enumerator;
@@ -418,7 +480,7 @@
 	const gchar     *extension;
 	glong            extension_len;
 
-	path = module_config_get_directory ();
+	path = get_directory ();
 	file = g_file_new_for_path (path);
 
 	enumerator = g_file_enumerate_children (file,
@@ -459,7 +521,7 @@
 
 		child = g_file_get_child (file, name);
 		filename = g_file_get_path (child);
-		mc = module_config_load_file (filename);
+		mc = load_file (filename);
 		g_free (filename);
 
 		if (mc) {
@@ -494,11 +556,11 @@
 }
 
 static void
-module_config_changed_cb (GFileMonitor     *monitor,
-			  GFile            *file,
-			  GFile            *other_file,
-			  GFileMonitorEvent event_type,
-			  gpointer          user_data)
+changed_cb (GFileMonitor     *monitor,
+	    GFile            *file,
+	    GFile            *other_file,
+	    GFileMonitorEvent event_type,
+	    gpointer          user_data)
 {
 	gchar *filename;
 
@@ -512,7 +574,7 @@
 			   filename);
 		g_free (filename);
 
-		module_config_load ();
+		load_directory ();
 		break;
 
 	default:
@@ -530,7 +592,7 @@
 		return TRUE;
 	}
 
-	path = module_config_get_directory ();
+	path = get_directory ();
 	if (!g_file_test (path, G_FILE_TEST_IS_DIR | G_FILE_TEST_EXISTS)) {
 		g_critical ("Module config directory:'%s' doesn't exist",
 			    path);
@@ -540,11 +602,11 @@
 
 	modules = g_hash_table_new_full (g_str_hash,
 					 g_str_equal,
-					 (GDestroyNotify) g_free,
-					 (GDestroyNotify) module_config_free);
+					 g_free,
+					 module_destroy_notify);
 
 	/* Get modules */
-	if (!module_config_load ()) {
+	if (!load_directory ()) {
 		g_hash_table_unref (modules);
 		g_free (path);
 		return FALSE;
@@ -561,7 +623,7 @@
 					    NULL);
 
 	g_signal_connect (monitor, "changed",
-			  G_CALLBACK (module_config_changed_cb),
+			  G_CALLBACK (changed_cb),
 			  NULL);
 
 	g_object_unref (file);
@@ -579,9 +641,7 @@
 		return;
 	}
 
-	g_signal_handlers_disconnect_by_func (monitor,
-					      module_config_changed_cb,
-					      NULL);
+	g_signal_handlers_disconnect_by_func (monitor, changed_cb, NULL);
 
 	g_object_unref (monitor);
 

Modified: branches/indexer-split/src/trackerd/tracker-crawler.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.c	Fri Jul 18 17:48:24 2008
@@ -60,6 +60,7 @@
 
 	guint           idle_id;
 	guint           files_queue_handle_id;
+	gboolean        can_send_yet;
 
 	/* Specific to each crawl ... */
 	GList          *ignored_directory_patterns;
@@ -81,7 +82,7 @@
 };
 
 enum {
-	ALL_SENT,
+	PROCESSING_DIRECTORY,
 	FINISHED,
 	LAST_SIGNAL
 };
@@ -111,15 +112,17 @@
 
 	object_class->finalize = crawler_finalize;
 
-	signals[ALL_SENT] = 
-		g_signal_new ("all-sent",
+	signals[PROCESSING_DIRECTORY] = 
+		g_signal_new ("processing-directory",
 			      G_TYPE_FROM_CLASS (klass),
 			      G_SIGNAL_RUN_LAST,
 			      0,
 			      NULL, NULL,
-			      g_cclosure_marshal_VOID__VOID,
+			      tracker_marshal_VOID__STRING_OBJECT,
 			      G_TYPE_NONE, 
-			      0);
+			      2,
+			      G_TYPE_STRING,
+			      G_TYPE_OBJECT);
 	signals[FINISHED] = 
 		g_signal_new ("finished",
 			      G_TYPE_FROM_CLASS (klass),
@@ -589,6 +592,10 @@
 
 	crawler = TRACKER_CRAWLER (user_data);
 	
+	if (!crawler->private->can_send_yet) {
+		return TRUE;
+	}
+
 	/* This is here so we don't try to send something if we are
 	 * still waiting for a response from the last send.
 	 */
@@ -602,10 +609,7 @@
 
 	if (!queue || !module_name) {
 		g_message ("No file queues to process");
-
-		g_signal_emit (crawler, signals[ALL_SENT], 0);
 		crawler->private->files_queue_handle_id = 0;
-
 		return FALSE;
 	}
 
@@ -655,7 +659,7 @@
 		   GFile          *file,
 		   const gchar    *module_name)
 {
-	tracker_monitor_add (file, module_name);
+	g_signal_emit (crawler, signals[PROCESSING_DIRECTORY], 0, module_name, file);
 
 	file_enumerate_children (crawler, file);
 }
@@ -1016,7 +1020,7 @@
 		file = g_file_new_for_path (sl->data);
 		g_message ("  Searching directory:'%s'", (gchar *) sl->data);
 
-		file_enumerate_children (crawler, file);
+		add_directory (crawler, file);
 		g_object_unref (file);
 		g_free (sl->data);
 	}
@@ -1081,3 +1085,12 @@
 		       priv->files_found,
 		       priv->files_ignored);
 }
+
+void
+tracker_crawler_set_can_send_yet (TrackerCrawler *crawler,
+				  gboolean        can_send_yet)
+{
+	g_return_if_fail (TRACKER_IS_CRAWLER (crawler));
+
+	crawler->private->can_send_yet = can_send_yet;
+}

Modified: branches/indexer-split/src/trackerd/tracker-crawler.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.h	(original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.h	Fri Jul 18 17:48:24 2008
@@ -48,12 +48,14 @@
 	GObjectClass           parent;
 };
 
-GType           tracker_crawler_get_type     (void);
-TrackerCrawler *tracker_crawler_new          (TrackerConfig  *config,
-					      TrackerHal     *hal);
-gboolean        tracker_crawler_start        (TrackerCrawler *crawler,
-					      const gchar    *module_name);
-void            tracker_crawler_stop         (TrackerCrawler *crawler);
+GType           tracker_crawler_get_type         (void);
+TrackerCrawler *tracker_crawler_new              (TrackerConfig  *config,
+						  TrackerHal     *hal);
+gboolean        tracker_crawler_start            (TrackerCrawler *crawler,
+						  const gchar    *module_name);
+void            tracker_crawler_stop             (TrackerCrawler *crawler);
+void            tracker_crawler_set_can_send_yet (TrackerCrawler *crawler,
+						  gboolean        can_send_yet);
 
 G_END_DECLS
 

Modified: branches/indexer-split/src/trackerd/tracker-marshal.list
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-marshal.list	(original)
+++ branches/indexer-split/src/trackerd/tracker-marshal.list	Fri Jul 18 17:48:24 2008
@@ -2,6 +2,7 @@
 VOID:STRING,STRING,INT,INT,INT
 VOID:STRING,STRING,STRING
 VOID:STRING,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN
+VOID:STRING,OBJECT
 
 # XESAM signals -- HitsRemoved, HitsModified
 VOID:STRING,BOXED

Modified: branches/indexer-split/src/trackerd/tracker-processor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-processor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-processor.c	Fri Jul 18 17:48:24 2008
@@ -26,6 +26,7 @@
 #include <gio/gio.h>
 
 #include <libtracker-common/tracker-module-config.h>
+#include <libtracker-common/tracker-file-utils.h>
 #include <libtracker-common/tracker-hal.h>
 #include <libtracker-common/tracker-utils.h>
 
@@ -42,7 +43,6 @@
 struct TrackerProcessorPrivate {
 	TrackerConfig  *config;
 	TrackerHal     *hal;
-
 	TrackerCrawler *crawler;
 
 	DBusGProxy     *indexer_proxy;
@@ -66,26 +66,28 @@
 	LAST_SIGNAL
 };
 
-static void tracker_processor_finalize (GObject          *object);
-static void process_next_module        (TrackerProcessor *processor);
-static void indexer_status_cb          (DBusGProxy       *proxy,
-					gdouble           seconds_elapsed,
-					const gchar      *current_module_name,
-					guint             items_done,
-					guint             items_remaining,
-					gpointer          user_data);
-static void indexer_finished_cb        (DBusGProxy       *proxy,
-					gdouble           seconds_elapsed,
-					guint             items_done,
-					gpointer          user_data);
-static void crawler_all_sent_cb        (TrackerCrawler   *crawler,
-					gpointer          user_data);
-static void crawler_finished_cb        (TrackerCrawler   *crawler,
-					guint             directories_found,
-					guint             directories_ignored,
-					guint             files_found,
-					guint             files_ignored,
-					gpointer          user_data);
+static void tracker_processor_finalize      (GObject          *object);
+static void process_next_module             (TrackerProcessor *processor);
+static void indexer_status_cb               (DBusGProxy       *proxy,
+					     gdouble           seconds_elapsed,
+					     const gchar      *current_module_name,
+					     guint             items_done,
+					     guint             items_remaining,
+					     gpointer          user_data);
+static void indexer_finished_cb             (DBusGProxy       *proxy,
+					     gdouble           seconds_elapsed,
+					     guint             items_done,
+					     gpointer          user_data);
+static void crawler_processing_directory_cb (TrackerCrawler   *crawler,
+					     const gchar      *module_name,
+					     GFile            *file,
+					     gpointer          user_data);
+static void crawler_finished_cb             (TrackerCrawler   *crawler,
+					     guint             directories_found,
+					     guint             directories_ignored,
+					     guint             files_found,
+					     guint             files_ignored,
+					     gpointer          user_data);
 
 #ifdef HAVE_HAL
 static void mount_point_added_cb       (TrackerHal       *hal,
@@ -151,9 +153,9 @@
 					G_CALLBACK (indexer_status_cb),
 					processor);
 	g_object_unref (priv->indexer_proxy);
-	
+
 	g_signal_handlers_disconnect_by_func (priv->crawler,
-					      G_CALLBACK (crawler_all_sent_cb),
+					      G_CALLBACK (crawler_processing_directory_cb),
 					      object);
 	g_signal_handlers_disconnect_by_func (priv->crawler,
 					      G_CALLBACK (crawler_finished_cb),
@@ -179,62 +181,6 @@
 }
 
 static void
-add_monitors (const gchar *module_name)
-{
-	GList *monitors;
-	GList *l;
-
-	monitors = tracker_module_config_get_monitor_directories (module_name);
-
-	for (l = monitors; l; l = l->next) {
-		GFile       *file;
-		const gchar *path;
-
-		path = l->data;
-
-		g_message ("  Adding specific directory monitor:'%s'", path);
-
-		file = g_file_new_for_path (path);
-		tracker_monitor_add (file, module_name);
-		g_object_unref (file);
-	}
-
-	g_list_free (monitors);
-
-	if (!monitors) {
-		g_message ("  No specific monitors to set up");
-	}
-}
-
-static void
-add_recurse_monitors (const gchar *module_name)
-{
-	GList *monitors;
-	GList *l;
-
-	monitors = tracker_module_config_get_monitor_recurse_directories (module_name);
-
-	for (l = monitors; l; l = l->next) {
-		GFile       *file;
-		const gchar *path;
-
-		path = l->data;
-
-		g_message ("  Adding recurse directory monitor:'%s' (FIXME: Not finished)", path);
-
-		file = g_file_new_for_path (path);
-		tracker_monitor_add (file, module_name);
-		g_object_unref (file);
-	}
-
-	g_list_free (monitors);
-
-	if (!monitors) {
-		g_message ("  No recurse monitors to set up");
-	}
-}
-
-static void
 process_module (TrackerProcessor *processor,
 		const gchar      *module_name)
 {
@@ -350,11 +296,54 @@
 }
 
 static void
-crawler_all_sent_cb (TrackerCrawler *crawler,
-		     gpointer        user_data)
+crawler_processing_directory_cb (TrackerCrawler *crawler,
+				 const gchar    *module_name,
+				 GFile          *file,
+				 gpointer        user_data)
 {
-	g_message ("All items have been sent to the indexer to process, "
-		   "waiting for indexer to finished");
+#if 1
+	/* FIXME: We are doing this for now because the code is really inefficient */
+	tracker_monitor_add (file, module_name);
+#else
+	GList    *directories, *l;
+	gchar    *path;
+	gboolean  add_monitor;
+
+	path = g_file_get_path (file);
+
+	g_debug ("Processing module:'%s' with for path:'%s'",
+		 module_name,
+		 path);
+
+	/* Is it a monitor directory? */
+	directories = tracker_module_config_get_monitor_directories (module_name);
+
+	for (l = directories; l && !add_monitor; l = l->next) {
+		if (strcmp (path, l->data) == 0) {
+			add_monitor = TRUE;
+		}
+	}
+
+	g_list_free (directories);
+
+	/* Is it underneath a monitor recurse directory? */
+	directories = tracker_module_config_get_monitor_directories (module_name);
+
+	for (l = directories; l && !add_monitor; l = l->next) {
+		if (tracker_path_is_in_path (path, l->data) == 0) {
+			add_monitor = TRUE;
+		}
+	}
+
+	g_list_free (directories);
+	
+	/* Should we add? */
+	if (add_monitor) {
+		tracker_monitor_add (file, module_name);
+	}
+
+	g_free (path);
+#endif
 }
 
 static void
@@ -440,8 +429,8 @@
 
 	priv->crawler = tracker_crawler_new (config, hal);
 
-	g_signal_connect (priv->crawler, "all-sent",
-			  G_CALLBACK (crawler_all_sent_cb),
+	g_signal_connect (priv->crawler, "processing-directory",
+			  G_CALLBACK (crawler_processing_directory_cb),
 			  processor);
 	g_signal_connect (priv->crawler, "finished",
 			  G_CALLBACK (crawler_finished_cb),
@@ -531,6 +520,7 @@
 		g_signal_emit (processor, signals[FINISHED], 0);
 	} else {
 		tracker_status_set_and_signal (TRACKER_STATUS_INDEXING);
+		tracker_crawler_set_can_send_yet (priv->crawler, TRUE);
 	}
 }