tracker r1912 - in branches/indexer-split: . src/trackerd

[mr svn gnome org]

Author: mr
Date: Wed Jul 23 10:50:58 2008
New Revision: 1912
URL: http://svn.gnome.org/viewvc/tracker?rev=1912&view=rev

Log:
	* src/trackerd/tracker-crawler.c: 
	* src/trackerd/tracker-marshal.list:
	* src/trackerd/tracker-monitor.c: 
	* src/trackerd/tracker-processor.c: Don't create a gchar* for
	EVERY file we crawl just to move it from one queue to another,
	instead, just make the queue use GFile pointers and ref the one we
	are passed. This has halved the memory consumption of the daemon
	and speed things up. It also shouldn't hang while crawling now.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/trackerd/tracker-crawler.c
   branches/indexer-split/src/trackerd/tracker-marshal.list
   branches/indexer-split/src/trackerd/tracker-monitor.c
   branches/indexer-split/src/trackerd/tracker-processor.c

Modified: branches/indexer-split/src/trackerd/tracker-crawler.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.c	Wed Jul 23 10:50:58 2008
@@ -86,6 +86,7 @@
 
 enum {
 	PROCESSING_DIRECTORY,
+	PROCESSING_FILE,
 	FINISHED,
 	LAST_SIGNAL
 };
@@ -125,17 +126,27 @@
 			      2,
 			      G_TYPE_STRING,
 			      G_TYPE_OBJECT);
+	signals[PROCESSING_FILE] =
+		g_signal_new ("processing-file",
+			      G_TYPE_FROM_CLASS (klass),
+			      G_SIGNAL_RUN_LAST,
+			      0,
+			      NULL, NULL,
+			      tracker_marshal_VOID__STRING_OBJECT,
+			      G_TYPE_NONE,
+			      2,
+			      G_TYPE_STRING,
+			      G_TYPE_OBJECT);
 	signals[FINISHED] =
 		g_signal_new ("finished",
 			      G_TYPE_FROM_CLASS (klass),
 			      G_SIGNAL_RUN_LAST,
 			      0,
 			      NULL, NULL,
-			      tracker_marshal_VOID__STRING_POINTER_UINT_UINT_UINT_UINT,
+			      tracker_marshal_VOID__STRING_UINT_UINT_UINT_UINT,
 			      G_TYPE_NONE,
-			      6,
+			      5,
 			      G_TYPE_STRING,
-			      G_TYPE_POINTER,
 			      G_TYPE_UINT,
 			      G_TYPE_UINT,
 			      G_TYPE_UINT,
@@ -479,6 +490,14 @@
 }
 
 static void
+process_file (TrackerCrawler *crawler,
+	      const gchar    *module_name,
+	      GFile          *file)
+{
+	g_signal_emit (crawler, signals[PROCESSING_FILE], 0, module_name, file);
+}
+
+static void
 process_directory (TrackerCrawler *crawler,
 		   const gchar    *module_name,
 		   GFile          *file)
@@ -498,7 +517,17 @@
 	crawler = TRACKER_CRAWLER (data);
 	priv = crawler->private;
 
-	/* Crawler directory contents */
+	/* Crawler files */
+	file = g_queue_pop_head (priv->files);
+
+	if (file) {
+		process_file (crawler, priv->module_name, file);
+		g_object_unref (file);
+
+		return TRUE;
+	}
+
+	/* Crawler directories */
 	file = g_queue_pop_head (priv->directories);
 
 	if (file) {
@@ -856,7 +885,6 @@
 
 	g_signal_emit (crawler, signals[FINISHED], 0,
 		       priv->module_name,
-		       priv->files,
 		       priv->directories_found,
 		       priv->directories_ignored,
 		       priv->files_found,

Modified: branches/indexer-split/src/trackerd/tracker-marshal.list
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-marshal.list	(original)
+++ branches/indexer-split/src/trackerd/tracker-marshal.list	Wed Jul 23 10:50:58 2008
@@ -1,4 +1,4 @@
-VOID:STRING,POINTER,UINT,UINT,UINT,UINT
+VOID:STRING,UINT,UINT,UINT,UINT
 VOID:STRING,STRING,INT,INT,INT
 VOID:STRING,STRING,STRING
 VOID:STRING,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN

Modified: branches/indexer-split/src/trackerd/tracker-monitor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-monitor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-monitor.c	Wed Jul 23 10:50:58 2008
@@ -369,14 +369,14 @@
 	if (other_file) {
 		str2 = g_file_get_path (other_file);
 	} else {
-		str2 = g_strdup ("");
+		str2 = NULL;
 	}
 
 	g_message ("Received monitor event:%d->'%s' for file:'%s' and other file:'%s'",
 		   event_type,
 		   monitor_event_to_string (event_type),
 		   str1,
-		   str2);
+		   str2 ? str2 : "");
 		   
 	switch (event_type) {
 	case G_FILE_MONITOR_EVENT_CHANGED:

Modified: branches/indexer-split/src/trackerd/tracker-processor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-processor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-processor.c	Wed Jul 23 10:50:58 2008
@@ -116,13 +116,16 @@
 					     const gchar      *module_name,
 					     GFile            *file,
 					     gpointer          user_data);
+static void crawler_processing_file_cb      (TrackerCrawler   *crawler,
+					     const gchar      *module_name,
+					     GFile            *file,
+					     gpointer          user_data);
 static void crawler_processing_directory_cb (TrackerCrawler   *crawler,
 					     const gchar      *module_name,
 					     GFile            *file,
 					     gpointer          user_data);
 static void crawler_finished_cb             (TrackerCrawler   *crawler,
 					     const gchar      *module_name,
-					     GQueue           *files,
 					     guint             directories_found,
 					     guint             directories_ignored,
 					     guint             files_found,
@@ -247,6 +250,9 @@
 
 	if (priv->crawler) {
 		g_signal_handlers_disconnect_by_func (priv->crawler,
+						      G_CALLBACK (crawler_processing_file_cb),
+						      object);
+		g_signal_handlers_disconnect_by_func (priv->crawler,
 						      G_CALLBACK (crawler_processing_directory_cb),
 						      object);
 		g_signal_handlers_disconnect_by_func (priv->crawler,
@@ -325,7 +331,7 @@
 
 	queue = (GQueue *) data;
 
-	g_queue_foreach (queue, (GFunc) g_free, NULL);
+	g_queue_foreach (queue, (GFunc) g_object_unref, NULL);
 	g_queue_free (queue);
 }
 
@@ -338,7 +344,7 @@
 		gint  i;
 		
 		for (p = strv, i = 0; *p; p++, i++) {
-			g_queue_push_nth (queue, g_strdup (*p), i);
+			g_queue_push_nth (queue, g_file_new_for_path (*p), i);
 		}
 	}
 }
@@ -416,7 +422,7 @@
 	queue = get_next_queue_with_data (priv->items_deleted_queues, &module_name);
 
 	if (queue && g_queue_get_length (queue) > 0) {
-		files = tracker_dbus_queue_str_to_strv (queue, ITEMS_QUEUE_PROCESS_MAX);
+		files = tracker_dbus_queue_gfile_to_strv (queue, ITEMS_QUEUE_PROCESS_MAX);
 		
 		g_message ("Queue for module:'%s' deleted items processed, sending first %d to the indexer", 
 			   module_name,
@@ -439,7 +445,7 @@
 	queue = get_next_queue_with_data (priv->items_created_queues, &module_name);
 
 	if (queue && g_queue_get_length (queue) > 0) {
-		files = tracker_dbus_queue_str_to_strv (queue, ITEMS_QUEUE_PROCESS_MAX);
+		files = tracker_dbus_queue_gfile_to_strv (queue, ITEMS_QUEUE_PROCESS_MAX);
 		
 		g_message ("Queue for module:'%s' created items processed, sending first %d to the indexer", 
 			   module_name,
@@ -462,7 +468,7 @@
 	queue = get_next_queue_with_data (priv->items_updated_queues, &module_name);
 
 	if (queue && g_queue_get_length (queue) > 0) {
-		files = tracker_dbus_queue_str_to_strv (queue, ITEMS_QUEUE_PROCESS_MAX);
+		files = tracker_dbus_queue_gfile_to_strv (queue, ITEMS_QUEUE_PROCESS_MAX);
 		
 		g_message ("Queue for module:'%s' updated items processed, sending first %d to the indexer", 
 			   module_name,
@@ -556,6 +562,9 @@
 	/* Clean up last module's work */
 	if (priv->crawler) {
 		g_signal_handlers_disconnect_by_func (priv->crawler,
+						      G_CALLBACK (crawler_processing_file_cb),
+						      processor);
+		g_signal_handlers_disconnect_by_func (priv->crawler,
 						      G_CALLBACK (crawler_processing_directory_cb),
 						      processor);
 		g_signal_handlers_disconnect_by_func (priv->crawler,
@@ -584,6 +593,9 @@
 					     priv->hal,
 					     priv->current_module->data);
 
+	g_signal_connect (priv->crawler, "processing-file",
+			  G_CALLBACK (crawler_processing_file_cb),
+			  processor);
 	g_signal_connect (priv->crawler, "processing-directory",
 			  G_CALLBACK (crawler_processing_directory_cb),
 			  processor);
@@ -661,13 +673,11 @@
 {
 	TrackerProcessorPrivate *priv;
 	GQueue                  *queue;
-	gchar                   *path;
 
 	priv = TRACKER_PROCESSOR_GET_PRIVATE (user_data);
 
 	queue = g_hash_table_lookup (priv->items_created_queues, module_name);
-	path = g_file_get_path (file);
-	g_queue_push_tail (queue, path);
+	g_queue_push_tail (queue, g_object_ref (file));
 
 	item_queue_handlers_set_up (user_data);
 }
@@ -680,13 +690,11 @@
 {
 	TrackerProcessorPrivate *priv;
 	GQueue                  *queue;
-	gchar                   *path;
 
 	priv = TRACKER_PROCESSOR_GET_PRIVATE (user_data);
 
 	queue = g_hash_table_lookup (priv->items_updated_queues, module_name);
-	path = g_file_get_path (file);
-	g_queue_push_tail (queue, path);
+	g_queue_push_tail (queue, g_object_ref (file));
 
 	item_queue_handlers_set_up (user_data);
 }
@@ -699,18 +707,33 @@
 {
 	TrackerProcessorPrivate *priv;
 	GQueue                  *queue;
-	gchar                   *path;
 
 	priv = TRACKER_PROCESSOR_GET_PRIVATE (user_data);
 
 	queue = g_hash_table_lookup (priv->items_deleted_queues, module_name);
-	path = g_file_get_path (file);
-	g_queue_push_tail (queue, path);
+	g_queue_push_tail (queue, g_object_ref (file));
 
 	item_queue_handlers_set_up (user_data);
 }
 
 static void
+crawler_processing_file_cb (TrackerCrawler *crawler,
+			    const gchar    *module_name,
+			    GFile          *file,
+			    gpointer        user_data)
+{
+	
+	TrackerProcessorPrivate *priv;
+	GQueue                  *queue;
+
+	priv = TRACKER_PROCESSOR_GET_PRIVATE (user_data);
+
+	/* Add files in queue to our queues to send to the indexer */
+	queue = g_hash_table_lookup (priv->items_created_queues, module_name);
+	g_queue_push_tail (queue, g_object_ref (file));
+}
+
+static void
 crawler_processing_directory_cb (TrackerCrawler *crawler,
 				 const gchar    *module_name,
 				 GFile          *file,
@@ -718,17 +741,10 @@
 {
 	
 	TrackerProcessorPrivate *priv;
-	gchar                   *path;
 	gboolean                 add_monitor;
 
 	priv = TRACKER_PROCESSOR_GET_PRIVATE (user_data);
 
-	path = g_file_get_path (file);
-
-	g_debug ("Processing module:'%s' with for path:'%s'",
-		 module_name,
-		 path);
-
 	/* FIXME: Get ignored directories from .cfg? We know that
 	 * normally these would have monitors because these
 	 * directories are those crawled based on the module config.
@@ -739,14 +755,11 @@
 	if (add_monitor) {
 		tracker_monitor_add (priv->monitor, file, module_name);
 	}
-
-	g_free (path);
 }
 
 static void
 crawler_finished_cb (TrackerCrawler *crawler,
 		     const gchar    *module_name,
-		     GQueue         *files,
 		     guint           directories_found,
 		     guint           directories_ignored,
 		     guint           files_found,
@@ -755,9 +768,6 @@
 {
 	TrackerProcessor        *processor;
 	TrackerProcessorPrivate *priv;
-	GQueue                  *queue;
-	GFile                   *file;
-	gchar                   *path;
 
 	processor = TRACKER_PROCESSOR (user_data);
 	priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
@@ -767,27 +777,6 @@
 	priv->files_found += files_found;
 	priv->files_ignored += files_ignored;
 
-	/* Add files in queue to our queues to send to the indexer */
-	queue = g_hash_table_lookup (priv->items_created_queues, module_name);
-
-	/* Not sure if this is the best way to do this, we are
-	 * effectively editing the queue in the signal handler, this
-	 * isn't recommended code practise, maybe we should be
-	 * g_queue_peek_nth() but this is much faster because when we
-	 * process the next module, we will only pop head until and
-	 * unref all items anyway.
-	 */
-	while ((file = g_queue_pop_head (files)) != NULL) {
-		path = g_file_get_path (file);
-		g_queue_push_tail (queue, path);
-		g_object_unref (file);
-
-		/* This is a tight loop, we should eleviate this */
-		while (g_main_context_pending (NULL)) {
-			g_main_context_iteration (NULL, FALSE);
-		}
-	}
-	
 	/* Proceed to next module */
 	process_next_module (processor);
 }