tracker r1814 - in branches/indexer-split: . src/libtracker-common src/tracker-indexer src/trackerd

[mr svn gnome org]

Author: mr
Date: Tue Jul  1 16:20:47 2008
New Revision: 1814
URL: http://svn.gnome.org/viewvc/tracker?rev=1814&view=rev

Log:
	* src/tracker-indexer/tracker-main.c: Don't quit immediately, wait
	10 seconds or so, otherwise if the daemon calls "are we running"
	it replies and quits before the daemon can send the next message.

	* src/trackerd/tracker-crawler.c: Added back the code to handle
	sending files to the indexer.

	* src/trackerd/tracker-processor.c: Removed commented out code.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/libtracker-common/tracker-dbus.c
   branches/indexer-split/src/libtracker-common/tracker-dbus.h
   branches/indexer-split/src/tracker-indexer/tracker-indexer.c
   branches/indexer-split/src/tracker-indexer/tracker-indexer.h
   branches/indexer-split/src/tracker-indexer/tracker-main.c
   branches/indexer-split/src/trackerd/tracker-crawler.c
   branches/indexer-split/src/trackerd/tracker-processor.c

Modified: branches/indexer-split/src/libtracker-common/tracker-dbus.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-dbus.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-dbus.c	Tue Jul  1 16:20:47 2008
@@ -21,6 +21,8 @@
 
 #include "tracker-dbus.h"
 
+#include <gio/gio.h>
+
 GValue *
 tracker_dbus_g_value_slice_new (GType type)
 {
@@ -127,6 +129,48 @@
 	return strv;
 }
 
+gchar **
+tracker_dbus_gfile_queue_to_strv (GQueue *queue, 
+				  gint    max)
+{
+	gchar **strv;
+	gchar  *str;
+	GFile  *file;
+	gint    i, j;
+	gint    length;
+
+	length = g_queue_get_length (queue);
+		
+	if (max > 0) {
+		length = MIN (max, length);
+	}
+
+	strv = g_new0 (gchar*, length + 1);
+	
+	for (i = 0, j = 0; i < length; i++) {
+		file = g_queue_pop_head (queue);
+
+		if (!file) {
+			break;
+		}
+
+		str = g_file_get_path (file);
+		g_object_unref (file);
+
+		if (!g_utf8_validate (str, -1, NULL)) {
+			g_message ("Could not add string:'%s' to GStrv, invalid UTF-8", str);
+			g_free (str);
+			continue;
+		}
+
+		strv[j++] = str;
+	}
+
+        strv[j] = NULL;
+
+	return strv;
+}
+
 guint
 tracker_dbus_get_next_request_id (void)
 {

Modified: branches/indexer-split/src/libtracker-common/tracker-dbus.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-dbus.h	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-dbus.h	Tue Jul  1 16:20:47 2008
@@ -85,6 +85,8 @@
 gchar **         tracker_dbus_slist_to_strv       (GSList          *list);
 gchar **         tracker_dbus_async_queue_to_strv (GAsyncQueue     *queue, 
 						   gint             max);
+gchar **         tracker_dbus_gfile_queue_to_strv (GQueue          *queue, 
+						   gint             max);
 
 /* Requests */
 guint            tracker_dbus_get_next_request_id (void);

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.c	Tue Jul  1 16:20:47 2008
@@ -284,8 +284,10 @@
 			      G_SIGNAL_RUN_LAST,
 			      G_STRUCT_OFFSET (TrackerIndexerClass, finished),
 			      NULL, NULL,
-			      g_cclosure_marshal_VOID__VOID,
-			      G_TYPE_NONE, 0);
+			      g_cclosure_marshal_VOID__UINT,
+			      G_TYPE_NONE, 
+			      1,
+			      G_TYPE_UINT);
 	signals [INDEX_UPDATED] = 
 		g_signal_new ("index-updated",
 			      G_OBJECT_CLASS_TYPE (object_class),
@@ -643,7 +645,7 @@
 				   g_timer_elapsed (priv->timer, NULL),
 				   priv->items_indexed);
 
-			g_signal_emit (indexer, signals[FINISHED], 0);
+			g_signal_emit (indexer, signals[FINISHED], 0, priv->items_indexed);
 			return FALSE;
 		}
 

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer.h
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer.h	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.h	Tue Jul  1 16:20:47 2008
@@ -47,30 +47,32 @@
 struct TrackerIndexerClass {
 	GObjectClass parent_class;
 
-	void (*finished)      (TrackerIndexer *indexer);
+	void (*finished)      (TrackerIndexer *indexer,
+			       guint           items_indexed);
 	void (*index_updated) (TrackerIndexer *indexer);
 };
 
-GType           tracker_indexer_get_type       (void) G_GNUC_CONST;
-TrackerIndexer *tracker_indexer_new              (void);
-gboolean        tracker_indexer_set_running      (TrackerIndexer  *indexer,
-						  gboolean         should_be_running,
-						  GError         **error);
-gboolean        tracker_indexer_get_running      (TrackerIndexer  *indexer,
-						  gboolean        *is_running,
-						  GError         **error);
-gboolean        tracker_indexer_files_check      (TrackerIndexer  *indexer,
-						  const gchar     *module,
-						  GStrv            files,
-						  GError         **error);
-gboolean        tracker_indexer_files_update     (TrackerIndexer  *indexer,
-						  const gchar     *module,
-						  GStrv            files,
-						  GError         **error);
-gboolean        tracker_indexer_files_delete     (TrackerIndexer  *indexer,
-						  const gchar     *module,
-						  GStrv            files,
-						  GError         **error);
+GType           tracker_indexer_get_type     (void) G_GNUC_CONST;
+TrackerIndexer *tracker_indexer_new          (void);
+gboolean        tracker_indexer_set_running  (TrackerIndexer  *indexer,
+					      gboolean         should_be_running,
+					      GError         **error);
+gboolean        tracker_indexer_get_running  (TrackerIndexer  *indexer,
+					      gboolean        *is_running,
+					      GError         **error);
+void            tracker_indexer_process_all  (TrackerIndexer  *indexer);
+gboolean        tracker_indexer_files_check  (TrackerIndexer  *indexer,
+					      const gchar     *module,
+					      GStrv            files,
+					      GError         **error);
+gboolean        tracker_indexer_files_update (TrackerIndexer  *indexer,
+					      const gchar     *module,
+					      GStrv            files,
+					      GError         **error);
+gboolean        tracker_indexer_files_delete (TrackerIndexer  *indexer,
+					      const gchar     *module,
+					      GStrv            files,
+					      GError         **error);
 
 G_END_DECLS
 

Modified: branches/indexer-split/src/tracker-indexer/tracker-main.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-main.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-main.c	Tue Jul  1 16:20:47 2008
@@ -54,8 +54,11 @@
         "\n"								\
 	"  http://www.gnu.org/licenses/gpl.txt\n"; 
 
+#define QUIT_TIMEOUT 10 /* Seconds */
+
 static GMainLoop    *main_loop;
- 
+static guint         quit_timeout_id;
+
 static gint          verbosity = -1;
 static gboolean      process_all = FALSE;
 
@@ -129,11 +132,47 @@
 #endif
 }
 
+static gboolean
+quit_timeout_cb (gpointer user_data)
+{
+        TrackerIndexer *indexer;
+        gboolean        running = FALSE;
+
+        indexer = TRACKER_INDEXER (user_data);
+
+        if (!tracker_indexer_get_running (indexer, &running, NULL) || !running) {
+                g_message ("Indexer is still not running after %d seconds, quitting...",
+                           QUIT_TIMEOUT);
+                g_main_loop_quit (main_loop);
+                quit_timeout_id = 0;
+        } else {
+                g_message ("Indexer is now running, staying alive until finished...");
+        }
+
+        g_object_unref (indexer);
+
+        return FALSE;
+}
+
 static void
 indexer_finished_cb (TrackerIndexer *indexer,
+                     guint           items_indexed,
 		     gpointer	     user_data)
 {
-	g_main_loop_quit (main_loop);
+        if (items_indexed > 0) {
+                g_main_loop_quit (main_loop);
+                return;
+        }
+
+        /* If we didn't index anything yet, wait for a minimum of 10
+         * seconds or so before quitting. 
+         */
+        g_message ("Nothing was indexed, waiting %d seconds before quitting...",
+                   QUIT_TIMEOUT);
+
+        quit_timeout_id = g_timeout_add_seconds (QUIT_TIMEOUT,
+                                                 quit_timeout_cb,
+                                                 g_object_ref (indexer));
 }
 
 static void
@@ -256,9 +295,9 @@
 	}
 
         /* Create the indexer and run the main loop */
-
-	g_signal_connect (indexer, "finished",
-			  G_CALLBACK (indexer_finished_cb), NULL);
+        g_signal_connect (indexer, "finished",
+			  G_CALLBACK (indexer_finished_cb), 
+                          NULL);
 
         if (process_all) {
                 /* Tell the indexer to process all configured modules */
@@ -270,6 +309,10 @@
 	main_loop = g_main_loop_new (NULL, FALSE);
 	g_main_loop_run (main_loop);
 
+        if (quit_timeout_id) {
+                g_source_remove (quit_timeout_id);
+        }
+
 	g_object_unref (indexer);
 	g_object_unref (config);
 

Modified: branches/indexer-split/src/trackerd/tracker-crawler.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.c	Tue Jul  1 16:20:47 2008
@@ -609,15 +609,98 @@
 	}
 }
 
+static void
+indexer_check_files_cb (DBusGProxy *proxy,
+			GError     *error,
+			gpointer    user_data)
+{
+	if (error) {
+		g_critical ("Could not send files to indexer to check, %s",
+			    error->message);
+		g_error_free (error);
+	} else {
+		g_debug ("Sent!");
+	}
+}
+
+static void
+indexer_get_running_cb (DBusGProxy *proxy,
+			gboolean    running,
+			GError     *error,
+			gpointer    user_data)
+{
+	TrackerCrawler *crawler;
+	GStrv           files;
+	guint           total;
+
+	crawler = TRACKER_CRAWLER (user_data);
+
+	if (error || !running) {
+		g_message ("%s",
+			   error ? error->message : "Indexer exists but is not available yet, waiting...");
+
+		g_object_unref (crawler);
+		g_clear_error (&error);
+
+		return;
+	}
+
+	total = g_queue_get_length (crawler->private->files);
+	files = tracker_dbus_gfile_queue_to_strv (crawler->private->files,
+						  FILES_QUEUE_PROCESS_MAX);
+	
+	g_debug ("File check queue processed, sending first %d/%d to the indexer",
+		 g_strv_length (files), 
+		 total);
+
+	org_freedesktop_Tracker_Indexer_files_check_async (proxy,
+							   g_strdup (crawler->private->current_module_name),
+							   (const gchar **) files,
+							   indexer_check_files_cb,
+							   NULL);
+
+	g_object_unref (crawler);
+}
+
 static gboolean
+file_queue_handler_cb (gpointer user_data)
+{
+	TrackerCrawler *crawler;
+
+	crawler = TRACKER_CRAWLER (user_data);
+
+	if (g_queue_get_length (crawler->private->files) < 1) {
+		g_debug ("File check queue is empty... nothing to do");
+		crawler->private->files_queue_handle_id = 0;
+		return FALSE;
+	}
+
+	/* Check we can actually talk to the indexer */
+	org_freedesktop_Tracker_Indexer_get_running_async (tracker_dbus_indexer_get_proxy (),
+							   indexer_get_running_cb,
+							   g_object_ref (crawler));
+
+	return TRUE;
+}
+
+static void
+file_queue_handler_set_up (TrackerCrawler *crawler)
+{
+	if (crawler->private->files_queue_handle_id != 0) {
+		return;
+	}
+
+	crawler->private->files_queue_handle_id =
+		g_timeout_add (FILES_QUEUE_PROCESS_INTERVAL,
+			       file_queue_handler_cb,
+			       crawler);
+}
+
+static void
 process_file (TrackerCrawler *crawler,
 	      GFile          *file)
 {
-	/* I guess here we check the queue length and send files to
-	 * the indexer? 
-	 */ 
-	
-	return TRUE;
+	file_queue_handler_set_up (crawler); 
 }
 
 static void
@@ -639,12 +722,12 @@
 	file = g_queue_peek_head (crawler->private->files);
 
 	if (file) {
-		if (process_file (crawler, file)) {
-			file = g_queue_pop_head (crawler->private->files);
-			g_object_unref (file);
-		}
-
-		return TRUE;
+		/* Only return here if we want to throttle the
+		 * directory crawling. I don't think we want to do
+		 * that. 
+		 */
+		process_file (crawler, file);
+		/* return TRUE; */
 	}
 
 	/* Crawler directory contents */

Modified: branches/indexer-split/src/trackerd/tracker-processor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-processor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-processor.c	Tue Jul  1 16:20:47 2008
@@ -241,104 +241,6 @@
 	process_module (processor, priv->current_module->data);
 }
 
-#if 0
-
-static void
-file_queue_handler_set_up (TrackerCrawler *crawler)
-{
-	if (crawler->priv->files_queue_handle_id != 0) {
-		return;
-	}
-
-	crawler->priv->files_queue_handle_id =
-		g_timeout_add (FILES_QUEUE_PROCESS_INTERVAL,
-			       file_queue_handler_cb,
-			       crawler);
-}
-
-static void
-indexer_check_files_cb (DBusGProxy *proxy,
-			GError     *error,
-			gpointer    user_data)
-{
-	GStrv files;
-
-	files = (GStrv) user_data;
-
-	if (error) {
-		g_critical ("Could not send files to indexer to check, %s",
-			    error->message);
-		g_error_free (error);
-	} else {
-		g_debug ("Sent!");
-	}
-}
-
-static void
-indexer_get_running_cb (DBusGProxy *proxy,
-			gboolean    running,
-			GError     *error,
-			gpointer    user_data)
-{
-	TrackerCrawler *crawler;
-	GStrv           files;
-
-	crawler = TRACKER_CRAWLER (user_data);
-
-	if (error || !running) {
-		g_message ("%s",
-			   error ? error->message : "Indexer exists but is not available yet, waiting...");
-
-		g_object_unref (crawler);
-		g_clear_error (&error);
-
-		return;
-	}
-
-	g_debug ("File check queue being processed...");
-	files = tracker_dbus_async_queue_to_strv (crawler->priv->files,
-						  FILES_QUEUE_PROCESS_MAX);
-
-	g_debug ("File check queue processed, sending first %d to the indexer",
-		 g_strv_length (files));
-
-	org_freedesktop_Tracker_Indexer_files_check_async (proxy,
-							   "files",
-							   (const gchar **) files,
-							   indexer_check_files_cb,
-							   files);
-
-	g_object_unref (crawler);
-}
-
-static gboolean
-file_queue_handler_cb (gpointer user_data)
-{
-	TrackerCrawler *crawler;
-	DBusGProxy     *proxy;
-	gint            length;
-
-	crawler = user_data;
-
-	length = g_async_queue_length (crawler->priv->files);
-	if (length < 1) {
-		g_debug ("File check queue is empty... nothing to do");
-		crawler->priv->files_queue_handle_id = 0;
-		return FALSE;
-	}
-
-	/* Check we can actually talk to the indexer */
-	proxy = tracker_dbus_indexer_get_proxy ();
-
-	org_freedesktop_Tracker_Indexer_get_running_async (proxy,
-							   indexer_get_running_cb,
-							   g_object_ref (crawler));
-
-	return TRUE;
-}
-
-#endif
-
 static void
 crawler_finished_cb (TrackerCrawler *crawler,
 		     guint           directories_found,