tracker r1697 - in branches/indexer-split: . data/dbus src/libtracker-common src/tracker-indexer src/trackerd

[mr svn gnome org]

Author: mr
Date: Tue Jun 17 13:36:44 2008
New Revision: 1697
URL: http://svn.gnome.org/viewvc/tracker?rev=1697&view=rev

Log:
	* data/dbus/tracker-indexer.xml:
	* src/libtracker-common/tracker-dbus.c:
	(tracker_dbus_async_queue_to_strv): Don't wait if there is no data
	in the async queue with g_async_queue_timed_pop().

	* src/trackerd/tracker-crawler.c: Don't block when shutting down
	by calling g_async_queue_pop() on an empty queue.

	* src/trackerd/tracker-monitor.c: Push all events to the indexer
	after queueing them first.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/data/dbus/tracker-indexer.xml
   branches/indexer-split/src/libtracker-common/tracker-dbus.c
   branches/indexer-split/src/tracker-indexer/tracker-indexer.c
   branches/indexer-split/src/tracker-indexer/tracker-indexer.h
   branches/indexer-split/src/trackerd/tracker-crawler.c
   branches/indexer-split/src/trackerd/tracker-monitor.c
   branches/indexer-split/src/trackerd/tracker-monitor.h

Modified: branches/indexer-split/data/dbus/tracker-indexer.xml
==============================================================================
--- branches/indexer-split/data/dbus/tracker-indexer.xml	(original)
+++ branches/indexer-split/data/dbus/tracker-indexer.xml	Tue Jun 17 13:36:44 2008
@@ -16,16 +16,21 @@
     <method name="SetRunning">
       <arg type="b" name="should_be_running" direction="in" />
     </method>
-
-    <method name="ProcessFiles">
+    <method name="CheckFiles">
+      <arg type="as" name="files" direction="in" />
+    </method>
+    <method name="UpdateFiles">
+      <arg type="as" name="files" direction="in" />
+    </method>
+    <method name="DeleteFiles">
       <arg type="as" name="files" direction="in" />
     </method>
     
     <!-- The old signal in the daemon used to send the time taken, we
 	 can add this later
       -->
-    <signal name="Finished"/>
     <signal name="IndexUpdated"/>
+    <signal name="Finished"/>
 
   </interface>
 </node>

Modified: branches/indexer-split/src/libtracker-common/tracker-dbus.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-dbus.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-dbus.c	Tue Jun 17 13:36:44 2008
@@ -102,14 +102,9 @@
 	strv = g_new0 (gchar*, length + 1);
 	
 	while (i <= length) {
-		GTimeVal  t;
-		gchar    *str;
+		gchar *str;
 		
-		g_get_current_time (&t);
-		g_time_val_add (&t, 100000);
-
-		/* Get next item and wait 0.1 seconds max per try */
-		str = g_async_queue_timed_pop (queue, &t);
+		str = g_async_queue_try_pop (queue);
 
 		if (str) {
 			if (!g_utf8_validate (str, -1, NULL)) {
@@ -120,7 +115,7 @@
 
 			strv[i++] = str;
 		} else {
-			/* We don't expect this */
+			/* Queue is empty and we don't expect this */
 			break;
 		}
 	}

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.c	Tue Jun 17 13:36:44 2008
@@ -264,7 +264,6 @@
 {
 	TrackerIndexerPrivate *priv;
 	gchar *index_file;
-	gint initial_sleep;
 	GSList *m;
 
 	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
@@ -315,8 +314,6 @@
 	tracker_indexer_set_running (indexer, TRUE, NULL);
 
 	g_free (index_file);
-
-	return FALSE;
 }
 
 static void
@@ -657,9 +654,95 @@
 }
 
 gboolean
-tracker_indexer_process_files (TrackerIndexer  *indexer,
-			       GStrv            files,
-			       GError         **error)
+tracker_indexer_check_files (TrackerIndexer  *indexer,
+			     GStrv            files,
+			     GError         **error)
+{
+	TrackerIndexerPrivate *priv;
+	GModule               *module;
+	guint                  request_id;
+	gint                   i;
+
+	tracker_dbus_return_val_if_fail (TRACKER_IS_INDEXER (indexer), FALSE, error);
+	tracker_dbus_return_val_if_fail (files != NULL, FALSE, error);
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+	request_id = tracker_dbus_get_next_request_id ();
+
+	tracker_dbus_request_new (request_id,
+                                  "DBus request to check %d files",
+				  g_strv_length (files));
+
+	/* Assume we're using always the files module, bail out if it's not available */
+	module = g_hash_table_lookup (priv->indexer_modules, "files");
+
+	if (!module) {
+		tracker_dbus_request_failed (request_id,
+					     error,
+					     "The files module is not loaded");
+		return FALSE;
+	}
+
+	/* Add files to the queue */
+	for (i = 0; files[i]; i++) {
+		PathInfo *info;
+
+		info = path_info_new (module, files[i]);
+		tracker_indexer_add_file (indexer, info);
+	}
+
+	tracker_dbus_request_success (request_id);
+
+	return TRUE;
+}
+
+gboolean
+tracker_indexer_update_files (TrackerIndexer  *indexer,
+			      GStrv            files,
+			      GError         **error)
+{
+	TrackerIndexerPrivate *priv;
+	GModule               *module;
+	guint                  request_id;
+	gint                   i;
+
+	tracker_dbus_return_val_if_fail (TRACKER_IS_INDEXER (indexer), FALSE, error);
+	tracker_dbus_return_val_if_fail (files != NULL, FALSE, error);
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+	request_id = tracker_dbus_get_next_request_id ();
+
+	tracker_dbus_request_new (request_id,
+                                  "DBus request to update %d files",
+				  g_strv_length (files));
+
+	/* Assume we're using always the files module, bail out if it's not available */
+	module = g_hash_table_lookup (priv->indexer_modules, "files");
+
+	if (!module) {
+		tracker_dbus_request_failed (request_id,
+					     error,
+					     "The files module is not loaded");
+		return FALSE;
+	}
+
+	/* Add files to the queue */
+	for (i = 0; files[i]; i++) {
+		PathInfo *info;
+
+		info = path_info_new (module, files[i]);
+		tracker_indexer_add_file (indexer, info);
+	}
+
+	tracker_dbus_request_success (request_id);
+
+	return TRUE;
+}
+
+gboolean
+tracker_indexer_delete_files (TrackerIndexer  *indexer,
+			      GStrv            files,
+			      GError         **error)
 {
 	TrackerIndexerPrivate *priv;
 	GModule               *module;
@@ -673,7 +756,7 @@
 	request_id = tracker_dbus_get_next_request_id ();
 
 	tracker_dbus_request_new (request_id,
-                                  "DBus request to process %d files",
+                                  "DBus request to delete %d files",
 				  g_strv_length (files));
 
 	/* Assume we're using always the files module, bail out if it's not available */

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer.h
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer.h	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.h	Tue Jun 17 13:36:44 2008
@@ -52,16 +52,22 @@
 };
 
 GType           tracker_indexer_get_type      (void) G_GNUC_CONST;
-TrackerIndexer *tracker_indexer_new           (gboolean         reindex);
-gboolean        tracker_indexer_set_running   (TrackerIndexer  *indexer,
-					       gboolean         should_be_running,
-					       GError         **error);
-gboolean        tracker_indexer_get_running   (TrackerIndexer  *indexer,
-					       gboolean        *is_running,
-					       GError         **error);
-gboolean        tracker_indexer_process_files (TrackerIndexer  *indexer,
-					       GStrv            files,
-					       GError         **error);
+TrackerIndexer *tracker_indexer_new          (gboolean         reindex);
+gboolean        tracker_indexer_set_running  (TrackerIndexer  *indexer,
+					      gboolean         should_be_running,
+					      GError         **error);
+gboolean        tracker_indexer_get_running  (TrackerIndexer  *indexer,
+					      gboolean        *is_running,
+					      GError         **error);
+gboolean        tracker_indexer_check_files  (TrackerIndexer  *indexer,
+					      GStrv            files,
+					      GError         **error);
+gboolean        tracker_indexer_update_files (TrackerIndexer  *indexer,
+					      GStrv            files,
+					      GError         **error);
+gboolean        tracker_indexer_delete_files (TrackerIndexer  *indexer,
+					      GStrv            files,
+					      GError         **error);
 
 G_END_DECLS
 

Modified: branches/indexer-split/src/trackerd/tracker-crawler.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.c	Tue Jun 17 13:36:44 2008
@@ -83,27 +83,27 @@
 	GFile          *parent;
 } EnumeratorData;
 
-static void crawler_finalize        (GObject         *object);
-static void crawler_set_property    (GObject         *object,
-				     guint            param_id,
-				     const GValue    *value,
-				     GParamSpec      *pspec);
-static void set_ignored_file_types  (TrackerCrawler  *crawler);
-
+static void crawler_finalize          (GObject         *object);
+static void crawler_set_property      (GObject         *object,
+				       guint            param_id,
+				       const GValue    *value,
+				       GParamSpec      *pspec);
+static void set_ignored_file_types    (TrackerCrawler  *crawler);
 
 #ifdef HAVE_HAL
-static void mount_point_added_cb    (TrackerHal      *hal,
-				     const gchar     *mount_point,
-				     gpointer         user_data);
-static void mount_point_removed_cb  (TrackerHal      *hal,
-				     const gchar     *mount_point,
-				     gpointer         user_data);
-
+static void mount_point_added_cb      (TrackerHal      *hal,
+				       const gchar     *mount_point,
+				       gpointer         user_data);
+static void mount_point_removed_cb    (TrackerHal      *hal,
+				       const gchar     *mount_point,
+				       gpointer         user_data);
 #endif /* HAVE_HAL */
-static void file_enumerate_next     (GFileEnumerator *enumerator,
-				     EnumeratorData  *ed);
-static void file_enumerate_children (TrackerCrawler  *crawler,
-				     GFile           *file);
+
+static void file_enumerate_next       (GFileEnumerator *enumerator,
+				       EnumeratorData  *ed);
+static void file_enumerate_children   (TrackerCrawler  *crawler,
+				       GFile           *file);
+static void file_queue_handler_set_up (TrackerCrawler  *crawler);
 
 G_DEFINE_TYPE(TrackerCrawler, tracker_crawler, G_TYPE_OBJECT)
 
@@ -240,9 +240,9 @@
 		priv->files_queue_handle_id = 0;
 	}
 
-        for (str = g_async_queue_pop (priv->files);
+	for (str = g_async_queue_try_pop (priv->files);
 	     str;
-	     str = g_async_queue_pop (priv->files)) {
+	     str = g_async_queue_try_pop (priv->files)) {
 		g_free (str);
 	}
 
@@ -771,6 +771,7 @@
 			g_free (path);
 		} else {
 			g_async_queue_push (crawler->priv->files, path);
+			file_queue_handler_set_up (crawler);
 		}
 	}	
 
@@ -837,16 +838,16 @@
 }
 
 static void
-indexer_process_files_cb (DBusGProxy *proxy, 
-			  GError     *error, 
-			  gpointer    user_data)
+indexer_check_files_cb (DBusGProxy *proxy, 
+			GError     *error, 
+			gpointer    user_data)
 {
 	GStrv files;
 	
 	files = (GStrv) user_data;
 
 	if (error) {
-		g_critical ("Could not send files to indexer to process, %s", 
+		g_critical ("Could not send files to indexer to check, %s", 
 			    error->message);
 		g_error_free (error);
 	} else {
@@ -875,23 +876,23 @@
 		return;
 	}
 
-	g_debug ("Processing file queue...");
+	g_debug ("File check queue being processed...");
 	files = tracker_dbus_async_queue_to_strv (crawler->priv->files,
 						  FILES_QUEUE_PROCESS_MAX);
 	
-	g_debug ("Sending %d files to indexer to process", 
+	g_debug ("File check queue processed, sending first %d to the indexer", 
 		 g_strv_length (files));
 	
-	org_freedesktop_Tracker_Indexer_process_files_async (proxy, 
-							     (const gchar **) files,
-							     indexer_process_files_cb,
-							     files);
+	org_freedesktop_Tracker_Indexer_check_files_async (proxy, 
+							   (const gchar **) files,
+							   indexer_check_files_cb,
+							   files);
 
 	g_object_unref (crawler);
 }
 
 static gboolean
-file_queue_handle_cb (gpointer user_data)
+file_queue_handler_cb (gpointer user_data)
 {
 	TrackerCrawler *crawler;
 	DBusGProxy     *proxy;
@@ -901,8 +902,9 @@
 
 	length = g_async_queue_length (crawler->priv->files);
 	if (length < 1) {
-		g_debug ("Processing file queue... nothing to do, queue empty");
-		return TRUE;
+		g_debug ("File check queue is empty... nothing to do");
+		crawler->priv->files_queue_handle_id = 0;
+		return FALSE;
 	}
 
 	/* Check we can actually talk to the indexer */
@@ -915,6 +917,19 @@
 	return TRUE;
 }
 
+static void
+file_queue_handler_set_up (TrackerCrawler *crawler)
+{
+	if (crawler->priv->files_queue_handle_id != 0) {
+		return;
+	}
+
+	crawler->priv->files_queue_handle_id = 
+		g_timeout_add (FILES_QUEUE_PROCESS_INTERVAL, 
+			       file_queue_handler_cb,
+			       crawler);
+}
+
 void
 tracker_crawler_start (TrackerCrawler *crawler)
 {
@@ -929,15 +944,6 @@
 
 	priv = crawler->priv;
 
-	/* Set up queue handler */
-	if (priv->files_queue_handle_id) {
-		g_source_remove (priv->files_queue_handle_id);	
-	}
-
-	priv->files_queue_handle_id = g_timeout_add (FILES_QUEUE_PROCESS_INTERVAL, 
-						     file_queue_handle_cb,
-						     crawler);
-
 	/* Get locations to index from config, if none are set, we use
 	 * $HOME as the default.
 	 */

Modified: branches/indexer-split/src/trackerd/tracker-monitor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-monitor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-monitor.c	Tue Jun 17 13:36:44 2008
@@ -20,11 +20,14 @@
 
 #include <string.h>
 
+#include <libtracker-common/tracker-dbus.h>
 #include <libtracker-common/tracker-file-utils.h>
 
 #include "tracker-monitor.h"
+#include "tracker-dbus.h"
+#include "tracker-indexer-client.h"
 
-/* #define TESTING */
+#define TESTING 
 
 /* This is the default inotify limit - 500 to allow some monitors for
  * other applications. 
@@ -33,10 +36,17 @@
  * /proc/sys/fs/inotify/max_user_watches when there is a possiblity
  * that we don't even use inotify?
  */
-#define MAX_MONITORS (guint) ((2 ^ 13) - 500)   
+#define MAX_MONITORS                 (guint) ((2 ^ 13) - 500)   
+
+#define FILES_QUEUE_PROCESS_INTERVAL 2000
+#define FILES_QUEUE_PROCESS_MAX      5000
 
-static GHashTable    *monitors;
 static TrackerConfig *config;
+static GHashTable    *monitors;
+static GAsyncQueue   *files_created;
+static GAsyncQueue   *files_updated;
+static GAsyncQueue   *files_deleted;
+static guint          files_queue_handlers_id;
 
 gboolean 
 tracker_monitor_init (TrackerConfig *_config) 
@@ -54,12 +64,55 @@
 						  (GDestroyNotify) g_file_monitor_cancel);
 	}
 
+	if (!files_created) {
+		files_created = g_async_queue_new ();
+	}
+
+	if (!files_updated) {
+		files_updated = g_async_queue_new ();
+	}
+
+	if (!files_deleted) {
+		files_deleted = g_async_queue_new ();
+	}
+
 	return TRUE;
 }
 
 void
 tracker_monitor_shutdown (void)
 {
+	gchar *str;
+
+	if (files_queue_handlers_id) {
+		g_source_remove (files_queue_handlers_id);
+		files_queue_handlers_id = 0;
+	}
+
+        for (str = g_async_queue_try_pop (files_deleted);
+	     str;
+	     str = g_async_queue_try_pop (files_deleted)) {
+		g_free (str);
+	}
+
+	g_async_queue_unref (files_deleted);
+
+        for (str = g_async_queue_try_pop (files_updated);
+	     str;
+	     str = g_async_queue_try_pop (files_updated)) {
+		g_free (str);
+	}
+
+	g_async_queue_unref (files_updated);
+
+        for (str = g_async_queue_try_pop (files_created);
+	     str;
+	     str = g_async_queue_try_pop (files_created)) {
+		g_free (str);
+	}
+
+	g_async_queue_unref (files_created);
+
 	if (monitors) {
 		g_hash_table_unref (monitors);
 		monitors = NULL;
@@ -71,6 +124,122 @@
 	}
 }
 
+static void
+indexer_files_processed_cb (DBusGProxy *proxy, 
+			    GError     *error, 
+			    gpointer    user_data)
+{
+	GStrv files;
+	
+	files = (GStrv) user_data;
+
+	if (error) {
+		g_critical ("Could not send %d files to indexer, %s", 
+			    g_strv_length (files),
+			    error->message);
+		g_error_free (error);
+	} else {
+		g_debug ("Sent!");
+	}
+}
+
+static void
+indexer_get_running_cb (DBusGProxy *proxy, 
+			gboolean    running, 
+			GError     *error, 
+			gpointer    user_data)
+{
+	GStrv files;
+
+	if (error || !running) {
+		g_message ("%s", 
+			   error ? error->message : "Indexer exists but is not available yet, waiting...");
+
+		g_clear_error (&error);
+
+		return;
+	}
+
+	/* First do the deleted queue */
+	g_debug ("Files deleted queue being processed...");
+	files = tracker_dbus_async_queue_to_strv (files_deleted,
+						  FILES_QUEUE_PROCESS_MAX);
+	
+	if (g_strv_length (files) > 0) {
+		g_debug ("Files deleted queue processed, sending first %d to the indexer", 
+			 g_strv_length (files));
+		org_freedesktop_Tracker_Indexer_delete_files_async (proxy, 
+								    (const gchar **) files,
+								    indexer_files_processed_cb,
+								    files);
+	}
+
+	/* Second do the created queue */
+	g_debug ("Files created queue being processed...");
+	files = tracker_dbus_async_queue_to_strv (files_created,
+						  FILES_QUEUE_PROCESS_MAX);
+	if (g_strv_length (files) > 0) {
+		g_debug ("Files created queue processed, sending first %d to the indexer", 
+			 g_strv_length (files));
+		org_freedesktop_Tracker_Indexer_check_files_async (proxy, 
+								   (const gchar **) files,
+								   indexer_files_processed_cb,
+								   files);
+	}
+
+	/* Second do the created queue */
+	g_debug ("Files updated queue being processed...");
+	files = tracker_dbus_async_queue_to_strv (files_updated,
+						  FILES_QUEUE_PROCESS_MAX);
+	
+	if (g_strv_length (files) > 0) {
+		g_debug ("Files updated queue processed, sending first %d to the indexer", 
+			 g_strv_length (files));
+		org_freedesktop_Tracker_Indexer_update_files_async (proxy, 
+								    (const gchar **) files,
+								    indexer_files_processed_cb,
+								    files);
+	}
+}
+
+static gboolean
+file_queue_handlers_cb (gpointer user_data)
+{
+	DBusGProxy *proxy;
+	gint        items_to_process = 0;
+
+	items_to_process += g_async_queue_length (files_created);
+	items_to_process += g_async_queue_length (files_updated);
+	items_to_process += g_async_queue_length (files_deleted);
+
+	if (items_to_process < 1) {
+		g_debug ("All queues are empty... nothing to do");
+		files_queue_handlers_id = 0;
+		return FALSE;
+	}
+
+	/* Check we can actually talk to the indexer */
+	proxy = tracker_dbus_indexer_get_proxy ();
+	
+	org_freedesktop_Tracker_Indexer_get_running_async (proxy, 
+							   indexer_get_running_cb,
+							   NULL);
+
+	return TRUE;
+}
+
+static void
+file_queue_handlers_set_up (void)
+{
+	if (files_queue_handlers_id) {
+		return;
+	}
+
+	files_queue_handlers_id = g_timeout_add (FILES_QUEUE_PROCESS_INTERVAL, 
+						 file_queue_handlers_cb,
+						 NULL);
+}
+
 static const gchar *
 monitor_event_to_string (GFileMonitorEvent event_type)
 {
@@ -118,7 +287,31 @@
 		   str1,
 		   str2);
 		   
-	g_free (str1);
+	switch (event_type) {
+	case G_FILE_MONITOR_EVENT_CHANGED:
+	case G_FILE_MONITOR_EVENT_CHANGES_DONE_HINT:
+	case G_FILE_MONITOR_EVENT_ATTRIBUTE_CHANGED:
+		g_async_queue_push (files_updated, str1);
+		file_queue_handlers_set_up ();
+		break;
+
+	case G_FILE_MONITOR_EVENT_DELETED:
+		g_async_queue_push (files_deleted, str1);
+		file_queue_handlers_set_up ();
+		break;
+
+	case G_FILE_MONITOR_EVENT_CREATED:
+		g_async_queue_push (files_created, str1);
+		file_queue_handlers_set_up ();
+		break;
+
+	case G_FILE_MONITOR_EVENT_PRE_UNMOUNT:
+	case G_FILE_MONITOR_EVENT_UNMOUNTED:
+		/* Do nothing */
+		g_free (str1);
+		break;
+	}
+
 	g_free (str2);
 }
 

Modified: branches/indexer-split/src/trackerd/tracker-monitor.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-monitor.h	(original)
+++ branches/indexer-split/src/trackerd/tracker-monitor.h	Tue Jun 17 13:36:44 2008
@@ -37,7 +37,6 @@
 gboolean tracker_monitor_is_watched_by_string (const gchar   *path);
 gint     tracker_monitor_get_count            (void);
 
-
 G_END_DECLS
 
 #endif /* __TRACKERD_MONITOR_H__ */