[tracker] Monitor cache time now 1s & processor queue time now 0s for < 50 items

[Jürg Billeter <juergbi src gnome org>]

commit 7edc8b0f1cdda11bea9d3f3110ddfa1a326ea11b
Author: Martyn Russell <martyn imendio com>
Date:   Thu May 28 13:22:28 2009 +0100

    Monitor cache time now 1s & processor queue time now 0s for < 50 items
    
    Now we only wait 1 second before sending files to the processor from
    the monitor module. The reason for this, is to be more responsive from
    monitor events. Now a DELETE event is handled within 1 to 2 seconds if
    the indexer is idle.
    
    The processor queue handler is set up using a 2 second timeout per
    1000 files. Now when we have finished the initial check of all files,
    we use g_idle_add() instead to make sure the indexer deals with items
    much quicker.

 src/trackerd/tracker-monitor.c   |   11 +++--
 src/trackerd/tracker-processor.c |  110 +++++++++++++++++++++++++++++++++-----
 2 files changed, 104 insertions(+), 17 deletions(-)
---
diff --git a/src/trackerd/tracker-monitor.c b/src/trackerd/tracker-monitor.c
index 4a77fb0..1fac077 100644
--- a/src/trackerd/tracker-monitor.c
+++ b/src/trackerd/tracker-monitor.c
@@ -40,11 +40,14 @@
 
 #define TRACKER_MONITOR_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_MONITOR, TrackerMonitorPrivate))
 
+/* The life time of an item in the cache */
+#define CACHE_LIFETIME_SECONDS 1
+
 /* When we receive IO monitor events, we pause sending information to
  * the indexer for a few seconds before continuing. We have to receive
  * NO events for at least a few seconds before unpausing.
  */
-#define PAUSE_ON_IO_SECONDS   5
+#define PAUSE_ON_IO_SECONDS    5
 
 /* If this is defined, we pause the indexer when we get events. If it
  * is not, we don't do any pausing.
@@ -898,7 +901,7 @@ libinotify_cached_events_timeout_cb (gpointer data)
 			force_emit = start_event_seconds > cache_timeout;
 		}
 
-		timed_out = last_event_seconds >= MAX (2, scan_timeout);
+		timed_out = last_event_seconds >= MAX (CACHE_LIFETIME_SECONDS, scan_timeout);
 
 		/* Make sure the item is in the cache for at least 2
 		 * seconds OR the time as stated by the module config
@@ -1114,7 +1117,7 @@ libinotify_monitor_event_cb (INotifyHandle *handle,
 			g_debug ("Setting up event pair timeout check");
 
 			monitor->private->event_pairs_timeout_id =
-				g_timeout_add_seconds (2,
+				g_timeout_add_seconds (CACHE_LIFETIME_SECONDS,
 						       libinotify_event_pairs_timeout_cb,
 						       monitor);
 		}
@@ -1307,7 +1310,7 @@ libinotify_monitor_event_cb (INotifyHandle *handle,
 		g_debug ("Setting up cached events timeout check");
 
 		monitor->private->cached_events_timeout_id =
-			g_timeout_add_seconds (2,
+			g_timeout_add_seconds (CACHE_LIFETIME_SECONDS,
 					       libinotify_cached_events_timeout_cb,
 					       monitor);
 	}
diff --git a/src/trackerd/tracker-processor.c b/src/trackerd/tracker-processor.c
index 248786e..d02bafc 100644
--- a/src/trackerd/tracker-processor.c
+++ b/src/trackerd/tracker-processor.c
@@ -41,10 +41,23 @@
 
 #define TRACKER_PROCESSOR_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_PROCESSOR, TrackerProcessorPrivate))
 
-#define ITEMS_QUEUE_PROCESS_INTERVAL 2
-#define ITEMS_QUEUE_PROCESS_MAX      1000
+/* This means, if we have <= 50 items in the queues, we will use a
+ * g_idle_add() call instead of g_timeout_add() with
+ * ITEMS_QUEUE_PROCESS_INTERVAL to make sure items get handled as
+ * quickly as possible. NOTE: This is only used AFTER an initial
+ * check of all files and index status.
+ */ 
+#define ITEMS_QUEUE_PROCESS_QUICK_COUNT 50
+
+/* This is the interval we wait before attempting to send items to
+ * the indexer (again).
+ */
+#define ITEMS_QUEUE_PROCESS_INTERVAL    2
 
-#define ITEMS_SIGNAL_TO_DAEMON_RATIO 500
+/* This is the maximum number of items we send at one time to the
+ * indexer 
+ */
+#define ITEMS_QUEUE_PROCESS_MAX         1000
 
 typedef enum {
 	SENT_TYPE_NONE,
@@ -527,6 +540,31 @@ item_queue_readd_items (GQueue *queue,
 	}
 }
 
+static guint 
+item_queue_count_all (TrackerProcessor *processor)
+{
+	GList *l;
+	guint  items = 0;
+
+	for (l = processor->private->modules; l; l = l->next) {
+		GQueue *q;
+
+		q = g_hash_table_lookup (processor->private->items_created_queues, l->data);
+		items += g_queue_get_length (q);
+
+		q = g_hash_table_lookup (processor->private->items_updated_queues, l->data);
+		items += g_queue_get_length (q);
+
+		q = g_hash_table_lookup (processor->private->items_deleted_queues, l->data);
+		items += g_queue_get_length (q);
+
+		q = g_hash_table_lookup (processor->private->items_moved_queues, l->data);
+		items += g_queue_get_length (q);
+	}
+
+	return items;
+}
+
 static void
 item_queue_processed_cb (DBusGProxy *proxy,
 			 GError     *error,
@@ -582,27 +620,57 @@ static gboolean
 item_queue_handlers_cb (gpointer user_data)
 {
 	TrackerProcessor *processor;
+	TrackerStatus     status;
 	GQueue		 *queue;
 	GStrv		  files;
 	gchar		 *module_name;
+	gboolean          should_repeat = FALSE;
+	GTimeVal          time_now;
+	static GTimeVal   time_last = { 0, 0 };
 
 	processor = user_data;
 
-	/* This way we don't send anything to the indexer from monitor
-	 * events but we still queue them ready to send when we are
-	 * unpaused.
-	 */
-	if (tracker_status_get () == TRACKER_STATUS_PAUSED) {
-		g_message ("We are paused, sending nothing to the index until we are unpaused");
+	status = tracker_status_get ();
+
+	/* Don't spam */
+	g_get_current_time (&time_now);
+
+	should_repeat = (time_now.tv_sec - time_last.tv_sec) >= 5;
+	if (should_repeat) {
+		time_last = time_now;
+	}
+
+	switch (status) {
+	case TRACKER_STATUS_PAUSED:
+		/* This way we don't send anything to the indexer from
+		 * monitor events but we still queue them ready to
+		 * send when we are unpaused.  
+		 */
+		if (should_repeat) {
+			g_message ("We are paused, sending nothing to the "
+				   "indexer until we are unpaused");
+		}
+
+	case TRACKER_STATUS_PENDING:
+	case TRACKER_STATUS_WATCHING:
+		/* Wait until we have finished crawling before
+		 * sending anything.
+		 */
 		return TRUE;
+
+	default:
+		break;
 	}
 
 	/* This is here so we don't try to send something if we are
 	 * still waiting for a response from the last send.
 	 */
 	if (processor->private->sent_type != SENT_TYPE_NONE) {
-		g_message ("Still waiting for response from indexer, "
-			   "not sending more files yet");
+		if (should_repeat) {
+			g_message ("Still waiting for response from indexer, "
+				   "not sending more files yet");
+		}
+
 		return TRUE;
 	}
 
@@ -753,10 +821,26 @@ item_queue_handlers_set_up (TrackerProcessor *processor)
 		return;
 	}
 
+	if (!tracker_status_get_is_initial_check ()) {
+		guint count;
+
+		/* Get items left to handle */
+		count = item_queue_count_all (processor);
+		
+		if (count <= ITEMS_QUEUE_PROCESS_QUICK_COUNT) {
+			g_message ("Only %d items queued currently, setting up quick handler", 
+				   count);
+			processor->private->item_queues_handler_id =
+				g_idle_add (item_queue_handlers_cb,
+					    processor);
+			return;
+		}
+	}
+
 	processor->private->item_queues_handler_id =
 		g_timeout_add_seconds (ITEMS_QUEUE_PROCESS_INTERVAL,
-			       item_queue_handlers_cb,
-			       processor);
+				       item_queue_handlers_cb,
+				       processor);
 }
 
 static gboolean