GNOME.org
 

tracker r1878 - in branches/indexer-split: . src/libtracker-common src/libtracker-db src/tracker-indexer src/trackerd

Author: mr
Date: Wed Jul 16 15:39:13 2008
New Revision: 1878
URL: http://svn.gnome.org/viewvc/tracker?rev=1878&view=rev

Log:
	* src/libtracker-db/tracker-db-manager.c: When setting up multiple
	interfaces, don't assume we ALWAYS want to "add_functions",
	instead use the array value for that database type (as we do
	normally).

	* src/tracker-indexer/tracker-indexer.c: Don't print a status
	message if there are 0 items remaining and change the transaction
	limit to 200.

	* src/trackerd/tracker-dbus.c: Added signal prototypes for the
	indexer for module-started, module-finished and started signals.

	* src/trackerd/tracker-processor.c: Use nicer debugging and
	include the time elapsed now in the status update.

	* src/trackerd/tracker-xesam-manager.c: Disable live search event
	updates during indexing. This causes havoc with the cache. The
	cache is already being written to by the indexer so the daemon
	becomes completely unresponsive. This is Enabled after indexing is
	finished.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/libtracker-common/tracker-utils.c
   branches/indexer-split/src/libtracker-common/tracker-utils.h
   branches/indexer-split/src/libtracker-db/tracker-db-manager.c
   branches/indexer-split/src/tracker-indexer/tracker-indexer.c
   branches/indexer-split/src/trackerd/tracker-dbus.c
   branches/indexer-split/src/trackerd/tracker-processor.c
   branches/indexer-split/src/trackerd/tracker-xesam-manager.c

Modified: branches/indexer-split/src/libtracker-common/tracker-utils.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-utils.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-utils.c	Wed Jul 16 15:39:13 2008
@@ -118,9 +118,10 @@
 }
 
 gchar *
-tracker_seconds_estimate_to_string (gdouble seconds_elapsed,
-				    guint   items_done,
-				    guint   items_remaining)
+tracker_seconds_estimate_to_string (gdouble  seconds_elapsed,
+				    gboolean short_string,
+				    guint    items_done,
+				    guint    items_remaining)
 {
 	gdouble per_item;
 	gdouble total;
@@ -138,13 +139,15 @@
 	per_item = seconds_elapsed / items_done;
 	total = per_item * items_remaining;
 
-	return tracker_seconds_to_string (total);
+	return tracker_seconds_to_string (total, short_string);
 }
 
 gchar *
-tracker_seconds_to_string (gdouble seconds_elapsed)
+tracker_seconds_to_string (gdouble  seconds_elapsed,
+			   gboolean short_string)
 {
 	GString *s;
+	gchar   *str;
 	gdouble  total;
 	gint     days, hours, minutes, seconds;
 
@@ -161,37 +164,55 @@
 
 	s = g_string_new ("");
 
-	if (days) {
-		g_string_append_printf (s, "%s%d day%s", 
-					s->len > 0 ? " " : "",
-					days, 
-					days == 1 ? "" : "s");
-	}
-	
-	if (hours) {
-		g_string_append_printf (s, "%s%2.2d hour%s", 
-					s->len > 0 ? " " : "",
-					hours, 
-					hours == 1 ? "" : "s");
-	}
-
-	if (minutes) {
-		g_string_append_printf (s, "%s%2.2d minute%s", 
-					s->len > 0 ? " " : "",
-					minutes, 
-					minutes == 1 ? "" : "s"); 
-	}
-
-	if (seconds) {
-		g_string_append_printf (s, "%s%2.2d second%s", 
-					s->len > 0 ? " " : "",
-					seconds, 
-					seconds == 1 ? "" : "s");
-	}
-
-	if (s->len < 1) {
-		g_string_append_printf (s, _("unknown time"));
+	if (short_string) {
+		if (days) {
+			g_string_append_printf (s, " %dd", days);
+		}
+		
+		if (hours) {
+			g_string_append_printf (s, " %2.2dh", hours);
+		}
+		
+		if (minutes) {
+			g_string_append_printf (s, " %2.2dm", minutes);
+		}
+		
+		if (seconds) {
+			g_string_append_printf (s, " %2.2ds", seconds);
+		}
+	} else {
+		if (days) {
+			g_string_append_printf (s, " %d day%s", 
+						days, 
+						days == 1 ? "" : "s");
+		}
+		
+		if (hours) {
+			g_string_append_printf (s, " %2.2d hour%s", 
+						hours, 
+						hours == 1 ? "" : "s");
+		}
+		
+		if (minutes) {
+			g_string_append_printf (s, " %2.2d minute%s", 
+						minutes, 
+						minutes == 1 ? "" : "s"); 
+		}
+		
+		if (seconds) {
+			g_string_append_printf (s, " %2.2d second%s", 
+						seconds, 
+						seconds == 1 ? "" : "s");
+		}
+	}
+
+	str = g_string_free (s, FALSE);
+
+	if (str[0] == '\0') {
+		str = g_strdup (_("unknown time"));
+	} else {
+		g_strchug (str);
 	}
 
-	return g_string_free (s, FALSE);
+	return str;
 }

Modified: branches/indexer-split/src/libtracker-common/tracker-utils.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-utils.h	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-utils.h	Wed Jul 16 15:39:13 2008
@@ -32,8 +32,10 @@
 					     const gchar *needle);
 gchar *  tracker_escape_string              (const gchar *in);
 gchar *  tracker_seconds_estimate_to_string (gdouble      seconds_elapsed,
+					     gboolean     short_string,
 					     guint        items_done,
 					     guint        items_remaining);
-gchar *  tracker_seconds_to_string          (gdouble      seconds_elapsed);
+gchar *  tracker_seconds_to_string          (gdouble      seconds_elapsed,
+					     gboolean     short_string);
 
 #endif /* __LIBTRACKER_COMMON_UTILS_H__ */

Modified: branches/indexer-split/src/libtracker-db/tracker-db-manager.c
==============================================================================
--- branches/indexer-split/src/libtracker-db/tracker-db-manager.c	(original)
+++ branches/indexer-split/src/libtracker-db/tracker-db-manager.c	Wed Jul 16 15:39:13 2008
@@ -2474,9 +2474,8 @@
 			db_set_params (connection,
 				       dbs[db].cache_size,
 				       dbs[db].page_size,
-				       TRUE,
+				       dbs[db].add_functions,
 				       FALSE);
-
 		} else {
 			db_exec_no_reply (connection, 
 					  "ATTACH '%s' as '%s'",

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.c	Wed Jul 16 15:39:13 2008
@@ -78,7 +78,7 @@
 #define FLUSH_FREQUENCY             5
 
 /* Transaction every 'x' items */
-#define TRANSACTION_MAX             100
+#define TRANSACTION_MAX             200
 
 /* Throttle defaults */
 #define THROTTLE_DEFAULT            0
@@ -217,20 +217,27 @@
 	items_remaining = g_queue_get_length (indexer->private->file_queue);
 	seconds_elapsed = g_timer_elapsed (indexer->private->timer, NULL);
 
-	if (indexer->private->items_indexed > 0) {
-		gchar *str;
-
-		str = tracker_seconds_estimate_to_string (seconds_elapsed, 
-							  indexer->private->items_indexed, 
-							  items_remaining);
+	if (indexer->private->items_indexed > 0 && 
+	    items_remaining > 0) {
+		gchar *str1;
+		gchar *str2;
+
+		str1 = tracker_seconds_estimate_to_string (seconds_elapsed, 
+							   TRUE,
+							   indexer->private->items_indexed, 
+							   items_remaining);
+		str2 = tracker_seconds_to_string (seconds_elapsed, TRUE);
 		
-		g_message ("Indexed %d, %d remaining, current module:'%s', %s est. left (%s)", 
+		g_message ("Indexed %d/%d, module:'%s', %s left, %s elapsed (%s)", 
 			   indexer->private->items_indexed,
-			   items_remaining,
+			   indexer->private->items_indexed + items_remaining,
 			   indexer->private->current_module_name,
-			   str,
+			   str1,
+			   str2,
 			   why);
-		g_free (str);
+
+		g_free (str2);
+		g_free (str1);
 	}
 	
 	g_signal_emit (indexer, signals[STATUS], 0, 
@@ -547,7 +554,7 @@
 	indexer->private->idle_id = 0;
 	
 	/* Print out how long it took us */
-	str = tracker_seconds_to_string (seconds_elapsed);
+	str = tracker_seconds_to_string (seconds_elapsed, FALSE);
 
 	g_message ("Indexer finished in %s, %d items indexed in total",
 		   str,

Modified: branches/indexer-split/src/trackerd/tracker-dbus.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-dbus.c	Wed Jul 16 15:39:13 2008
@@ -367,10 +367,21 @@
 					 G_TYPE_UINT,
 					 G_TYPE_INVALID);
 		dbus_g_proxy_add_signal (proxy_for_indexer,
+					 "Started",
+					 G_TYPE_INVALID);
+		dbus_g_proxy_add_signal (proxy_for_indexer,
 					 "Finished",
 					 G_TYPE_DOUBLE,
 					 G_TYPE_UINT,
 					 G_TYPE_INVALID);
+		dbus_g_proxy_add_signal (proxy_for_indexer,
+					 "ModuleStarted",
+					 G_TYPE_STRING,
+					 G_TYPE_INVALID);
+		dbus_g_proxy_add_signal (proxy_for_indexer,
+					 "ModuleFinished",
+					 G_TYPE_STRING,
+					 G_TYPE_INVALID);
 	}
 
 	return proxy_for_indexer;

Modified: branches/indexer-split/src/trackerd/tracker-processor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-processor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-processor.c	Wed Jul 16 15:39:13 2008
@@ -298,18 +298,28 @@
 		   guint        items_remaining,
 		   gpointer     user_data)
 {
-	gchar *str;
+	gchar *str1;
+	gchar *str2;
 
-	str = tracker_seconds_estimate_to_string (seconds_elapsed, 
-						  items_done, 
-						  items_remaining);
+	if (items_remaining < 1) {
+		return;
+	}
 
-	g_message ("Indexed %d, %d remaining, current module:'%s', %s est. left", 
+	str1 = tracker_seconds_estimate_to_string (seconds_elapsed, 
+						   TRUE,
+						   items_done, 
+						   items_remaining);
+	str2 = tracker_seconds_to_string (seconds_elapsed, TRUE);
+	
+	g_message ("Indexed %d/%d, module:'%s', %s left, %s elapsed", 
 		   items_done,
-		   items_remaining,
+		   items_done + items_remaining,
 		   current_module_name,
-		   str);
-	g_free (str);
+		   str1,
+		   str2);
+	
+	g_free (str2);
+	g_free (str1);
 }
 
 static void
@@ -321,7 +331,7 @@
 	TrackerProcessor *processor;
 	gchar            *str;
 
-	str = tracker_seconds_to_string (seconds_elapsed);
+	str = tracker_seconds_to_string (seconds_elapsed, FALSE);
 
 	g_message ("Indexer finished in %s, %d items indexed in total",
 		   str,

Modified: branches/indexer-split/src/trackerd/tracker-xesam-manager.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-xesam-manager.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-xesam-manager.c	Wed Jul 16 15:39:13 2008
@@ -33,9 +33,12 @@
 #include "tracker-dbus.h"
 #include "tracker-main.h"
 
-static GHashTable *xesam_sessions; 
-static gchar      *xesam_dir;
-static gboolean    live_search_handler_running = FALSE;
+static gboolean            initialized;
+static TrackerDBInterface *xesam_db_iface;
+static GHashTable         *xesam_sessions; 
+static gchar              *xesam_dir;
+static gboolean            indexing_finished;
+static guint               live_search_handler_id;
 
 static void
 indexer_status_cb (DBusGProxy  *proxy,
@@ -48,6 +51,30 @@
 	tracker_xesam_manager_wakeup ();	
 }
 
+static void
+indexer_started_cb (DBusGProxy *proxy,
+		    gpointer    user_data)
+{
+	/* So now when we get status updates we DO NOT process live
+	 * events and update live searches. The indexer is using the cache.
+	 */
+	g_message ("Disabling live search event updates (indexer started)");
+	indexing_finished = FALSE;
+}
+
+static void
+indexer_finished_cb (DBusGProxy *proxy,
+		     gdouble     seconds_elapsed,
+		     guint       items_done,
+		     gpointer    user_data)
+{
+	/* So now when we get status updates we can process live
+	 * events and update live searches.
+	 */
+	g_message ("Enabling live search event updates (indexer finished)");
+	indexing_finished = TRUE;
+}
+
 GQuark
 tracker_xesam_manager_error_quark (void)
 {
@@ -65,22 +92,47 @@
 {
 	DBusGProxy *proxy;
 
-	if (xesam_sessions) {
+	if (initialized) {
 		return;
 	}
 
+	/* Set up sessions hash table */
 	xesam_sessions = g_hash_table_new_full (g_str_hash, 
 						g_str_equal, 
 						(GDestroyNotify) g_free, 
 						(GDestroyNotify) g_object_unref);
 
+	/* Set up locations */
 	xesam_dir = g_build_filename (g_get_home_dir (), ".xesam", NULL);
 
+	/* Set up DBus proxy to the indexer process */
 	proxy = tracker_dbus_indexer_get_proxy ();
+	g_object_ref (proxy);
+
 	dbus_g_proxy_connect_signal (proxy, "Status",
 				     G_CALLBACK (indexer_status_cb),
-				     g_object_ref (proxy),
-				     (GClosureNotify) g_object_unref);
+				     NULL,
+				     NULL);
+	dbus_g_proxy_connect_signal (proxy, "Started",
+				     G_CALLBACK (indexer_started_cb),
+				     NULL,
+				     NULL);
+	dbus_g_proxy_connect_signal (proxy, "Finished",
+				     G_CALLBACK (indexer_finished_cb),
+				     NULL,
+				     NULL);
+
+	/* Set the indexing finished state back to unfinished */
+	indexing_finished = FALSE;
+
+	/* Get the DB interface now instead of later when the database
+	 * is potentially being hammered with new information by the
+	 * indexer. Before, if we just got it in the live update from
+	 * the indexer, we couldn't create the interface quickly
+	 * because the database is being used heavily by the indexer
+	 * already. It is best to do this initially to avoid that.
+	 */ 
+	xesam_db_iface = tracker_db_manager_get_db_interface_by_service (TRACKER_DB_FOR_XESAM_SERVICE);
 }
 
 void
@@ -88,14 +140,31 @@
 {
 	DBusGProxy *proxy;
 
-	if (!xesam_sessions) {
+	if (!initialized) {
 		return;
 	}
 
+	g_object_unref (xesam_db_iface);
+	xesam_db_iface = NULL;
+
 	proxy = tracker_dbus_indexer_get_proxy ();
 	dbus_g_proxy_disconnect_signal (proxy, "Status",
 					G_CALLBACK (indexer_status_cb),
 					NULL);
+	dbus_g_proxy_disconnect_signal (proxy, "Started",
+					G_CALLBACK (indexer_started_cb),
+					NULL);
+	dbus_g_proxy_disconnect_signal (proxy, "Finished",
+					G_CALLBACK (indexer_finished_cb),
+					NULL);
+	g_object_unref (proxy);
+
+	indexing_finished = FALSE;
+
+	if (live_search_handler_id != 0) {
+		g_source_remove (live_search_handler_id);
+		live_search_handler_id = 0;
+	}
 
 	g_free (xesam_dir);
 	xesam_dir = NULL;
@@ -239,13 +308,8 @@
 static gboolean 
 live_search_handler (gpointer data)
 {
-	TrackerDBInterface *iface;
-	GList              *sessions;
-	gboolean            reason_to_live = FALSE;
-
-	iface = tracker_db_manager_get_db_interface_by_service (TRACKER_DB_FOR_XESAM_SERVICE);
-
-	g_return_val_if_fail (iface != NULL, FALSE);
+	GList    *sessions;
+	gboolean  reason_to_live = FALSE;
 
 	sessions = g_hash_table_get_values (xesam_sessions);
 
@@ -315,8 +379,7 @@
 
 	g_list_free (sessions);
 
-	tracker_db_xesam_delete_handled_events (iface);
-
+	tracker_db_xesam_delete_handled_events (xesam_db_iface);
 
 	return reason_to_live;
 }
@@ -324,29 +387,42 @@
 static void 
 live_search_handler_destroy (gpointer data)
 {
-	live_search_handler_running = FALSE;
+	live_search_handler_id = 0;
 }
 
 void 
 tracker_xesam_manager_wakeup (void)
 {
-	/* This happens each time a new event is created */
-
-	/* We could do this in a thread too, in case blocking the GMainLoop is
-	 * not ideal (it's not, because during these blocks of code, no DBus
-	 * request handler can run). 
+	/* This happens each time a new event is created:
+	 *
+	 * We could do this in a thread too, in case blocking the
+	 * GMainLoop is not ideal (it's not, because during these
+	 * blocks of code, no DBus request handler can run).  
 	 * 
-	 * In case of a thread we could use usleep() and stop the thread if
-	 * we didn't get a wakeup-call nor we had items to process this loop
+	 * In case of a thread we could use usleep() and stop the
+	 * thread if we didn't get a wakeup-call nor we had items to
+	 * process this loop 
+	 *
+	 * There are problems with this. Right now we WAIT until
+	 * after indexing has completed otherwise we are in a
+	 * situation where a "status" signal from the indexer makes us
+	 * delete events from the Events table. This requires the
+	 * cache db and means we end up waiting for the indexer to
+	 * finish doing what it is doing first. The daemon then stops
+	 * pretty much and blocks. This is bad. So we wait for the
+	 * indexing to be finished before doing this.
 	 */
+	if (!indexing_finished) {
+		return;
+	}
 
-	if (!live_search_handler_running) {
-		live_search_handler_running = TRUE;
-		g_timeout_add_full (G_PRIORITY_DEFAULT_IDLE,
-				    2000, /* 2 seconds */
-				    live_search_handler,
-				    NULL,
-				    live_search_handler_destroy);
+	if (live_search_handler_id == 0) {
+		live_search_handler_id = 
+			g_timeout_add_full (G_PRIORITY_DEFAULT_IDLE,
+					    2000, /* 2 seconds */
+					    live_search_handler,
+					    NULL,
+					    live_search_handler_destroy);
 	}
 }
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]