tracker r2247 - in branches/indexer-split: . src/tracker-indexer src/trackerd

[mr svn gnome org]

Author: mr
Date: Fri Sep 19 09:54:24 2008
New Revision: 2247
URL: http://svn.gnome.org/viewvc/tracker?rev=2247&view=rev

Log:
	* src/trackerd/tracker-processor.c: Reduce the size of the list of
	files we send to the indexer down to 1000 from 5000. DBus isn't
	supposed to be optimized for > 4kb chunks.

	* src/tracker-indexer/tracker-indexer.c: Make sure we remove all
	entries in the hash table we use for mtime cache on parent
	directories when we have finished indexing. Use Quarks to know
	what the current module is instead of strdup() every time we
	process a file. Actually free the TrackerMetadata on move events.

	* src/tracker-indexer/tracker-metadata-utils.c: (get_file_content):
	Don't write until the end of the buf, use a length, we know it and it
	should be faster if the GString function doesn't need to calculate the
	length too.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/tracker-indexer/tracker-indexer.c
   branches/indexer-split/src/tracker-indexer/tracker-metadata-utils.c
   branches/indexer-split/src/trackerd/tracker-processor.c

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.c	Fri Sep 19 09:54:24 2008
@@ -106,7 +106,7 @@
 	GHashTable *mtime_cache;
 
 	GList *module_names;
-	gchar *current_module_name;
+	GQuark current_module;
 	GHashTable *indexer_modules;
 
 	gchar *db_dir;
@@ -293,7 +293,7 @@
 		g_message ("Indexed %d/%d, module:'%s', %s left, %s elapsed (%s)",
 			   indexer->private->files_indexed,
 			   indexer->private->files_indexed + files_remaining,
-			   indexer->private->current_module_name,
+			   g_quark_to_string (indexer->private->current_module),
 			   str1,
 			   str2,
 			   why);
@@ -304,7 +304,7 @@
 
 	g_signal_emit (indexer, signals[STATUS], 0,
 		       seconds_elapsed,
-		       indexer->private->current_module_name,
+		       g_quark_to_string (indexer->private->current_module),
 		       indexer->private->files_indexed,
 		       files_remaining);
 }
@@ -466,7 +466,6 @@
 	g_free (priv->db_dir);
 
 	g_hash_table_unref (priv->indexer_modules);
-	g_free (priv->current_module_name);
 	g_list_free (priv->module_names);
 
 	g_queue_foreach (priv->modules_queue, (GFunc) g_free, NULL);
@@ -641,6 +640,9 @@
 
 	state_set_flags (indexer, TRACKER_INDEXER_STATE_STOPPED);
 
+	/* Clean up temporary data */
+	g_hash_table_remove_all (indexer->private->mtime_cache);
+
 	/* Print out how long it took us */
 	str = tracker_seconds_to_string (seconds_elapsed, FALSE);
 
@@ -807,6 +809,9 @@
 					 NULL);
 
 	priv->module_names = tracker_module_config_get_modules ();
+	for (l = priv->module_names; l; l = l->next) {
+ 		g_quark_from_string (l->data);
+	}
 
 	priv->indexer_modules = g_hash_table_new_full (g_str_hash,
 						       g_str_equal,
@@ -1850,10 +1855,10 @@
 						   NULL, 
 						   &mtime);
 		if (!exists) {
-			g_critical ("Expected path '%s/%s' to exist, not in database?",
-				    parent_dirname,
-				    parent_basename);
-
+			g_message ("Expected path '%s/%s' to exist, not in database?",
+				   parent_dirname,
+				   parent_basename);
+			
 			g_free (parent_basename);
 			g_free (parent_dirname);
 
@@ -1861,8 +1866,8 @@
 		}
 
 		if (g_lstat (dirname, &st) == -1) {
-			g_critical ("Expected path '%s' to exist, could not stat()",
-				    parent_dirname);
+			g_message ("Expected path '%s' to exist, could not stat()",
+				   parent_dirname);
 
 			g_free (parent_basename);
 			g_free (parent_dirname);
@@ -1906,8 +1911,7 @@
 	 */
 
 	/* Set the current module */
-	g_free (indexer->private->current_module_name);
-	indexer->private->current_module_name = g_strdup (info->module_name);
+	indexer->private->current_module = g_quark_from_string (info->module_name);
 	
 	if (!tracker_indexer_module_file_get_uri (info->module,
 						  info->file,
@@ -1915,7 +1919,7 @@
 						  &basename)) {
 		return TRUE;
 	}
-	
+
 	/* 
 	 * FIRST:
 	 * ======
@@ -1957,11 +1961,11 @@
 	 * metadata. For move PathInfo we use the db function to move
 	 * a service and set the metadata.
 	 */
-	metadata = tracker_indexer_module_file_get_metadata (info->module, info->file);
-
 	if (G_UNLIKELY (info->other_file)) {
 		item_move (indexer, info, dirname, basename);
 	} else {
+		metadata = tracker_indexer_module_file_get_metadata (info->module, info->file);
+
 		if (metadata) {
 			item_create_or_update (indexer, info, dirname, basename, metadata);
 			tracker_metadata_free (metadata);
@@ -2024,11 +2028,10 @@
 {
 	/* Signal the last module as finished */
 	g_signal_emit (indexer, signals[MODULE_FINISHED], 0,
-		       indexer->private->current_module_name);
+		       g_quark_to_string (indexer->private->current_module));
 
 	/* Set current module */
-	g_free (indexer->private->current_module_name);
-	indexer->private->current_module_name = g_strdup (next_module_name);
+	indexer->private->current_module = g_quark_from_string (next_module_name);
 
 	/* Signal the next module as started */
 	if (next_module_name) {

Modified: branches/indexer-split/src/tracker-indexer/tracker-metadata-utils.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-metadata-utils.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-metadata-utils.c	Fri Sep 19 09:54:24 2008
@@ -557,7 +557,8 @@
 			 has_more_data ? "yes" : "no",
 			 has_reached_max ? "yes" : "no");
 
-		s = g_string_append (s, buf);
+		/* The + 1 is for the NULL terminating byte */
+		s = g_string_append_len (s, buf, bytes_read + 1);
 	}
 
 	if (has_reached_max) {

Modified: branches/indexer-split/src/trackerd/tracker-processor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-processor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-processor.c	Fri Sep 19 09:54:24 2008
@@ -43,7 +43,7 @@
 #define TRACKER_PROCESSOR_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_PROCESSOR, TrackerProcessorPrivate))
 
 #define ITEMS_QUEUE_PROCESS_INTERVAL 2000
-#define ITEMS_QUEUE_PROCESS_MAX      5000
+#define ITEMS_QUEUE_PROCESS_MAX      1000
 
 #define ITEMS_SIGNAL_TO_DAEMON_RATIO 500