tracker r1811 - in branches/indexer-split: . src/libtracker-common src/trackerd

[mr svn gnome org]

Author: mr
Date: Tue Jul  1 14:54:19 2008
New Revision: 1811
URL: http://svn.gnome.org/viewvc/tracker?rev=1811&view=rev

Log:
	* src/trackerd/tracker-crawler.c: Make sure we check the
	file_index_patterns now. Improve statistics.

	* src/trackerd/tracker-marshal.list: Added to extend finished
	signal to include some statistics.

	* src/trackerd/tracker-processor.c: Make use of the enable/disable
	flag in the module config.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/libtracker-common/tracker-module-config.c
   branches/indexer-split/src/libtracker-common/tracker-module-config.h
   branches/indexer-split/src/trackerd/tracker-crawler.c
   branches/indexer-split/src/trackerd/tracker-marshal.list
   branches/indexer-split/src/trackerd/tracker-processor.c

Modified: branches/indexer-split/src/libtracker-common/tracker-module-config.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-module-config.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-module-config.c	Tue Jul  1 14:54:19 2008
@@ -53,9 +53,10 @@
 	GList      *ignored_file_patterns;
 
 	/* Index */
-	gchar      *service;
-	GHashTable *mime_types;
-	GHashTable *files;
+	gchar      *index_service;
+	GHashTable *index_mime_types;
+	GHashTable *index_files;
+	GList      *index_file_patterns;
 
 	/* Specific Options, FIXME: Finish */
 
@@ -68,9 +69,14 @@
 static void
 module_config_free (ModuleConfig *mc)
 {
-	g_hash_table_unref (mc->files);
-	g_hash_table_unref (mc->mime_types);
-	g_free (mc->service);
+	g_list_foreach (mc->index_file_patterns,
+			(GFunc) g_pattern_spec_free,
+			NULL);
+	g_list_free (mc->index_file_patterns);
+
+	g_hash_table_unref (mc->index_files);
+	g_hash_table_unref (mc->index_mime_types);
+	g_free (mc->index_service);
 
 	g_list_foreach (mc->ignored_file_patterns,
 			(GFunc) g_pattern_spec_free,
@@ -153,6 +159,33 @@
 	mc->ignored_directory_patterns = g_list_reverse (patterns);
 }
 
+static void
+module_config_set_index_file_patterns (ModuleConfig *mc)
+{
+	GPatternSpec *spec;
+	GList        *index_files;
+	GList        *l;
+	GList        *patterns = NULL;
+
+	g_list_foreach (mc->index_file_patterns,
+			(GFunc) g_pattern_spec_free,
+			NULL);
+	g_list_free (mc->index_file_patterns);
+
+	index_files = g_hash_table_get_keys (mc->index_files);
+
+	for (l = index_files; l; l = l->next) {
+		g_message ("  Adding file index pattern:'%s'", 
+			   (gchar *) l->data);
+		spec = g_pattern_spec_new (l->data);
+		patterns = g_list_prepend (patterns, spec);
+	}
+
+	g_list_free (index_files);
+
+	mc->index_file_patterns = g_list_reverse (patterns);
+}
+
 static gboolean
 module_config_load_boolean (GKeyFile    *key_file,
 			    const gchar *group,
@@ -345,17 +378,17 @@
 						FALSE);
 
 	/* Index */
-	mc->service =
+	mc->index_service =
 		module_config_load_string (key_file,
 					   GROUP_INDEX,
 					   "Service",
 					   FALSE);
-	mc->mime_types =
+	mc->index_mime_types =
 		module_config_load_string_list (key_file,
 						GROUP_INDEX,
 						"MimeTypes",
 						FALSE);
-	mc->files =
+	mc->index_files =
 		module_config_load_string_list (key_file,
 						GROUP_INDEX,
 						"Files",
@@ -365,6 +398,7 @@
 
 	module_config_set_ignored_file_patterns (mc);
 	module_config_set_ignored_directory_patterns (mc);
+	module_config_set_index_file_patterns (mc);
 
 	return mc;
 }
@@ -636,7 +670,7 @@
 }
 
 const gchar *
-tracker_module_config_get_service (const gchar *name)
+tracker_module_config_get_index_service (const gchar *name)
 {
 	ModuleConfig *mc;
 
@@ -645,11 +679,11 @@
 	mc = g_hash_table_lookup (modules, name);
 	g_return_val_if_fail (mc, NULL);
 
-	return mc->service;
+	return mc->index_service;
 }
 
 GList *
-tracker_module_config_get_mime_types (const gchar *name)
+tracker_module_config_get_index_mime_types (const gchar *name)
 {
 	ModuleConfig *mc;
 
@@ -658,11 +692,11 @@
 	mc = g_hash_table_lookup (modules, name);
 	g_return_val_if_fail (mc, NULL);
 
-	return g_hash_table_get_keys (mc->mime_types);
+	return g_hash_table_get_keys (mc->index_mime_types);
 }
 
 GList *
-tracker_module_config_get_files (const gchar *name)
+tracker_module_config_get_index_files (const gchar *name)
 {
 	ModuleConfig *mc;
 
@@ -671,7 +705,7 @@
 	mc = g_hash_table_lookup (modules, name);
 	g_return_val_if_fail (mc, NULL);
 
-	return g_hash_table_get_keys (mc->files);
+	return g_hash_table_get_keys (mc->index_files);
 }
 
 /*
@@ -703,3 +737,16 @@
 
 	return g_list_copy (mc->ignored_directory_patterns);
 }
+
+GList *
+tracker_module_config_get_index_file_patterns (const gchar *name)
+{
+	ModuleConfig *mc;
+
+	g_return_val_if_fail (name != NULL, NULL);
+
+	mc = g_hash_table_lookup (modules, name);
+	g_return_val_if_fail (mc, NULL);
+
+	return g_list_copy (mc->index_file_patterns);
+}

Modified: branches/indexer-split/src/libtracker-common/tracker-module-config.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-module-config.h	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-module-config.h	Tue Jul  1 14:54:19 2008
@@ -27,21 +27,26 @@
 
 gboolean     tracker_module_config_init                            (void);
 void         tracker_module_config_shutdown                        (void);
+
 GList *      tracker_module_config_get_modules                     (void);
+
 const gchar *tracker_module_config_get_description                 (const gchar *name);
 gboolean     tracker_module_config_get_enabled                     (const gchar *name);
+
 GList *      tracker_module_config_get_monitor_directories         (const gchar *name);
 GList *      tracker_module_config_get_monitor_recurse_directories (const gchar *name);
+
 GList *      tracker_module_config_get_ignored_directories         (const gchar *name);
 GList *      tracker_module_config_get_ignored_files               (const gchar *name);
-const gchar *tracker_module_config_get_service                     (const gchar *name);
-const gchar *tracker_module_config_get_service                     (const gchar *name);
-GList *      tracker_module_config_get_mime_types                  (const gchar *name);
-GList *      tracker_module_config_get_files                       (const gchar *name);
+
+const gchar *tracker_module_config_get_index_service               (const gchar *name);
+GList *      tracker_module_config_get_index_mime_types            (const gchar *name);
+GList *      tracker_module_config_get_index_files                 (const gchar *name);
 
 /* Convenience functions */
 GList *      tracker_module_config_get_ignored_directory_patterns  (const gchar *name);
 GList *      tracker_module_config_get_ignored_file_patterns       (const gchar *name);
+GList *      tracker_module_config_get_index_file_patterns         (const gchar *name);
 
 G_END_DECLS
 

Modified: branches/indexer-split/src/trackerd/tracker-crawler.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.c	Tue Jul  1 14:54:19 2008
@@ -33,10 +33,11 @@
 #include "tracker-dbus.h"
 #include "tracker-indexer-client.h"
 #include "tracker-monitor.h"
+#include "tracker-marshal.h"
 
 #define TRACKER_CRAWLER_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_CRAWLER, TrackerCrawlerPrivate))
 
-/*#define TESTING*/
+/* #define TESTING */
 
 #define FILE_ATTRIBUTES				\
 	G_FILE_ATTRIBUTE_STANDARD_NAME ","	\
@@ -62,6 +63,7 @@
 	/* Specific to each crawl ... */
 	GList          *ignored_directory_patterns;
 	GList          *ignored_file_patterns;
+	GList          *index_file_patterns;
 	gchar          *current_module_name;
 
 	/* Statistics */
@@ -70,6 +72,8 @@
 	guint           directories_ignored;
 	guint           files_found;
 	guint           files_ignored;
+	guint           monitors_added;
+	guint           monitors_ignored;
 
 	gboolean        running;
 	gboolean        finished;
@@ -155,9 +159,13 @@
 			      G_SIGNAL_RUN_LAST,
 			      0,
 			      NULL, NULL,
-			      g_cclosure_marshal_VOID__VOID,
+			      tracker_marshal_VOID__UINT_UINT_UINT_UINT,
 			      G_TYPE_NONE, 
-			      0);
+			      4,
+			      G_TYPE_UINT,
+			      G_TYPE_UINT,
+			      G_TYPE_UINT,
+			      G_TYPE_UINT);
 
 	g_type_class_add_private (object_class, sizeof (TrackerCrawlerPrivate));
 }
@@ -188,6 +196,10 @@
 
 	g_free (priv->current_module_name);
 
+	if (priv->index_file_patterns) {
+		g_list_free (priv->index_file_patterns);
+	}
+
 	if (priv->ignored_file_patterns) {
 		g_list_free (priv->ignored_file_patterns);
 	}
@@ -503,14 +515,22 @@
 
 	/* Test ignore types */
 	if (is_directory) {
-		l = crawler->private->ignored_directory_patterns;
+		for (l = crawler->private->ignored_directory_patterns; l; l = l->next) {
+			if (g_pattern_match_string (l->data, basename)) {
+				goto done;
+			}
+		}
 	} else {
-		l = crawler->private->ignored_file_patterns;
-	}
-	
-	for (; l; l = l->next) {
-		if (g_pattern_match_string (l->data, basename)) {
-			goto done;
+		for (l = crawler->private->ignored_file_patterns; l; l = l->next) {
+			if (g_pattern_match_string (l->data, basename)) {
+				goto done;
+			}
+		}
+
+		for (l = crawler->private->index_file_patterns; l; l = l->next) {
+			if (!g_pattern_match_string (l->data, basename)) {
+				goto done;
+			}
 		}
 	}
 
@@ -870,10 +890,16 @@
 		g_list_free (priv->ignored_file_patterns);
 	}
 
-	priv->ignored_file_patterns = 
-		tracker_module_config_get_ignored_file_patterns (module_name);
+	if (priv->index_file_patterns) {
+		g_list_free (priv->index_file_patterns);
+	}
+
 	priv->ignored_directory_patterns = 
 		tracker_module_config_get_ignored_directory_patterns (module_name);
+	priv->ignored_file_patterns = 
+		tracker_module_config_get_ignored_file_patterns (module_name);
+	priv->index_file_patterns = 
+		tracker_module_config_get_index_file_patterns (module_name);
 
 	priv->current_module_name = g_strdup (module_name);
 
@@ -889,6 +915,8 @@
 	priv->directories_ignored = 0;
 	priv->files_found = 0;
 	priv->files_ignored = 0;
+	priv->monitors_added = tracker_monitor_get_count ();
+	priv->monitors_ignored = tracker_monitor_get_ignored ();
 
 	for (sl = paths; sl; sl = sl->next) {
 		file = g_file_new_for_path (sl->data);
@@ -922,6 +950,11 @@
 	g_free (priv->current_module_name);
 	priv->current_module_name = NULL;
 
+	if (priv->index_file_patterns) {
+		g_list_free (priv->index_file_patterns);
+		priv->index_file_patterns = NULL;
+	}
+
 	if (priv->ignored_file_patterns) {
 		g_list_free (priv->ignored_file_patterns);
 		priv->ignored_file_patterns = NULL;
@@ -944,11 +977,15 @@
 		   priv->files_found,
 		   priv->files_ignored);
 	g_message ("  Added %d monitors, ignored %d monitors",
-		   tracker_monitor_get_count (),
-		   tracker_monitor_get_ignored ());
+		   tracker_monitor_get_count () - priv->monitors_added,
+		   tracker_monitor_get_ignored () - priv->monitors_ignored);
 
 	g_timer_destroy (priv->timer);
 	priv->timer = NULL;
 
-	g_signal_emit (crawler, signals[FINISHED], 0);
+	g_signal_emit (crawler, signals[FINISHED], 0,
+		       priv->directories_found,
+		       priv->directories_ignored,
+		       priv->files_found,
+		       priv->files_ignored);
 }

Modified: branches/indexer-split/src/trackerd/tracker-marshal.list
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-marshal.list	(original)
+++ branches/indexer-split/src/trackerd/tracker-marshal.list	Tue Jul  1 14:54:19 2008
@@ -1,3 +1,4 @@
+VOID:UINT,UINT,UINT,UINT
 VOID:STRING,STRING,INT,INT,INT
 VOID:STRING,STRING,STRING
 VOID:STRING,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN,BOOLEAN

Modified: branches/indexer-split/src/trackerd/tracker-processor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-processor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-processor.c	Tue Jul  1 14:54:19 2008
@@ -1,5 +1,5 @@
 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
-/* 
+/*
  * Copyright (C) 2008, Nokia
  *
  * This library is free software; you can redistribute it and/or
@@ -37,18 +37,24 @@
 typedef struct TrackerProcessorPrivate TrackerProcessorPrivate;
 
 struct TrackerProcessorPrivate {
-	TrackerConfig  *config; 
+	TrackerConfig  *config;
 #ifdef HAVE_HAL
-	TrackerHal     *hal; 
+	TrackerHal     *hal;
 #endif  /* HAVE_HAL */
-	TrackerCrawler *crawler; 
-	
-	GList          *modules; 
-	GList          *current_module; 
+	TrackerCrawler *crawler;
+
+	GList          *modules;
+	GList          *current_module;
 
 	GTimer         *timer;
 
 	gboolean        finished;
+
+	/* Statistics */
+	guint           directories_found;
+	guint           directories_ignored;
+	guint           files_found;
+	guint           files_ignored;
 };
 
 enum {
@@ -59,6 +65,10 @@
 static void tracker_processor_finalize (GObject          *object);
 static void process_next_module        (TrackerProcessor *processor);
 static void crawler_finished_cb        (TrackerCrawler   *crawler,
+					guint             directories_found,
+					guint             directories_ignored,
+					guint             files_found,
+					guint             files_ignored,
 					gpointer          user_data);
 
 static guint signals[LAST_SIGNAL] = { 0, };
@@ -72,7 +82,7 @@
 
 	object_class->finalize = tracker_processor_finalize;
 
-	signals [FINISHED] = 
+	signals [FINISHED] =
 		g_signal_new ("finished",
 			      G_OBJECT_CLASS_TYPE (object_class),
 			      G_SIGNAL_RUN_LAST,
@@ -128,7 +138,7 @@
 	GList *l;
 
 	monitors = tracker_module_config_get_monitor_directories (name);
-	
+
 	for (l = monitors; l; l = l->next) {
 		GFile       *file;
 		const gchar *path;
@@ -156,7 +166,7 @@
 	GList *l;
 
 	monitors = tracker_module_config_get_monitor_recurse_directories (name);
-	
+
 	for (l = monitors; l; l = l->next) {
 		GFile       *file;
 		const gchar *path;
@@ -187,6 +197,12 @@
 
 	g_message ("Processing module:'%s'", module_name);
 
+	/* Check it is enabled */
+	if (!tracker_module_config_get_enabled (module_name)) {
+		process_next_module (processor);
+		return;
+	}
+
 	/* Set up monitors */
 
 	/* Set up recursive monitors */
@@ -197,7 +213,10 @@
 		 * the next module.
 		 */
 		process_next_module (processor);
+		return;
 	}
+
+	/* Do soemthing else? */
 }
 
 static void
@@ -322,11 +341,22 @@
 
 static void
 crawler_finished_cb (TrackerCrawler *crawler,
+		     guint           directories_found,
+		     guint           directories_ignored,
+		     guint           files_found,
+		     guint           files_ignored,
 		     gpointer        user_data)
 {
-	TrackerProcessor *processor;
-	
+	TrackerProcessor        *processor;
+	TrackerProcessorPrivate *priv;
+
 	processor = TRACKER_PROCESSOR (user_data);
+	priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
+
+	priv->directories_found += directories_found;
+	priv->directories_ignored += directories_ignored;
+	priv->files_found += files_found;
+	priv->files_ignored += files_ignored;
 
 	process_next_module (processor);
 }
@@ -394,11 +424,21 @@
 		tracker_crawler_stop (priv->crawler);
 	}
 
+	g_message ("Process %s\n",
+		   priv->finished ? "has finished" : "been stopped");
+
 	g_timer_stop (priv->timer);
-	
-	g_message ("Process %s %4.4f seconds",
-		   priv->finished ? "finished in" : "stopped after",
+
+	g_message ("Total time taken : %4.4f seconds",
 		   g_timer_elapsed (priv->timer, NULL));
-	
+	g_message ("Total directories: %d (%d ignored)", 
+		   priv->directories_found,
+		   priv->directories_ignored);
+	g_message ("Total files      : %d (%d ignored)",
+		   priv->files_found,
+		   priv->files_ignored);
+	g_message ("Total monitors   : %d\n",
+		   tracker_monitor_get_count ());
+
 	g_signal_emit (processor, signals[FINISHED], 0);
 }