[tracker/porting/processor-rework: 1/5] Rework the processors device and module logic so it is clearer
- From: Philip Van Hoof <pvanhoof src gnome org>
- To: svn-commits-list gnome org
- Subject: [tracker/porting/processor-rework: 1/5] Rework the processors device and module logic so it is clearer
- Date: Fri, 5 Jun 2009 06:49:23 -0400 (EDT)
commit 3bedb3d3594f5c65a1e53a0ce8a60b0c6119de99
Author: Martyn Russell <martyn imendio com>
Date: Wed May 27 17:34:57 2009 +0100
Rework the processors device and module logic so it is clearer
This also cleans up some of the debugging so there is less of it. It
is also now less likely we end up in recursive loops because of real
time updates from removable devices being inserted and removed (the
logic here is much simpler and comprehensible
Ported to master by Philip
---
src/trackerd/tracker-daemon.c | 5 +-
src/trackerd/tracker-main.c | 59 ++--
src/trackerd/tracker-processor.c | 860 +++++++++++++++++++++-----------------
src/trackerd/tracker-status.c | 28 ++-
src/trackerd/tracker-status.h | 3 +
5 files changed, 544 insertions(+), 411 deletions(-)
diff --git a/src/trackerd/tracker-daemon.c b/src/trackerd/tracker-daemon.c
index aa0200a..47b124f 100644
--- a/src/trackerd/tracker-daemon.c
+++ b/src/trackerd/tracker-daemon.c
@@ -158,7 +158,7 @@ indexer_started_cb (DBusGProxy *proxy,
return;
}
- g_message ("Starting statistics cache timeout");
+ g_debug ("Starting statistics cache timeout");
priv->stats_cache_timeout_id =
g_timeout_add_seconds (STATS_CACHE_LIFETIME,
stats_cache_timeout,
@@ -183,7 +183,7 @@ indexer_finished_cb (DBusGProxy *proxy,
return;
}
- g_message ("Stopping statistics cache timeout");
+ g_debug ("Stopping statistics cache timeout");
g_source_remove (priv->stats_cache_timeout_id);
priv->stats_cache_timeout_id = 0;
}
@@ -400,6 +400,7 @@ stats_cache_get_latest (void)
GPtrArray *stats;
/* Set up empty list of services because SQL queries won't give us 0 items. */
+
g_message ("Requesting statistics from database for an accurate signal");
values = g_hash_table_new_full (g_str_hash,
diff --git a/src/trackerd/tracker-main.c b/src/trackerd/tracker-main.c
index 52a41a8..01f80e0 100644
--- a/src/trackerd/tracker-main.c
+++ b/src/trackerd/tracker-main.c
@@ -591,9 +591,13 @@ initialize_directories (void)
static gboolean
initialize_databases (void)
{
- /*
- * Create SQLite databases
+ /* This means we doing the initial check that our dbs are up
+ * to date. Once we get finished from the indexer, we set
+ * this to FALSE.
*/
+ tracker_status_set_is_initial_check (TRUE);
+
+ /* We set our first time indexing state here */
if (!tracker_status_get_is_readonly () && force_reindex) {
tracker_status_set_is_first_time_index (TRUE);
}
@@ -693,20 +697,19 @@ backup_user_metadata (TrackerConfig *config, TrackerLanguage *language)
* Saving the last backup file to help with debugging.
*/
static void
-crawling_finished_cb (TrackerProcessor *processor,
+processor_finished_cb (TrackerProcessor *processor,
gpointer user_data)
{
GError *error = NULL;
- gulong *callback_id;
- static gint counter = 0;
- callback_id = user_data;
-
- if (++counter >= 2) {
+ if (!tracker_status_get_is_initial_check ()) {
gchar *rebackup;
- g_debug ("Uninstalling initial crawling callback");
- g_signal_handler_disconnect (processor, *callback_id);
+ g_debug ("Uninstalling initial processor finished callback");
+
+ g_signal_handlers_disconnect_by_func (processor,
+ processor_finished_cb,
+ user_data);
if (g_file_test (get_ttl_backup_filename (), G_FILE_TEST_EXISTS)) {
org_freedesktop_Tracker_Indexer_restore_backup (tracker_dbus_indexer_get_proxy (),
@@ -725,22 +728,9 @@ crawling_finished_cb (TrackerProcessor *processor,
g_rename (get_ttl_backup_filename (), rebackup);
g_free (rebackup);
}
-
- } else {
- g_debug ("%d finished signal", counter);
}
}
-static void
-backup_restore_on_crawling_finished (TrackerProcessor *processor)
-{
- static gulong restore_cb_id = 0;
-
- g_debug ("Setting callback for crawling finish detection");
- restore_cb_id = g_signal_connect (processor, "finished",
- G_CALLBACK (crawling_finished_cb),
- &restore_cb_id);
-}
#ifdef HAVE_HAL
@@ -1019,8 +1009,23 @@ main (gint argc, gchar *argv[])
set_up_mount_points (hal_storage);
#endif /* HAVE_HAL */
+ if (private->shutdown) {
+ goto shutdown;
+ }
+
+ /*
+ * Start public interfaces (DBus, push modules, etc)
+ */
private->processor = tracker_processor_new (config, hal_storage);
+ if (force_reindex &&
+ g_file_test (get_ttl_backup_filename (), G_FILE_TEST_EXISTS)) {
+ g_debug ("Setting callback for crawling finish detection");
+ g_signal_connect (private->processor, "finished",
+ G_CALLBACK (processor_finished_cb),
+ NULL);
+ }
+
/* Make Tracker available for introspection */
if (!tracker_dbus_register_objects (config,
language,
@@ -1057,16 +1062,12 @@ main (gint argc, gchar *argv[])
tracker_status_set_and_signal (TRACKER_STATUS_IDLE);
}
- if (flags & TRACKER_DB_MANAGER_FORCE_REINDEX ||
- g_file_test (get_ttl_backup_filename (), G_FILE_TEST_EXISTS)) {
- backup_restore_on_crawling_finished (private->processor);
- }
-
- if (!private->shutdown && tracker_status_get_is_ready ()) {
+ if (!private->shutdown) {
private->main_loop = g_main_loop_new (NULL, FALSE);
g_main_loop_run (private->main_loop);
}
+shutdown:
/*
* Shutdown the daemon
*/
diff --git a/src/trackerd/tracker-processor.c b/src/trackerd/tracker-processor.c
index fa7a7e2..25ad22a 100644
--- a/src/trackerd/tracker-processor.c
+++ b/src/trackerd/tracker-processor.c
@@ -79,27 +79,34 @@ struct TrackerProcessorPrivate {
/* Status */
GList *modules;
GList *current_module;
- gboolean iterated_modules;
- GList *removable_devices_completed;
- GList *removable_devices_current;
- GList *removable_devices;
+ GList *devices;
+ GList *current_device;
GTimer *timer;
gboolean interrupted;
- gboolean finished;
+
+ gboolean finished_modules;
+ gboolean finished_devices;
+ gboolean finished_sending;
+ gboolean finished_indexer;
/* Statistics */
+ guint total_directories_found;
+ guint total_directories_ignored;
+ guint total_files_found;
+ guint total_files_ignored;
+
guint directories_found;
guint directories_ignored;
guint files_found;
guint files_ignored;
- guint items_done;
- guint items_remaining;
+ guint indexer_items_done;
+ guint indexer_items_remaining;
- gdouble seconds_elapsed;
+ gdouble indexer_seconds_elapsed;
};
enum {
@@ -107,60 +114,67 @@ enum {
LAST_SIGNAL
};
-static void tracker_processor_finalize (GObject *object);
-static void crawler_destroy_notify (gpointer data);
-static void item_queue_destroy_notify (gpointer data);
-static void process_module_next (TrackerProcessor *processor);
-static void indexer_status_cb (DBusGProxy *proxy,
- gdouble seconds_elapsed,
- const gchar *current_module_name,
- guint items_processed,
- guint items_indexed,
- guint items_remaining,
- gpointer user_data);
-static void indexer_finished_cb (DBusGProxy *proxy,
- gdouble seconds_elapsed,
- guint items_processed,
- guint items_indexed,
- gboolean interrupted,
- gpointer user_data);
-static void monitor_item_created_cb (TrackerMonitor *monitor,
- const gchar *module_name,
- GFile *file,
- gboolean is_directory,
- gpointer user_data);
-static void monitor_item_updated_cb (TrackerMonitor *monitor,
- const gchar *module_name,
- GFile *file,
- gboolean is_directory,
- gpointer user_data);
-static void monitor_item_deleted_cb (TrackerMonitor *monitor,
- const gchar *module_name,
- GFile *file,
- gboolean is_directory,
- gpointer user_data);
-static void monitor_item_moved_cb (TrackerMonitor *monitor,
- const gchar *module_name,
- GFile *file,
- GFile *other_file,
- gboolean is_directory,
- gboolean is_source_monitored,
- gpointer user_data);
-static void crawler_processing_file_cb (TrackerCrawler *crawler,
- const gchar *module_name,
- GFile *file,
- gpointer user_data);
-static void crawler_processing_directory_cb (TrackerCrawler *crawler,
- const gchar *module_name,
- GFile *file,
- gpointer user_data);
-static void crawler_finished_cb (TrackerCrawler *crawler,
- const gchar *module_name,
- guint directories_found,
- guint directories_ignored,
- guint files_found,
- guint files_ignored,
- gpointer user_data);
+static void tracker_processor_finalize (GObject *object);
+static void crawler_destroy_notify (gpointer data);
+static void item_queue_destroy_notify (gpointer data);
+static void process_module_next (TrackerProcessor *processor);
+static void process_modules_stop (TrackerProcessor *processor);
+static void process_device_next (TrackerProcessor *processor);
+static void process_devices_stop (TrackerProcessor *processor);
+static void process_check_completely_finished (TrackerProcessor *processor);
+static void process_next (TrackerProcessor *processor);
+static void indexer_status_cb (DBusGProxy *proxy,
+ gdouble seconds_elapsed,
+ const gchar *current_module_name,
+ guint items_processed,
+ guint items_indexed,
+ guint items_remaining,
+ gpointer user_data);
+static void indexer_started_cb (DBusGProxy *proxy,
+ gpointer user_data);
+static void indexer_finished_cb (DBusGProxy *proxy,
+ gdouble seconds_elapsed,
+ guint items_processed,
+ guint items_indexed,
+ gboolean interrupted,
+ gpointer user_data);
+static void monitor_item_created_cb (TrackerMonitor *monitor,
+ const gchar *module_name,
+ GFile *file,
+ gboolean is_directory,
+ gpointer user_data);
+static void monitor_item_updated_cb (TrackerMonitor *monitor,
+ const gchar *module_name,
+ GFile *file,
+ gboolean is_directory,
+ gpointer user_data);
+static void monitor_item_deleted_cb (TrackerMonitor *monitor,
+ const gchar *module_name,
+ GFile *file,
+ gboolean is_directory,
+ gpointer user_data);
+static void monitor_item_moved_cb (TrackerMonitor *monitor,
+ const gchar *module_name,
+ GFile *file,
+ GFile *other_file,
+ gboolean is_directory,
+ gboolean is_source_monitored,
+ gpointer user_data);
+static void crawler_processing_file_cb (TrackerCrawler *crawler,
+ const gchar *module_name,
+ GFile *file,
+ gpointer user_data);
+static void crawler_processing_directory_cb (TrackerCrawler *crawler,
+ const gchar *module_name,
+ GFile *file,
+ gpointer user_data);
+static void crawler_finished_cb (TrackerCrawler *crawler,
+ const gchar *module_name,
+ guint directories_found,
+ guint directories_ignored,
+ guint files_found,
+ guint files_ignored,
+ gpointer user_data);
#ifdef HAVE_HAL
static void mount_point_added_cb (TrackerStorage *hal,
@@ -298,6 +312,9 @@ tracker_processor_finalize (GObject *object)
g_list_free (priv->modules);
+ dbus_g_proxy_disconnect_signal (priv->indexer_proxy, "Started",
+ G_CALLBACK (indexer_started_cb),
+ NULL);
dbus_g_proxy_disconnect_signal (priv->indexer_proxy, "Finished",
G_CALLBACK (indexer_finished_cb),
NULL);
@@ -321,14 +338,9 @@ tracker_processor_finalize (GObject *object)
g_object_unref (priv->monitor);
#ifdef HAVE_HAL
- if (priv->removable_devices) {
- g_list_foreach (priv->removable_devices, (GFunc) g_free, NULL);
- g_list_free (priv->removable_devices);
- }
-
- if (priv->removable_devices_completed) {
- g_list_foreach (priv->removable_devices_completed, (GFunc) g_free, NULL);
- g_list_free (priv->removable_devices_completed);
+ if (priv->devices) {
+ g_list_foreach (priv->devices, (GFunc) g_free, NULL);
+ g_list_free (priv->devices);
}
if (priv->hal) {
@@ -574,8 +586,6 @@ item_queue_processed_cb (DBusGProxy *proxy,
}
item_queue_readd_items (queue, processor->private->sent_items);
- } else {
- g_debug ("Sent!");
}
g_strfreev (processor->private->sent_items);
@@ -626,6 +636,8 @@ item_queue_handlers_cb (gpointer user_data)
module_name,
g_strv_length (files));
+ processor->private->finished_indexer = FALSE;
+
processor->private->sent_type = SENT_TYPE_DELETED;
processor->private->sent_module_name = module_name;
processor->private->sent_items = files;
@@ -654,6 +666,8 @@ item_queue_handlers_cb (gpointer user_data)
module_name,
g_strv_length (files));
+ processor->private->finished_indexer = FALSE;
+
processor->private->sent_type = SENT_TYPE_CREATED;
processor->private->sent_module_name = module_name;
processor->private->sent_items = files;
@@ -682,6 +696,8 @@ item_queue_handlers_cb (gpointer user_data)
module_name,
g_strv_length (files));
+ processor->private->finished_indexer = FALSE;
+
processor->private->sent_type = SENT_TYPE_UPDATED;
processor->private->sent_module_name = module_name;
processor->private->sent_items = files;
@@ -721,6 +737,8 @@ item_queue_handlers_cb (gpointer user_data)
module_name,
g_strv_length (files));
+ processor->private->finished_indexer = FALSE;
+
processor->private->sent_type = SENT_TYPE_MOVED;
processor->private->sent_module_name = module_name;
processor->private->sent_items = files;
@@ -738,12 +756,17 @@ item_queue_handlers_cb (gpointer user_data)
g_message ("No items in any queues to process, doing nothing");
processor->private->item_queues_handler_id = 0;
+ processor->private->finished_sending = TRUE;
+ process_check_completely_finished (processor);
+
return FALSE;
}
static void
item_queue_handlers_set_up (TrackerProcessor *processor)
{
+ processor->private->finished_sending = FALSE;
+
if (processor->private->item_queues_handler_id != 0) {
return;
}
@@ -774,95 +797,6 @@ is_path_on_ignore_list (GSList *ignore_list,
}
static void
-process_module_files_add_removable_media (TrackerProcessor *processor)
-{
- TrackerCrawler *crawler;
- const gchar *module_name = "files";
- gboolean nothing_to_do;
-
-#ifndef HAVE_HAL
- nothing_to_do = TRUE;
-#else
- if (!processor->private->removable_devices) {
- nothing_to_do = TRUE;
- } else {
- nothing_to_do = FALSE;
- }
-#endif
-
- if (nothing_to_do) {
- g_message (" Removable media being added (one at a time):");
- g_message (" NONE");
-
- return;
- }
-
- crawler = g_hash_table_lookup (processor->private->crawlers, module_name);
-
- tracker_crawler_use_module_paths (crawler, FALSE);
- tracker_crawler_special_paths_clear (crawler);
-
- /* So, that we do is:
- *
- * 1. Create a list of the removable device roots
- * 2. Process one of these roots
- * 3. Move that root to new "completed" list.
- * 4. Process the next item (goto #1).
- *
- * Conditions:
- *
- * a) An MMC is added:
- * - We add the root to the list of removable device roots
- * - We check the root is not already on the "completed" list.
- * - We start the process_module if not running.
- * b) An MMC is removed:
- * - If we are running not running, do nothing.
- * - If we are running and root is current root, FIXME: mr?
- * - If we are running and root is not current root,
- * remove it from list to of removable device roots.
- */
-
- for (;
- processor->private->removable_devices_current;
- processor->private->removable_devices_current = processor->private->removable_devices_current->next) {
- GFile *file;
- const gchar *root;
-
- root = processor->private->removable_devices_current->data;
-
- /* Don't iterate a device we have already crawled. */
- if (g_list_find_custom (processor->private->removable_devices_completed,
- root,
- (GCompareFunc) g_strcmp0)) {
- continue;
- }
-
- g_message (" Removable media being added (one at a time):");
-
- if (path_should_be_ignored_for_media (processor, root)) {
- g_message (" %s (ignored due to config)", root);
-
- /* Add to completed list */
- processor->private->removable_devices_completed =
- g_list_append (processor->private->removable_devices_completed,
- g_strdup (root));
-
- continue;
- }
-
- g_message (" %s", root);
-
- file = g_file_new_for_path (root);
- tracker_monitor_add (processor->private->monitor, module_name, file);
- g_object_unref (file);
-
- tracker_crawler_special_paths_add (crawler, root);
-
- break;
- }
-}
-
-static void
process_module_files_add_legacy_options (TrackerProcessor *processor)
{
TrackerCrawler *crawler;
@@ -960,12 +894,14 @@ process_module_is_disabled (TrackerProcessor *processor,
GSList *disabled_modules;
if (!tracker_module_config_get_enabled (module_name)) {
+ g_message (" Module disabled by module config");
return TRUE;
}
disabled_modules = tracker_config_get_disabled_modules (processor->private->config);
- if (g_slist_find_custom (disabled_modules, module_name, (GCompareFunc) strcmp)) {
+ if (g_slist_find_custom (disabled_modules, module_name, (GCompareFunc) g_strcmp0)) {
+ g_message (" Module disabled by user");
return TRUE;
}
@@ -974,27 +910,13 @@ process_module_is_disabled (TrackerProcessor *processor,
static void
process_module (TrackerProcessor *processor,
- const gchar *module_name,
- gboolean is_removable_media)
+ const gchar *module_name)
{
TrackerCrawler *crawler;
- GSList *disabled_modules;
-
- g_message ("Processing module:'%s' %s",
- module_name,
- is_removable_media ? "(for removable media)" : "");
- /* Check it is not disabled by the module config */
- if (!tracker_module_config_get_enabled (module_name)) {
- g_message (" Module disabled by module config");
- process_module_next (processor);
- return;
- }
+ g_message ("Processing module:'%s'", module_name);
- /* Check it is not disabled by the user locally */
- disabled_modules = tracker_config_get_disabled_modules (processor->private->config);
- if (g_slist_find_custom (disabled_modules, module_name, (GCompareFunc) strcmp)) {
- g_message (" Module disabled by user");
+ if (process_module_is_disabled (processor, module_name)) {
process_module_next (processor);
return;
}
@@ -1003,11 +925,7 @@ process_module (TrackerProcessor *processor,
tracker_status_set_and_signal (TRACKER_STATUS_WATCHING);
if (strcmp (module_name, "files") == 0) {
- if (is_removable_media) {
- process_module_files_add_removable_media (processor);
- } else {
- process_module_files_add_legacy_options (processor);
- }
+ process_module_files_add_legacy_options (processor);
}
/* Gets all files and directories */
@@ -1016,9 +934,6 @@ process_module (TrackerProcessor *processor,
crawler = g_hash_table_lookup (processor->private->crawlers, module_name);
if (!tracker_crawler_start (crawler)) {
- /* If there is nothing to crawl, we are done, process
- * the next module.
- */
process_module_next (processor);
}
}
@@ -1026,9 +941,6 @@ process_module (TrackerProcessor *processor,
static void
process_module_next (TrackerProcessor *processor)
{
- const gchar *module_name;
- gboolean is_removable_media;
-
if (tracker_status_get_is_readonly ()) {
/* Block any request to process
* modules if indexing is not enabled
@@ -1036,11 +948,9 @@ process_module_next (TrackerProcessor *processor)
return;
}
- /* Don't recursively iterate the modules if this function is
- * called, check first.
- */
+ /* Don't recursively iterate the modules */
if (!processor->private->current_module) {
- if (!processor->private->iterated_modules) {
+ if (!processor->private->finished_modules) {
processor->private->current_module = processor->private->modules;
}
} else {
@@ -1049,32 +959,310 @@ process_module_next (TrackerProcessor *processor)
/* If we have no further modules to iterate */
if (!processor->private->current_module) {
- processor->private->iterated_modules = TRUE;
+ process_modules_stop (processor);
+ process_next (processor);
+ return;
+ }
+
+ process_module (processor, processor->private->current_module->data);
+}
+
+static void
+process_device (TrackerProcessor *processor,
+ const gchar *device_root)
+{
+ TrackerCrawler *crawler;
+ GFile *file;
+ const gchar *module_name = "files";
- /* Handle removable media */
- module_name = "files";
- is_removable_media = TRUE;
+ g_message ("Processing device with root:'%s'", device_root);
- /* Only if the module is not disabled. Otherwise we
- * get into a recursive loop. Also we make sure that
- * we haven't already handled all removable devices
- * already.
+ if (process_module_is_disabled (processor, module_name)) {
+ process_device_next (processor);
+ return;
+ }
+
+ /* Here we set up legacy .cfg options like watch roots */
+ tracker_status_set_and_signal (TRACKER_STATUS_WATCHING);
+
+ /* Gets all files and directories */
+ tracker_status_set_and_signal (TRACKER_STATUS_PENDING);
+
+ crawler = g_hash_table_lookup (processor->private->crawlers, module_name);
+
+ tracker_crawler_use_module_paths (crawler, FALSE);
+ tracker_crawler_special_paths_clear (crawler);
+
+ if (path_should_be_ignored_for_media (processor, device_root)) {
+ g_message (" Ignored due to config");
+ process_device_next (processor);
+ return;
+ }
+
+ file = g_file_new_for_path (device_root);
+ tracker_monitor_add (processor->private->monitor, module_name, file);
+ g_object_unref (file);
+
+ tracker_crawler_special_paths_add (crawler, device_root);
+
+ if (!tracker_crawler_start (crawler)) {
+ process_device_next (processor);
+ }
+}
+
+static void
+process_device_next (TrackerProcessor *processor)
+{
+ if (tracker_status_get_is_readonly ()) {
+ /* Block any request to process
+ * modules if indexing is not enabled
*/
+ return;
+ }
+
+ /* Don't recursively iterate the devices */
+ if (!processor->private->current_device) {
+ if (!processor->private->finished_devices) {
+ processor->private->current_device = processor->private->devices;
+ }
+ } else {
+ processor->private->current_device = processor->private->current_device->next;
+ }
+
+ /* If we have no further devices to iterate */
+ if (!processor->private->current_device) {
+ process_devices_stop (processor);
+ process_next (processor);
+ return;
+ }
+
+ process_device (processor, processor->private->current_device->data);
+}
+
+static void
+process_modules_start (TrackerProcessor *processor)
+{
+ if (processor->private->timer) {
+ g_timer_destroy (processor->private->timer);
+ }
+
+ processor->private->timer = g_timer_new ();
+
+ processor->private->finished_modules = FALSE;
+
+ processor->private->directories_found = 0;
+ processor->private->directories_ignored = 0;
+ processor->private->files_found = 0;
+ processor->private->files_ignored = 0;
+
+ process_module_next (processor);
+}
+
+static void
+process_modules_stop (TrackerProcessor *processor)
+{
+ if (processor->private->finished_modules) {
+ return;
+ }
+
+ g_message ("--------------------------------------------------");
+ g_message ("Processor %s crawling modules",
+ processor->private->interrupted ? "been stopped" : "has finished");
+
+ processor->private->finished_modules = TRUE;
+
+ if (processor->private->interrupted) {
+ TrackerCrawler *crawler;
+
+ crawler = g_hash_table_lookup (processor->private->crawlers,
+ processor->private->current_module->data);
+ if (crawler) {
+ tracker_crawler_stop (crawler);
+ }
+
+ if (processor->private->timer) {
+ g_timer_destroy (processor->private->timer);
+ processor->private->timer = NULL;
+ }
+ } else {
+ gdouble elapsed;
+
+ if (processor->private->timer) {
+ g_timer_stop (processor->private->timer);
+ elapsed = g_timer_elapsed (processor->private->timer, NULL);
+ } else {
+ elapsed = 0;
+ }
- if (process_module_is_disabled (processor, module_name) ||
- (g_list_length (processor->private->removable_devices) <=
- g_list_length (processor->private->removable_devices_completed))) {
- processor->private->interrupted = FALSE;
- tracker_processor_stop (processor);
- return;
+ g_message ("Module time taken : %4.4f seconds",
+ elapsed);
+ g_message ("Module directories: %d (%d ignored)",
+ processor->private->directories_found,
+ processor->private->directories_ignored);
+ g_message ("Module files : %d (%d ignored)",
+ processor->private->files_found,
+ processor->private->files_ignored);
+ }
+
+ g_message ("--------------------------------------------------\n");
+}
+
+static void
+process_devices_start (TrackerProcessor *processor)
+{
+ if (processor->private->timer) {
+ g_timer_destroy (processor->private->timer);
+ }
+
+ processor->private->timer = g_timer_new ();
+
+ processor->private->finished_devices = FALSE;
+
+ processor->private->directories_found = 0;
+ processor->private->directories_ignored = 0;
+ processor->private->files_found = 0;
+ processor->private->files_ignored = 0;
+
+ process_device_next (processor);
+}
+
+static void
+process_devices_stop (TrackerProcessor *processor)
+{
+ if (processor->private->finished_devices) {
+ return;
+ }
+
+ g_message ("--------------------------------------------------");
+ g_message ("Processor %s crawling devices",
+ processor->private->interrupted ? "been stopped" : "has finished");
+
+ processor->private->finished_devices = TRUE;
+
+ if (processor->private->interrupted) {
+ TrackerCrawler *crawler;
+
+ crawler = g_hash_table_lookup (processor->private->crawlers, "files");
+ if (crawler) {
+ tracker_crawler_stop (crawler);
+ }
+
+ if (processor->private->timer) {
+ g_timer_destroy (processor->private->timer);
+ processor->private->timer = NULL;
}
} else {
- module_name = processor->private->current_module->data;
- is_removable_media = FALSE;
+ gdouble elapsed;
+
+ if (processor->private->timer) {
+ g_timer_stop (processor->private->timer);
+ elapsed = g_timer_elapsed (processor->private->timer, NULL);
+ } else {
+ elapsed = 0;
+ }
+
+ g_message ("Device time taken : %4.4f seconds",
+ elapsed);
+ g_message ("Device directories: %d (%d ignored)",
+ processor->private->directories_found,
+ processor->private->directories_ignored);
+ g_message ("Device files : %d (%d ignored)",
+ processor->private->files_found,
+ processor->private->files_ignored);
+ }
+
+ g_message ("--------------------------------------------------\n");
+}
+
+static void
+process_continue (TrackerProcessor *processor)
+{
+ if (!processor->private->finished_modules) {
+ process_module_next (processor);
+ return;
+ }
+
+ if (!processor->private->finished_devices) {
+ process_device_next (processor);
+ return;
}
- /* Set up new crawler for new module */
- process_module (processor, module_name, is_removable_media);
+ /* Nothing to do */
+}
+
+static void
+process_next (TrackerProcessor *processor)
+{
+ if (!processor->private->finished_modules) {
+ process_modules_start (processor);
+ return;
+ }
+
+ if (!processor->private->finished_devices) {
+ process_devices_start (processor);
+ return;
+ }
+
+ /* Only do this the first time, otherwise the results are
+ * likely to be inaccurate. Devices can be added or removed so
+ * we can't assume stats are correct.
+ */
+ if (tracker_status_get_is_initial_check ()) {
+ g_message ("--------------------------------------------------");
+ g_message ("Total directories : %d (%d ignored)",
+ processor->private->total_directories_found,
+ processor->private->total_directories_ignored);
+ g_message ("Total files : %d (%d ignored)",
+ processor->private->total_files_found,
+ processor->private->total_files_ignored);
+ g_message ("Total monitors : %d",
+ tracker_monitor_get_count (processor->private->monitor, NULL));
+ g_message ("--------------------------------------------------\n");
+ }
+
+ /* Now we have finished crawling, we enable monitor events */
+ g_message ("Enabling monitor events");
+ tracker_monitor_set_enabled (processor->private->monitor, TRUE);
+
+ /* Now we set the state to IDLE, the reason we do this, is it
+ * allows us to either return to an idle state if there was
+ * nothing to do, OR it allows us to start handling the
+ * queues of files we just crawled. The queue handler won't
+ * send files to the indexer while we are PENDING or WATCHING.
+ */
+ tracker_status_set_and_signal (TRACKER_STATUS_IDLE);
+}
+
+static void
+process_finish (TrackerProcessor *processor)
+{
+ /* Optimize DBs */
+ tracker_status_set_and_signal (TRACKER_STATUS_OPTIMIZING);
+ tracker_db_manager_optimize ();
+
+ /* All done */
+ tracker_status_set_and_signal (TRACKER_STATUS_IDLE);
+
+ /* Signal to the applet we are finished */
+ g_signal_emit_by_name (tracker_dbus_get_object (TRACKER_TYPE_DAEMON),
+ "index-finished",
+ processor->private->indexer_seconds_elapsed);
+
+ /* Set our internal state */
+ tracker_status_set_is_initial_check (FALSE);
+
+ g_signal_emit (processor, signals[FINISHED], 0);
+}
+
+static void
+process_check_completely_finished (TrackerProcessor *processor)
+{
+ if (!processor->private->finished_sending ||
+ !processor->private->finished_indexer) {
+ return;
+ }
+
+ process_finish (processor);
}
static void
@@ -1096,9 +1284,9 @@ indexer_status_cb (DBusGProxy *proxy,
processor = user_data;
/* Update our local copy */
- processor->private->items_done = items_processed;
- processor->private->items_remaining = items_remaining;
- processor->private->seconds_elapsed = seconds_elapsed;
+ processor->private->indexer_items_done = items_processed;
+ processor->private->indexer_items_remaining = items_remaining;
+ processor->private->indexer_seconds_elapsed = seconds_elapsed;
if (items_remaining < 1 ||
current_module_name == NULL ||
@@ -1143,26 +1331,35 @@ indexer_status_cb (DBusGProxy *proxy,
}
static void
+indexer_started_cb (DBusGProxy *proxy,
+ gpointer user_data)
+{
+ TrackerProcessor *processor;
+
+ processor = user_data;
+
+ processor->private->finished_indexer = FALSE;
+}
+
+static void
indexer_finished_cb (DBusGProxy *proxy,
- gdouble seconds_elapsed,
- guint items_processed,
- guint items_indexed,
- gboolean interrupted,
- gpointer user_data)
+ gdouble seconds_elapsed,
+ guint items_processed,
+ guint items_indexed,
+ gboolean interrupted,
+ gpointer user_data)
{
TrackerProcessor *processor;
- GObject *object;
gchar *str;
processor = user_data;
- object = tracker_dbus_get_object (TRACKER_TYPE_DAEMON);
- processor->private->items_done = items_processed;
- processor->private->items_remaining = 0;
- processor->private->seconds_elapsed = seconds_elapsed;
+ processor->private->indexer_items_done = items_processed;
+ processor->private->indexer_items_remaining = 0;
+ processor->private->indexer_seconds_elapsed = seconds_elapsed;
/* Signal to any applications */
- g_signal_emit_by_name (object,
+ g_signal_emit_by_name (tracker_dbus_get_object (TRACKER_TYPE_DAEMON),
"index-progress",
"", /* Service */
"", /* Path */
@@ -1174,27 +1371,15 @@ indexer_finished_cb (DBusGProxy *proxy,
/* Message to the console about state */
str = tracker_seconds_to_string (seconds_elapsed, FALSE);
- g_message ("Indexer finished in %s, %d items processed in total (%d indexed)",
+ g_message ("Indexer finished last batch in %s, %d items processed in total (%d indexed)",
str,
items_processed,
items_indexed);
g_free (str);
- /* Do we even need this step Optimizing ? */
- tracker_status_set_and_signal (TRACKER_STATUS_OPTIMIZING);
- tracker_db_manager_optimize ();
-
- /* Now the indexer is done, we can signal our status as IDLE */
- tracker_status_set_and_signal (TRACKER_STATUS_IDLE);
-
- /* Signal to the applet we are finished */
- g_signal_emit_by_name (object,
- "index-finished",
- seconds_elapsed);
-
- /* Signal the processor is now finished */
- processor->private->finished = TRUE;
- g_signal_emit (processor, signals[FINISHED], 0);
+ /* Save indexer's state */
+ processor->private->finished_indexer = TRUE;
+ process_check_completely_finished (processor);
}
static void
@@ -1221,6 +1406,7 @@ processor_files_check (TrackerProcessor *processor,
if (is_directory) {
tracker_crawler_add_unexpected_path (crawler, path);
}
+
queue = g_hash_table_lookup (processor->private->items_created_queues, module_name);
g_queue_push_tail (queue, g_object_ref (file));
@@ -1420,6 +1606,8 @@ crawler_processing_file_cb (TrackerCrawler *crawler,
/* Add files in queue to our queues to send to the indexer */
queue = g_hash_table_lookup (processor->private->items_created_queues, module_name);
g_queue_push_tail (queue, g_object_ref (file));
+
+ item_queue_handlers_set_up (processor);
}
static void
@@ -1448,6 +1636,8 @@ crawler_processing_directory_cb (TrackerCrawler *crawler,
/* Add files in queue to our queues to send to the indexer */
queue = g_hash_table_lookup (processor->private->items_created_queues, module_name);
g_queue_push_tail (queue, g_object_ref (file));
+
+ item_queue_handlers_set_up (processor);
}
static void
@@ -1468,34 +1658,27 @@ crawler_finished_cb (TrackerCrawler *crawler,
processor->private->directories_ignored += directories_ignored;
processor->private->files_found += files_found;
processor->private->files_ignored += files_ignored;
-
- /* If we have iterated all other modules, we know we are
- * working on removable devices.
- */
- if (processor->private->iterated_modules &&
- processor->private->removable_devices_current) {
- const gchar *root;
- root = processor->private->removable_devices_current->data;
+ processor->private->total_directories_found += directories_found;
+ processor->private->total_directories_ignored += directories_ignored;
+ processor->private->total_files_found += files_found;
+ processor->private->total_files_ignored += files_ignored;
+
+ if (processor->private->finished_modules == TRUE &&
+ processor->private->finished_devices == FALSE) {
+ GList *l;
+
+ l = processor->private->current_device;
- /* Don't add to the list if *already* on it. How can
- * this happen I hear you ask? :) Well, the crawler
- * will emit finished if a new directory is added and
- * it isn't monitored and has to be crawled. This can
- * happen on a removable device that we have already
- * scanned.
- */
- if (!g_list_find_custom (processor->private->removable_devices_completed,
- root,
- (GCompareFunc) g_strcmp0)) {
- processor->private->removable_devices_completed =
- g_list_append (processor->private->removable_devices_completed,
- g_strdup (root));
+ /* Now free that device so we don't recrawl it */
+ if (l) {
+ g_free (l->data);
+ processor->private->devices = g_list_delete_link (processor->private->devices, l);
}
}
-
- /* Proceed to next module */
- process_module_next (processor);
+
+ /* Proceed to next thing to process */
+ process_continue (processor);
}
#ifdef HAVE_HAL
@@ -1519,7 +1702,6 @@ mount_point_added_cb (TrackerStorage *hal,
TrackerProcessor *processor;
TrackerProcessorPrivate *priv;
TrackerStatus status;
- GList *l;
gchar *mp;
processor = user_data;
@@ -1529,21 +1711,12 @@ mount_point_added_cb (TrackerStorage *hal,
mp = normalize_mount_point (mount_point);
/* Add removable device to list of known devices to iterate */
- priv->removable_devices = g_list_append (priv->removable_devices, mp);
-
- /* Remove from completed list so we don't ignore it */
- l = g_list_find_custom (priv->removable_devices_completed,
- mp,
- (GCompareFunc) g_strcmp0);
-
- if (l) {
- g_free (l->data);
- priv->removable_devices_completed =
- g_list_delete_link (priv->removable_devices_completed, l);
+ if (!g_list_find_custom (priv->devices, mp, (GCompareFunc) g_strcmp0)) {
+ priv->devices = g_list_append (priv->devices, mp);
}
- /* Reset the current removable device */
- processor->private->removable_devices_current = processor->private->removable_devices;
+ /* Reset finished devices flag */
+ processor->private->finished_devices = FALSE;
/* If we are idle/not doing anything, start up the processor
* again so we handle the new location.
@@ -1555,7 +1728,7 @@ mount_point_added_cb (TrackerStorage *hal,
* crawled all locations so we need to start up the
* processor again for the removable media once more.
*/
- process_module_next (processor);
+ process_next (processor);
}
}
@@ -1578,24 +1751,11 @@ mount_point_removed_cb (TrackerStorage *hal,
/* Remove directory from list of iterated_removable_media, so
* we don't traverse it.
*/
- l = g_list_find_custom (priv->removable_devices,
- mp,
- (GCompareFunc) g_strcmp0);
+ l = g_list_find_custom (priv->devices, mp, (GCompareFunc) g_strcmp0);
if (l) {
g_free (l->data);
- priv->removable_devices =
- g_list_delete_link (priv->removable_devices, l);
- }
-
- l = g_list_find_custom (priv->removable_devices_completed,
- mp,
- (GCompareFunc) g_strcmp0);
-
- if (l) {
- g_free (l->data);
- priv->removable_devices_completed =
- g_list_delete_link (priv->removable_devices_completed, l);
+ priv->devices = g_list_delete_link (priv->devices, l);
}
/* Remove the monitor, the volumes are updated somewhere else
@@ -1638,16 +1798,7 @@ tracker_processor_new (TrackerConfig *config,
/* Set up hal */
priv->hal = g_object_ref (hal);
- priv->removable_devices = tracker_storage_get_removable_device_roots (priv->hal);
- priv->removable_devices_current = priv->removable_devices;
- priv->removable_devices_completed = NULL;
-
- g_signal_connect (priv->hal, "mount-point-added",
- G_CALLBACK (mount_point_added_cb),
- processor);
- g_signal_connect (priv->hal, "mount-point-removed",
- G_CALLBACK (mount_point_removed_cb),
- processor);
+ priv->devices = tracker_storage_get_removable_device_roots (priv->hal);
#endif /* HAVE_HAL */
/* Set up the crawlers now we have config and hal */
@@ -1698,6 +1849,10 @@ tracker_processor_new (TrackerConfig *config,
G_CALLBACK (indexer_status_cb),
processor,
NULL);
+ dbus_g_proxy_connect_signal (proxy, "Started",
+ G_CALLBACK (indexer_started_cb),
+ processor,
+ NULL);
dbus_g_proxy_connect_signal (proxy, "Finished",
G_CALLBACK (indexer_finished_cb),
processor,
@@ -1711,82 +1866,29 @@ tracker_processor_start (TrackerProcessor *processor)
{
g_return_if_fail (TRACKER_IS_PROCESSOR (processor));
- g_message ("Starting to process %d modules...",
- g_list_length (processor->private->modules));
-
- if (processor->private->timer) {
- g_timer_destroy (processor->private->timer);
- }
-
- processor->private->timer = g_timer_new ();
+ processor->private->interrupted = FALSE;
- processor->private->interrupted = TRUE;
+ processor->private->finished_modules = FALSE;
+ processor->private->finished_devices = FALSE;
+ processor->private->finished_sending = FALSE;
+ processor->private->finished_indexer = FALSE;
- process_module_next (processor);
+ process_next (processor);
}
-void
+void
tracker_processor_stop (TrackerProcessor *processor)
{
- gdouble elapsed;
-
g_return_if_fail (TRACKER_IS_PROCESSOR (processor));
- if (processor->private->interrupted) {
- TrackerCrawler *crawler;
-
- crawler = g_hash_table_lookup (processor->private->crawlers,
- processor->private->current_module->data);
- tracker_crawler_stop (crawler);
-
- }
-
-
- /* Now we have finished crawling, we enable monitor events */
- g_message ("Enabling monitor events");
- tracker_monitor_set_enabled (processor->private->monitor, TRUE);
-
- g_message ("Process %s\n",
- processor->private->finished ? "has finished" : "been stopped");
-
- if (processor->private->timer) {
- g_timer_stop (processor->private->timer);
- elapsed = g_timer_elapsed (processor->private->timer, NULL);
- } else {
- elapsed = 0;
- }
-
- g_message ("Total time taken : %4.4f seconds",
- elapsed);
- g_message ("Total directories: %d (%d ignored)",
- processor->private->directories_found,
- processor->private->directories_ignored);
- g_message ("Total files : %d (%d ignored)",
- processor->private->files_found,
- processor->private->files_ignored);
- g_message ("Total monitors : %d\n",
- tracker_monitor_get_count (processor->private->monitor, NULL));
-
- /* Here we set to IDLE when we were stopped, otherwise, we
- * we are currently in the process of sending files to the
- * indexer and we set the state to INDEXING
- */
- if (processor->private->interrupted) {
- tracker_status_set_and_signal (TRACKER_STATUS_OPTIMIZING);
- tracker_db_manager_optimize ();
+ processor->private->interrupted = TRUE;
- /* All done */
- tracker_status_set_and_signal (TRACKER_STATUS_IDLE);
+ process_modules_stop (processor);
+ process_devices_stop (processor);
+
+ /* Queues? */
- processor->private->finished = TRUE;
- g_signal_emit (processor, signals[FINISHED], 0);
- } else {
- /* Set status to IDLE, so it isn't left to any other state
- * if there are no files to be sent to the indexer.
- */
- tracker_status_set_and_signal (TRACKER_STATUS_IDLE);
- item_queue_handlers_set_up (processor);
- }
+ process_finish (processor);
}
void
@@ -1848,7 +1950,7 @@ tracker_processor_get_directories_found (TrackerProcessor *processor)
{
g_return_val_if_fail (TRACKER_IS_PROCESSOR (processor), 0);
- return processor->private->directories_found;
+ return processor->private->total_directories_found;
}
guint
@@ -1856,7 +1958,7 @@ tracker_processor_get_directories_ignored (TrackerProcessor *processor)
{
g_return_val_if_fail (TRACKER_IS_PROCESSOR (processor), 0);
- return processor->private->directories_ignored;
+ return processor->private->total_directories_ignored;
}
guint
@@ -1864,7 +1966,7 @@ tracker_processor_get_directories_total (TrackerProcessor *processor)
{
g_return_val_if_fail (TRACKER_IS_PROCESSOR (processor), 0);
- return processor->private->directories_found + processor->private->directories_ignored;
+ return processor->private->total_directories_found + processor->private->total_directories_ignored;
}
guint
@@ -1872,7 +1974,7 @@ tracker_processor_get_files_found (TrackerProcessor *processor)
{
g_return_val_if_fail (TRACKER_IS_PROCESSOR (processor), 0);
- return processor->private->files_found;
+ return processor->private->total_files_found;
}
guint
@@ -1880,7 +1982,7 @@ tracker_processor_get_files_ignored (TrackerProcessor *processor)
{
g_return_val_if_fail (TRACKER_IS_PROCESSOR (processor), 0);
- return processor->private->files_ignored;
+ return processor->private->total_files_ignored;
}
guint
@@ -1888,7 +1990,7 @@ tracker_processor_get_files_total (TrackerProcessor *processor)
{
g_return_val_if_fail (TRACKER_IS_PROCESSOR (processor), 0);
- return processor->private->files_found + processor->private->files_ignored;
+ return processor->private->total_files_found + processor->private->total_files_ignored;
}
gdouble
@@ -1896,5 +1998,5 @@ tracker_processor_get_seconds_elapsed (TrackerProcessor *processor)
{
g_return_val_if_fail (TRACKER_IS_PROCESSOR (processor), 0);
- return processor->private->seconds_elapsed;
+ return processor->private->indexer_seconds_elapsed;
}
diff --git a/src/trackerd/tracker-status.c b/src/trackerd/tracker-status.c
index 704c6bd..db355b6 100644
--- a/src/trackerd/tracker-status.c
+++ b/src/trackerd/tracker-status.c
@@ -59,6 +59,7 @@ typedef struct {
gboolean is_ready;
gboolean is_running;
gboolean is_first_time_index;
+ gboolean is_initial_check;
gboolean is_paused_manually;
gboolean is_paused_for_batt;
gboolean is_paused_for_io;
@@ -535,6 +536,7 @@ tracker_status_init (TrackerConfig *config,
private->is_ready = FALSE;
private->is_running = FALSE;
private->is_first_time_index = FALSE;
+ private->is_initial_check = FALSE;
private->is_paused_manually = FALSE;
private->is_paused_for_batt = FALSE;
private->is_paused_for_io = FALSE;
@@ -1007,6 +1009,31 @@ tracker_status_set_is_first_time_index (gboolean value)
}
gboolean
+tracker_status_get_is_initial_check (void)
+{
+ TrackerStatusPrivate *private;
+
+ private = g_static_private_get (&private_key);
+ g_return_val_if_fail (private != NULL, FALSE);
+
+ return private->is_initial_check;
+}
+
+void
+tracker_status_set_is_initial_check (gboolean value)
+{
+ TrackerStatusPrivate *private;
+
+ private = g_static_private_get (&private_key);
+ g_return_if_fail (private != NULL);
+
+ /* Set value */
+ private->is_initial_check = value;
+
+ /* We don't need to signal this */
+}
+
+gboolean
tracker_status_get_in_merge (void)
{
TrackerStatusPrivate *private;
@@ -1178,4 +1205,3 @@ tracker_status_set_is_paused_for_dbus (gboolean value)
/* Set indexer state and our state to paused or not */
indexer_recheck (TRUE, TRUE, emit);
}
-
diff --git a/src/trackerd/tracker-status.h b/src/trackerd/tracker-status.h
index 8c86fb7..d4d8202 100644
--- a/src/trackerd/tracker-status.h
+++ b/src/trackerd/tracker-status.h
@@ -69,6 +69,9 @@ void tracker_status_set_is_running (gboolean value);
void tracker_status_set_is_first_time_index (gboolean value);
gboolean tracker_status_get_is_first_time_index (void);
+void tracker_status_set_is_initial_check (gboolean value);
+gboolean tracker_status_get_is_initial_check (void);
+
gboolean tracker_status_get_in_merge (void);
void tracker_status_set_in_merge (gboolean value);
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
