tracker r1807 - in branches/indexer-split: . data/modules src/libtracker-common src/trackerd
- From: mr svn gnome org
- To: svn-commits-list gnome org
- Subject: tracker r1807 - in branches/indexer-split: . data/modules src/libtracker-common src/trackerd
- Date: Tue, 1 Jul 2008 11:44:34 +0000 (UTC)
Author: mr
Date: Tue Jul 1 11:44:34 2008
New Revision: 1807
URL: http://svn.gnome.org/viewvc/tracker?rev=1807&view=rev
Log:
* data/modules/files.module: Added $HOME to the list of recursive
monitored directorys.
* src/libtracker-common/tracker-file-utils.c: Add comment to the
source so we know if memory is created or if a new list is the
only thing that is created from the
tracker_path_list_filter_duplicates() function.
* src/trackerd/tracker-processor.c: Added code to iterate all
modules and crawl each of them in turn. More work is needed here
to do other things like setting up monitors. This shouldn't be
done in the crawler on EVERY directory as it currently is.
Modified:
branches/indexer-split/ChangeLog
branches/indexer-split/data/modules/files.module
branches/indexer-split/src/libtracker-common/tracker-file-utils.c
branches/indexer-split/src/libtracker-common/tracker-module-config.c
branches/indexer-split/src/libtracker-common/tracker-module-config.h
branches/indexer-split/src/trackerd/tracker-crawler.c
branches/indexer-split/src/trackerd/tracker-crawler.h
branches/indexer-split/src/trackerd/tracker-processor.c
Modified: branches/indexer-split/data/modules/files.module
==============================================================================
--- branches/indexer-split/data/modules/files.module (original)
+++ branches/indexer-split/data/modules/files.module Tue Jul 1 11:44:34 2008
@@ -4,7 +4,7 @@
[Monitors]
Directories=
-RecurseDirectories=
+RecurseDirectories=$HOME;
[Ignored]
Directories=po;CVS;.svn;.git
Modified: branches/indexer-split/src/libtracker-common/tracker-file-utils.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-file-utils.c (original)
+++ branches/indexer-split/src/libtracker-common/tracker-file-utils.c Tue Jul 1 11:44:34 2008
@@ -476,6 +476,11 @@
GSList *checked_roots = NULL;
GSList *l1, *l2;
+ /* This function CREATES a new list and the data in the list
+ * is new too! g_free() must be called on the list data and
+ * g_slist_free() on the list too when done with.
+ */
+
/* ONLY HERE do we add separators on each location we check.
* The reason for this is that these locations are user
* entered in the configuration and we need to make sure we
@@ -511,6 +516,7 @@
if (g_str_has_prefix (l2->data, path)) {
checked_roots = g_slist_remove_link (checked_roots, l2);
g_free (l2->data);
+
l2 = checked_roots;
continue;
}
Modified: branches/indexer-split/src/libtracker-common/tracker-module-config.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-module-config.c (original)
+++ branches/indexer-split/src/libtracker-common/tracker-module-config.c Tue Jul 1 11:44:34 2008
@@ -38,22 +38,25 @@
typedef struct {
/* General */
- gchar *description;
- gboolean enabled;
-
+ gchar *description;
+ gboolean enabled;
+
/* Monitors */
- GSList *monitor_directories;
- GSList *monitor_recurse_directories;
-
+ GHashTable *monitor_directories;
+ GHashTable *monitor_recurse_directories;
+
/* Ignored */
- GSList *ignored_directories;
- GSList *ignored_files;
+ GHashTable *ignored_directories;
+ GHashTable *ignored_files;
+
+ GList *ignored_directory_patterns;
+ GList *ignored_file_patterns;
/* Index */
- gchar *service;
- GSList *mime_types;
- GSList *files;
-
+ gchar *service;
+ GHashTable *mime_types;
+ GHashTable *files;
+
/* Specific Options, FIXME: Finish */
} ModuleConfig;
@@ -65,27 +68,27 @@
static void
module_config_free (ModuleConfig *mc)
{
- g_free (mc->description);
-
- g_slist_foreach (mc->monitor_directories, (GFunc) g_free, NULL);
- g_slist_free (mc->monitor_directories);
-
- g_slist_foreach (mc->monitor_recurse_directories, (GFunc) g_free, NULL);
- g_slist_free (mc->monitor_recurse_directories);
-
- g_slist_foreach (mc->ignored_directories, (GFunc) g_free, NULL);
- g_slist_free (mc->ignored_directories);
+ g_hash_table_unref (mc->files);
+ g_hash_table_unref (mc->mime_types);
+ g_free (mc->service);
- g_slist_foreach (mc->ignored_files, (GFunc) g_free, NULL);
- g_slist_free (mc->ignored_files);
+ g_list_foreach (mc->ignored_file_patterns,
+ (GFunc) g_pattern_spec_free,
+ NULL);
+ g_list_free (mc->ignored_file_patterns);
+
+ g_list_foreach (mc->ignored_directory_patterns,
+ (GFunc) g_pattern_spec_free,
+ NULL);
+ g_list_free (mc->ignored_directory_patterns);
- g_free (mc->service);
+ g_hash_table_unref (mc->ignored_files);
+ g_hash_table_unref (mc->ignored_directories);
- g_slist_foreach (mc->mime_types, (GFunc) g_free, NULL);
- g_slist_free (mc->mime_types);
+ g_hash_table_unref (mc->monitor_recurse_directories);
+ g_hash_table_unref (mc->monitor_directories);
- g_slist_foreach (mc->files, (GFunc) g_free, NULL);
- g_slist_free (mc->files);
+ g_free (mc->description);
g_slice_free (ModuleConfig, mc);
}
@@ -96,7 +99,61 @@
return g_build_path (G_DIR_SEPARATOR_S, SHAREDIR, "tracker", "modules", NULL);
}
-gboolean
+static void
+module_config_set_ignored_file_patterns (ModuleConfig *mc)
+{
+ GPatternSpec *spec;
+ GList *ignored_files;
+ GList *l;
+ GList *patterns = NULL;
+
+ g_list_foreach (mc->ignored_file_patterns,
+ (GFunc) g_pattern_spec_free,
+ NULL);
+ g_list_free (mc->ignored_file_patterns);
+
+ ignored_files = g_hash_table_get_keys (mc->ignored_files);
+
+ for (l = ignored_files; l; l = l->next) {
+ g_message (" Adding file ignore pattern:'%s'",
+ (gchar *) l->data);
+ spec = g_pattern_spec_new (l->data);
+ patterns = g_list_prepend (patterns, spec);
+ }
+
+ g_list_free (ignored_files);
+
+ mc->ignored_file_patterns = g_list_reverse (patterns);
+}
+
+static void
+module_config_set_ignored_directory_patterns (ModuleConfig *mc)
+{
+ GPatternSpec *spec;
+ GList *ignored_directories;
+ GList *l;
+ GList *patterns = NULL;
+
+ g_list_foreach (mc->ignored_directory_patterns,
+ (GFunc) g_pattern_spec_free,
+ NULL);
+ g_list_free (mc->ignored_directory_patterns);
+
+ ignored_directories = g_hash_table_get_keys (mc->ignored_directories);
+
+ for (l = ignored_directories; l; l = l->next) {
+ g_message (" Adding directory ignore pattern:'%s'",
+ (gchar *) l->data);
+ spec = g_pattern_spec_new (l->data);
+ patterns = g_list_prepend (patterns, spec);
+ }
+
+ g_list_free (ignored_directories);
+
+ mc->ignored_directory_patterns = g_list_reverse (patterns);
+}
+
+static gboolean
module_config_load_boolean (GKeyFile *key_file,
const gchar *group,
const gchar *key)
@@ -108,11 +165,11 @@
if (error) {
g_message ("Couldn't load module config boolean in "
- "group:'%s' with key:'%s', %s",
+ "group:'%s' with key:'%s', %s",
group,
- key,
+ key,
error->message);
-
+
g_error_free (error);
g_key_file_free (key_file);
@@ -122,7 +179,7 @@
return boolean;
}
-gchar *
+static gchar *
module_config_load_string (GKeyFile *key_file,
const gchar *group,
const gchar *key,
@@ -135,11 +192,11 @@
if (error) {
g_message ("Couldn't load module config string in "
- "group:'%s' with key:'%s', %s",
+ "group:'%s' with key:'%s', %s",
group,
- key,
+ key,
error->message);
-
+
g_error_free (error);
g_key_file_free (key_file);
@@ -158,52 +215,71 @@
return str;
}
-GSList *
+static GHashTable *
module_config_load_string_list (GKeyFile *key_file,
const gchar *group,
const gchar *key,
gboolean expand_strings_as_paths)
{
- GError *error = NULL;
- GSList *list;
- gchar **str;
- gchar **p;
- gsize size;
+ GError *error = NULL;
+ GHashTable *table;
+ gchar **str;
+ gchar **p;
+ gsize size;
+
+ table = g_hash_table_new_full (g_str_hash,
+ g_str_equal,
+ g_free,
+ NULL);
str = g_key_file_get_string_list (key_file, group, key, &size, &error);
if (error) {
g_message ("Couldn't load module config string list in "
- "group:'%s' with key:'%s', %s",
+ "group:'%s' with key:'%s', %s",
group,
- key,
+ key,
error->message);
-
+
g_error_free (error);
g_key_file_free (key_file);
- return NULL;
+ return table;
}
- if (!expand_strings_as_paths) {
- list = tracker_string_list_to_gslist (str, size);
- } else {
- list = NULL;
+ for (p = str; *p; p++) {
+ gchar *real_path;
- for (p = str; *p; p++) {
- gchar *real_path;
-
+ if (!expand_strings_as_paths) {
+ if (g_hash_table_lookup (table, *p)) {
+ continue;
+ }
+
+ g_hash_table_insert (table,
+ g_strdup (*p),
+ GINT_TO_POINTER (1));
+ } else {
+ if (g_hash_table_lookup (table, *p)) {
+ continue;
+ }
+
real_path = tracker_path_evaluate_name (*p);
- list = g_slist_prepend (list, real_path);
+ if (g_hash_table_lookup (table, real_path)) {
+ g_free (real_path);
+ continue;
+ }
+
+ g_hash_table_insert (table,
+ real_path,
+ GINT_TO_POINTER (1));
g_debug ("Got real path:'%s' for '%s'", real_path, *p);
}
- list = g_slist_reverse (list);
}
g_strfreev (str);
- return list;
+ return table;
}
static ModuleConfig *
@@ -214,79 +290,82 @@
ModuleConfig *mc;
key_file = g_key_file_new ();
-
+
/* Load options */
g_key_file_load_from_file (key_file, filename, G_KEY_FILE_NONE, &error);
if (error) {
- g_message ("Couldn't load module config for '%s', %s",
- filename,
+ g_message ("Couldn't load module config for '%s', %s",
+ filename,
error->message);
-
+
g_error_free (error);
g_key_file_free (key_file);
return NULL;
}
+ g_message ("Loading module config:'%s'", filename);
+
mc = g_slice_new0 (ModuleConfig);
/* General */
- mc->description =
+ mc->description =
module_config_load_string (key_file,
GROUP_GENERAL,
- "Description",
+ "Description",
FALSE);
- mc->enabled =
+ mc->enabled =
module_config_load_boolean (key_file,
GROUP_GENERAL,
"Enabled");
/* Monitors */
- mc->monitor_directories =
- module_config_load_string_list (key_file,
- GROUP_MONITORS,
+ mc->monitor_directories =
+ module_config_load_string_list (key_file,
+ GROUP_MONITORS,
"Directories",
TRUE);
- mc->monitor_recurse_directories =
- module_config_load_string_list (key_file,
- GROUP_MONITORS,
+ mc->monitor_recurse_directories =
+ module_config_load_string_list (key_file,
+ GROUP_MONITORS,
"RecurseDirectories",
TRUE);
/* Ignored */
- mc->ignored_directories =
- module_config_load_string_list (key_file,
- GROUP_IGNORED,
+ mc->ignored_directories =
+ module_config_load_string_list (key_file,
+ GROUP_IGNORED,
"Directories",
TRUE);
- mc->ignored_files =
- module_config_load_string_list (key_file,
- GROUP_IGNORED,
+ mc->ignored_files =
+ module_config_load_string_list (key_file,
+ GROUP_IGNORED,
"Files",
FALSE);
/* Index */
- mc->service =
+ mc->service =
module_config_load_string (key_file,
GROUP_INDEX,
"Service",
FALSE);
- mc->mime_types =
- module_config_load_string_list (key_file,
- GROUP_INDEX,
+ mc->mime_types =
+ module_config_load_string_list (key_file,
+ GROUP_INDEX,
"MimeTypes",
FALSE);
- mc->files =
- module_config_load_string_list (key_file,
- GROUP_INDEX,
+ mc->files =
+ module_config_load_string_list (key_file,
+ GROUP_INDEX,
"Files",
FALSE);
-
+
/* FIXME: Specific options */
- g_message ("Loaded module config:'%s'", filename);
-
+ module_config_set_ignored_file_patterns (mc);
+ module_config_set_ignored_directory_patterns (mc);
+
return mc;
}
@@ -310,7 +389,7 @@
G_FILE_ATTRIBUTE_STANDARD_NAME ","
G_FILE_ATTRIBUTE_STANDARD_TYPE,
G_PRIORITY_DEFAULT,
- NULL,
+ NULL,
&error);
if (error) {
@@ -328,7 +407,7 @@
extension = ".module";
extension_len = g_utf8_strlen (extension, -1);
- /* We should probably do this async */
+ /* We should probably do this async */
for (info = g_file_enumerator_next_file (enumerator, NULL, &error);
info && !error;
info = g_file_enumerator_next_file (enumerator, NULL, &error)) {
@@ -368,7 +447,7 @@
}
g_message ("Loaded module config, %d found",
- g_hash_table_size (modules));
+ g_hash_table_size (modules));
g_object_unref (enumerator);
g_object_unref (file);
@@ -382,7 +461,7 @@
GFile *file,
GFile *other_file,
GFileMonitorEvent event_type,
- gpointer user_data)
+ gpointer user_data)
{
gchar *filename;
@@ -392,8 +471,8 @@
case G_FILE_MONITOR_EVENT_CHANGED:
case G_FILE_MONITOR_EVENT_CHANGES_DONE_HINT:
filename = g_file_get_path (file);
- g_message ("Config file changed:'%s', reloading settings...",
- filename);
+ g_message ("Config file changed:'%s', reloading settings...",
+ filename);
g_free (filename);
module_config_load ();
@@ -434,17 +513,17 @@
}
/* Add file monitoring for changes */
- g_message ("Setting up monitor for changes to modules directory:'%s'",
+ g_message ("Setting up monitor for changes to modules directory:'%s'",
path);
-
+
file = g_file_new_for_path (path);
monitor = g_file_monitor_directory (file,
G_FILE_MONITOR_NONE,
NULL,
NULL);
-
+
g_signal_connect (monitor, "changed",
- G_CALLBACK (module_config_changed_cb),
+ G_CALLBACK (module_config_changed_cb),
NULL);
g_object_unref (file);
@@ -454,17 +533,17 @@
return TRUE;
}
-void
+void
tracker_module_config_shutdown (void)
{
if (!initiated) {
return;
}
-
+
g_signal_handlers_disconnect_by_func (monitor,
module_config_changed_cb,
NULL);
-
+
g_object_unref (monitor);
g_hash_table_unref (modules);
@@ -495,7 +574,7 @@
tracker_module_config_get_enabled (const gchar *name)
{
ModuleConfig *mc;
-
+
g_return_val_if_fail (name != NULL, FALSE);
mc = g_hash_table_lookup (modules, name);
@@ -504,56 +583,56 @@
return mc->enabled;
}
-GSList *
+GList *
tracker_module_config_get_monitor_directories (const gchar *name)
{
ModuleConfig *mc;
-
+
g_return_val_if_fail (name != NULL, FALSE);
mc = g_hash_table_lookup (modules, name);
g_return_val_if_fail (mc, NULL);
- return mc->monitor_directories;
+ return g_hash_table_get_keys (mc->monitor_directories);
}
-GSList *
+GList *
tracker_module_config_get_monitor_recurse_directories (const gchar *name)
{
ModuleConfig *mc;
-
+
g_return_val_if_fail (name != NULL, FALSE);
mc = g_hash_table_lookup (modules, name);
g_return_val_if_fail (mc, NULL);
- return mc->monitor_recurse_directories;
+ return g_hash_table_get_keys (mc->monitor_recurse_directories);
}
-GSList *
+GList *
tracker_module_config_get_ignored_directories (const gchar *name)
{
ModuleConfig *mc;
-
+
g_return_val_if_fail (name != NULL, FALSE);
mc = g_hash_table_lookup (modules, name);
g_return_val_if_fail (mc, NULL);
- return mc->ignored_directories;
+ return g_hash_table_get_keys (mc->ignored_directories);
}
-GSList *
+GList *
tracker_module_config_get_ignored_files (const gchar *name)
{
ModuleConfig *mc;
-
+
g_return_val_if_fail (name != NULL, FALSE);
mc = g_hash_table_lookup (modules, name);
g_return_val_if_fail (mc, NULL);
- return mc->ignored_files;
+ return g_hash_table_get_keys (mc->ignored_files);
}
const gchar *
@@ -569,28 +648,58 @@
return mc->service;
}
-GSList *
+GList *
tracker_module_config_get_mime_types (const gchar *name)
{
ModuleConfig *mc;
-
+
g_return_val_if_fail (name != NULL, FALSE);
mc = g_hash_table_lookup (modules, name);
g_return_val_if_fail (mc, NULL);
- return mc->mime_types;
+ return g_hash_table_get_keys (mc->mime_types);
}
-GSList *
+GList *
tracker_module_config_get_files (const gchar *name)
{
ModuleConfig *mc;
-
+
g_return_val_if_fail (name != NULL, FALSE);
mc = g_hash_table_lookup (modules, name);
g_return_val_if_fail (mc, NULL);
- return mc->files;
+ return g_hash_table_get_keys (mc->files);
+}
+
+/*
+ * Convenience functions
+ */
+
+GList *
+tracker_module_config_get_ignored_file_patterns (const gchar *name)
+{
+ ModuleConfig *mc;
+
+ g_return_val_if_fail (name != NULL, NULL);
+
+ mc = g_hash_table_lookup (modules, name);
+ g_return_val_if_fail (mc, NULL);
+
+ return g_list_copy (mc->ignored_file_patterns);
+}
+
+GList *
+tracker_module_config_get_ignored_directory_patterns (const gchar *name)
+{
+ ModuleConfig *mc;
+
+ g_return_val_if_fail (name != NULL, NULL);
+
+ mc = g_hash_table_lookup (modules, name);
+ g_return_val_if_fail (mc, NULL);
+
+ return g_list_copy (mc->ignored_directory_patterns);
}
Modified: branches/indexer-split/src/libtracker-common/tracker-module-config.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-module-config.h (original)
+++ branches/indexer-split/src/libtracker-common/tracker-module-config.h Tue Jul 1 11:44:34 2008
@@ -27,19 +27,21 @@
gboolean tracker_module_config_init (void);
void tracker_module_config_shutdown (void);
-
GList * tracker_module_config_get_modules (void);
-
const gchar *tracker_module_config_get_description (const gchar *name);
gboolean tracker_module_config_get_enabled (const gchar *name);
-GSList * tracker_module_config_get_monitor_directories (const gchar *name);
-GSList * tracker_module_config_get_monitor_recurse_directories (const gchar *name);
-GSList * tracker_module_config_get_ignored_directories (const gchar *name);
-GSList * tracker_module_config_get_ignored_files (const gchar *name);
+GList * tracker_module_config_get_monitor_directories (const gchar *name);
+GList * tracker_module_config_get_monitor_recurse_directories (const gchar *name);
+GList * tracker_module_config_get_ignored_directories (const gchar *name);
+GList * tracker_module_config_get_ignored_files (const gchar *name);
const gchar *tracker_module_config_get_service (const gchar *name);
const gchar *tracker_module_config_get_service (const gchar *name);
-GSList * tracker_module_config_get_mime_types (const gchar *name);
-GSList * tracker_module_config_get_files (const gchar *name);
+GList * tracker_module_config_get_mime_types (const gchar *name);
+GList * tracker_module_config_get_files (const gchar *name);
+
+/* Convenience functions */
+GList * tracker_module_config_get_ignored_directory_patterns (const gchar *name);
+GList * tracker_module_config_get_ignored_file_patterns (const gchar *name);
G_END_DECLS
Modified: branches/indexer-split/src/trackerd/tracker-crawler.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.c (original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.c Tue Jul 1 11:44:34 2008
@@ -1,5 +1,5 @@
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
-/*
+/*
* Copyright (C) 2008, Nokia
*
* This library is free software; you can redistribute it and/or
@@ -27,15 +27,16 @@
#include <libtracker-common/tracker-dbus.h>
#include <libtracker-common/tracker-file-utils.h>
#include <libtracker-common/tracker-utils.h>
+#include <libtracker-common/tracker-module-config.h>
#include "tracker-crawler.h"
#include "tracker-dbus.h"
#include "tracker-indexer-client.h"
#include "tracker-monitor.h"
-#define GET_PRIV(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_CRAWLER, TrackerCrawlerPriv))
+#define TRACKER_CRAWLER_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_CRAWLER, TrackerCrawlerPrivate))
-/*#define TESTING*/
+#define TESTING
#define FILE_ATTRIBUTES \
G_FILE_ATTRIBUTE_STANDARD_NAME "," \
@@ -44,7 +45,7 @@
#define FILES_QUEUE_PROCESS_INTERVAL 2000
#define FILES_QUEUE_PROCESS_MAX 5000
-struct _TrackerCrawlerPriv {
+struct _TrackerCrawlerPrivate {
TrackerConfig *config;
#ifdef HAVE_HAL
TrackerHal *hal;
@@ -52,32 +53,41 @@
GTimer *timer;
- GAsyncQueue *files;
- guint files_queue_handle_id;
-
- GSList *ignored_file_types;
+ GQueue *directories;
+ GQueue *files;
- GHashTable *ignored_names;
- GHashTable *temp_black_list;
+ guint idle_id;
+ guint files_queue_handle_id;
- gchar **ignored_suffixes;
- gchar **ignored_prefixes;
+ /* Specific to each crawl ... */
+ GList *ignored_directory_patterns;
+ GList *ignored_file_patterns;
+ gchar *current_module_name;
+ /* Statistics */
guint enumerations;
- guint files_found;
- guint files_ignored;
-
+ guint directories_found;
+ guint directories_ignored;
+ guint files_found;
+ guint files_ignored;
+
gboolean running;
+ gboolean finished;
};
enum {
PROP_0,
PROP_CONFIG,
-#ifdef HAVE_HAL
+#ifdef HAVE_HAL
PROP_HAL
#endif
};
+enum {
+ FINISHED,
+ LAST_SIGNAL
+};
+
typedef struct {
TrackerCrawler *crawler;
GFile *parent;
@@ -88,7 +98,6 @@
guint param_id,
const GValue *value,
GParamSpec *pspec);
-static void set_ignored_file_types (TrackerCrawler *crawler);
#ifdef HAVE_HAL
static void mount_point_added_cb (TrackerHal *hal,
@@ -103,7 +112,12 @@
EnumeratorData *ed);
static void file_enumerate_children (TrackerCrawler *crawler,
GFile *file);
+
+#if 0
static void file_queue_handler_set_up (TrackerCrawler *crawler);
+#endif
+
+static guint signals[LAST_SIGNAL] = { 0, };
G_DEFINE_TYPE(TrackerCrawler, tracker_crawler, G_TYPE_OBJECT)
@@ -125,7 +139,7 @@
tracker_config_get_type (),
G_PARAM_WRITABLE));
-#ifdef HAVE_HAL
+#ifdef HAVE_HAL
g_object_class_install_property (object_class,
PROP_HAL,
g_param_spec_object ("hal",
@@ -135,129 +149,59 @@
G_PARAM_WRITABLE));
#endif /* HAVE_HAL */
- g_type_class_add_private (object_class, sizeof (TrackerCrawlerPriv));
+ signals[FINISHED] =
+ g_signal_new ("finished",
+ G_TYPE_FROM_CLASS (klass),
+ G_SIGNAL_RUN_LAST,
+ 0,
+ NULL, NULL,
+ g_cclosure_marshal_VOID__VOID,
+ G_TYPE_NONE,
+ 0);
+
+ g_type_class_add_private (object_class, sizeof (TrackerCrawlerPrivate));
}
static void
tracker_crawler_init (TrackerCrawler *object)
{
- TrackerCrawlerPriv *priv;
- GPtrArray *ptr_array;
+ TrackerCrawlerPrivate *priv;
- object->priv = GET_PRIV (object);
+ object->private = TRACKER_CRAWLER_GET_PRIVATE (object);
- priv = object->priv;
+ priv = object->private;
- /* Create async queue for handling file pushes to the indexer */
- priv->files = g_async_queue_new ();
-
- /* Whole file names to ignore */
- priv->ignored_names = g_hash_table_new (g_str_hash, g_str_equal);
- g_hash_table_insert (priv->ignored_names, "po", NULL);
- g_hash_table_insert (priv->ignored_names, "CVS", NULL);
- g_hash_table_insert (priv->ignored_names, "Makefile", NULL);
- g_hash_table_insert (priv->ignored_names, "SCCS", NULL);
- g_hash_table_insert (priv->ignored_names, "ltmain.sh", NULL);
- g_hash_table_insert (priv->ignored_names, "libtool", NULL);
- g_hash_table_insert (priv->ignored_names, "config.status", NULL);
- g_hash_table_insert (priv->ignored_names, "conftest", NULL);
- g_hash_table_insert (priv->ignored_names, "confdefs.h", NULL);
-
- /* Temporary black list */
- priv->temp_black_list = g_hash_table_new_full (g_str_hash,
- g_str_equal,
- g_free,
- NULL);
-
- /* Suffixes to ignore */
- ptr_array = g_ptr_array_new ();
- g_ptr_array_add (ptr_array, g_strdup ("~"));
- g_ptr_array_add (ptr_array, g_strdup (".o"));
- g_ptr_array_add (ptr_array, g_strdup (".la"));
- g_ptr_array_add (ptr_array, g_strdup (".lo"));
- g_ptr_array_add (ptr_array, g_strdup (".loT"));
- g_ptr_array_add (ptr_array, g_strdup (".in"));
- g_ptr_array_add (ptr_array, g_strdup (".csproj"));
- g_ptr_array_add (ptr_array, g_strdup (".m4"));
- g_ptr_array_add (ptr_array, g_strdup (".rej"));
- g_ptr_array_add (ptr_array, g_strdup (".gmo"));
- g_ptr_array_add (ptr_array, g_strdup (".orig"));
- g_ptr_array_add (ptr_array, g_strdup (".pc"));
- g_ptr_array_add (ptr_array, g_strdup (".omf"));
- g_ptr_array_add (ptr_array, g_strdup (".aux"));
- g_ptr_array_add (ptr_array, g_strdup (".tmp"));
- g_ptr_array_add (ptr_array, g_strdup (".po"));
- g_ptr_array_add (ptr_array, g_strdup (".vmdk"));
- g_ptr_array_add (ptr_array, g_strdup (".vmx"));
- g_ptr_array_add (ptr_array, g_strdup (".vmxf"));
- g_ptr_array_add (ptr_array, g_strdup (".vmsd"));
- g_ptr_array_add (ptr_array, g_strdup (".nvram"));
- g_ptr_array_add (ptr_array, g_strdup (".part"));
- g_ptr_array_add (ptr_array, NULL);
- priv->ignored_suffixes = (gchar **) g_ptr_array_free (ptr_array, FALSE);
-
- /* Prefixes to ignore */
- ptr_array = g_ptr_array_new ();
- g_ptr_array_add (ptr_array, g_strdup ("autom4te"));
- g_ptr_array_add (ptr_array, g_strdup ("conftest."));
- g_ptr_array_add (ptr_array, g_strdup ("confstat"));
- g_ptr_array_add (ptr_array, g_strdup ("config."));
- g_ptr_array_add (ptr_array, NULL);
- priv->ignored_prefixes = (gchar **) g_ptr_array_free (ptr_array, FALSE);
+ priv->directories = g_queue_new ();
+ priv->files = g_queue_new ();
}
static void
crawler_finalize (GObject *object)
{
- TrackerCrawlerPriv *priv;
- gchar *str;
- gint i;
-
- priv = GET_PRIV (object);
-
- for (i = 0; priv->ignored_prefixes[i] != NULL; i++) {
- g_free (priv->ignored_prefixes[i]);
- }
-
- g_free (priv->ignored_prefixes);
-
- for (i = 0; priv->ignored_suffixes[i] != NULL; i++) {
- g_free (priv->ignored_suffixes[i]);
- }
+ TrackerCrawlerPrivate *priv;
- g_free (priv->ignored_suffixes);
+ priv = TRACKER_CRAWLER_GET_PRIVATE (object);
- g_hash_table_unref (priv->temp_black_list);
- g_hash_table_unref (priv->ignored_names);
- g_slist_foreach (priv->ignored_file_types,
- (GFunc) g_pattern_spec_free,
- NULL);
- g_slist_free (priv->ignored_file_types);
+ tracker_crawler_stop (TRACKER_CRAWLER (object));
if (priv->files_queue_handle_id) {
- g_source_remove (priv->files_queue_handle_id);
+ g_source_remove (priv->files_queue_handle_id);
priv->files_queue_handle_id = 0;
}
- for (str = g_async_queue_try_pop (priv->files);
- str;
- str = g_async_queue_try_pop (priv->files)) {
- g_free (str);
- }
+ g_queue_foreach (priv->files, (GFunc) g_free, NULL);
+ g_queue_free (priv->files);
- g_async_queue_unref (priv->files);
+ g_queue_foreach (priv->directories, (GFunc) g_free, NULL);
+ g_queue_free (priv->directories);
- if (priv->timer) {
- g_timer_destroy (priv->timer);
- }
-
-#ifdef HAVE_HAL
+#ifdef HAVE_HAL
if (priv->hal) {
- g_signal_handlers_disconnect_by_func (priv->hal,
+ g_signal_handlers_disconnect_by_func (priv->hal,
mount_point_added_cb,
object);
- g_signal_handlers_disconnect_by_func (priv->hal,
+ g_signal_handlers_disconnect_by_func (priv->hal,
mount_point_removed_cb,
object);
@@ -278,9 +222,9 @@
const GValue *value,
GParamSpec *pspec)
{
- TrackerCrawlerPriv *priv;
+ TrackerCrawlerPrivate *priv;
- priv = GET_PRIV (object);
+ priv = TRACKER_CRAWLER_GET_PRIVATE (object);
switch (param_id) {
case PROP_CONFIG:
@@ -306,21 +250,21 @@
{
g_return_val_if_fail (TRACKER_IS_CONFIG (config), NULL);
- return g_object_new (TRACKER_TYPE_CRAWLER,
+ return g_object_new (TRACKER_TYPE_CRAWLER,
"config", config,
- NULL);
+ NULL);
}
void
tracker_crawler_set_config (TrackerCrawler *object,
TrackerConfig *config)
{
- TrackerCrawlerPriv *priv;
+ TrackerCrawlerPrivate *priv;
g_return_if_fail (TRACKER_IS_CRAWLER (object));
g_return_if_fail (TRACKER_IS_CONFIG (config));
- priv = GET_PRIV (object);
+ priv = TRACKER_CRAWLER_GET_PRIVATE (object);
if (config) {
g_object_ref (config);
@@ -332,40 +276,37 @@
priv->config = config;
- /* The ignored file types depend on the config */
- set_ignored_file_types (object);
-
g_object_notify (G_OBJECT (object), "config");
}
-#ifdef HAVE_HAL
+#ifdef HAVE_HAL
void
tracker_crawler_set_hal (TrackerCrawler *object,
TrackerHal *hal)
{
- TrackerCrawlerPriv *priv;
+ TrackerCrawlerPrivate *priv;
g_return_if_fail (TRACKER_IS_CRAWLER (object));
g_return_if_fail (TRACKER_IS_HAL (hal));
- priv = GET_PRIV (object);
+ priv = TRACKER_CRAWLER_GET_PRIVATE (object);
if (hal) {
- g_signal_connect (hal, "mount-point-added",
+ g_signal_connect (hal, "mount-point-added",
G_CALLBACK (mount_point_added_cb),
object);
- g_signal_connect (hal, "mount-point-removed",
+ g_signal_connect (hal, "mount-point-removed",
G_CALLBACK (mount_point_removed_cb),
object);
g_object_ref (hal);
}
if (priv->hal) {
- g_signal_handlers_disconnect_by_func (hal,
+ g_signal_handlers_disconnect_by_func (hal,
mount_point_added_cb,
object);
- g_signal_handlers_disconnect_by_func (hal,
+ g_signal_handlers_disconnect_by_func (hal,
mount_point_removed_cb,
object);
g_object_unref (priv->hal);
@@ -381,36 +322,6 @@
/*
* Functions
*/
-static void
-set_ignored_file_types (TrackerCrawler *crawler)
-{
- TrackerCrawlerPriv *priv;
- GPatternSpec *spec;
- GSList *ignored_file_types;
- GSList *patterns = NULL;
- GSList *l;
-
- priv = crawler->priv;
-
- g_slist_foreach (priv->ignored_file_types,
- (GFunc) g_pattern_spec_free,
- NULL);
- g_slist_free (priv->ignored_file_types);
-
- if (!priv->config) {
- return;
- }
-
- /* File types as configured to ignore, using pattern matching */
- ignored_file_types = tracker_config_get_no_index_file_types (priv->config);
-
- for (l = ignored_file_types; l; l = l->next) {
- spec = g_pattern_spec_new (l->data);
- patterns = g_slist_prepend (patterns, spec);
- }
-
- priv->ignored_file_types = g_slist_reverse (patterns);
-}
#ifdef HAVE_HAL
@@ -420,7 +331,7 @@
gpointer user_data)
{
g_message ("** TRAWLING THROUGH NEW MOUNT POINT:'%s'", mount_point);
-
+
/* list = g_slist_prepend (NULL, (gchar*) mount_point); */
/* process_directory_list (list, TRUE, iface); */
/* g_slist_free (list); */
@@ -432,7 +343,7 @@
gpointer user_data)
{
g_message ("** CLEANING UP OLD MOUNT POINT:'%s'", mount_point);
-
+
/* process_index_delete_directory_check (mount_point, iface); */
}
@@ -440,7 +351,7 @@
static void
get_remote_roots (TrackerCrawler *crawler,
- GSList **mounted_directory_roots,
+ GSList **mounted_directory_roots,
GSList **removable_device_roots)
{
GSList *l1 = NULL;
@@ -450,11 +361,11 @@
* so we make this process faster?
*/
-#ifdef HAVE_HAL
- l1 = tracker_hal_get_mounted_directory_roots (crawler->priv->hal);
- l2 = tracker_hal_get_removable_device_roots (crawler->priv->hal);
+#ifdef HAVE_HAL
+ l1 = tracker_hal_get_mounted_directory_roots (crawler->private->hal);
+ l2 = tracker_hal_get_removable_device_roots (crawler->private->hal);
#endif /* HAVE_HAL */
-
+
/* The options to index removable media and the index mounted
* directories are both mutually exclusive even though
* removable media is mounted on a directory.
@@ -465,12 +376,12 @@
if (l2) {
GSList *l;
GSList *list = NULL;
-
+
for (l = l1; l; l = l->next) {
if (g_slist_find_custom (l2, l->data, (GCompareFunc) strcmp)) {
continue;
- }
-
+ }
+
list = g_slist_prepend (list, l->data);
}
@@ -495,14 +406,14 @@
gboolean ignore = FALSE;
ignore_mounted_directories =
- !tracker_config_get_index_mounted_directories (crawler->priv->config);
- ignore_removable_devices =
- !tracker_config_get_index_removable_devices (crawler->priv->config);
+ !tracker_config_get_index_mounted_directories (crawler->private->config);
+ ignore_removable_devices =
+ !tracker_config_get_index_removable_devices (crawler->private->config);
if (ignore_mounted_directories || ignore_removable_devices) {
- get_remote_roots (crawler,
- &mounted_directory_roots,
- &removable_device_roots);
+ get_remote_roots (crawler,
+ &mounted_directory_roots,
+ &removable_device_roots);
}
if (ignore_mounted_directories) {
@@ -523,7 +434,7 @@
}
/* FIXME: Should we add a DIR_SEPARATOR on the end of
- * these before comparing them?
+ * these before comparing them?
*/
if (g_str_has_prefix (path, l->data)) {
ignore = TRUE;
@@ -537,34 +448,34 @@
static gboolean
path_should_be_ignored (TrackerCrawler *crawler,
- const gchar *path)
+ const gchar *path,
+ gboolean is_directory)
{
- GSList *l;
- gchar *basename;
- gchar **p;
- gboolean ignore;
+ GList *l;
+ gchar *basename;
+ gboolean ignore;
if (tracker_is_empty_string (path)) {
return TRUE;
}
+ if (!g_utf8_validate (path, -1, NULL)) {
+ g_message ("Ignoring path:'%s', not valid UTF-8", path);
+ return TRUE;
+ }
+
/* Most common things to ignore */
if (strcmp (path, "/boot") == 0 ||
strcmp (path, "/dev") == 0 ||
strcmp (path, "/lib") == 0 ||
strcmp (path, "/proc") == 0 ||
strcmp (path, "/sys") == 0 ||
- strcmp (path, "/tmp") == 0 ||
+ strcmp (path, "/tmp") == 0 ||
strcmp (path, "/var") == 0) {
return TRUE;
}
-
- if (g_str_has_prefix (path, g_get_tmp_dir ())) {
- return TRUE;
- }
- if (!g_utf8_validate (path, -1, NULL)) {
- g_message ("Ignoring path:'%s', not valid UTF-8", path);
+ if (g_str_has_prefix (path, g_get_tmp_dir ())) {
return TRUE;
}
@@ -575,34 +486,14 @@
goto done;
}
- /* Test exact basenames */
- if (g_hash_table_lookup (crawler->priv->ignored_names,
- basename)) {
- goto done;
+ /* Test ignore types */
+ if (is_directory) {
+ l = crawler->private->ignored_directory_patterns;
+ } else {
+ l = crawler->private->ignored_file_patterns;
}
- /* Test temporary black list */
- if (g_hash_table_lookup (crawler->priv->temp_black_list,
- basename)) {
- goto done;
- }
-
- /* Test suffixes */
- for (p = crawler->priv->ignored_suffixes; *p; p++) {
- if (g_str_has_suffix (basename, *p)) {
- goto done;
- }
- }
-
- /* Test prefixes */
- for (p = crawler->priv->ignored_prefixes; *p; p++) {
- if (g_str_has_prefix (basename, *p)) {
- goto done;
- }
- }
-
- /* Test ignore types */
- for (l = crawler->priv->ignored_file_types; l; l = l->next) {
+ for (; l; l = l->next) {
if (g_pattern_match_string (l->data, basename)) {
goto done;
}
@@ -612,7 +503,7 @@
if (path_should_be_ignored_for_media (crawler, path)) {
goto done;
}
-
+
ignore = FALSE;
done:
@@ -621,73 +512,150 @@
return ignore;
}
-static EnumeratorData *
-enumerator_data_new (TrackerCrawler *crawler,
- GFile *parent)
+static void
+add_file (TrackerCrawler *crawler,
+ GFile *file)
{
- EnumeratorData *ed;
+ gchar *path;
- ed = g_slice_new0 (EnumeratorData);
- ed->crawler = g_object_ref (crawler);
- ed->parent = g_object_ref (parent);
+ g_return_if_fail (G_IS_FILE (file));
- return ed;
-}
+ path = g_file_get_path (file);
-static void
-enumerator_data_free (EnumeratorData *ed)
-{
- g_object_unref (ed->parent);
- g_object_unref (ed->crawler);
- g_slice_free (EnumeratorData, ed);
+ if (path_should_be_ignored (crawler, path, FALSE)) {
+ crawler->private->files_ignored++;
+
+#ifdef TESTING
+ g_debug ("Ignored:'%s' (%d)",
+ path,
+ crawler->private->enumerations);
+#endif /* TESTING */
+ } else {
+ crawler->private->files_found++;
+
+#ifdef TESTING
+ g_debug ("Found :'%s' (%d)",
+ path,
+ crawler->private->enumerations);
+#endif /* TESTING */
+
+ g_queue_push_tail (crawler->private->files, g_object_ref (file));
+ }
}
static void
-file_enumerators_increment (TrackerCrawler *crawler)
+add_directory (TrackerCrawler *crawler,
+ GFile *file)
{
- TrackerCrawlerPriv *priv;
+ gchar *path;
- priv = crawler->priv;
+ g_return_if_fail (G_IS_FILE (file));
- if (priv->enumerations == 0) {
- g_message ("Starting to crawl file system...");
+ path = g_file_get_path (file);
- if (!priv->timer) {
- priv->timer = g_timer_new ();
- } else {
- g_timer_reset (priv->timer);
- }
+ if (path_should_be_ignored (crawler, path, TRUE)) {
+ crawler->private->directories_ignored++;
+
+#ifdef TESTING
+ g_debug ("Ignored:'%s' (%d)",
+ path,
+ crawler->private->enumerations);
+#endif /* TESTING */
+ } else {
+ crawler->private->directories_found++;
- priv->files_found = 0;
- priv->files_ignored = 0;
+#ifdef TESTING
+ g_debug ("Found :'%s' (%d)",
+ path,
+ crawler->private->enumerations);
+#endif /* TESTING */
+
+ g_queue_push_tail (crawler->private->directories, g_object_ref (file));
}
+}
- priv->enumerations++;
+static gboolean
+process_file (TrackerCrawler *crawler,
+ GFile *file)
+{
+ /* I guess here we check the queue length and send files to
+ * the indexer?
+ */
+
+ return TRUE;
}
static void
-file_enumerators_decrement (TrackerCrawler *crawler)
+process_directory (TrackerCrawler *crawler,
+ GFile *file)
{
- TrackerCrawlerPriv *priv;
+ file_enumerate_children (crawler, file);
+}
- priv = crawler->priv;
+static gboolean
+process_func (gpointer data)
+{
+ TrackerCrawler *crawler;
+ GFile *file;
- priv->enumerations--;
+ crawler = TRACKER_CRAWLER (data);
- if (priv->enumerations == 0) {
- g_timer_stop (priv->timer);
+ /* Crawler file */
+ file = g_queue_peek_head (crawler->private->files);
- g_message ("%s crawling files in %4.4f seconds, %d found, %d ignored, "
- "%d monitors, %d monitors ignored",
- priv->running ? "Finished" : "Stopped",
- g_timer_elapsed (priv->timer, NULL),
- priv->files_found,
- priv->files_ignored,
- tracker_monitor_get_count (),
- tracker_monitor_get_ignored ());
+ if (file) {
+ if (process_file (crawler, file)) {
+ file = g_queue_pop_head (crawler->private->files);
+ g_object_unref (file);
+ }
+
+ return TRUE;
+ }
+
+ /* Crawler directory contents */
+ file = g_queue_pop_head (crawler->private->directories);
+
+ if (file) {
+ process_directory (crawler, file);
+ g_object_unref (file);
- priv->running = FALSE;
+ return TRUE;
}
+
+ /* If we still have some async operations in progress, wait
+ * for them to finish, if not, we are truly done.
+ */
+ if (crawler->private->enumerations > 0) {
+ return TRUE;
+ }
+
+ crawler->private->idle_id = 0;
+ crawler->private->finished = TRUE;
+
+ tracker_crawler_stop (crawler);
+
+ return FALSE;
+}
+
+static EnumeratorData *
+enumerator_data_new (TrackerCrawler *crawler,
+ GFile *parent)
+{
+ EnumeratorData *ed;
+
+ ed = g_slice_new0 (EnumeratorData);
+ ed->crawler = g_object_ref (crawler);
+ ed->parent = g_object_ref (parent);
+
+ return ed;
+}
+
+static void
+enumerator_data_free (EnumeratorData *ed)
+{
+ g_object_unref (ed->parent);
+ g_object_unref (ed->crawler);
+ g_slice_free (EnumeratorData, ed);
}
static void
@@ -698,12 +666,12 @@
TrackerCrawler *crawler;
crawler = TRACKER_CRAWLER (user_data);
- file_enumerators_decrement (crawler);
+ crawler->private->enumerations--;
- if (!g_file_enumerator_close_finish (G_FILE_ENUMERATOR (enumerator),
- result,
+ if (!g_file_enumerator_close_finish (G_FILE_ENUMERATOR (enumerator),
+ result,
NULL)) {
- g_warning ("Couldn't close GFileEnumerator:%p",
+ g_warning ("Couldn't close GFileEnumerator:%p",
enumerator);
}
}
@@ -719,8 +687,7 @@
GFile *parent, *child;
GFileInfo *info;
GList *files;
- gchar *path;
-
+
enumerator = G_FILE_ENUMERATOR (object);
ed = (EnumeratorData*) user_data;
@@ -731,16 +698,12 @@
result,
NULL);
- if (!crawler->priv->running) {
- return;
- }
-
- if (!files || !crawler->priv->running) {
+ if (!files || !crawler->private->running) {
/* No more files or we are stopping anyway, so clean
* up and close all file enumerators.
*/
enumerator_data_free (ed);
- g_file_enumerator_close_async (enumerator,
+ g_file_enumerator_close_async (enumerator,
G_PRIORITY_DEFAULT,
NULL,
file_enumerator_close_cb,
@@ -751,37 +714,14 @@
/* Files should only have 1 item in it */
info = files->data;
child = g_file_get_child (parent, g_file_info_get_name (info));
- path = g_file_get_path (child);
-
- if (path_should_be_ignored (crawler, path)) {
- crawler->priv->files_ignored++;
-#ifdef TESTING
- g_debug ("Ignored:'%s' (%d)",
- path,
- crawler->priv->enumerations);
-#endif /* TESTING */
-
- g_free (path);
+ if (g_file_info_get_file_type (info) == G_FILE_TYPE_DIRECTORY) {
+ add_directory (crawler, child);
} else {
- crawler->priv->files_found++;
-
-#ifdef TESTING
- g_debug ("Found :'%s' (%d)",
- path,
- crawler->priv->enumerations);
-#endif /* TESTING */
-
- if (g_file_info_get_file_type (info) == G_FILE_TYPE_DIRECTORY) {
- file_enumerate_children (crawler, child);
- g_free (path);
- } else {
- g_async_queue_push (crawler->priv->files, path);
- file_queue_handler_set_up (crawler);
- }
- }
+ add_file (crawler, child);
+ }
- g_object_unref (child);
+ g_object_unref (child);
g_list_free (files);
/* Get next file */
@@ -792,7 +732,7 @@
file_enumerate_next (GFileEnumerator *enumerator,
EnumeratorData *ed)
{
- g_file_enumerator_next_files_async (enumerator,
+ g_file_enumerator_next_files_async (enumerator,
1,
G_PRIORITY_DEFAULT,
NULL,
@@ -815,7 +755,7 @@
enumerator = g_file_enumerate_children_finish (parent, result, NULL);
if (!enumerator) {
- file_enumerators_decrement (crawler);
+ crawler->private->enumerations--;
return;
}
@@ -830,178 +770,170 @@
file_enumerate_children (TrackerCrawler *crawler,
GFile *file)
{
- file_enumerators_increment (crawler);
+ crawler->private->enumerations++;
tracker_monitor_add (file);
- g_file_enumerate_children_async (file,
- FILE_ATTRIBUTES,
+ g_file_enumerate_children_async (file,
+ FILE_ATTRIBUTES,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
G_PRIORITY_DEFAULT,
- NULL,
+ NULL,
file_enumerate_children_cb,
crawler);
}
-static void
-indexer_check_files_cb (DBusGProxy *proxy,
- GError *error,
- gpointer user_data)
-{
- GStrv files;
-
- files = (GStrv) user_data;
-
- if (error) {
- g_critical ("Could not send files to indexer to check, %s",
- error->message);
- g_error_free (error);
- } else {
- g_debug ("Sent!");
+gboolean
+tracker_crawler_start (TrackerCrawler *crawler,
+ const gchar *module_name)
+{
+ TrackerCrawlerPrivate *priv;
+ GFile *file;
+ GSList *paths = NULL;
+ GSList *sl;
+ GList *directories;
+ GList *l;
+ gchar *path;
+ gboolean exists;
+
+ g_return_val_if_fail (TRACKER_IS_CRAWLER (crawler), FALSE);
+ g_return_val_if_fail (module_name != NULL, FALSE);
+
+ priv = crawler->private;
+
+ g_message ("Crawling directories for module:'%s'",
+ module_name);
+
+ directories = tracker_module_config_get_monitor_recurse_directories (module_name);
+ if (!directories) {
+ g_message (" No directories to iterate, doing nothing");
+ return FALSE;
}
-}
-static void
-indexer_get_running_cb (DBusGProxy *proxy,
- gboolean running,
- GError *error,
- gpointer user_data)
-{
- TrackerCrawler *crawler;
- GStrv files;
+ for (l = directories; l; l = l->next) {
+ /* Check location exists before we do anything */
+ file = g_file_new_for_path (path);
+ exists = g_file_query_exists (file, NULL);
- crawler = TRACKER_CRAWLER (user_data);
+ if (!exists) {
+ g_message (" Directory:'%s' does not exist",
+ path);
+ g_object_unref (file);
+ continue;
+ }
- if (error || !running) {
- g_message ("%s",
- error ? error->message : "Indexer exists but is not available yet, waiting...");
+ paths = g_slist_prepend (paths, g_strdup (l->data));
+ g_object_unref (file);
+ }
- g_object_unref (crawler);
- g_clear_error (&error);
+ g_list_free (directories);
- return;
+ if (!paths) {
+ g_message (" No directories that actually exist to iterate, doing nothing");
+ return FALSE;
}
- g_debug ("File check queue being processed...");
- files = tracker_dbus_async_queue_to_strv (crawler->priv->files,
- FILES_QUEUE_PROCESS_MAX);
-
- g_debug ("File check queue processed, sending first %d to the indexer",
- g_strv_length (files));
-
- org_freedesktop_Tracker_Indexer_files_check_async (proxy,
- "files",
- (const gchar **) files,
- indexer_check_files_cb,
- files);
+ paths = g_slist_reverse (paths);
+ sl = tracker_path_list_filter_duplicates (paths);
+ g_slist_foreach (paths, (GFunc) g_free, NULL);
+ g_slist_free (paths);
+ paths = sl;
- g_object_unref (crawler);
-}
+ /* Time the event */
+ if (priv->timer) {
+ g_timer_destroy (priv->timer);
+ }
-static gboolean
-file_queue_handler_cb (gpointer user_data)
-{
- TrackerCrawler *crawler;
- DBusGProxy *proxy;
- gint length;
+ priv->timer = g_timer_new ();
- crawler = user_data;
+ /* Set up all the important data to start this crawl */
+ if (priv->ignored_directory_patterns) {
+ g_list_free (priv->ignored_directory_patterns);
+ }
- length = g_async_queue_length (crawler->priv->files);
- if (length < 1) {
- g_debug ("File check queue is empty... nothing to do");
- crawler->priv->files_queue_handle_id = 0;
- return FALSE;
+ if (priv->ignored_file_patterns) {
+ g_list_free (priv->ignored_file_patterns);
}
- /* Check we can actually talk to the indexer */
- proxy = tracker_dbus_indexer_get_proxy ();
+ priv->ignored_file_patterns =
+ tracker_module_config_get_ignored_file_patterns (module_name);
+ priv->ignored_directory_patterns =
+ tracker_module_config_get_ignored_directory_patterns (module_name);
- org_freedesktop_Tracker_Indexer_get_running_async (proxy,
- indexer_get_running_cb,
- g_object_ref (crawler));
+ priv->current_module_name = g_strdup (module_name);
- return TRUE;
-}
+ /* Set idle handler to process directories and files found */
+ priv->idle_id = g_idle_add (process_func, crawler);
+
+ /* Set as running now */
+ priv->running = TRUE;
+ priv->finished = FALSE;
-static void
-file_queue_handler_set_up (TrackerCrawler *crawler)
-{
- if (crawler->priv->files_queue_handle_id != 0) {
- return;
+ /* Reset stats */
+ priv->directories_found = 0;
+ priv->directories_ignored = 0;
+ priv->files_found = 0;
+ priv->files_ignored = 0;
+
+ for (sl = paths; sl; sl = sl->next) {
+ file = g_file_new_for_path (sl->data);
+ g_message (" Searching directory:'%s'", (gchar *) sl->data);
+
+ file_enumerate_children (crawler, file);
+ g_object_unref (file);
+ g_free (sl->data);
}
- crawler->priv->files_queue_handle_id =
- g_timeout_add (FILES_QUEUE_PROCESS_INTERVAL,
- file_queue_handler_cb,
- crawler);
+ g_slist_free (paths);
+
+ return TRUE;
}
void
-tracker_crawler_start (TrackerCrawler *crawler)
+tracker_crawler_stop (TrackerCrawler *crawler)
{
- TrackerCrawlerPriv *priv;
- GFile *file;
- GSList *config_roots;
- GSList *roots = NULL;
- GSList *l;
- gboolean exists;
+ TrackerCrawlerPrivate *priv;
g_return_if_fail (TRACKER_IS_CRAWLER (crawler));
- priv = crawler->priv;
-
- /* Get locations to index from config, if none are set, we use
- * $HOME as the default.
- */
- config_roots = tracker_config_get_crawl_directory_roots (priv->config);
- if (config_roots) {
- /* Make sure none of the roots co-inside each other */
- roots = tracker_path_list_filter_duplicates (config_roots);
- }
+ priv = crawler->private;
- if (!roots) {
- const gchar *home;
-
- home = g_get_home_dir ();
- roots = g_slist_prepend (roots, g_strdup (home));
+ priv->running = FALSE;
- g_message ("No locations are configured to crawl, "
- "using default location (home directory)");
+ if (priv->idle_id) {
+ g_source_remove (priv->idle_id);
}
- /* Set as running now */
- priv->running = TRUE;
+ g_free (priv->current_module_name);
+ priv->current_module_name = NULL;
- /* Start iterating roots */
- for (l = roots; l; l = l->next) {
- file = g_file_new_for_path (l->data);
- exists = g_file_query_exists (file, NULL);
-
- if (exists) {
- g_message ("Searching directory:'%s'",
- (gchar*) l->data);
- file_enumerate_children (crawler, file);
- } else {
- g_message ("Searching directory:'%s' failed, does not exist",
- (gchar*) l->data);
- }
-
- g_object_unref (file);
- g_free (l->data);
+ if (priv->ignored_file_patterns) {
+ g_list_free (priv->ignored_file_patterns);
+ priv->ignored_file_patterns = NULL;
}
- g_slist_free (roots);
-}
+ if (priv->ignored_directory_patterns) {
+ g_list_free (priv->ignored_directory_patterns);
+ priv->ignored_directory_patterns = NULL;
+ }
-void
-tracker_crawler_stop (TrackerCrawler *crawler)
-{
- TrackerCrawlerPriv *priv;
+ g_timer_stop (priv->timer);
- g_return_if_fail (TRACKER_IS_CRAWLER (crawler));
+ g_message (" %s crawling files in %4.4f seconds",
+ priv->finished ? "Finished" : "Stopped",
+ g_timer_elapsed (priv->timer, NULL));
+ g_message (" Found %d directories, ignored %d directories",
+ priv->directories_found,
+ priv->directories_ignored);
+ g_message (" Found %d files, ignored %d files",
+ priv->files_found,
+ priv->files_ignored);
+ g_message (" Added %d monitors, ignored %d monitors",
+ tracker_monitor_get_count (),
+ tracker_monitor_get_ignored ());
- priv = crawler->priv;
+ g_timer_destroy (priv->timer);
+ priv->timer = NULL;
- priv->running = FALSE;
+ g_signal_emit (crawler, signals[FINISHED], 0);
}
Modified: branches/indexer-split/src/trackerd/tracker-crawler.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.h (original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.h Tue Jul 1 11:44:34 2008
@@ -38,17 +38,17 @@
#define TRACKER_IS_CRAWLER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), TRACKER_TYPE_CRAWLER))
#define TRACKER_CRAWLER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), TRACKER_TYPE_CRAWLER, TrackerCrawlerClass))
-typedef struct _TrackerCrawler TrackerCrawler;
-typedef struct _TrackerCrawlerClass TrackerCrawlerClass;
-typedef struct _TrackerCrawlerPriv TrackerCrawlerPriv;
+typedef struct _TrackerCrawler TrackerCrawler;
+typedef struct _TrackerCrawlerClass TrackerCrawlerClass;
+typedef struct _TrackerCrawlerPrivate TrackerCrawlerPrivate;
struct _TrackerCrawler {
- GObject parent;
- TrackerCrawlerPriv *priv;
+ GObject parent;
+ TrackerCrawlerPrivate *private;
};
struct _TrackerCrawlerClass {
- GObjectClass parent;
+ GObjectClass parent;
};
GType tracker_crawler_get_type (void);
@@ -61,7 +61,8 @@
TrackerHal *hal);
#endif /* HAVE_HAL */
-void tracker_crawler_start (TrackerCrawler *crawler);
+gboolean tracker_crawler_start (TrackerCrawler *crawler,
+ const gchar *module_name);
void tracker_crawler_stop (TrackerCrawler *crawler);
G_END_DECLS
Modified: branches/indexer-split/src/trackerd/tracker-processor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-processor.c (original)
+++ branches/indexer-split/src/trackerd/tracker-processor.c Tue Jul 1 11:44:34 2008
@@ -43,30 +43,23 @@
#endif /* HAVE_HAL */
TrackerCrawler *crawler;
- GQueue *dir_queue;
- GQueue *file_queue;
GList *modules;
GList *current_module;
- guint idle_id;
-
GTimer *timer;
gboolean finished;
};
-typedef struct {
- gchar *module_name;
- gchar *path;
-} ProcessInfo;
-
enum {
FINISHED,
LAST_SIGNAL
};
-static void tracker_processor_finalize (GObject *object);
-static void info_free (ProcessInfo *info);
+static void tracker_processor_finalize (GObject *object);
+static void process_next_module (TrackerProcessor *processor);
+static void crawler_finished_cb (TrackerCrawler *crawler,
+ gpointer user_data);
static guint signals[LAST_SIGNAL] = { 0, };
@@ -98,9 +91,6 @@
priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
- priv->dir_queue = g_queue_new ();
- priv->file_queue = g_queue_new ();
-
priv->modules = tracker_module_config_get_modules ();
}
@@ -115,19 +105,11 @@
g_timer_destroy (priv->timer);
}
- if (priv->idle_id) {
- g_source_remove (priv->idle_id);
- priv->idle_id = 0;
- }
-
g_list_free (priv->modules);
- g_queue_foreach (priv->file_queue, (GFunc) info_free, NULL);
- g_queue_free (priv->file_queue);
-
- g_queue_foreach (priv->dir_queue, (GFunc) info_free, NULL);
- g_queue_free (priv->dir_queue);
-
+ g_signal_handlers_disconnect_by_func (priv->crawler,
+ G_CALLBACK (crawler_finished_cb),
+ object);
g_object_unref (priv->crawler);
#ifdef HAVE_HAL
@@ -139,80 +121,11 @@
G_OBJECT_CLASS (tracker_processor_parent_class)->finalize (object);
}
-static ProcessInfo *
-info_new (const gchar *module_name,
- const gchar *path)
-{
- ProcessInfo *info;
-
- info = g_slice_new (ProcessInfo);
-
- info->module_name = g_strdup (module_name);
- info->path = g_strdup (path);
-
- return info;
-}
-
-static void
-info_free (ProcessInfo *info)
-{
- g_free (info->module_name);
- g_free (info->path);
- g_slice_free (ProcessInfo, info);
-}
-
-static void
-add_file (TrackerProcessor *processor,
- ProcessInfo *info)
-{
- TrackerProcessorPrivate *priv;
-
- g_return_if_fail (info != NULL);
-
- priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
-
- g_queue_push_tail (priv->file_queue, info);
-}
-
-static void
-add_directory (TrackerProcessor *processor,
- ProcessInfo *info)
-{
- TrackerProcessorPrivate *priv;
- gboolean ignore = FALSE;
- gchar **ignore_dirs = NULL;
- gint i;
-
- g_return_if_fail (info != NULL);
-
- priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
-
- /* ignore_dirs = tracker_processor_module_get_ignore_directories (info->module_name); */
-
- if (ignore_dirs) {
- for (i = 0; ignore_dirs[i]; i++) {
- if (strcmp (info->path, ignore_dirs[i]) == 0) {
- ignore = TRUE;
- break;
- }
- }
- }
-
- if (!ignore) {
- g_queue_push_tail (priv->dir_queue, info);
- } else {
- g_message (" Ignoring directory:'%s'", info->path);
- info_free (info);
- }
-
- g_strfreev (ignore_dirs);
-}
-
static void
add_monitors (const gchar *name)
{
- GSList *monitors;
- GSList *l;
+ GList *monitors;
+ GList *l;
monitors = tracker_module_config_get_monitor_directories (name);
@@ -229,6 +142,8 @@
g_object_unref (file);
}
+ g_list_free (monitors);
+
if (!monitors) {
g_message (" No specific monitors to set up");
}
@@ -237,8 +152,8 @@
static void
add_recurse_monitors (const gchar *name)
{
- GSList *monitors;
- GSList *l;
+ GList *monitors;
+ GList *l;
monitors = tracker_module_config_get_monitor_recurse_directories (name);
@@ -255,144 +170,165 @@
g_object_unref (file);
}
+ g_list_free (monitors);
+
if (!monitors) {
g_message (" No recurse monitors to set up");
}
}
-static gboolean
-process_file (TrackerProcessor *processor,
- ProcessInfo *info)
+static void
+process_module (TrackerProcessor *processor,
+ const gchar *module_name)
{
- g_message (" Processing file:'%s'", info->path);
- return TRUE;
+ TrackerProcessorPrivate *priv;
+
+ priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
+
+ g_message ("Processing module:'%s'", module_name);
+
+ /* Set up monitors */
+
+ /* Set up recursive monitors */
+
+ /* Gets all files and directories */
+ if (!tracker_crawler_start (priv->crawler, module_name)) {
+ /* If there is nothing to crawl, we are done, process
+ * the next module.
+ */
+ process_next_module (processor);
+ }
}
static void
-process_directory (TrackerProcessor *processor,
- ProcessInfo *info,
- gboolean recurse)
+process_next_module (TrackerProcessor *processor)
{
- GDir *dir;
- const gchar *name;
+ TrackerProcessorPrivate *priv;
- g_message (" Processing directory:'%s'", info->path);
+ priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
- dir = g_dir_open (info->path, 0, NULL);
+ if (!priv->current_module) {
+ priv->current_module = priv->modules;
+ } else {
+ priv->current_module = priv->current_module->next;
+ }
- if (!dir) {
+ if (!priv->current_module) {
+ priv->finished = TRUE;
+ tracker_processor_stop (processor);
return;
}
- while ((name = g_dir_read_name (dir)) != NULL) {
- ProcessInfo *new_info;
- gchar *path;
-
- path = g_build_filename (info->path, name, NULL);
-
- new_info = info_new (info->module_name, path);
- add_file (processor, new_info);
+ process_module (processor, priv->current_module->data);
+}
- if (recurse && g_file_test (path, G_FILE_TEST_IS_DIR)) {
- new_info = info_new (info->module_name, path);
- add_directory (processor, new_info);
- }
+#if 0
- g_free (path);
+static void
+file_queue_handler_set_up (TrackerCrawler *crawler)
+{
+ if (crawler->priv->files_queue_handle_id != 0) {
+ return;
}
- g_dir_close (dir);
+ crawler->priv->files_queue_handle_id =
+ g_timeout_add (FILES_QUEUE_PROCESS_INTERVAL,
+ file_queue_handler_cb,
+ crawler);
}
static void
-process_module (TrackerProcessor *processor,
- const gchar *module_name)
-{
- TrackerProcessorPrivate *priv;
- GSList *dirs, *l;
+indexer_check_files_cb (DBusGProxy *proxy,
+ GError *error,
+ gpointer user_data)
+{
+ GStrv files;
+
+ files = (GStrv) user_data;
+
+ if (error) {
+ g_critical ("Could not send files to indexer to check, %s",
+ error->message);
+ g_error_free (error);
+ } else {
+ g_debug ("Sent!");
+ }
+}
- priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
+static void
+indexer_get_running_cb (DBusGProxy *proxy,
+ gboolean running,
+ GError *error,
+ gpointer user_data)
+{
+ TrackerCrawler *crawler;
+ GStrv files;
+
+ crawler = TRACKER_CRAWLER (user_data);
+
+ if (error || !running) {
+ g_message ("%s",
+ error ? error->message : "Indexer exists but is not available yet, waiting...");
- g_message ("Processing module:'%s'", module_name);
+ g_object_unref (crawler);
+ g_clear_error (&error);
- dirs = tracker_module_config_get_monitor_recurse_directories (module_name);
- if (!dirs) {
- g_message (" No directories to iterate, doing nothing");
return;
}
- for (l = dirs; l; l = l->next) {
- ProcessInfo *info;
+ g_debug ("File check queue being processed...");
+ files = tracker_dbus_async_queue_to_strv (crawler->priv->files,
+ FILES_QUEUE_PROCESS_MAX);
+
+ g_debug ("File check queue processed, sending first %d to the indexer",
+ g_strv_length (files));
+
+ org_freedesktop_Tracker_Indexer_files_check_async (proxy,
+ "files",
+ (const gchar **) files,
+ indexer_check_files_cb,
+ files);
- info = info_new (module_name, l->data);
- add_directory (processor, info);
- }
+ g_object_unref (crawler);
}
static gboolean
-process_func (gpointer data)
+file_queue_handler_cb (gpointer user_data)
{
- TrackerProcessor *processor;
- TrackerProcessorPrivate *priv;
- ProcessInfo *info;
+ TrackerCrawler *crawler;
+ DBusGProxy *proxy;
+ gint length;
+
+ crawler = user_data;
+
+ length = g_async_queue_length (crawler->priv->files);
+ if (length < 1) {
+ g_debug ("File check queue is empty... nothing to do");
+ crawler->priv->files_queue_handle_id = 0;
+ return FALSE;
+ }
- processor = TRACKER_PROCESSOR (data);
- priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
+ /* Check we can actually talk to the indexer */
+ proxy = tracker_dbus_indexer_get_proxy ();
-#if 0
- /* Process monitors first */
- for (l = modules; l; l = l->next) {
- gchar *name;
+ org_freedesktop_Tracker_Indexer_get_running_async (proxy,
+ indexer_get_running_cb,
+ g_object_ref (crawler));
- name = l->data;
- g_message ("Processoring module:'%s'", name);
-
- add_monitors (name);
- add_recurse_monitors (name);
+ return TRUE;
+}
- /* FIXME: Finish, start crawling? */
- }
#endif
- /* Processor file */
- info = g_queue_peek_head (priv->file_queue);
-
- if (info) {
- if (process_file (processor, info)) {
- info = g_queue_pop_head (priv->file_queue);
- info_free (info);
- }
-
- return TRUE;
- }
-
- /* Processor directory contents */
- info = g_queue_pop_head (priv->dir_queue);
-
- if (info) {
- process_directory (processor, info, TRUE);
- info_free (info);
- return TRUE;
- }
-
- /* Dirs/files queues are empty, processor the next module */
- if (!priv->current_module) {
- priv->current_module = priv->modules;
- } else {
- priv->current_module = priv->current_module->next;
- }
+static void
+crawler_finished_cb (TrackerCrawler *crawler,
+ gpointer user_data)
+{
+ TrackerProcessor *processor;
- if (!priv->current_module) {
- priv->finished = TRUE;
+ processor = TRACKER_PROCESSOR (user_data);
- tracker_processor_stop (processor);
-
- return FALSE;
- }
-
- process_module (processor, priv->current_module->data);
-
- return TRUE;
+ process_next_module (processor);
}
TrackerProcessor *
@@ -415,6 +351,10 @@
tracker_crawler_set_hal (priv->crawler, priv->hal);
#endif /* HAVE_HAL */
+ g_signal_connect (priv->crawler, "finished",
+ G_CALLBACK (crawler_finished_cb),
+ processor);
+
return processor;
}
@@ -430,15 +370,15 @@
g_message ("Starting to process %d modules...",
g_list_length (priv->modules));
- priv->finished = FALSE;
-
if (priv->timer) {
g_timer_destroy (priv->timer);
}
priv->timer = g_timer_new ();
- priv->idle_id = g_idle_add (process_func, processor);
+ priv->finished = FALSE;
+
+ process_next_module (processor);
}
void
@@ -450,19 +390,13 @@
priv = TRACKER_PROCESSOR_GET_PRIVATE (processor);
- if (priv->crawler) {
+ if (!priv->finished) {
tracker_crawler_stop (priv->crawler);
}
- if (priv->idle_id) {
- g_source_remove (priv->idle_id);
- priv->idle_id = 0;
- }
-
- /* No more modules to query, we're done */
g_timer_stop (priv->timer);
- g_message ("Processed %s %4.4f seconds",
+ g_message ("Process %s %4.4f seconds",
priv->finished ? "finished in" : "stopped after",
g_timer_elapsed (priv->timer, NULL));
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
