tracker r1657 - in branches/indexer-split: . src/libtracker-common src/trackerd
- From: mr svn gnome org
- To: svn-commits-list gnome org
- Subject: tracker r1657 - in branches/indexer-split: . src/libtracker-common src/trackerd
- Date: Thu, 12 Jun 2008 13:11:16 +0000 (UTC)
Author: mr
Date: Thu Jun 12 13:11:15 2008
New Revision: 1657
URL: http://svn.gnome.org/viewvc/tracker?rev=1657&view=rev
Log:
* src/libtracker-common/tracker-hal.c: Always initialise the hash
tables to avoid errors when we can't contact the HAL DBus service.
* src/trackerd/linux-inotify-syscalls.h: Removed now we don't need
it thanks to the GIO file monitoring.
* src/trackerd/tracker-crawler.c: Get the locations to crawl from
the config now and add a default of $HOME. Also added function to
check that we don't crawl the same location more than once and
added more locations to the ignore list.
* src/trackerd/tracker-db.c:
* src/trackerd/tracker-email-kmail.c:
* src/trackerd/tracker-process-files.c: Removed the need for the
TrackerDBInterface when adding or removing monitors.
* src/trackerd/tracker-main.c: Added back the initial_sleep
command line argument so we can at least inspect the startup
details and logging before the console is spammed with file
crawling information.
Removed:
branches/indexer-split/src/trackerd/linux-inotify-syscalls.h
Modified:
branches/indexer-split/ChangeLog
branches/indexer-split/src/libtracker-common/tracker-hal.c
branches/indexer-split/src/trackerd/tracker-crawler.c
branches/indexer-split/src/trackerd/tracker-db.c
branches/indexer-split/src/trackerd/tracker-email-kmail.c
branches/indexer-split/src/trackerd/tracker-main.c
branches/indexer-split/src/trackerd/tracker-monitor.c
branches/indexer-split/src/trackerd/tracker-monitor.h
branches/indexer-split/src/trackerd/tracker-process-files.c
Modified: branches/indexer-split/src/libtracker-common/tracker-hal.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-hal.c (original)
+++ branches/indexer-split/src/libtracker-common/tracker-hal.c Thu Jun 12 13:11:15 2008
@@ -152,8 +152,25 @@
DBusError error;
DBusConnection *connection;
+ g_message ("Initializing HAL...");
+
priv = GET_PRIV (hal);
+ priv->all_devices = g_hash_table_new_full (g_str_hash,
+ g_str_equal,
+ (GDestroyNotify) g_free,
+ (GDestroyNotify) g_free);
+
+ priv->mounted_devices = g_hash_table_new_full (g_str_hash,
+ g_str_equal,
+ (GDestroyNotify) g_free,
+ (GDestroyNotify) g_free);
+
+ priv->removable_devices = g_hash_table_new_full (g_str_hash,
+ g_str_equal,
+ (GDestroyNotify) g_free,
+ (GDestroyNotify) g_free);
+
dbus_error_init (&error);
connection = dbus_bus_get (DBUS_BUS_SYSTEM, &error);
@@ -166,7 +183,6 @@
dbus_connection_setup_with_g_main (connection, NULL);
- g_message ("Initializing HAL...");
priv->context = libhal_ctx_new ();
if (!priv->context) {
@@ -191,21 +207,7 @@
return;
}
- priv->all_devices = g_hash_table_new_full (g_str_hash,
- g_str_equal,
- (GDestroyNotify) g_free,
- (GDestroyNotify) g_free);
-
- priv->mounted_devices = g_hash_table_new_full (g_str_hash,
- g_str_equal,
- (GDestroyNotify) g_free,
- (GDestroyNotify) g_free);
-
- priv->removable_devices = g_hash_table_new_full (g_str_hash,
- g_str_equal,
- (GDestroyNotify) g_free,
- (GDestroyNotify) g_free);
-
+
/* Volume and property notification callbacks */
g_message ("HAL monitors set for devices added/removed/mounted/umounted...");
libhal_ctx_set_device_added (priv->context, hal_device_added_cb);
Modified: branches/indexer-split/src/trackerd/tracker-crawler.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.c (original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.c Thu Jun 12 13:11:15 2008
@@ -32,6 +32,8 @@
#define GET_PRIV(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_CRAWLER, TrackerCrawlerPriv))
+#define TESTING
+
#define FILES_QUEUE_PROCESS_INTERVAL 2000
#define FILES_QUEUE_PROCESS_MAX 5000
@@ -409,6 +411,10 @@
GSList *l1 = NULL;
GSList *l2 = NULL;
+ /* FIXME: Shouldn't we keep this static for a period of time
+ * so we make this process faster?
+ */
+
#ifdef HAVE_HAL
l1 = tracker_hal_get_mounted_directory_roots (crawler->priv->hal);
l2 = tracker_hal_get_removable_device_roots (crawler->priv->hal);
@@ -442,6 +448,59 @@
}
static gboolean
+path_should_be_ignored_for_media (TrackerCrawler *crawler,
+ const gchar *path)
+{
+ GSList *roots = NULL;
+ GSList *mounted_directory_roots = NULL;
+ GSList *removable_device_roots = NULL;
+ GSList *l;
+ gboolean ignore_mounted_directories;
+ gboolean ignore_removable_devices;
+ gboolean ignore = FALSE;
+
+ ignore_mounted_directories =
+ !tracker_config_get_index_mounted_directories (crawler->priv->config);
+ ignore_removable_devices =
+ !tracker_config_get_index_removable_devices (crawler->priv->config);
+
+ if (ignore_mounted_directories || ignore_removable_devices) {
+ get_remote_roots (crawler,
+ &mounted_directory_roots,
+ &removable_device_roots);
+ }
+
+ if (ignore_mounted_directories) {
+ roots = g_slist_concat (roots, mounted_directory_roots);
+ }
+
+ if (ignore_removable_devices) {
+ roots = g_slist_concat (roots, removable_device_roots);
+ }
+
+ for (l = roots; l && !ignore; l = l->next) {
+ /* If path matches a mounted or removable device by
+ * prefix then we should ignore it since we don't
+ * crawl those by choice in the config.
+ */
+ if (strcmp (path, l->data) == 0) {
+ ignore = TRUE;
+ }
+
+ /* FIXME: Should we add a DIR_SEPARATOR on the end of
+ * these before comparing them?
+ */
+ if (g_str_has_prefix (path, l->data)) {
+ ignore = TRUE;
+ }
+ }
+
+ g_slist_free (roots);
+
+ return ignore;
+}
+
+static gboolean
path_should_be_ignored (TrackerCrawler *crawler,
const gchar *path)
{
@@ -455,10 +514,13 @@
}
/* Most common things to ignore */
- if (g_str_has_prefix (path, "/proc/") ||
- g_str_has_prefix (path, "/dev/") ||
- g_str_has_prefix (path, "/tmp/")) {
- /* FIXME: What about /boot, /sys, /sbin, /bin ? */
+ if (strcmp (path, "/boot") == 0 ||
+ strcmp (path, "/dev") == 0 ||
+ strcmp (path, "/lib") == 0 ||
+ strcmp (path, "/proc") == 0 ||
+ strcmp (path, "/sys") == 0 ||
+ strcmp (path, "/tmp") == 0 ||
+ strcmp (path, "/var") == 0) {
return TRUE;
}
@@ -478,7 +540,7 @@
basename)) {
goto done;
}
-
+
/* Test temporary black list */
if (g_hash_table_lookup (crawler->priv->temp_black_list,
basename)) {
@@ -505,12 +567,23 @@
goto done;
}
}
+
+ /* Should we crawl mounted or removable media */
+ if (path_should_be_ignored_for_media (crawler, path)) {
+ goto done;
+ }
ignore = FALSE;
done:
g_free (basename);
+#ifdef TESTING
+ /* g_debug ("%s:'%s'", */
+ /* ignore ? "Block " : "Crawl ", */
+ /* path); */
+#endif /* TESTING */
+
return ignore;
}
@@ -580,6 +653,8 @@
return;
}
+ context = g_main_context_default ();
+
for (info = g_file_enumerator_next_file (enumerator, NULL, NULL);
info && crawler->priv->running;
info = g_file_enumerator_next_file (enumerator, NULL, NULL)) {
@@ -588,15 +663,19 @@
if (path_should_be_ignored (crawler, relative_path)) {
crawler->priv->files_ignored++;
+#ifdef TESTING
g_debug ("Ignored:'%s' (%d)",
relative_path,
crawler->priv->enumerations);
+#endif /* TESTING */
g_free (relative_path);
} else {
crawler->priv->files_found++;
+#ifdef TESTING
g_debug ("Found :'%s' (%d)",
relative_path,
crawler->priv->enumerations);
+#endif /* TESTING */
if (g_file_info_get_file_type (info) == G_FILE_TYPE_DIRECTORY) {
file_enumerate (crawler, child);
@@ -610,7 +689,6 @@
/* Iterate pending events between each file in case
* there are requests waiting from DBus, etc
*/
- context = g_main_context_default ();
while (g_main_context_pending (context)) {
g_main_context_iteration (context, FALSE);
}
@@ -720,18 +798,91 @@
return TRUE;
}
+static GSList *
+check_roots_dont_coinside (TrackerCrawler *crawler,
+ GSList *roots)
+{
+ GSList *checked_roots = NULL;
+ GSList *l1, *l2;
+
+ /* ONLY HERE do we add separators on each location we check.
+ * The reason for this is that these locations are user
+ * entered in the configuration and we need to make sure we
+ * don't include the same location more than once.
+ */
+
+ for (l1 = roots; l1; l1 = l1->next) {
+ gchar *path;
+ gboolean should_add = TRUE;
+
+ if (!g_str_has_suffix (l1->data, G_DIR_SEPARATOR_S)) {
+ path = g_strconcat (l1->data, G_DIR_SEPARATOR_S, NULL);
+ } else {
+ path = g_strdup (l1->data);
+ }
+
+ l2 = checked_roots;
+
+ while (l2 && should_add) {
+ /* If the new path exists as a lower level
+ * path or is the same as an existing checked
+ * root we disgard it, it will be checked
+ * anyway.
+ */
+ if (g_str_has_prefix (path, l2->data)) {
+ should_add = FALSE;
+ }
+
+ /* If the new path exists as a higher level
+ * path to one already in the checked roots,
+ * we remove the checked roots version
+ */
+ if (g_str_has_prefix (l2->data, path)) {
+ checked_roots = g_slist_remove_link (checked_roots, l2);
+ g_free (l2->data);
+ l2 = checked_roots;
+ continue;
+ }
+
+ l2 = l2->next;
+ }
+
+ if (should_add) {
+ checked_roots = g_slist_prepend (checked_roots, path);
+ continue;
+ }
+
+ g_free (path);
+ }
+
+ checked_roots = g_slist_reverse (checked_roots);
+
+#ifdef TESTING
+ g_debug ("Using the following roots to crawl:");
+
+ for (l1 = checked_roots; l1; l1 = l1->next) {
+ g_debug (" %s", (gchar*) l1->data);
+ }
+#endif /* TESTING */
+
+ return checked_roots;
+}
+
void
tracker_crawler_start (TrackerCrawler *crawler)
{
TrackerCrawlerPriv *priv;
GFile *file;
- const gchar *path;
+ GSList *config_roots;
+ GSList *roots = NULL;
+ GSList *l;
gboolean exists;
g_return_if_fail (TRACKER_IS_CRAWLER (crawler));
priv = crawler->priv;
-
+
+ /* Set up queue handler */
if (priv->files_queue_handle_id) {
g_source_remove (priv->files_queue_handle_id);
}
@@ -739,28 +890,48 @@
priv->files_queue_handle_id = g_timeout_add (FILES_QUEUE_PROCESS_INTERVAL,
file_queue_handle_cb,
crawler);
-
- priv->running = TRUE;
- if (0) {
- get_remote_roots (crawler, NULL, NULL);
+ /* Get locations to index from config, if none are set, we use
+ * $HOME as the default.
+ */
+ config_roots = tracker_config_get_crawl_directory_roots (priv->config);
+ if (config_roots) {
+ /* Make sure none of the roots co-inside each other */
+ roots = check_roots_dont_coinside (crawler, config_roots);
}
- /* path = "/home/martyn/Documents"; */
- path = g_get_home_dir ();
+ if (!roots) {
+ const gchar *home;
- file = g_file_new_for_path (path);
-
- exists = g_file_query_exists (file, NULL);
-
- if (exists) {
- g_message ("Searching directory:'%s'", path);
- file_enumerate (crawler, file);
- } else {
- g_message ("Searching directory:'%s' failed, does not exist", path);
+ home = g_get_home_dir ();
+ roots = g_slist_prepend (roots, g_strdup (home));
+
+ g_message ("No locations are configured to crawl, "
+ "using default location (home directory)");
+ }
+
+ /* Set as running now */
+ priv->running = TRUE;
+
+ /* Start iterating roots */
+ for (l = roots; l; l = l->next) {
+ file = g_file_new_for_path (l->data);
+ exists = g_file_query_exists (file, NULL);
+
+ if (exists) {
+ g_message ("Searching directory:'%s'",
+ (gchar*) l->data);
+ file_enumerate (crawler, file);
+ } else {
+ g_message ("Searching directory:'%s' failed, does not exist",
+ (gchar*) l->data);
+ }
+
+ g_object_unref (file);
+ g_free (l->data);
}
- g_object_unref (file);
+ g_slist_free (roots);
}
void
Modified: branches/indexer-split/src/trackerd/tracker-db.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db.c (original)
+++ branches/indexer-split/src/trackerd/tracker-db.c Thu Jun 12 13:11:15 2008
@@ -556,12 +556,12 @@
const gchar *moved_to_uri)
{
/* Stop watching old dir, start watching new dir */
- tracker_monitor_remove (moved_from_uri, TRUE, iface);
+ tracker_monitor_remove (moved_from_uri, TRUE);
tracker_db_file_move (iface, moved_from_uri, moved_to_uri);
directory_move_files (iface, moved_from_uri, moved_to_uri);
- tracker_monitor_add (moved_to_uri, iface);
+ tracker_monitor_add (moved_to_uri);
}
static gint
Modified: branches/indexer-split/src/trackerd/tracker-email-kmail.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-email-kmail.c (original)
+++ branches/indexer-split/src/trackerd/tracker-email-kmail.c Thu Jun 12 13:11:15 2008
@@ -663,7 +663,7 @@
(( in_imap_dir && dir_name[0] == '.' && g_str_has_suffix (dir_name, ".directory")) ||
!in_imap_dir )
) {
- if (!tracker_monitor_is_watched (dir_path, iface)) {
+ if (!tracker_monitor_is_watched (dir_path)) {
/* if we are in a maildir directory, we will only index emails in directory "cur" */
gchar *dir_cur = g_build_filename (dir_path, "cur", NULL);
Modified: branches/indexer-split/src/trackerd/tracker-main.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-main.c (original)
+++ branches/indexer-split/src/trackerd/tracker-main.c Thu Jun 12 13:11:15 2008
@@ -139,6 +139,7 @@
static gboolean low_memory;
static gint throttle = -1;
static gint verbosity = -1;
+static gint initial_sleep = -1;
static GOptionEntry entries[] = {
{ "exclude-dir", 'e', 0,
@@ -181,6 +182,10 @@
G_OPTION_ARG_NONE, &reindex,
N_("Force a re-index of all content"),
NULL },
+ { "initial-sleep", 's', 0,
+ G_OPTION_ARG_INT, &initial_sleep,
+ N_("Time to wait before crawling the file system"),
+ NULL },
{ NULL }
};
@@ -668,6 +673,23 @@
}
}
+static gboolean
+start_cb (gpointer user_data)
+{
+ DBusGProxy *proxy;
+
+ if (!tracker->is_running) {
+ return FALSE;
+ }
+
+ /* Get files first */
+ tracker_crawler_start (tracker->crawler);
+
+ proxy = tracker_dbus_indexer_get_proxy ();
+ tracker_xesam_subscribe_indexer_updated (proxy);
+
+ return FALSE;
+}
gint
main (gint argc, gchar *argv[])
@@ -836,21 +858,19 @@
g_message ("Waiting for DBus requests...");
- if (!tracker->readonly) {
- if (tracker_config_get_enable_indexing (tracker->config)) {
- /* Get files first */
- tracker_crawler_start (tracker->crawler);
-
- if (tracker->is_running) {
- DBusGProxy *proxy;
-
- proxy = tracker_dbus_indexer_get_proxy ();
- tracker_xesam_subscribe_indexer_updated (proxy);
- }
-
+ if (!tracker->readonly &&
+ tracker_config_get_enable_indexing (tracker->config)) {
+ if (initial_sleep > 0) {
+ g_message ("Waiting %d seconds before starting",
+ initial_sleep);
+ g_timeout_add (initial_sleep * 1000,
+ start_cb,
+ NULL);
} else {
- g_message ("Indexing disabled, not starting");
+ g_idle_add (start_cb, tracker);
}
+ } else {
+ g_message ("Indexing disabled, not starting");
}
if (tracker->is_running) {
@@ -860,10 +880,6 @@
g_message ("Shutting down...\n");
- if (tracker->crawler) {
- g_object_unref (tracker->crawler);
- }
-
/*
* Shutdown the daemon
*/
@@ -897,6 +913,9 @@
g_object_unref (tracker->hal);
}
#endif
+ if (tracker->crawler) {
+ g_object_unref (tracker->crawler);
+ }
if (tracker->language) {
g_object_unref (tracker->language);
Modified: branches/indexer-split/src/trackerd/tracker-monitor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-monitor.c (original)
+++ branches/indexer-split/src/trackerd/tracker-monitor.c Thu Jun 12 13:11:15 2008
@@ -18,8 +18,21 @@
* Boston, MA 02110-1301, USA.
*/
+#include <gio/gio.h>
+
#include "tracker-monitor.h"
+/* This is the default inotify limit - 500 to allow some monitors for
+ * other applications.
+ *
+ * FIXME: Should we try reading
+ * /proc/sys/fs/inotify/max_user_watches when there is a possiblity
+ * that we don't even use inotify?
+ */
+#define MAX_MONITORS (guint) ((2 ^ 13) - 500)
+
+static GHashTable *monitors;
+
#if 0
static void
get_monitor_roots (GSList **included,
@@ -69,46 +82,111 @@
gboolean
tracker_monitor_init (void)
{
+ if (monitors) {
+ return TRUE;
+ }
+
+ monitors = g_hash_table_new_full (g_str_hash,
+ g_str_equal,
+ g_free,
+ (GDestroyNotify) g_file_monitor_cancel);
return TRUE;
}
void
tracker_monitor_shutdown (void)
{
+ if (!monitors) {
+ return;
+ }
+
+ g_hash_table_unref (monitors);
+ monitors = NULL;
}
gboolean
-tracker_monitor_add (const gchar *uri,
- TrackerDBInterface *iface)
+tracker_monitor_add (const gchar *path)
{
- g_return_val_if_fail (uri != NULL, FALSE);
-
+ GFile *file;
+ GFileMonitor *monitor;
+ GError *error = NULL;
+
+ g_return_val_if_fail (path != NULL, FALSE);
+
+ if (g_hash_table_lookup (monitors, path)) {
+ return TRUE;
+ }
+
+ /* Cap the number of monitors */
+ if (g_hash_table_size (monitors) >= MAX_MONITORS) {
+ g_warning ("The maximum number of monitors to set (%d) "
+ "has been reached, not adding any new ones",
+ MAX_MONITORS);
+ return FALSE;
+ }
+
+ /* We don't check if a file exists or not since we might want
+ * to monitor locations which don't exist yet.
+ */
+ file = g_file_new_for_path (path);
+ monitor = g_file_monitor_directory (file,
+ G_FILE_MONITOR_WATCH_MOUNTS,
+ NULL,
+ &error);
+ g_object_unref (file);
+
+ if (error) {
+ g_warning ("Could not add monitor for path:'%s', %s",
+ path,
+ error->message);
+ g_error_free (error);
+ return FALSE;
+ }
+
+ g_debug ("Added monitor for:'%s', total monitors:%d",
+ path,
+ g_hash_table_size (monitors));
+
+ g_hash_table_insert (monitors,
+ g_strdup (path),
+ monitor);
- return FALSE;
+ return TRUE;
}
gboolean
-tracker_monitor_remove (const gchar *uri,
- gboolean delete_subdirs,
- TrackerDBInterface *iface)
+tracker_monitor_remove (const gchar *path,
+ gboolean delete_subdirs)
{
- g_return_val_if_fail (uri != NULL, FALSE);
+ GFileMonitor *monitor;
+
+ g_return_val_if_fail (path != NULL, FALSE);
+
+ monitor = g_hash_table_lookup (monitors, path);
+ if (!monitor) {
+ return TRUE;
+ }
- return FALSE;
+ g_hash_table_remove (monitors, path);
+
+ g_debug ("Removed monitor for:'%s', total monitors:%d",
+ path,
+ g_hash_table_size (monitors));
+
+ return TRUE;
}
gboolean
-tracker_monitor_is_watched (const gchar *uri,
- TrackerDBInterface *iface)
+tracker_monitor_is_watched (const gchar *path)
{
- g_return_val_if_fail (uri != NULL, FALSE);
+ g_return_val_if_fail (path != NULL, FALSE);
- return FALSE;
+ return g_hash_table_lookup (monitors, path) != NULL;
}
gint
tracker_monitor_get_count (void)
{
- return 0;
+ return g_hash_table_size (monitors);
}
Modified: branches/indexer-split/src/trackerd/tracker-monitor.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-monitor.h (original)
+++ branches/indexer-split/src/trackerd/tracker-monitor.h Thu Jun 12 13:11:15 2008
@@ -28,13 +28,10 @@
gboolean tracker_monitor_init (void);
void tracker_monitor_shutdown (void);
-gboolean tracker_monitor_add (const gchar *uri,
- TrackerDBInterface *iface);
-gboolean tracker_monitor_remove (const gchar *uri,
- gboolean delete_subdirs,
- TrackerDBInterface *iface);
-gboolean tracker_monitor_is_watched (const gchar *uri,
- TrackerDBInterface *iface);
+gboolean tracker_monitor_add (const gchar *path);
+gboolean tracker_monitor_remove (const gchar *path,
+ gboolean delete_subdirs);
+gboolean tracker_monitor_is_watched (const gchar *path);
gint tracker_monitor_get_count (void);
G_END_DECLS
Modified: branches/indexer-split/src/trackerd/tracker-process-files.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-process-files.c (original)
+++ branches/indexer-split/src/trackerd/tracker-process-files.c Thu Jun 12 13:11:15 2008
@@ -251,9 +251,7 @@
continue;
}
- if (!tracker_monitor_add (dir, iface)) {
- g_warning ("Watch failed for:'%s'", dir);
- }
+ tracker_monitor_add (dir);
}
for (l = list; l; l = l->next) {
@@ -400,7 +398,7 @@
tracker_db_directory_delete (iface, info->file_id, info->uri);
- tracker_monitor_remove (info->uri, TRUE, iface);
+ tracker_monitor_remove (info->uri, TRUE);
g_message ("Deleting directory:'%s' and subdirs", info->uri);
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
