[tracker/crawling-interval] Add CrawlingInterval configuration option.
- From: Carlos Garnacho <carlosg src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/crawling-interval] Add CrawlingInterval configuration option.
- Date: Wed, 28 Apr 2010 13:32:35 +0000 (UTC)
commit eecb80cd7e67608446abe4f12785b14782469f7a
Author: Carlos Garnacho <carlosg gnome org>
Date: Wed Apr 28 15:23:37 2010 +0200
Add CrawlingInterval configuration option.
This either forever enables mtime queries for initial crawling, disables it,
or specified an interval in days until the next crawling.
The other use of crawling (setting up monitors) is unavoidable.
src/libtracker-miner/tracker-miner-fs.c | 48 ++++++++++++++++++++-
src/libtracker-miner/tracker-miner-fs.h | 5 ++
src/miners/fs/tracker-config.c | 60 +++++++++++++++++++++++++-
src/miners/fs/tracker-config.h | 3 +
src/miners/fs/tracker-main.c | 72 +++++++++++++++++++++++++++++++
src/miners/fs/tracker-miner-files.c | 2 -
6 files changed, 185 insertions(+), 5 deletions(-)
---
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index 2d59aa7..fb6edad 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -139,6 +139,7 @@ struct TrackerMinerFSPrivate {
guint shown_totals : 1;
guint is_paused : 1;
guint is_crawling : 1;
+ guint initial_crawling : 1;
/* Statistics */
guint total_directories_found;
@@ -180,7 +181,8 @@ enum {
enum {
PROP_0,
PROP_THROTTLE,
- PROP_POOL_LIMIT
+ PROP_POOL_LIMIT,
+ PROP_INITIAL_CRAWLING
};
static void fs_finalize (GObject *object);
@@ -304,6 +306,13 @@ tracker_miner_fs_class_init (TrackerMinerFSClass *klass)
"Number of files that can be concurrently processed",
1, G_MAXUINT, 1,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+ g_object_class_install_property (object_class,
+ PROP_INITIAL_CRAWLING,
+ g_param_spec_boolean ("initial-crawling",
+ "Initial crawling",
+ "Whether to perform initial crawling or not",
+ TRUE,
+ G_PARAM_READWRITE));
/**
* TrackerMinerFS::check-file:
* @miner_fs: the #TrackerMinerFS
@@ -535,6 +544,8 @@ tracker_miner_fs_init (TrackerMinerFS *object)
(GEqualFunc) g_file_equal,
(GDestroyNotify) g_object_unref,
(GDestroyNotify) g_free);
+
+ priv->initial_crawling = TRUE;
}
static ProcessData *
@@ -683,6 +694,9 @@ fs_set_property (GObject *object,
fs->private->pool_limit = g_value_get_uint (value);
g_message ("Miner process pool limit is set to %d", fs->private->pool_limit);
break;
+ case PROP_INITIAL_CRAWLING:
+ fs->private->initial_crawling = g_value_get_boolean (value);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@@ -706,6 +720,9 @@ fs_get_property (GObject *object,
case PROP_POOL_LIMIT:
g_value_set_uint (value, fs->private->pool_limit);
break;
+ case PROP_INITIAL_CRAWLING:
+ g_value_set_boolean (value, fs->private->initial_crawling);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@@ -2495,6 +2512,11 @@ crawler_check_file_cb (TrackerCrawler *crawler,
{
TrackerMinerFS *fs = user_data;
+ if (!fs->private->been_crawled &&
+ !fs->private->initial_crawling) {
+ return FALSE;
+ }
+
return should_process_file (fs, file, FALSE);
}
@@ -2514,7 +2536,12 @@ crawler_check_directory_cb (TrackerCrawler *crawler,
} else {
gboolean should_change_index;
- should_change_index = should_change_index_for_file (fs, file);
+ if (!fs->private->been_crawled &&
+ !fs->private->initial_crawling) {
+ should_change_index = FALSE;
+ } else {
+ should_change_index = should_change_index_for_file (fs, file);
+ }
if (!should_change_index) {
/* Mark the file as ignored, we still want the crawler
@@ -3224,3 +3251,20 @@ tracker_miner_fs_force_recheck (TrackerMinerFS *fs)
crawl_directories_start (fs);
}
+
+void
+tracker_miner_fs_set_initial_crawling (TrackerMinerFS *fs,
+ gboolean do_initial_crawling)
+{
+ g_return_if_fail (TRACKER_IS_MINER_FS (fs));
+
+ fs->private->initial_crawling = do_initial_crawling;
+}
+
+gboolean
+tracker_miner_fs_get_initial_crawling (TrackerMinerFS *fs)
+{
+ g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), FALSE);
+
+ return fs->private->initial_crawling;
+}
diff --git a/src/libtracker-miner/tracker-miner-fs.h b/src/libtracker-miner/tracker-miner-fs.h
index 61b628d..5105e1e 100644
--- a/src/libtracker-miner/tracker-miner-fs.h
+++ b/src/libtracker-miner/tracker-miner-fs.h
@@ -112,6 +112,11 @@ G_CONST_RETURN gchar *tracker_miner_fs_get_parent_urn (TrackerMinerFS *fs,
GFile *file);
void tracker_miner_fs_force_recheck (TrackerMinerFS *fs);
+void tracker_miner_fs_set_initial_crawling (TrackerMinerFS *fs,
+ gboolean do_initial_crawling);
+gboolean tracker_miner_fs_get_initial_crawling (TrackerMinerFS *fs);
+
+
G_END_DECLS
#endif /* __LIBTRACKER_MINER_MINER_FS_H__ */
diff --git a/src/miners/fs/tracker-config.c b/src/miners/fs/tracker-config.c
index da0a387..4b8874d 100644
--- a/src/miners/fs/tracker-config.c
+++ b/src/miners/fs/tracker-config.c
@@ -37,6 +37,7 @@
#define GROUP_GENERAL "General"
#define GROUP_MONITORS "Monitors"
#define GROUP_INDEXING "Indexing"
+#define GROUP_CRAWLING "Crawling"
/* Default values */
#define DEFAULT_VERBOSITY 0
@@ -51,6 +52,7 @@
#define DEFAULT_INDEX_ON_BATTERY FALSE
#define DEFAULT_INDEX_ON_BATTERY_FIRST_TIME TRUE
#define DEFAULT_LOW_DISK_SPACE_LIMIT 1 /* 0->100 / -1 */
+#define DEFAULT_CRAWLING_INTERVAL 0 /* 0->7 / -1 */
typedef struct {
/* General */
@@ -76,6 +78,9 @@ typedef struct {
GSList *ignored_directories_with_content;
GSList *ignored_files;
+ /* Crawling */
+ gint crawling_interval;
+
/* Convenience data */
GSList *ignored_directory_patterns;
GSList *ignored_directory_paths;
@@ -133,6 +138,9 @@ enum {
PROP_IGNORED_DIRECTORIES,
PROP_IGNORED_DIRECTORIES_WITH_CONTENT,
PROP_IGNORED_FILES,
+
+ /* Crawling */
+ PROP_CRAWLING_INTERVAL
};
static ObjectToKeyFile conversions[] = {
@@ -156,6 +164,8 @@ static ObjectToKeyFile conversions[] = {
{ G_TYPE_POINTER, "ignored-directories", GROUP_INDEXING, "IgnoredDirectories" },
{ G_TYPE_POINTER, "ignored-directories-with-content", GROUP_INDEXING, "IgnoredDirectoriesWithContent" },
{ G_TYPE_POINTER, "ignored-files", GROUP_INDEXING, "IgnoredFiles" },
+ /* Crawling */
+ { G_TYPE_INT, "crawling-interval", GROUP_CRAWLING, "CrawlingInterval" }
};
G_DEFINE_TYPE (TrackerConfig, tracker_config, TRACKER_TYPE_CONFIG_FILE);
@@ -320,6 +330,18 @@ tracker_config_class_init (TrackerConfigClass *klass)
" List of files to NOT index (separator=;)",
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+ /* Crawling */
+ g_object_class_install_property (object_class,
+ PROP_CRAWLING_INTERVAL,
+ g_param_spec_int ("crawling-interval",
+ "Crawling interval",
+ " Interval at which startup crawling may happen. 0 is always, -1 is never,"
+ " and any number > 0 is the crawling interval in number of days.",
+ -1,
+ G_MAXINT,
+ DEFAULT_CRAWLING_INTERVAL,
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+
g_type_class_add_private (object_class, sizeof (TrackerConfigPrivate));
}
@@ -408,6 +430,10 @@ config_set_property (GObject *object,
tracker_config_set_ignored_files (TRACKER_CONFIG (object),
g_value_get_pointer (value));
break;
+ case PROP_CRAWLING_INTERVAL:
+ tracker_config_set_crawling_interval (TRACKER_CONFIG (object),
+ g_value_get_int (value));
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
@@ -482,7 +508,9 @@ config_get_property (GObject *object,
case PROP_IGNORED_FILES:
g_value_set_pointer (value, priv->ignored_files);
break;
-
+ case PROP_CRAWLING_INTERVAL:
+ g_value_set_int (value, priv->crawling_interval);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
break;
@@ -1278,6 +1306,18 @@ tracker_config_get_ignored_files (TrackerConfig *config)
return priv->ignored_files;
}
+gint
+tracker_config_get_crawling_interval (TrackerConfig *config)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_val_if_fail (TRACKER_IS_CONFIG (config), 0);
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ return priv->crawling_interval;
+}
+
void
tracker_config_set_verbosity (TrackerConfig *config,
gint value)
@@ -1642,6 +1682,24 @@ tracker_config_set_ignored_files (TrackerConfig *config,
g_object_notify (G_OBJECT (config), "ignored-files");
}
+void
+tracker_config_set_crawling_interval (TrackerConfig *config,
+ gint interval)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+ if (!tracker_keyfile_object_validate_int (config, "crawling-interval", interval)) {
+ return;
+ }
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ priv->crawling_interval = interval;
+ g_object_notify (G_OBJECT (config), "crawling-interval");
+}
+
/*
* Convenience functions
*/
diff --git a/src/miners/fs/tracker-config.h b/src/miners/fs/tracker-config.h
index 57ffacf..6108f76 100644
--- a/src/miners/fs/tracker-config.h
+++ b/src/miners/fs/tracker-config.h
@@ -68,6 +68,7 @@ GSList * tracker_config_get_index_single_directories (TrackerConfi
GSList * tracker_config_get_ignored_directories (TrackerConfig *config);
GSList * tracker_config_get_ignored_directories_with_content (TrackerConfig *config);
GSList * tracker_config_get_ignored_files (TrackerConfig *config);
+gint tracker_config_get_crawling_interval (TrackerConfig *config);
void tracker_config_set_verbosity (TrackerConfig *config,
gint value);
@@ -103,6 +104,8 @@ void tracker_config_set_ignored_directories_with_content (TrackerConfi
GSList *files);
void tracker_config_set_ignored_files (TrackerConfig *config,
GSList *files);
+void tracker_config_set_crawling_interval (TrackerConfig *config,
+ gint interval);
/*
* Convenience functions:
diff --git a/src/miners/fs/tracker-main.c b/src/miners/fs/tracker-main.c
index f661570..9016b0c 100644
--- a/src/miners/fs/tracker-main.c
+++ b/src/miners/fs/tracker-main.c
@@ -62,6 +62,8 @@
"\n" \
" http://www.gnu.org/licenses/gpl.txt\n";
+#define SECONDS_PER_DAY 60 * 60 * 24
+
static GMainLoop *main_loop;
static GSList *miners;
static GSList *current_miner;
@@ -72,6 +74,7 @@ static gint initial_sleep = -1;
static gchar *eligible;
static gchar *add_file;
static gboolean version;
+static gchar *crawl_timestamp_file;
static GOptionEntry entries[] = {
{ "verbosity", 'v', 0,
@@ -191,6 +194,68 @@ initialize_priority (void)
}
}
+static gboolean
+should_crawl (TrackerConfig *config)
+{
+ gint crawling_interval;
+
+ if (G_UNLIKELY (!crawl_timestamp_file)) {
+ crawl_timestamp_file = g_build_filename (g_get_user_cache_dir (),
+ "tracker",
+ "crawling-timestamp.txt",
+ NULL);
+ }
+
+ crawling_interval = tracker_config_get_crawling_interval (config);
+
+ g_message ("Checking whether to perform mtime checks during crawling:");
+
+ if (crawling_interval == -1) {
+ g_message (" Disabled");
+ return FALSE;
+ } else if (crawling_interval == 0) {
+ g_message (" Enabled");
+ return TRUE;
+ } else {
+ guint64 then, now;
+ gchar *content;
+
+ if (!g_file_get_contents (crawl_timestamp_file, &content, NULL, NULL)) {
+ g_message (" No previous timestamp, crawling forced");
+ return TRUE;
+ }
+
+ now = (guint64) time (NULL);
+
+ then = g_ascii_strtoull (content, NULL, 10);
+ g_free (content);
+
+ if (now < then + (crawling_interval * SECONDS_PER_DAY)) {
+ g_message (" Postponed");
+ return FALSE;
+ } else {
+ g_message (" (More than) %d days after last crawling, enabled", crawling_interval);
+ return FALSE;
+ }
+ }
+}
+
+static void
+save_crawling_time (void)
+{
+ GError *error = NULL;
+ gchar *content;
+
+ content = g_strdup_printf ("%" G_GUINT64_FORMAT, (guint64) time (NULL));
+
+ g_file_set_contents (crawl_timestamp_file, content, -1, &error);
+
+ if (error) {
+ g_critical ("Could not save crawling timestamp: %s", error->message);
+ g_error_free (error);
+ }
+}
+
static void
miner_handle_next (void)
{
@@ -241,6 +306,11 @@ miner_finished_cb (TrackerMinerFS *fs,
return;
}
+ if (TRACKER_IS_MINER_FILES (fs) &&
+ tracker_miner_fs_get_initial_crawling (fs)) {
+ save_crawling_time ();
+ }
+
miner_handle_next ();
}
@@ -585,6 +655,8 @@ main (gint argc, gchar *argv[])
if (!add_file) {
miner_files = tracker_miner_files_new (config);
+ tracker_miner_fs_set_initial_crawling (TRACKER_MINER_FS (miner_files),
+ should_crawl (config));
} else {
GFile *file;
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index c38d56d..ed60bc5 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -1222,8 +1222,6 @@ miner_files_check_directory (TrackerMinerFS *fs,
tracker_config_get_index_single_directories (mf->private->config),
tracker_config_get_ignored_directory_paths (mf->private->config),
tracker_config_get_ignored_directory_patterns (mf->private->config));
-
-
}
static gboolean
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
