[tracker/crawling-interval] Add CrawlingInterval configuration option.
- From: Carlos Garnacho <carlosg src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/crawling-interval] Add CrawlingInterval configuration option.
- Date: Thu, 29 Apr 2010 15:52:52 +0000 (UTC)
commit d32156426039b06affd51fa6f483bd1120a4de9f
Author: Carlos Garnacho <carlosg gnome org>
Date: Wed Apr 28 15:23:37 2010 +0200
Add CrawlingInterval configuration option.
This either forever enables mtime queries for initial crawling, disables it,
or specified an interval in days until the next crawling.
The other use of crawling (setting up monitors) is unavoidable.
src/libtracker-miner/tracker-miner-fs.c | 31 ++++++++++++-
src/miners/fs/tracker-config.c | 53 +++++++++++++++++++++-
src/miners/fs/tracker-config.h | 3 +
src/miners/fs/tracker-main.c | 3 +
src/miners/fs/tracker-miner-files.c | 76 ++++++++++++++++++++++++++++++-
5 files changed, 161 insertions(+), 5 deletions(-)
---
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index 2d59aa7..3e22a10 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -139,6 +139,7 @@ struct TrackerMinerFSPrivate {
guint shown_totals : 1;
guint is_paused : 1;
guint is_crawling : 1;
+ guint mtime_checking : 1;
/* Statistics */
guint total_directories_found;
@@ -180,7 +181,8 @@ enum {
enum {
PROP_0,
PROP_THROTTLE,
- PROP_POOL_LIMIT
+ PROP_POOL_LIMIT,
+ PROP_MTIME_CHECKING
};
static void fs_finalize (GObject *object);
@@ -304,6 +306,13 @@ tracker_miner_fs_class_init (TrackerMinerFSClass *klass)
"Number of files that can be concurrently processed",
1, G_MAXUINT, 1,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+ g_object_class_install_property (object_class,
+ PROP_MTIME_CHECKING,
+ g_param_spec_boolean ("mtime-checking",
+ "Mtime checking",
+ "Whether to perform mtime checks during initial crawling or not",
+ TRUE,
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
/**
* TrackerMinerFS::check-file:
* @miner_fs: the #TrackerMinerFS
@@ -535,6 +544,8 @@ tracker_miner_fs_init (TrackerMinerFS *object)
(GEqualFunc) g_file_equal,
(GDestroyNotify) g_object_unref,
(GDestroyNotify) g_free);
+
+ priv->mtime_checking = TRUE;
}
static ProcessData *
@@ -683,6 +694,9 @@ fs_set_property (GObject *object,
fs->private->pool_limit = g_value_get_uint (value);
g_message ("Miner process pool limit is set to %d", fs->private->pool_limit);
break;
+ case PROP_MTIME_CHECKING:
+ fs->private->mtime_checking = g_value_get_boolean (value);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@@ -706,6 +720,9 @@ fs_get_property (GObject *object,
case PROP_POOL_LIMIT:
g_value_set_uint (value, fs->private->pool_limit);
break;
+ case PROP_MTIME_CHECKING:
+ g_value_set_boolean (value, fs->private->mtime_checking);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@@ -2495,6 +2512,11 @@ crawler_check_file_cb (TrackerCrawler *crawler,
{
TrackerMinerFS *fs = user_data;
+ if (!fs->private->been_crawled &&
+ !fs->private->mtime_checking) {
+ return FALSE;
+ }
+
return should_process_file (fs, file, FALSE);
}
@@ -2514,7 +2536,12 @@ crawler_check_directory_cb (TrackerCrawler *crawler,
} else {
gboolean should_change_index;
- should_change_index = should_change_index_for_file (fs, file);
+ if (!fs->private->been_crawled &&
+ !fs->private->mtime_checking) {
+ should_change_index = FALSE;
+ } else {
+ should_change_index = should_change_index_for_file (fs, file);
+ }
if (!should_change_index) {
/* Mark the file as ignored, we still want the crawler
diff --git a/src/miners/fs/tracker-config.c b/src/miners/fs/tracker-config.c
index da0a387..92a6c91 100644
--- a/src/miners/fs/tracker-config.c
+++ b/src/miners/fs/tracker-config.c
@@ -51,6 +51,7 @@
#define DEFAULT_INDEX_ON_BATTERY FALSE
#define DEFAULT_INDEX_ON_BATTERY_FIRST_TIME TRUE
#define DEFAULT_LOW_DISK_SPACE_LIMIT 1 /* 0->100 / -1 */
+#define DEFAULT_CRAWLING_INTERVAL 0 /* 0->7 / -1 */
typedef struct {
/* General */
@@ -75,6 +76,7 @@ typedef struct {
GSList *ignored_directories;
GSList *ignored_directories_with_content;
GSList *ignored_files;
+ gint crawling_interval;
/* Convenience data */
GSList *ignored_directory_patterns;
@@ -133,6 +135,7 @@ enum {
PROP_IGNORED_DIRECTORIES,
PROP_IGNORED_DIRECTORIES_WITH_CONTENT,
PROP_IGNORED_FILES,
+ PROP_CRAWLING_INTERVAL
};
static ObjectToKeyFile conversions[] = {
@@ -156,6 +159,7 @@ static ObjectToKeyFile conversions[] = {
{ G_TYPE_POINTER, "ignored-directories", GROUP_INDEXING, "IgnoredDirectories" },
{ G_TYPE_POINTER, "ignored-directories-with-content", GROUP_INDEXING, "IgnoredDirectoriesWithContent" },
{ G_TYPE_POINTER, "ignored-files", GROUP_INDEXING, "IgnoredFiles" },
+ { G_TYPE_INT, "crawling-interval", GROUP_INDEXING, "CrawlingInterval" }
};
G_DEFINE_TYPE (TrackerConfig, tracker_config, TRACKER_TYPE_CONFIG_FILE);
@@ -319,6 +323,17 @@ tracker_config_class_init (TrackerConfigClass *klass)
"Ignored files",
" List of files to NOT index (separator=;)",
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+ g_object_class_install_property (object_class,
+ PROP_CRAWLING_INTERVAL,
+ g_param_spec_int ("crawling-interval",
+ "Crawling interval",
+ " Interval in days to check the filesystem is up to date in the database."
+ " If set to 0, crawling always occurs on startup, if -1 crawling is"
+ " disabled entirely.",
+ -1,
+ G_MAXINT,
+ DEFAULT_CRAWLING_INTERVAL,
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
g_type_class_add_private (object_class, sizeof (TrackerConfigPrivate));
}
@@ -408,6 +423,10 @@ config_set_property (GObject *object,
tracker_config_set_ignored_files (TRACKER_CONFIG (object),
g_value_get_pointer (value));
break;
+ case PROP_CRAWLING_INTERVAL:
+ tracker_config_set_crawling_interval (TRACKER_CONFIG (object),
+ g_value_get_int (value));
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
@@ -482,7 +501,9 @@ config_get_property (GObject *object,
case PROP_IGNORED_FILES:
g_value_set_pointer (value, priv->ignored_files);
break;
-
+ case PROP_CRAWLING_INTERVAL:
+ g_value_set_int (value, priv->crawling_interval);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
break;
@@ -1278,6 +1299,18 @@ tracker_config_get_ignored_files (TrackerConfig *config)
return priv->ignored_files;
}
+gint
+tracker_config_get_crawling_interval (TrackerConfig *config)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_val_if_fail (TRACKER_IS_CONFIG (config), 0);
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ return priv->crawling_interval;
+}
+
void
tracker_config_set_verbosity (TrackerConfig *config,
gint value)
@@ -1642,6 +1675,24 @@ tracker_config_set_ignored_files (TrackerConfig *config,
g_object_notify (G_OBJECT (config), "ignored-files");
}
+void
+tracker_config_set_crawling_interval (TrackerConfig *config,
+ gint interval)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+ if (!tracker_keyfile_object_validate_int (config, "crawling-interval", interval)) {
+ return;
+ }
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ priv->crawling_interval = interval;
+ g_object_notify (G_OBJECT (config), "crawling-interval");
+}
+
/*
* Convenience functions
*/
diff --git a/src/miners/fs/tracker-config.h b/src/miners/fs/tracker-config.h
index 57ffacf..6108f76 100644
--- a/src/miners/fs/tracker-config.h
+++ b/src/miners/fs/tracker-config.h
@@ -68,6 +68,7 @@ GSList * tracker_config_get_index_single_directories (TrackerConfi
GSList * tracker_config_get_ignored_directories (TrackerConfig *config);
GSList * tracker_config_get_ignored_directories_with_content (TrackerConfig *config);
GSList * tracker_config_get_ignored_files (TrackerConfig *config);
+gint tracker_config_get_crawling_interval (TrackerConfig *config);
void tracker_config_set_verbosity (TrackerConfig *config,
gint value);
@@ -103,6 +104,8 @@ void tracker_config_set_ignored_directories_with_content (TrackerConfi
GSList *files);
void tracker_config_set_ignored_files (TrackerConfig *config,
GSList *files);
+void tracker_config_set_crawling_interval (TrackerConfig *config,
+ gint interval);
/*
* Convenience functions:
diff --git a/src/miners/fs/tracker-main.c b/src/miners/fs/tracker-main.c
index f661570..0e3e50c 100644
--- a/src/miners/fs/tracker-main.c
+++ b/src/miners/fs/tracker-main.c
@@ -62,6 +62,8 @@
"\n" \
" http://www.gnu.org/licenses/gpl.txt\n";
+#define SECONDS_PER_DAY 60 * 60 * 24
+
static GMainLoop *main_loop;
static GSList *miners;
static GSList *current_miner;
@@ -72,6 +74,7 @@ static gint initial_sleep = -1;
static gchar *eligible;
static gchar *add_file;
static gboolean version;
+static gchar *crawl_timestamp_file;
static GOptionEntry entries[] = {
{ "verbosity", 'v', 0,
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index c38d56d..6836183 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -42,6 +42,7 @@
#include "tracker-marshal.h"
#define DISK_SPACE_CHECK_FREQUENCY 10
+#define SECONDS_PER_DAY 60 * 60 * 24
#define TRACKER_MINER_FILES_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_MINER_FILES, TrackerMinerFilesPrivate))
@@ -90,6 +91,8 @@ enum {
PROP_CONFIG
};
+static gchar *crawl_timestamp_file = NULL;
+
static void miner_files_set_property (GObject *object,
guint param_id,
const GValue *value,
@@ -152,6 +155,8 @@ static gboolean miner_files_ignore_next_update_file (TrackerMinerFS *f
GFile *file,
TrackerSparqlBuilder *sparql,
GCancellable *cancellable);
+static void miner_files_finished (TrackerMinerFS *fs);
+
static void extractor_get_embedded_metadata_cancel (GCancellable *cancellable,
ProcessFileData *data);
@@ -175,6 +180,7 @@ tracker_miner_files_class_init (TrackerMinerFilesClass *klass)
miner_fs_class->monitor_directory = miner_files_monitor_directory;
miner_fs_class->process_file = miner_files_process_file;
miner_fs_class->ignore_next_update_file = miner_files_ignore_next_update_file;
+ miner_fs_class->finished = miner_files_finished;
g_object_class_install_property (object_class,
PROP_CONFIG,
@@ -1222,8 +1228,6 @@ miner_files_check_directory (TrackerMinerFS *fs,
tracker_config_get_index_single_directories (mf->private->config),
tracker_config_get_ignored_directory_paths (mf->private->config),
tracker_config_get_ignored_directory_patterns (mf->private->config));
-
-
}
static gboolean
@@ -1691,6 +1695,73 @@ miner_files_ignore_next_update_file (TrackerMinerFS *fs,
return TRUE;
}
+static gboolean
+should_check_mtime (TrackerConfig *config)
+{
+ gint crawling_interval;
+
+ if (G_UNLIKELY (!crawl_timestamp_file)) {
+ crawl_timestamp_file = g_build_filename (g_get_user_cache_dir (),
+ "tracker",
+ "crawling-timestamp.txt",
+ NULL);
+ }
+
+ crawling_interval = tracker_config_get_crawling_interval (config);
+
+ g_message ("Checking whether to perform mtime checks during crawling:");
+
+ if (crawling_interval == -1) {
+ g_message (" Disabled");
+ return FALSE;
+ } else if (crawling_interval == 0) {
+ g_message (" Enabled");
+ return TRUE;
+ } else {
+ guint64 then, now;
+ gchar *content;
+
+ if (!g_file_get_contents (crawl_timestamp_file, &content, NULL, NULL)) {
+ g_message (" No previous timestamp, crawling forced");
+ return TRUE;
+ }
+
+ now = (guint64) time (NULL);
+
+ then = g_ascii_strtoull (content, NULL, 10);
+ g_free (content);
+
+ if (now < then + (crawling_interval * SECONDS_PER_DAY)) {
+ g_message (" Postponed");
+ return FALSE;
+ } else {
+ g_message ("Not occurred for %d days, crawling forced", crawling_interval);
+ return FALSE;
+ }
+ }
+}
+
+static void
+save_crawling_time (void)
+{
+ GError *error = NULL;
+ gchar *content;
+
+ content = g_strdup_printf ("%" G_GUINT64_FORMAT, (guint64) time (NULL));
+
+ g_file_set_contents (crawl_timestamp_file, content, -1, &error);
+
+ if (error) {
+ g_critical ("Could not save crawling timestamp: %s", error->message);
+ g_error_free (error);
+ }
+}
+
+static void
+miner_files_finished (TrackerMinerFS *fs)
+{
+ save_crawling_time ();
+}
TrackerMiner *
tracker_miner_files_new (TrackerConfig *config)
@@ -1699,6 +1770,7 @@ tracker_miner_files_new (TrackerConfig *config)
"name", "Files",
"config", config,
"process-pool-limit", 10,
+ "mtime-checking", should_check_mtime (config),
NULL);
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
