[tracker/crawling-interval: 1/3] Add CrawlingInterval configuration option.
- From: Martyn James Russell <mr src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/crawling-interval: 1/3] Add CrawlingInterval configuration option.
- Date: Tue, 1 Jun 2010 16:00:37 +0000 (UTC)
commit 0dfac5d27f371d4694f0a6818f86d8b02966f9b4
Author: Carlos Garnacho <carlosg gnome org>
Date: Wed Apr 28 15:23:37 2010 +0200
Add CrawlingInterval configuration option.
This either forever enables mtime queries for initial crawling, disables it,
or specified an interval in days until the next crawling.
The other use of crawling (setting up monitors) is unavoidable.
src/libtracker-miner/tracker-miner-fs.c | 31 ++++++++++++-
src/miners/fs/tracker-config.c | 53 +++++++++++++++++++++-
src/miners/fs/tracker-config.h | 3 +
src/miners/fs/tracker-main.c | 3 +
src/miners/fs/tracker-miner-files.c | 76 ++++++++++++++++++++++++++++++-
5 files changed, 161 insertions(+), 5 deletions(-)
---
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index d1259a1..764d794 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -139,6 +139,7 @@ struct TrackerMinerFSPrivate {
guint shown_totals : 1;
guint is_paused : 1;
guint is_crawling : 1;
+ guint mtime_checking : 1;
/* Statistics */
guint total_directories_found;
@@ -180,7 +181,8 @@ enum {
enum {
PROP_0,
PROP_THROTTLE,
- PROP_POOL_LIMIT
+ PROP_POOL_LIMIT,
+ PROP_MTIME_CHECKING
};
static void fs_finalize (GObject *object);
@@ -304,6 +306,13 @@ tracker_miner_fs_class_init (TrackerMinerFSClass *klass)
"Number of files that can be concurrently processed",
1, G_MAXUINT, 1,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+ g_object_class_install_property (object_class,
+ PROP_MTIME_CHECKING,
+ g_param_spec_boolean ("mtime-checking",
+ "Mtime checking",
+ "Whether to perform mtime checks during initial crawling or not",
+ TRUE,
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
/**
* TrackerMinerFS::check-file:
* @miner_fs: the #TrackerMinerFS
@@ -535,6 +544,8 @@ tracker_miner_fs_init (TrackerMinerFS *object)
(GEqualFunc) g_file_equal,
(GDestroyNotify) g_object_unref,
(GDestroyNotify) g_free);
+
+ priv->mtime_checking = TRUE;
}
static ProcessData *
@@ -687,6 +698,9 @@ fs_set_property (GObject *object,
fs->private->pool_limit = g_value_get_uint (value);
g_message ("Miner process pool limit is set to %d", fs->private->pool_limit);
break;
+ case PROP_MTIME_CHECKING:
+ fs->private->mtime_checking = g_value_get_boolean (value);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@@ -710,6 +724,9 @@ fs_get_property (GObject *object,
case PROP_POOL_LIMIT:
g_value_set_uint (value, fs->private->pool_limit);
break;
+ case PROP_MTIME_CHECKING:
+ g_value_set_boolean (value, fs->private->mtime_checking);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@@ -2566,6 +2583,11 @@ crawler_check_file_cb (TrackerCrawler *crawler,
{
TrackerMinerFS *fs = user_data;
+ if (!fs->private->been_crawled &&
+ !fs->private->mtime_checking) {
+ return FALSE;
+ }
+
return should_process_file (fs, file, FALSE);
}
@@ -2585,7 +2607,12 @@ crawler_check_directory_cb (TrackerCrawler *crawler,
} else {
gboolean should_change_index;
- should_change_index = should_change_index_for_file (fs, file);
+ if (!fs->private->been_crawled &&
+ !fs->private->mtime_checking) {
+ should_change_index = FALSE;
+ } else {
+ should_change_index = should_change_index_for_file (fs, file);
+ }
if (!should_change_index) {
/* Mark the file as ignored, we still want the crawler
diff --git a/src/miners/fs/tracker-config.c b/src/miners/fs/tracker-config.c
index 0bea3b4..8f80d11 100644
--- a/src/miners/fs/tracker-config.c
+++ b/src/miners/fs/tracker-config.c
@@ -51,6 +51,7 @@
#define DEFAULT_INDEX_ON_BATTERY FALSE
#define DEFAULT_INDEX_ON_BATTERY_FIRST_TIME TRUE
#define DEFAULT_LOW_DISK_SPACE_LIMIT 1 /* 0->100 / -1 */
+#define DEFAULT_CRAWLING_INTERVAL 0 /* 0->7 / -1 */
typedef struct {
/* General */
@@ -77,6 +78,7 @@ typedef struct {
GSList *ignored_directories;
GSList *ignored_directories_with_content;
GSList *ignored_files;
+ gint crawling_interval;
/* Convenience data */
GSList *ignored_directory_patterns;
@@ -135,6 +137,7 @@ enum {
PROP_IGNORED_DIRECTORIES,
PROP_IGNORED_DIRECTORIES_WITH_CONTENT,
PROP_IGNORED_FILES,
+ PROP_CRAWLING_INTERVAL
};
static ObjectToKeyFile conversions[] = {
@@ -158,6 +161,7 @@ static ObjectToKeyFile conversions[] = {
{ G_TYPE_POINTER, "ignored-directories", GROUP_INDEXING, "IgnoredDirectories" },
{ G_TYPE_POINTER, "ignored-directories-with-content", GROUP_INDEXING, "IgnoredDirectoriesWithContent" },
{ G_TYPE_POINTER, "ignored-files", GROUP_INDEXING, "IgnoredFiles" },
+ { G_TYPE_INT, "crawling-interval", GROUP_INDEXING, "CrawlingInterval" }
};
G_DEFINE_TYPE (TrackerConfig, tracker_config, TRACKER_TYPE_CONFIG_FILE);
@@ -321,6 +325,17 @@ tracker_config_class_init (TrackerConfigClass *klass)
"Ignored files",
" List of files to NOT index (separator=;)",
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+ g_object_class_install_property (object_class,
+ PROP_CRAWLING_INTERVAL,
+ g_param_spec_int ("crawling-interval",
+ "Crawling interval",
+ " Interval in days to check the filesystem is up to date in the database."
+ " If set to 0, crawling always occurs on startup, if -1 crawling is"
+ " disabled entirely.",
+ -1,
+ G_MAXINT,
+ DEFAULT_CRAWLING_INTERVAL,
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
g_type_class_add_private (object_class, sizeof (TrackerConfigPrivate));
}
@@ -410,6 +425,10 @@ config_set_property (GObject *object,
tracker_config_set_ignored_files (TRACKER_CONFIG (object),
g_value_get_pointer (value));
break;
+ case PROP_CRAWLING_INTERVAL:
+ tracker_config_set_crawling_interval (TRACKER_CONFIG (object),
+ g_value_get_int (value));
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
@@ -484,7 +503,9 @@ config_get_property (GObject *object,
case PROP_IGNORED_FILES:
g_value_set_pointer (value, priv->ignored_files);
break;
-
+ case PROP_CRAWLING_INTERVAL:
+ g_value_set_int (value, priv->crawling_interval);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
break;
@@ -1302,6 +1323,18 @@ tracker_config_get_ignored_files (TrackerConfig *config)
return priv->ignored_files;
}
+gint
+tracker_config_get_crawling_interval (TrackerConfig *config)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_val_if_fail (TRACKER_IS_CONFIG (config), 0);
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ return priv->crawling_interval;
+}
+
void
tracker_config_set_verbosity (TrackerConfig *config,
gint value)
@@ -1730,6 +1763,24 @@ tracker_config_set_ignored_files (TrackerConfig *config,
g_object_notify (G_OBJECT (config), "ignored-files");
}
+void
+tracker_config_set_crawling_interval (TrackerConfig *config,
+ gint interval)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+ if (!tracker_keyfile_object_validate_int (config, "crawling-interval", interval)) {
+ return;
+ }
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ priv->crawling_interval = interval;
+ g_object_notify (G_OBJECT (config), "crawling-interval");
+}
+
/*
* Convenience functions
*/
diff --git a/src/miners/fs/tracker-config.h b/src/miners/fs/tracker-config.h
index e72e1c1..6599c51 100644
--- a/src/miners/fs/tracker-config.h
+++ b/src/miners/fs/tracker-config.h
@@ -70,6 +70,7 @@ GSList * tracker_config_get_index_single_directories_unfiltered (Tracke
GSList * tracker_config_get_ignored_directories (TrackerConfig *config);
GSList * tracker_config_get_ignored_directories_with_content (TrackerConfig *config);
GSList * tracker_config_get_ignored_files (TrackerConfig *config);
+gint tracker_config_get_crawling_interval (TrackerConfig *config);
void tracker_config_set_verbosity (TrackerConfig *config,
gint value);
@@ -105,6 +106,8 @@ void tracker_config_set_ignored_directories_with_content (TrackerConfi
GSList *files);
void tracker_config_set_ignored_files (TrackerConfig *config,
GSList *files);
+void tracker_config_set_crawling_interval (TrackerConfig *config,
+ gint interval);
/*
* Convenience functions:
diff --git a/src/miners/fs/tracker-main.c b/src/miners/fs/tracker-main.c
index 6246f2e..1470c40 100644
--- a/src/miners/fs/tracker-main.c
+++ b/src/miners/fs/tracker-main.c
@@ -62,6 +62,8 @@
"\n" \
" http://www.gnu.org/licenses/gpl.txt\n";
+#define SECONDS_PER_DAY 60 * 60 * 24
+
static GMainLoop *main_loop;
static GSList *miners;
static GSList *current_miner;
@@ -72,6 +74,7 @@ static gint initial_sleep = -1;
static gchar *eligible;
static gchar *add_file;
static gboolean version;
+static gchar *crawl_timestamp_file;
static GOptionEntry entries[] = {
{ "verbosity", 'v', 0,
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index 296e33f..68e0321 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -44,6 +44,7 @@
#include "tracker-marshal.h"
#define DISK_SPACE_CHECK_FREQUENCY 10
+#define SECONDS_PER_DAY 60 * 60 * 24
#define TRACKER_MINER_FILES_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_MINER_FILES, TrackerMinerFilesPrivate))
@@ -96,6 +97,8 @@ enum {
PROP_CONFIG
};
+static gchar *crawl_timestamp_file = NULL;
+
static void miner_files_set_property (GObject *object,
guint param_id,
const GValue *value,
@@ -160,6 +163,8 @@ static gboolean miner_files_ignore_next_update_file (TrackerMinerFS *f
GFile *file,
TrackerSparqlBuilder *sparql,
GCancellable *cancellable);
+static void miner_files_finished (TrackerMinerFS *fs);
+
static void extractor_get_embedded_metadata_cancel (GCancellable *cancellable,
ProcessFileData *data);
@@ -190,6 +195,7 @@ tracker_miner_files_class_init (TrackerMinerFilesClass *klass)
miner_fs_class->monitor_directory = miner_files_monitor_directory;
miner_fs_class->process_file = miner_files_process_file;
miner_fs_class->ignore_next_update_file = miner_files_ignore_next_update_file;
+ miner_fs_class->finished = miner_files_finished;
g_object_class_install_property (object_class,
PROP_CONFIG,
@@ -1306,8 +1312,6 @@ miner_files_check_directory (TrackerMinerFS *fs,
tracker_config_get_index_single_directories (mf->private->config),
tracker_config_get_ignored_directory_paths (mf->private->config),
tracker_config_get_ignored_directory_patterns (mf->private->config));
-
-
}
static gboolean
@@ -1770,6 +1774,73 @@ miner_files_ignore_next_update_file (TrackerMinerFS *fs,
return TRUE;
}
+static gboolean
+should_check_mtime (TrackerConfig *config)
+{
+ gint crawling_interval;
+
+ if (G_UNLIKELY (!crawl_timestamp_file)) {
+ crawl_timestamp_file = g_build_filename (g_get_user_cache_dir (),
+ "tracker",
+ "crawling-timestamp.txt",
+ NULL);
+ }
+
+ crawling_interval = tracker_config_get_crawling_interval (config);
+
+ g_message ("Checking whether to perform mtime checks during crawling:");
+
+ if (crawling_interval == -1) {
+ g_message (" Disabled");
+ return FALSE;
+ } else if (crawling_interval == 0) {
+ g_message (" Enabled");
+ return TRUE;
+ } else {
+ guint64 then, now;
+ gchar *content;
+
+ if (!g_file_get_contents (crawl_timestamp_file, &content, NULL, NULL)) {
+ g_message (" No previous timestamp, crawling forced");
+ return TRUE;
+ }
+
+ now = (guint64) time (NULL);
+
+ then = g_ascii_strtoull (content, NULL, 10);
+ g_free (content);
+
+ if (now < then + (crawling_interval * SECONDS_PER_DAY)) {
+ g_message (" Postponed");
+ return FALSE;
+ } else {
+ g_message ("Not occurred for %d days, crawling forced", crawling_interval);
+ return FALSE;
+ }
+ }
+}
+
+static void
+save_crawling_time (void)
+{
+ GError *error = NULL;
+ gchar *content;
+
+ content = g_strdup_printf ("%" G_GUINT64_FORMAT, (guint64) time (NULL));
+
+ g_file_set_contents (crawl_timestamp_file, content, -1, &error);
+
+ if (error) {
+ g_critical ("Could not save crawling timestamp: %s", error->message);
+ g_error_free (error);
+ }
+}
+
+static void
+miner_files_finished (TrackerMinerFS *fs)
+{
+ save_crawling_time ();
+}
TrackerMiner *
tracker_miner_files_new (TrackerConfig *config)
@@ -1778,6 +1849,7 @@ tracker_miner_files_new (TrackerConfig *config)
"name", "Files",
"config", config,
"process-pool-limit", 10,
+ "mtime-checking", should_check_mtime (config),
NULL);
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
