[tracker/journal-rotating] libtracker-db, libtracker-data: Support for journal rotation
- From: Philip Van Hoof <pvanhoof src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [tracker/journal-rotating] libtracker-db, libtracker-data: Support for journal rotation
- Date: Thu, 6 May 2010 12:42:33 +0000 (UTC)
commit d8294f5bae3d5ecbb385477c05415b6e4a03d50a
Author: Philip Van Hoof <philip codeminded be>
Date: Wed May 5 17:10:04 2010 +0200
libtracker-db, libtracker-data: Support for journal rotation
src/libtracker-common/tracker-keyfile-object.c | 157 ++++++++++---
src/libtracker-common/tracker-keyfile-object.h | 15 ++
src/libtracker-data/tracker-data-backup.c | 8 +-
src/libtracker-data/tracker-data-manager.c | 8 +-
src/libtracker-data/tracker-data-manager.h | 2 +
src/libtracker-data/tracker-data-update.c | 6 +-
src/libtracker-db/tracker-db-journal.c | 259 +++++++++++++++++----
src/libtracker-db/tracker-db-journal.h | 6 +-
src/libtracker-db/tracker-db-manager.c | 38 +++-
src/tracker-control/tracker-control.c | 2 +-
src/tracker-store/tracker-config.c | 129 ++++++++++-
src/tracker-store/tracker-config.h | 19 +-
src/tracker-store/tracker-main.c | 2 +
tests/libtracker-data/tracker-backup-test.c | 2 +
tests/libtracker-data/tracker-ontology-test.c | 8 +-
tests/libtracker-data/tracker-sparql-blank-test.c | 1 +
tests/libtracker-data/tracker-sparql-test.c | 1 +
tests/libtracker-db/tracker-db-journal.c | 6 +-
tests/libtracker-fts/tracker-fts-test.c | 1 +
19 files changed, 572 insertions(+), 98 deletions(-)
---
diff --git a/src/libtracker-common/tracker-keyfile-object.c b/src/libtracker-common/tracker-keyfile-object.c
index c9abbed..2d17932 100644
--- a/src/libtracker-common/tracker-keyfile-object.c
+++ b/src/libtracker-common/tracker-keyfile-object.c
@@ -114,6 +114,28 @@ tracker_keyfile_object_default_int (gpointer object,
return ispec->default_value;
}
+
+guint64
+tracker_keyfile_object_default_uint64 (gpointer object,
+ const gchar *property)
+{
+ GObjectClass *klass;
+ GParamSpec *spec;
+ GParamSpecUInt64 *ispec;
+
+ g_return_val_if_fail (G_IS_OBJECT (object), 0);
+ g_return_val_if_fail (property != NULL, 0);
+
+ klass = G_OBJECT_GET_CLASS (object);
+ spec = g_object_class_find_property (G_OBJECT_CLASS (klass), property);
+ g_return_val_if_fail (spec != NULL, 0);
+
+ ispec = G_PARAM_SPEC_UINT64 (spec);
+ g_return_val_if_fail (ispec != NULL, 0);
+
+ return ispec->default_value;
+}
+
gboolean
tracker_keyfile_object_validate_int (gpointer object,
const gchar *property,
@@ -143,6 +165,36 @@ tracker_keyfile_object_validate_int (gpointer object,
return TRUE;
}
+
+gboolean
+tracker_keyfile_object_validate_uint64 (gpointer object,
+ const gchar *property,
+ guint64 value)
+{
+#ifdef G_DISABLE_CHECKS
+ GParamSpec *spec;
+ GValue gvalue = { 0 };
+ gboolean valid;
+#endif
+
+ g_return_val_if_fail (G_IS_OBJECT (object), FALSE);
+ g_return_val_if_fail (property != NULL, FALSE);
+
+#ifdef G_DISABLE_CHECKS
+ spec = g_object_class_find_property (G_OBJECT_CLASS (object), property);
+ g_return_val_if_fail (spec != NULL, FALSE);
+
+ g_value_init (&gvalue, spec->value_type);
+ g_value_set_uint64 (&gvalue, value);
+ valid = g_param_value_validate (spec, &gvalue);
+ g_value_unset (&gvalue);
+
+ g_return_val_if_fail (valid != TRUE, FALSE);
+#endif
+
+ return TRUE;
+}
+
void
tracker_keyfile_object_load_int (gpointer object,
const gchar *property,
@@ -170,6 +222,33 @@ tracker_keyfile_object_load_int (gpointer object,
}
void
+tracker_keyfile_object_load_uint64 (gpointer object,
+ const gchar *property,
+ GKeyFile *key_file,
+ const gchar *group,
+ const gchar *key)
+{
+ GError *error = NULL;
+ gchar *str;
+
+ g_return_if_fail (G_IS_OBJECT (object));
+ g_return_if_fail (property != NULL);
+ g_return_if_fail (key_file != NULL);
+ g_return_if_fail (group != NULL);
+ g_return_if_fail (key != NULL);
+
+ str = g_key_file_get_string (key_file, group, key, &error);
+ if (!error) {
+ g_object_set (G_OBJECT (object), property, g_ascii_strtoull (str, NULL, 10), NULL);
+ } else {
+ g_message ("Couldn't load object property '%s' (int) in group '%s', %s",
+ property, group, error->message);
+ g_error_free (error);
+ }
+ g_free (str);
+}
+
+void
tracker_keyfile_object_load_boolean (gpointer object,
const gchar *property,
GKeyFile *key_file,
@@ -241,18 +320,18 @@ tracker_keyfile_object_load_string_list (gpointer object,
g_return_if_fail (key != NULL);
value = g_key_file_get_string_list (key_file, group, key, NULL, NULL);
- l = tracker_string_list_to_gslist (value, -1);
+ l = tracker_string_list_to_gslist (value, -1);
g_strfreev (value);
- if (G_LIKELY (!return_instead)) {
- g_object_set (G_OBJECT (object), property, l, NULL);
+ if (G_LIKELY (!return_instead)) {
+ g_object_set (G_OBJECT (object), property, l, NULL);
- /* List is copied internally */
- g_slist_foreach (l, (GFunc) g_free, NULL);
- g_slist_free (l);
- } else {
- *return_instead = l;
- }
+ /* List is copied internally */
+ g_slist_foreach (l, (GFunc) g_free, NULL);
+ g_slist_free (l);
+ } else {
+ *return_instead = l;
+ }
}
void
@@ -274,32 +353,32 @@ tracker_keyfile_object_load_directory_list (gpointer object,
g_return_if_fail (key != NULL);
value = g_key_file_get_string_list (key_file, group, key, NULL, NULL);
- l = directory_string_list_to_gslist ((const gchar **) value);
+ l = directory_string_list_to_gslist ((const gchar **) value);
g_strfreev (value);
- if (l) {
- GSList *filtered;
+ if (l) {
+ GSList *filtered;
- /* Should we make the basename (2nd argument) here
- * part of this function's API?
- */
- filtered = tracker_path_list_filter_duplicates (l, ".", is_recursive);
+ /* Should we make the basename (2nd argument) here
+ * part of this function's API?
+ */
+ filtered = tracker_path_list_filter_duplicates (l, ".", is_recursive);
- g_slist_foreach (l, (GFunc) g_free, NULL);
- g_slist_free (l);
+ g_slist_foreach (l, (GFunc) g_free, NULL);
+ g_slist_free (l);
- l = filtered;
- }
+ l = filtered;
+ }
- if (G_LIKELY (!return_instead)) {
- g_object_set (G_OBJECT (object), property, l, NULL);
+ if (G_LIKELY (!return_instead)) {
+ g_object_set (G_OBJECT (object), property, l, NULL);
- /* List is copied internally */
- g_slist_foreach (l, (GFunc) g_free, NULL);
- g_slist_free (l);
- } else {
- *return_instead = l;
- }
+ /* List is copied internally */
+ g_slist_foreach (l, (GFunc) g_free, NULL);
+ g_slist_free (l);
+ } else {
+ *return_instead = l;
+ }
}
void
@@ -322,6 +401,28 @@ tracker_keyfile_object_save_int (gpointer object,
}
void
+tracker_keyfile_object_save_uint64 (gpointer object,
+ const gchar *property,
+ GKeyFile *key_file,
+ const gchar *group,
+ const gchar *key)
+{
+ guint64 value;
+ gchar *str;
+
+ g_return_if_fail (G_IS_OBJECT (object));
+ g_return_if_fail (property != NULL);
+ g_return_if_fail (key_file != NULL);
+ g_return_if_fail (group != NULL);
+ g_return_if_fail (key != NULL);
+
+ g_object_get (G_OBJECT (object), property, &value, NULL);
+ str = g_strdup_printf ("%" G_GUINT64_FORMAT, value);
+ g_key_file_set_string (key_file, group, key, str);
+ g_free (str);
+}
+
+void
tracker_keyfile_object_save_boolean (gpointer object,
const gchar *property,
GKeyFile *key_file,
diff --git a/src/libtracker-common/tracker-keyfile-object.h b/src/libtracker-common/tracker-keyfile-object.h
index c06cfcb..94df5ee 100644
--- a/src/libtracker-common/tracker-keyfile-object.h
+++ b/src/libtracker-common/tracker-keyfile-object.h
@@ -34,14 +34,24 @@ gboolean tracker_keyfile_object_default_boolean (gpointer object,
const gchar *property);
gint tracker_keyfile_object_default_int (gpointer object,
const gchar *property);
+guint64 tracker_keyfile_object_default_uint64 (gpointer object,
+ const gchar *property);
gboolean tracker_keyfile_object_validate_int (gpointer object,
const gchar *property,
gint value);
+gboolean tracker_keyfile_object_validate_uint64 (gpointer object,
+ const gchar *property,
+ guint64 value);
void tracker_keyfile_object_load_int (gpointer object,
const gchar *property,
GKeyFile *key_file,
const gchar *group,
const gchar *key);
+void tracker_keyfile_object_load_uint64 (gpointer object,
+ const gchar *property,
+ GKeyFile *key_file,
+ const gchar *group,
+ const gchar *key);
void tracker_keyfile_object_load_boolean (gpointer object,
const gchar *property,
GKeyFile *key_file,
@@ -71,6 +81,11 @@ void tracker_keyfile_object_save_int (gpointer object,
GKeyFile *key_file,
const gchar *group,
const gchar *key);
+void tracker_keyfile_object_save_uint64 (gpointer object,
+ const gchar *property,
+ GKeyFile *key_file,
+ const gchar *group,
+ const gchar *key);
void tracker_keyfile_object_save_boolean (gpointer object,
const gchar *property,
GKeyFile *key_file,
diff --git a/src/libtracker-data/tracker-data-backup.c b/src/libtracker-data/tracker-data-backup.c
index 9e3f298..3c5c2f5 100644
--- a/src/libtracker-data/tracker-data-backup.c
+++ b/src/libtracker-data/tracker-data-backup.c
@@ -143,6 +143,8 @@ tracker_data_backup_restore (GFile *journal,
if (g_file_query_exists (info->journal, NULL)) {
TrackerDBManagerFlags flags = tracker_db_manager_get_flags ();
gboolean is_first;
+ gsize chunk_size = 0;
+ gboolean do_rotating = FALSE;
tracker_db_manager_move_to_temp ();
tracker_data_manager_shutdown ();
@@ -156,7 +158,8 @@ tracker_data_backup_restore (GFile *journal,
&info->error);
tracker_db_manager_init_locations ();
- tracker_db_journal_init (NULL, FALSE);
+ tracker_db_journal_get_rotating (&do_rotating, &chunk_size);
+ tracker_db_journal_init (NULL, FALSE, do_rotating, chunk_size);
if (info->error) {
tracker_db_manager_restore_from_temp ();
@@ -166,7 +169,8 @@ tracker_data_backup_restore (GFile *journal,
tracker_db_journal_shutdown ();
- tracker_data_manager_init (flags, test_schemas, &is_first, TRUE,
+ tracker_data_manager_init (flags, do_rotating, chunk_size,
+ test_schemas, &is_first, TRUE,
busy_callback, busy_user_data,
"Restoring backup");
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index c82deeb..ddc5ea6 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -2028,6 +2028,8 @@ get_new_service_id (TrackerDBInterface *iface)
gboolean
tracker_data_manager_init (TrackerDBManagerFlags flags,
+ gboolean do_journal_rotating,
+ gsize chunk_size,
const gchar **test_schemas,
gboolean *first_time,
gboolean journal_check,
@@ -2119,7 +2121,7 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
in_journal_replay = FALSE;
/* open journal for writing */
- tracker_db_journal_init (NULL, FALSE);
+ tracker_db_journal_init (NULL, FALSE, do_journal_rotating, chunk_size);
check_ontology = TRUE;
@@ -2132,7 +2134,7 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
/* Truncate journal as it does not even contain a single valid transaction
* or is explicitly ignored (journal_check == FALSE, only for test cases) */
- tracker_db_journal_init (NULL, TRUE);
+ tracker_db_journal_init (NULL, TRUE, do_journal_rotating, chunk_size);
/* load ontology from files into memory (max_id starts at zero: first-time) */
@@ -2193,7 +2195,7 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
/* First time, no need to check ontology */
check_ontology = FALSE;
} else {
- tracker_db_journal_init (NULL, FALSE);
+ tracker_db_journal_init (NULL, FALSE, do_journal_rotating, chunk_size);
/* Load ontology from database into memory */
db_get_static_data (iface);
diff --git a/src/libtracker-data/tracker-data-manager.h b/src/libtracker-data/tracker-data-manager.h
index e83d6c7..e8876fb 100644
--- a/src/libtracker-data/tracker-data-manager.h
+++ b/src/libtracker-data/tracker-data-manager.h
@@ -40,6 +40,8 @@ G_BEGIN_DECLS
#endif
gboolean tracker_data_manager_init (TrackerDBManagerFlags flags,
+ gboolean do_journal_rotating,
+ gsize chunk_size,
const gchar **test_schema,
gboolean *first_time,
gboolean journal_check,
diff --git a/src/libtracker-data/tracker-data-update.c b/src/libtracker-data/tracker-data-update.c
index 3cdc383..485eed6 100644
--- a/src/libtracker-data/tracker-data-update.c
+++ b/src/libtracker-data/tracker-data-update.c
@@ -2798,11 +2798,15 @@ tracker_data_replay_journal (GHashTable *classes,
if (journal_error) {
gsize size;
+ gsize chunk_size = 0;
+ gboolean do_rotating = FALSE;
size = tracker_db_journal_reader_get_size_of_correct ();
tracker_db_journal_reader_shutdown ();
- tracker_db_journal_init (NULL, FALSE);
+ tracker_db_journal_get_rotating (&do_rotating, &chunk_size);
+
+ tracker_db_journal_init (NULL, FALSE, do_rotating, chunk_size);
tracker_db_journal_truncate (size);
tracker_db_journal_shutdown ();
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index 0b572ab..8fc34f5 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -31,6 +31,7 @@
#include <unistd.h>
#include <errno.h>
#include <fcntl.h>
+#include <stdlib.h>
#include <glib/gstdio.h>
@@ -84,6 +85,7 @@ static struct {
gint p_id;
gint o_id;
const gchar *object;
+ guint current_file;
} reader;
static struct {
@@ -95,6 +97,8 @@ static struct {
gchar *cur_block;
guint cur_entry_amount;
guint cur_pos;
+ gsize chunk_size;
+ gboolean do_rotating;
} writer;
static guint32
@@ -201,50 +205,19 @@ tracker_db_journal_error_quark (void)
return g_quark_from_static_string (TRACKER_DB_JOURNAL_ERROR_DOMAIN);
}
-gboolean
-tracker_db_journal_init (const gchar *filename, gboolean truncate)
+static gboolean
+tracker_db_journal_init_file (gboolean truncate)
{
- gchar *directory;
struct stat st;
int flags;
int mode;
- g_return_val_if_fail (writer.journal == 0, FALSE);
-
writer.cur_block_len = 0;
writer.cur_pos = 0;
writer.cur_entry_amount = 0;
writer.cur_block_alloc = 0;
writer.cur_block = NULL;
- /* Used mostly for testing */
- if (G_UNLIKELY (filename)) {
- writer.journal_filename = g_strdup (filename);
- } else {
- writer.journal_filename = g_build_filename (g_get_user_data_dir (),
- "tracker",
- "data",
- JOURNAL_FILENAME,
- NULL);
- }
-
- directory = g_path_get_dirname (writer.journal_filename);
- if (g_strcmp0 (directory, ".")) {
- mode = S_IRWXU | S_IRWXG | S_IRWXO;
- if (g_mkdir_with_parents (directory, mode)) {
- g_critical ("tracker data directory does not exist and "
- "could not be created: %s",
- g_strerror (errno));
-
- g_free (directory);
- g_free (writer.journal_filename);
- writer.journal_filename = NULL;
-
- return FALSE;
- }
- }
- g_free (directory);
-
mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
flags = O_WRONLY | O_APPEND | O_CREAT | O_LARGEFILE;
if (truncate) {
@@ -285,7 +258,7 @@ tracker_db_journal_init (const gchar *filename, gboolean truncate)
writer.cur_block[4] = 'g';
writer.cur_block[5] = '\0';
writer.cur_block[6] = '0';
- writer.cur_block[7] = '2';
+ writer.cur_block[7] = '3';
if (!write_all_data (writer.journal, writer.cur_block, 8)) {
g_free (writer.journal_filename);
@@ -301,6 +274,51 @@ tracker_db_journal_init (const gchar *filename, gboolean truncate)
}
gboolean
+tracker_db_journal_init (const gchar *filename,
+ gboolean truncate,
+ gboolean do_rotating,
+ gsize chunk_size)
+{
+ gchar *directory;
+ int mode;
+
+ g_return_val_if_fail (writer.journal == 0, FALSE);
+
+ /* Used mostly for testing */
+ if (G_UNLIKELY (filename)) {
+ writer.journal_filename = g_strdup (filename);
+ } else {
+ writer.journal_filename = g_build_filename (g_get_user_data_dir (),
+ "tracker",
+ "data",
+ JOURNAL_FILENAME,
+ NULL);
+ }
+
+ directory = g_path_get_dirname (writer.journal_filename);
+ if (g_strcmp0 (directory, ".")) {
+ mode = S_IRWXU | S_IRWXG | S_IRWXO;
+ if (g_mkdir_with_parents (directory, mode)) {
+ g_critical ("tracker data directory does not exist and "
+ "could not be created: %s",
+ g_strerror (errno));
+
+ g_free (directory);
+ g_free (writer.journal_filename);
+ writer.journal_filename = NULL;
+
+ return FALSE;
+ }
+ }
+ g_free (directory);
+
+ writer.do_rotating = do_rotating;
+ writer.chunk_size = chunk_size;
+
+ return tracker_db_journal_init_file (truncate);
+}
+
+gboolean
tracker_db_journal_shutdown (void)
{
if (writer.journal == 0) {
@@ -606,6 +624,74 @@ tracker_db_journal_truncate (gsize new_size)
return (ftruncate (writer.journal, new_size) != -1);
}
+void
+tracker_db_journal_get_rotating (gboolean *do_rotating,
+ gsize *chunk_size)
+{
+ *do_rotating = writer.do_rotating;
+ *chunk_size = writer.chunk_size;
+}
+
+static gboolean
+tracker_db_journal_rotate (void)
+{
+ gchar *fullpath;
+ static guint max = 0;
+
+ if (max == 0) {
+ gchar *directory;
+ GDir *journal_dir;
+ const gchar *f_name;
+
+ directory = g_path_get_dirname (writer.journal_filename);
+
+ journal_dir = g_dir_open (directory, 0, NULL);
+
+ f_name = g_dir_read_name (journal_dir);
+
+ while (f_name) {
+ gchar *ptr;
+ guint cur;
+
+ if (f_name) {
+
+ if (!g_str_has_prefix (f_name, "tracker-store.journal.")) {
+ f_name = g_dir_read_name (journal_dir);
+ continue;
+ }
+
+ ptr = strrchr (f_name, '.');
+ if (ptr) {
+ ptr++;
+ cur = atoi (ptr);
+ max = MAX (cur, max);
+ }
+ }
+
+ f_name = g_dir_read_name (journal_dir);
+ }
+
+ g_dir_close (journal_dir);
+ g_free (directory);
+ }
+
+ tracker_db_journal_fsync ();
+
+ if (close (writer.journal) != 0) {
+ g_warning ("Could not close journal, %s",
+ g_strerror (errno));
+ return FALSE;
+ }
+
+ fullpath = g_strdup_printf ("%s.%d", writer.journal_filename, ++max);
+
+ g_rename (writer.journal_filename, fullpath);
+
+ g_free (fullpath);
+
+ return tracker_db_journal_init_file (TRUE);
+}
+
gboolean
tracker_db_journal_commit_db_transaction (void)
{
@@ -651,6 +737,13 @@ tracker_db_journal_commit_db_transaction (void)
/* Clean up for next transaction */
cur_block_kill ();
+ if (writer.do_rotating && (writer.cur_size > writer.chunk_size)) {
+ if (!tracker_db_journal_rotate ()) {
+ g_critical ("Could not rotate journal, %s", g_strerror (errno));
+ return FALSE;
+ }
+ }
+
return TRUE;
}
@@ -670,6 +763,8 @@ tracker_db_journal_reader_init (const gchar *filename)
{
GError *error = NULL;
gchar *filename_used;
+ gchar *filename_open;
+ gchar *test;
g_return_val_if_fail (reader.file == NULL, FALSE);
@@ -684,16 +779,29 @@ tracker_db_journal_reader_init (const gchar *filename)
NULL);
}
+ test = g_strdup_printf ("%s.1", filename_used);
+
+ if (g_file_test (test, G_FILE_TEST_EXISTS)) {
+ filename_open = test;
+ reader.current_file = 1;
+ } else {
+ g_free (test);
+ filename_open = g_strdup (filename_used);
+ reader.current_file = 0;
+ }
+
reader.type = TRACKER_DB_JOURNAL_START;
reader.filename = filename_used;
- reader.file = g_mapped_file_new (reader.filename, FALSE, &error);
+ reader.file = g_mapped_file_new (filename_open, FALSE, &error);
+
+ g_free (filename_open);
if (error) {
if (!g_error_matches (error, G_FILE_ERROR, G_FILE_ERROR_NOENT)) {
/* do not warn if the file does not exist, just return FALSE */
g_warning ("Could not create TrackerDBJournalReader for file '%s', %s",
- reader.filename,
- error->message ? error->message : "no error given");
+ reader.filename,
+ error->message ? error->message : "no error given");
}
g_error_free (error);
g_free (reader.filename);
@@ -713,7 +821,7 @@ tracker_db_journal_reader_init (const gchar *filename)
return FALSE;
}
- if (memcmp (reader.current, "trlog\00002", 8)) {
+ if (memcmp (reader.current, "trlog\00003", 8)) {
tracker_db_journal_reader_shutdown ();
return FALSE;
}
@@ -731,6 +839,70 @@ tracker_db_journal_reader_get_size_of_correct (void)
return (gsize) (reader.last_success - reader.start);
}
+static gboolean
+reader_next_file (GError **error)
+{
+ gchar *filename_open;
+ gchar *test;
+ GError *new_error = NULL;
+
+ test = g_strdup_printf ("%s.%d", reader.filename, ++reader.current_file);
+
+ if (g_file_test (test, G_FILE_TEST_EXISTS)) {
+ filename_open = test;
+ } else {
+ g_free (test);
+ filename_open = g_strdup (reader.filename);
+ /* Last file is the active journal file */
+ reader.current_file = 0;
+ }
+
+#if GLIB_CHECK_VERSION(2,22,0)
+ g_mapped_file_unref (reader.file);
+#else
+ g_mapped_file_free (reader.file);
+#endif
+
+ reader.file = g_mapped_file_new (filename_open, FALSE, &new_error);
+
+ if (new_error) {
+ g_propagate_error (error, new_error);
+ return FALSE;
+ }
+
+ reader.last_success = reader.start = reader.current =
+ g_mapped_file_get_contents (reader.file);
+
+ reader.end = reader.current + g_mapped_file_get_length (reader.file);
+
+ /* verify journal file header */
+ if (reader.end - reader.current < 8) {
+ g_set_error (error, TRACKER_DB_JOURNAL_ERROR, 0,
+ "Damaged journal entry at begin of journal");
+ tracker_db_journal_reader_shutdown ();
+ return FALSE;
+ }
+
+ if (memcmp (reader.current, "trlog\00003", 8)) {
+ g_set_error (error, TRACKER_DB_JOURNAL_ERROR, 0,
+ "Damaged journal entry at begin of journal");
+ tracker_db_journal_reader_shutdown ();
+ return FALSE;
+ }
+
+ reader.current += 8;
+
+ reader.type = TRACKER_DB_JOURNAL_END_TRANSACTION;
+
+ reader.entry_begin = NULL;
+ reader.entry_end = NULL;
+ reader.amount_of_triples = 0;
+
+ g_free (filename_open);
+
+ return TRUE;
+}
+
gboolean
tracker_db_journal_reader_shutdown (void)
{
@@ -840,9 +1012,11 @@ tracker_db_journal_reader_next (GError **error)
/* Check the end is not before where we currently are */
if (reader.current >= reader.end) {
/* Return FALSE as there is no further entry but
- * do not set error as it's not an error case.
- */
- return FALSE;
+ * do not set error as it's not an error case. */
+ if (reader.current_file != 0)
+ return reader_next_file (error);
+ else
+ return FALSE;
}
/* Check the end is not smaller than the first uint32
@@ -1154,6 +1328,7 @@ gdouble
tracker_db_journal_reader_get_progress (void)
{
gdouble percent = ((gdouble)(reader.end - reader.start));
- return ((gdouble)(reader.current - reader.start)) / percent;
+ /* TODO: Fix this now that multiple chunks can exist */
+ return (((gdouble)(reader.current - reader.start)) / percent);
}
diff --git a/src/libtracker-db/tracker-db-journal.h b/src/libtracker-db/tracker-db-journal.h
index 254f670..4ef0ca5 100644
--- a/src/libtracker-db/tracker-db-journal.h
+++ b/src/libtracker-db/tracker-db-journal.h
@@ -48,11 +48,15 @@ GQuark tracker_db_journal_error_quark (void);
* Writer API
*/
gboolean tracker_db_journal_init (const gchar *filename,
- gboolean truncate);
+ gboolean truncate,
+ gboolean do_rotating,
+ gsize chunk_size);
gboolean tracker_db_journal_shutdown (void);
const gchar* tracker_db_journal_get_filename (void);
gsize tracker_db_journal_get_size (void);
+void tracker_db_journal_get_rotating (gboolean *do_rotating,
+ gsize *chunk_size);
gboolean tracker_db_journal_start_transaction (time_t time);
gboolean tracker_db_journal_start_ontology_transaction (time_t time);
diff --git a/src/libtracker-db/tracker-db-manager.c b/src/libtracker-db/tracker-db-manager.c
index 51aa979..fcc8d82 100644
--- a/src/libtracker-db/tracker-db-manager.c
+++ b/src/libtracker-db/tracker-db-manager.c
@@ -422,11 +422,45 @@ db_manager_remove_all (gboolean rm_journal)
if (opath) {
GFile *file;
gchar *cpath;
+ gchar *directory;
+ GDir *journal_dir;
+ const gchar *f_name;
cpath = g_strdup (opath);
tracker_db_journal_shutdown ();
- g_message (" Removing journal:'%s'",
- cpath);
+
+ g_message (" Removing journal:'%s'", cpath);
+
+ directory = g_path_get_dirname (cpath);
+ journal_dir = g_dir_open (directory, 0, NULL);
+ f_name = g_dir_read_name (journal_dir);
+
+ /* Remove rotated chunks */
+ while (f_name) {
+ gchar *fullpath;
+
+ if (f_name) {
+ if (!g_str_has_prefix (f_name, "tracker-store.journal.")) {
+ f_name = g_dir_read_name (journal_dir);
+ continue;
+ }
+
+ }
+
+ fullpath = g_build_filename (directory, f_name, NULL);
+
+ file = g_file_new_for_path (fullpath);
+ g_file_delete (file, NULL, NULL);
+ g_object_unref (file);
+ g_free (fullpath);
+
+ f_name = g_dir_read_name (journal_dir);
+ }
+
+ g_dir_close (journal_dir);
+ g_free (directory);
+
+ /* Remove active journal */
file = g_file_new_for_path (cpath);
g_file_delete (file, NULL, NULL);
g_object_unref (file);
diff --git a/src/tracker-control/tracker-control.c b/src/tracker-control/tracker-control.c
index ec4ab44..380b121 100644
--- a/src/tracker-control/tracker-control.c
+++ b/src/tracker-control/tracker-control.c
@@ -411,7 +411,7 @@ main (int argc, char **argv)
g_log_set_default_handler (log_handler, NULL);
/* This call is needed to set the journal's filename */
- tracker_db_journal_init (NULL, FALSE);
+ tracker_db_journal_init (NULL, FALSE, FALSE, G_MAXSIZE);
/* Clean up */
if (!tracker_db_manager_init (TRACKER_DB_MANAGER_REMOVE_ALL, NULL, FALSE)) {
diff --git a/src/tracker-store/tracker-config.c b/src/tracker-store/tracker-config.c
index f18b15e..d8736f0 100644
--- a/src/tracker-store/tracker-config.c
+++ b/src/tracker-store/tracker-config.c
@@ -33,16 +33,21 @@
/* GKeyFile defines */
#define GROUP_GENERAL "General"
-#define GROUP_INDEXING "Indexing"
+#define GROUP_JOURNAL "Journal"
/* Default values */
-#define DEFAULT_VERBOSITY 2
+#define DEFAULT_VERBOSITY 2
+#define DEFAULT_JOURNAL_CHUNK_SIZE 500
+#define DEFAULT_ENABLE_JOURNAL_ROTATING TRUE
/* typedef struct TrackerConfigPrivate TrackerConfigPrivate; */
typedef struct {
/* General */
gint verbosity;
+ /* Journal */
+ gsize journal_chunk_size;
+ gboolean enable_journal_rotating;
} TrackerConfigPrivate;
typedef struct {
@@ -71,11 +76,16 @@ enum {
PROP_0,
/* General */
- PROP_VERBOSITY
+ PROP_VERBOSITY,
+ /* Journal */
+ PROP_JOURNAL_CHUNK_SIZE,
+ PROP_ENABLE_JOURNAL_ROTATING
};
static ObjectToKeyFile conversions[] = {
- { G_TYPE_INT, "verbosity", GROUP_GENERAL, "Verbosity" },
+ { G_TYPE_INT, "verbosity", GROUP_GENERAL, "Verbosity" },
+ { G_TYPE_BOOLEAN, "enable-journal-rotating", GROUP_JOURNAL, "EnableJournalRotating" },
+ { G_TYPE_UINT64, "journal-chunk-size", GROUP_JOURNAL, "JournalChunkSize" },
};
G_DEFINE_TYPE (TrackerConfig, tracker_config, TRACKER_TYPE_CONFIG_FILE);
@@ -101,6 +111,24 @@ tracker_config_class_init (TrackerConfigClass *klass)
DEFAULT_VERBOSITY,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+ g_object_class_install_property (object_class,
+ PROP_JOURNAL_CHUNK_SIZE,
+ g_param_spec_uint64 ("journal-chunk-size",
+ "Journal chunk size",
+ " Size of a journal-chunk before rotation",
+ 0,
+ G_MAXSIZE,
+ DEFAULT_JOURNAL_CHUNK_SIZE,
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+
+ g_object_class_install_property (object_class,
+ PROP_ENABLE_JOURNAL_ROTATING,
+ g_param_spec_boolean ("enable-journal-rotating",
+ "Enable journal rotating",
+ " Enable journal rotating",
+ DEFAULT_ENABLE_JOURNAL_ROTATING,
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+
g_type_class_add_private (object_class, sizeof (TrackerConfigPrivate));
}
@@ -121,7 +149,15 @@ config_set_property (GObject *object,
tracker_config_set_verbosity (TRACKER_CONFIG (object),
g_value_get_int (value));
break;
-
+ /* Journal */
+ case PROP_JOURNAL_CHUNK_SIZE:
+ tracker_config_set_journal_chunk_size (TRACKER_CONFIG (object),
+ (gsize) g_value_get_uint64 (value));
+ break;
+ case PROP_ENABLE_JOURNAL_ROTATING:
+ tracker_config_set_enable_journal_rotating (TRACKER_CONFIG (object),
+ g_value_get_boolean (value));
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
break;
@@ -143,7 +179,12 @@ config_get_property (GObject *object,
case PROP_VERBOSITY:
g_value_set_int (value, priv->verbosity);
break;
-
+ case PROP_JOURNAL_CHUNK_SIZE:
+ g_value_set_uint64 (value, (guint64) priv->journal_chunk_size);
+ break;
+ case PROP_ENABLE_JOURNAL_ROTATING:
+ g_value_set_boolean (value, priv->enable_journal_rotating);
+ break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
break;
@@ -179,6 +220,7 @@ config_create_with_defaults (TrackerConfig *config,
for (i = 0; i < G_N_ELEMENTS (conversions); i++) {
gboolean has_key;
+ gchar *str;
has_key = g_key_file_has_key (key_file,
conversions[i].group,
@@ -197,6 +239,17 @@ config_create_with_defaults (TrackerConfig *config,
conversions[i].property));
break;
+ case G_TYPE_UINT64:
+ str = g_strdup_printf ("%" G_GUINT64_FORMAT,
+ tracker_keyfile_object_default_uint64 (config,
+ conversions[i].property));
+ g_key_file_set_string (key_file,
+ conversions[i].group,
+ conversions[i].key,
+ str);
+ g_free (str);
+ break;
+
case G_TYPE_BOOLEAN:
g_key_file_set_boolean (key_file,
conversions[i].group,
@@ -249,6 +302,14 @@ config_load (TrackerConfig *config)
conversions[i].key);
break;
+ case G_TYPE_UINT64:
+ tracker_keyfile_object_load_uint64 (G_OBJECT (file),
+ conversions[i].property,
+ file->key_file,
+ conversions[i].group,
+ conversions[i].key);
+ break;
+
case G_TYPE_BOOLEAN:
tracker_keyfile_object_load_boolean (G_OBJECT (file),
conversions[i].property,
@@ -333,6 +394,62 @@ tracker_config_get_verbosity (TrackerConfig *config)
return priv->verbosity;
}
+gboolean
+tracker_config_get_enable_journal_rotating (TrackerConfig *config)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_val_if_fail (TRACKER_IS_CONFIG (config), DEFAULT_ENABLE_JOURNAL_ROTATING);
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ return priv->enable_journal_rotating;
+}
+
+gsize
+tracker_config_get_journal_chunk_size (TrackerConfig *config)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_val_if_fail (TRACKER_IS_CONFIG (config), DEFAULT_JOURNAL_CHUNK_SIZE);
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ return priv->journal_chunk_size;
+}
+
+void
+tracker_config_set_enable_journal_rotating (TrackerConfig *config,
+ gboolean value)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ priv->enable_journal_rotating = value;
+ g_object_notify (G_OBJECT (config), "enable-journal-rotating");
+}
+
+void
+tracker_config_set_journal_chunk_size (TrackerConfig *config,
+ gsize value)
+{
+ TrackerConfigPrivate *priv;
+
+ g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+ if (!tracker_keyfile_object_validate_uint64 (config, "journal-chunk-size", (guint64) value)) {
+ return;
+ }
+
+ priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+ priv->journal_chunk_size = value;
+ g_object_notify (G_OBJECT (config), "journal-chunk-size");
+}
+
void
tracker_config_set_verbosity (TrackerConfig *config,
gint value)
diff --git a/src/tracker-store/tracker-config.h b/src/tracker-store/tracker-config.h
index beb85f3..27f0695 100644
--- a/src/tracker-store/tracker-config.h
+++ b/src/tracker-store/tracker-config.h
@@ -44,13 +44,20 @@ struct TrackerConfigClass {
TrackerConfigFileClass parent_class;
};
-GType tracker_config_get_type (void) G_GNUC_CONST;
+GType tracker_config_get_type (void) G_GNUC_CONST;
-TrackerConfig *tracker_config_new (void);
-gboolean tracker_config_save (TrackerConfig *config);
-gint tracker_config_get_verbosity (TrackerConfig *config);
-void tracker_config_set_verbosity (TrackerConfig *config,
- gint value);
+TrackerConfig *tracker_config_new (void);
+gboolean tracker_config_save (TrackerConfig *config);
+gint tracker_config_get_verbosity (TrackerConfig *config);
+gboolean tracker_config_get_enable_journal_rotating (TrackerConfig *config);
+gsize tracker_config_get_journal_chunk_size (TrackerConfig *config);
+
+void tracker_config_set_verbosity (TrackerConfig *config,
+ gint value);
+void tracker_config_set_enable_journal_rotating (TrackerConfig *config,
+ gboolean value);
+void tracker_config_set_journal_chunk_size (TrackerConfig *config,
+ gsize value);
G_END_DECLS
diff --git a/src/tracker-store/tracker-main.c b/src/tracker-store/tracker-main.c
index d18b99d..b98d5cb 100644
--- a/src/tracker-store/tracker-main.c
+++ b/src/tracker-store/tracker-main.c
@@ -438,6 +438,8 @@ main (gint argc, gchar *argv[])
&busy_user_data);
if (!tracker_data_manager_init (flags,
+ tracker_config_get_enable_journal_rotating (config),
+ tracker_config_get_journal_chunk_size (config),
NULL,
&is_first_time_index,
TRUE,
diff --git a/tests/libtracker-data/tracker-backup-test.c b/tests/libtracker-data/tracker-backup-test.c
index d5e3a2a..01e0277 100644
--- a/tests/libtracker-data/tracker-backup-test.c
+++ b/tests/libtracker-data/tracker-backup-test.c
@@ -106,6 +106,7 @@ test_backup_and_restore_helper (gboolean journal)
test_schemas[3] = data_prefix;
tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+ FALSE, G_MAXSIZE,
(const gchar **) test_schemas,
NULL, FALSE, NULL, NULL, NULL);
@@ -158,6 +159,7 @@ test_backup_and_restore_helper (gboolean journal)
g_free (meta_db);
tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+ FALSE, G_MAXSIZE,
(const gchar **) test_schemas,
NULL, FALSE, NULL, NULL, NULL);
check_content_in_db (0, 0);
diff --git a/tests/libtracker-data/tracker-ontology-test.c b/tests/libtracker-data/tracker-ontology-test.c
index 44da5ca..86ddf44 100644
--- a/tests/libtracker-data/tracker-ontology-test.c
+++ b/tests/libtracker-data/tracker-ontology-test.c
@@ -264,7 +264,7 @@ test_ontology_change (void)
g_chmod (ontology_file, 0666);
- tracker_data_manager_init (0, (const gchar **) test_schemas,
+ tracker_data_manager_init (0, FALSE, G_MAXSIZE, (const gchar **) test_schemas,
NULL, FALSE, NULL, NULL, NULL);
if (g_file_get_contents (update, &queries, NULL, NULL)) {
@@ -290,7 +290,7 @@ test_ontology_change (void)
delete_db (FALSE);
- tracker_data_manager_init (0, (const gchar **) test_schemas,
+ tracker_data_manager_init (0, FALSE, G_MAXSIZE, (const gchar **) test_schemas,
NULL, TRUE, NULL, NULL, NULL);
for (i = 0; change_tests[i].test_name != NULL; i++) {
@@ -326,6 +326,7 @@ test_ontology_init (void)
{
/* first-time initialization */
tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+ FALSE, G_MAXSIZE,
NULL,
NULL,
FALSE,
@@ -336,7 +337,7 @@ test_ontology_init (void)
tracker_data_manager_shutdown ();
/* initialization from existing database */
- tracker_data_manager_init (0,
+ tracker_data_manager_init (0, FALSE, G_MAXSIZE,
NULL,
NULL,
FALSE,
@@ -368,6 +369,7 @@ test_query (gconstpointer test_data)
/* initialization */
tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+ FALSE, G_MAXSIZE,
NULL,
NULL,
FALSE,
diff --git a/tests/libtracker-data/tracker-sparql-blank-test.c b/tests/libtracker-data/tracker-sparql-blank-test.c
index 5b2f9b7..ecdbf6c 100644
--- a/tests/libtracker-data/tracker-sparql-blank-test.c
+++ b/tests/libtracker-data/tracker-sparql-blank-test.c
@@ -42,6 +42,7 @@ test_blank (void)
/* initialization */
tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+ FALSE, G_MAXSIZE,
NULL,
NULL,
FALSE,
diff --git a/tests/libtracker-data/tracker-sparql-test.c b/tests/libtracker-data/tracker-sparql-test.c
index 2ce7805..da9b78b 100644
--- a/tests/libtracker-data/tracker-sparql-test.c
+++ b/tests/libtracker-data/tracker-sparql-test.c
@@ -235,6 +235,7 @@ test_sparql_query (gconstpointer test_data)
test_schemas[0] = data_prefix;
tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+ FALSE, G_MAXSIZE,
test_schemas,
NULL, FALSE, NULL, NULL, NULL);
diff --git a/tests/libtracker-db/tracker-db-journal.c b/tests/libtracker-db/tracker-db-journal.c
index 6614ef9..5a2c623 100644
--- a/tests/libtracker-db/tracker-db-journal.c
+++ b/tests/libtracker-db/tracker-db-journal.c
@@ -29,13 +29,13 @@ test_init_and_shutdown (void)
gboolean result;
/* check double init/shutdown */
- result = tracker_db_journal_init (NULL, FALSE);
+ result = tracker_db_journal_init (NULL, FALSE, FALSE, G_MAXSIZE);
g_assert (result == TRUE);
result = tracker_db_journal_shutdown ();
g_assert (result == TRUE);
- result = tracker_db_journal_init (NULL, FALSE);
+ result = tracker_db_journal_init (NULL, FALSE, FALSE, G_MAXSIZE);
g_assert (result == TRUE);
result = tracker_db_journal_shutdown ();
@@ -53,7 +53,7 @@ test_write_functions (void)
path = g_build_filename (TOP_BUILDDIR, "tests", "libtracker-db", "tracker-store.journal", NULL);
g_unlink (path);
- tracker_db_journal_init (path, FALSE);
+ tracker_db_journal_init (path, FALSE, FALSE, G_MAXSIZE);
filename = tracker_db_journal_get_filename ();
g_assert (filename != NULL);
diff --git a/tests/libtracker-fts/tracker-fts-test.c b/tests/libtracker-fts/tracker-fts-test.c
index 84ec1fa..503dfdb 100644
--- a/tests/libtracker-fts/tracker-fts-test.c
+++ b/tests/libtracker-fts/tracker-fts-test.c
@@ -73,6 +73,7 @@ test_sparql_query (gconstpointer test_data)
test_schemas[0] = data_prefix;
tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+ FALSE, G_MAXSIZE,
test_schemas,
NULL, FALSE, NULL, NULL, NULL);
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
