[tracker/journal-rotating] libtracker-db, libtracker-data: Support for journal rotation



commit d8294f5bae3d5ecbb385477c05415b6e4a03d50a
Author: Philip Van Hoof <philip codeminded be>
Date:   Wed May 5 17:10:04 2010 +0200

    libtracker-db, libtracker-data: Support for journal rotation

 src/libtracker-common/tracker-keyfile-object.c    |  157 ++++++++++---
 src/libtracker-common/tracker-keyfile-object.h    |   15 ++
 src/libtracker-data/tracker-data-backup.c         |    8 +-
 src/libtracker-data/tracker-data-manager.c        |    8 +-
 src/libtracker-data/tracker-data-manager.h        |    2 +
 src/libtracker-data/tracker-data-update.c         |    6 +-
 src/libtracker-db/tracker-db-journal.c            |  259 +++++++++++++++++----
 src/libtracker-db/tracker-db-journal.h            |    6 +-
 src/libtracker-db/tracker-db-manager.c            |   38 +++-
 src/tracker-control/tracker-control.c             |    2 +-
 src/tracker-store/tracker-config.c                |  129 ++++++++++-
 src/tracker-store/tracker-config.h                |   19 +-
 src/tracker-store/tracker-main.c                  |    2 +
 tests/libtracker-data/tracker-backup-test.c       |    2 +
 tests/libtracker-data/tracker-ontology-test.c     |    8 +-
 tests/libtracker-data/tracker-sparql-blank-test.c |    1 +
 tests/libtracker-data/tracker-sparql-test.c       |    1 +
 tests/libtracker-db/tracker-db-journal.c          |    6 +-
 tests/libtracker-fts/tracker-fts-test.c           |    1 +
 19 files changed, 572 insertions(+), 98 deletions(-)
---
diff --git a/src/libtracker-common/tracker-keyfile-object.c b/src/libtracker-common/tracker-keyfile-object.c
index c9abbed..2d17932 100644
--- a/src/libtracker-common/tracker-keyfile-object.c
+++ b/src/libtracker-common/tracker-keyfile-object.c
@@ -114,6 +114,28 @@ tracker_keyfile_object_default_int (gpointer     object,
 	return ispec->default_value;
 }
 
+
+guint64
+tracker_keyfile_object_default_uint64 (gpointer     object,
+                                       const gchar *property)
+{
+	GObjectClass *klass;
+	GParamSpec *spec;
+	GParamSpecUInt64 *ispec;
+
+	g_return_val_if_fail (G_IS_OBJECT (object), 0);
+	g_return_val_if_fail (property != NULL, 0);
+
+	klass = G_OBJECT_GET_CLASS (object);
+	spec = g_object_class_find_property (G_OBJECT_CLASS (klass), property);
+	g_return_val_if_fail (spec != NULL, 0);
+
+	ispec = G_PARAM_SPEC_UINT64 (spec);
+	g_return_val_if_fail (ispec != NULL, 0);
+
+	return ispec->default_value;
+}
+
 gboolean
 tracker_keyfile_object_validate_int (gpointer     object,
                                      const gchar *property,
@@ -143,6 +165,36 @@ tracker_keyfile_object_validate_int (gpointer     object,
 	return TRUE;
 }
 
+
+gboolean
+tracker_keyfile_object_validate_uint64 (gpointer     object,
+                                        const gchar *property,
+                                        guint64      value)
+{
+#ifdef G_DISABLE_CHECKS
+	GParamSpec *spec;
+	GValue      gvalue = { 0 };
+	gboolean    valid;
+#endif
+
+	g_return_val_if_fail (G_IS_OBJECT (object), FALSE);
+	g_return_val_if_fail (property != NULL, FALSE);
+
+#ifdef G_DISABLE_CHECKS
+	spec = g_object_class_find_property (G_OBJECT_CLASS (object), property);
+	g_return_val_if_fail (spec != NULL, FALSE);
+
+	g_value_init (&gvalue, spec->value_type);
+	g_value_set_uint64 (&gvalue, value);
+	valid = g_param_value_validate (spec, &gvalue);
+	g_value_unset (&gvalue);
+
+	g_return_val_if_fail (valid != TRUE, FALSE);
+#endif
+
+	return TRUE;
+}
+
 void
 tracker_keyfile_object_load_int (gpointer     object,
                                  const gchar *property,
@@ -170,6 +222,33 @@ tracker_keyfile_object_load_int (gpointer     object,
 }
 
 void
+tracker_keyfile_object_load_uint64 (gpointer     object,
+                                    const gchar *property,
+                                    GKeyFile    *key_file,
+                                    const gchar *group,
+                                    const gchar *key)
+{
+	GError *error = NULL;
+	gchar *str;
+
+	g_return_if_fail (G_IS_OBJECT (object));
+	g_return_if_fail (property != NULL);
+	g_return_if_fail (key_file != NULL);
+	g_return_if_fail (group != NULL);
+	g_return_if_fail (key != NULL);
+
+	str = g_key_file_get_string (key_file, group, key, &error);
+	if (!error) {
+		g_object_set (G_OBJECT (object), property, g_ascii_strtoull (str, NULL, 10), NULL);
+	} else {
+		g_message ("Couldn't load object property '%s' (int) in group '%s', %s",
+		           property, group, error->message);
+		g_error_free (error);
+	}
+	g_free (str);
+}
+
+void
 tracker_keyfile_object_load_boolean (gpointer     object,
                                      const gchar *property,
                                      GKeyFile    *key_file,
@@ -241,18 +320,18 @@ tracker_keyfile_object_load_string_list (gpointer      object,
 	g_return_if_fail (key != NULL);
 
 	value = g_key_file_get_string_list (key_file, group, key, NULL, NULL);
-        l = tracker_string_list_to_gslist (value, -1);
+	l = tracker_string_list_to_gslist (value, -1);
 	g_strfreev (value);
 
-        if (G_LIKELY (!return_instead)) {
-                g_object_set (G_OBJECT (object), property, l, NULL);
+	if (G_LIKELY (!return_instead)) {
+		g_object_set (G_OBJECT (object), property, l, NULL);
 
-                /* List is copied internally */
-                g_slist_foreach (l, (GFunc) g_free, NULL);
-                g_slist_free (l);
-        } else {
-                *return_instead = l;
-        }
+		/* List is copied internally */
+		g_slist_foreach (l, (GFunc) g_free, NULL);
+		g_slist_free (l);
+	} else {
+		*return_instead = l;
+	}
 }
 
 void
@@ -274,32 +353,32 @@ tracker_keyfile_object_load_directory_list (gpointer      object,
 	g_return_if_fail (key != NULL);
 
 	value = g_key_file_get_string_list (key_file, group, key, NULL, NULL);
-        l = directory_string_list_to_gslist ((const gchar **) value);
+	l = directory_string_list_to_gslist ((const gchar **) value);
 	g_strfreev (value);
 
-        if (l) {
-                GSList *filtered;
+	if (l) {
+		GSList *filtered;
 
-                /* Should we make the basename (2nd argument) here
-                 * part of this function's API?
-                 */
-                filtered = tracker_path_list_filter_duplicates (l, ".", is_recursive);
+		/* Should we make the basename (2nd argument) here
+		 * part of this function's API?
+		 */
+		filtered = tracker_path_list_filter_duplicates (l, ".", is_recursive);
 
-                g_slist_foreach (l, (GFunc) g_free, NULL);
-                g_slist_free (l);
+		g_slist_foreach (l, (GFunc) g_free, NULL);
+		g_slist_free (l);
 
-                l = filtered;
-        }
+		l = filtered;
+	}
 
-        if (G_LIKELY (!return_instead)) {
-                g_object_set (G_OBJECT (object), property, l, NULL);
+	if (G_LIKELY (!return_instead)) {
+		g_object_set (G_OBJECT (object), property, l, NULL);
 
-                /* List is copied internally */
-                g_slist_foreach (l, (GFunc) g_free, NULL);
-                g_slist_free (l);
-        } else {
-                *return_instead = l;
-        }
+		/* List is copied internally */
+		g_slist_foreach (l, (GFunc) g_free, NULL);
+		g_slist_free (l);
+	} else {
+		*return_instead = l;
+	}
 }
 
 void
@@ -322,6 +401,28 @@ tracker_keyfile_object_save_int (gpointer     object,
 }
 
 void
+tracker_keyfile_object_save_uint64 (gpointer     object,
+                                    const gchar *property,
+                                    GKeyFile    *key_file,
+                                    const gchar *group,
+                                    const gchar *key)
+{
+	guint64 value;
+	gchar *str;
+
+	g_return_if_fail (G_IS_OBJECT (object));
+	g_return_if_fail (property != NULL);
+	g_return_if_fail (key_file != NULL);
+	g_return_if_fail (group != NULL);
+	g_return_if_fail (key != NULL);
+
+	g_object_get (G_OBJECT (object), property, &value, NULL);
+	str = g_strdup_printf ("%" G_GUINT64_FORMAT, value);
+	g_key_file_set_string (key_file, group, key, str);
+	g_free (str);
+}
+
+void
 tracker_keyfile_object_save_boolean (gpointer     object,
                                      const gchar *property,
                                      GKeyFile    *key_file,
diff --git a/src/libtracker-common/tracker-keyfile-object.h b/src/libtracker-common/tracker-keyfile-object.h
index c06cfcb..94df5ee 100644
--- a/src/libtracker-common/tracker-keyfile-object.h
+++ b/src/libtracker-common/tracker-keyfile-object.h
@@ -34,14 +34,24 @@ gboolean     tracker_keyfile_object_default_boolean  (gpointer      object,
                                                       const gchar  *property);
 gint         tracker_keyfile_object_default_int      (gpointer      object,
                                                       const gchar  *property);
+guint64      tracker_keyfile_object_default_uint64   (gpointer      object,
+                                                      const gchar  *property);
 gboolean     tracker_keyfile_object_validate_int     (gpointer      object,
                                                       const gchar  *property,
                                                       gint          value);
+gboolean     tracker_keyfile_object_validate_uint64  (gpointer     object,
+                                                      const gchar *property,
+                                                      guint64      value);
 void         tracker_keyfile_object_load_int         (gpointer      object,
                                                       const gchar  *property,
                                                       GKeyFile     *key_file,
                                                       const gchar  *group,
                                                       const gchar  *key);
+void         tracker_keyfile_object_load_uint64      (gpointer      object,
+                                                      const gchar  *property,
+                                                      GKeyFile     *key_file,
+                                                      const gchar  *group,
+                                                      const gchar  *key);
 void         tracker_keyfile_object_load_boolean     (gpointer      object,
                                                       const gchar  *property,
                                                       GKeyFile     *key_file,
@@ -71,6 +81,11 @@ void         tracker_keyfile_object_save_int         (gpointer      object,
                                                       GKeyFile     *key_file,
                                                       const gchar  *group,
                                                       const gchar  *key);
+void         tracker_keyfile_object_save_uint64      (gpointer      object,
+                                                      const gchar  *property,
+                                                      GKeyFile     *key_file,
+                                                      const gchar  *group,
+                                                      const gchar  *key);
 void         tracker_keyfile_object_save_boolean     (gpointer      object,
                                                       const gchar  *property,
                                                       GKeyFile     *key_file,
diff --git a/src/libtracker-data/tracker-data-backup.c b/src/libtracker-data/tracker-data-backup.c
index 9e3f298..3c5c2f5 100644
--- a/src/libtracker-data/tracker-data-backup.c
+++ b/src/libtracker-data/tracker-data-backup.c
@@ -143,6 +143,8 @@ tracker_data_backup_restore (GFile *journal,
 	if (g_file_query_exists (info->journal, NULL)) {
 		TrackerDBManagerFlags flags = tracker_db_manager_get_flags ();
 		gboolean is_first;
+		gsize chunk_size = 0;
+		gboolean do_rotating = FALSE;
 
 		tracker_db_manager_move_to_temp ();
 		tracker_data_manager_shutdown ();
@@ -156,7 +158,8 @@ tracker_data_backup_restore (GFile *journal,
 		             &info->error);
 
 		tracker_db_manager_init_locations ();
-		tracker_db_journal_init (NULL, FALSE);
+		tracker_db_journal_get_rotating (&do_rotating, &chunk_size);
+		tracker_db_journal_init (NULL, FALSE, do_rotating, chunk_size);
 
 		if (info->error) {
 			tracker_db_manager_restore_from_temp ();
@@ -166,7 +169,8 @@ tracker_data_backup_restore (GFile *journal,
 
 		tracker_db_journal_shutdown ();
 
-		tracker_data_manager_init (flags, test_schemas, &is_first, TRUE,
+		tracker_data_manager_init (flags, do_rotating, chunk_size,
+		                           test_schemas, &is_first, TRUE,
 		                           busy_callback, busy_user_data,
 		                           "Restoring backup");
 
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index c82deeb..ddc5ea6 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -2028,6 +2028,8 @@ get_new_service_id (TrackerDBInterface *iface)
 
 gboolean
 tracker_data_manager_init (TrackerDBManagerFlags  flags,
+                           gboolean               do_journal_rotating,
+                           gsize                  chunk_size,
                            const gchar          **test_schemas,
                            gboolean              *first_time,
                            gboolean               journal_check,
@@ -2119,7 +2121,7 @@ tracker_data_manager_init (TrackerDBManagerFlags  flags,
 		in_journal_replay = FALSE;
 
 		/* open journal for writing */
-		tracker_db_journal_init (NULL, FALSE);
+		tracker_db_journal_init (NULL, FALSE, do_journal_rotating, chunk_size);
 
 		check_ontology = TRUE;
 
@@ -2132,7 +2134,7 @@ tracker_data_manager_init (TrackerDBManagerFlags  flags,
 
 		/* Truncate journal as it does not even contain a single valid transaction
 		 * or is explicitly ignored (journal_check == FALSE, only for test cases) */
-		tracker_db_journal_init (NULL, TRUE);
+		tracker_db_journal_init (NULL, TRUE, do_journal_rotating, chunk_size);
 
 		/* load ontology from files into memory (max_id starts at zero: first-time) */
 
@@ -2193,7 +2195,7 @@ tracker_data_manager_init (TrackerDBManagerFlags  flags,
 		/* First time, no need to check ontology */
 		check_ontology = FALSE;
 	} else {
-		tracker_db_journal_init (NULL, FALSE);
+		tracker_db_journal_init (NULL, FALSE, do_journal_rotating, chunk_size);
 
 		/* Load ontology from database into memory */
 		db_get_static_data (iface);
diff --git a/src/libtracker-data/tracker-data-manager.h b/src/libtracker-data/tracker-data-manager.h
index e83d6c7..e8876fb 100644
--- a/src/libtracker-data/tracker-data-manager.h
+++ b/src/libtracker-data/tracker-data-manager.h
@@ -40,6 +40,8 @@ G_BEGIN_DECLS
 #endif
 
 gboolean tracker_data_manager_init                (TrackerDBManagerFlags  flags,
+                                                   gboolean               do_journal_rotating,
+                                                   gsize                  chunk_size,
                                                    const gchar          **test_schema,
                                                    gboolean              *first_time,
                                                    gboolean               journal_check,
diff --git a/src/libtracker-data/tracker-data-update.c b/src/libtracker-data/tracker-data-update.c
index 3cdc383..485eed6 100644
--- a/src/libtracker-data/tracker-data-update.c
+++ b/src/libtracker-data/tracker-data-update.c
@@ -2798,11 +2798,15 @@ tracker_data_replay_journal (GHashTable          *classes,
 
 	if (journal_error) {
 		gsize size;
+		gsize chunk_size = 0;
+		gboolean do_rotating = FALSE;
 
 		size = tracker_db_journal_reader_get_size_of_correct ();
 		tracker_db_journal_reader_shutdown ();
 
-		tracker_db_journal_init (NULL, FALSE);
+		tracker_db_journal_get_rotating (&do_rotating, &chunk_size);
+
+		tracker_db_journal_init (NULL, FALSE, do_rotating, chunk_size);
 		tracker_db_journal_truncate (size);
 		tracker_db_journal_shutdown ();
 
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index 0b572ab..8fc34f5 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -31,6 +31,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <stdlib.h>
 
 #include <glib/gstdio.h>
 
@@ -84,6 +85,7 @@ static struct {
 	gint p_id;
 	gint o_id;
 	const gchar *object;
+	guint current_file;
 } reader;
 
 static struct {
@@ -95,6 +97,8 @@ static struct {
 	gchar *cur_block;
 	guint cur_entry_amount;
 	guint cur_pos;
+	gsize chunk_size;
+	gboolean do_rotating;
 } writer;
 
 static guint32
@@ -201,50 +205,19 @@ tracker_db_journal_error_quark (void)
 	return g_quark_from_static_string (TRACKER_DB_JOURNAL_ERROR_DOMAIN);
 }
 
-gboolean
-tracker_db_journal_init (const gchar *filename, gboolean truncate)
+static gboolean
+tracker_db_journal_init_file (gboolean truncate)
 {
-	gchar *directory;
 	struct stat st;
 	int flags;
 	int mode;
 
-	g_return_val_if_fail (writer.journal == 0, FALSE);
-
 	writer.cur_block_len = 0;
 	writer.cur_pos = 0;
 	writer.cur_entry_amount = 0;
 	writer.cur_block_alloc = 0;
 	writer.cur_block = NULL;
 
-	/* Used mostly for testing */
-	if (G_UNLIKELY (filename)) {
-		writer.journal_filename = g_strdup (filename);
-	} else {
-		writer.journal_filename = g_build_filename (g_get_user_data_dir (),
-		                                            "tracker",
-		                                            "data",
-		                                            JOURNAL_FILENAME,
-		                                            NULL);
-	}
-
-	directory = g_path_get_dirname (writer.journal_filename);
-	if (g_strcmp0 (directory, ".")) {
-		mode = S_IRWXU | S_IRWXG | S_IRWXO;
-		if (g_mkdir_with_parents (directory, mode)) {
-			g_critical ("tracker data directory does not exist and "
-				    "could not be created: %s",
-				    g_strerror (errno));
-
-			g_free (directory);
-			g_free (writer.journal_filename);
-			writer.journal_filename = NULL;
-
-			return FALSE;
-		}
-	}
-	g_free (directory);
-
 	mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
 	flags = O_WRONLY | O_APPEND | O_CREAT | O_LARGEFILE;
 	if (truncate) {
@@ -285,7 +258,7 @@ tracker_db_journal_init (const gchar *filename, gboolean truncate)
 		writer.cur_block[4] = 'g';
 		writer.cur_block[5] = '\0';
 		writer.cur_block[6] = '0';
-		writer.cur_block[7] = '2';
+		writer.cur_block[7] = '3';
 
 		if (!write_all_data (writer.journal, writer.cur_block, 8)) {
 			g_free (writer.journal_filename);
@@ -301,6 +274,51 @@ tracker_db_journal_init (const gchar *filename, gboolean truncate)
 }
 
 gboolean
+tracker_db_journal_init (const gchar *filename,
+                         gboolean     truncate,
+                         gboolean     do_rotating,
+                         gsize        chunk_size)
+{
+	gchar *directory;
+	int mode;
+
+	g_return_val_if_fail (writer.journal == 0, FALSE);
+
+	/* Used mostly for testing */
+	if (G_UNLIKELY (filename)) {
+		writer.journal_filename = g_strdup (filename);
+	} else {
+		writer.journal_filename = g_build_filename (g_get_user_data_dir (),
+		                                            "tracker",
+		                                            "data",
+		                                            JOURNAL_FILENAME,
+		                                            NULL);
+	}
+
+	directory = g_path_get_dirname (writer.journal_filename);
+	if (g_strcmp0 (directory, ".")) {
+		mode = S_IRWXU | S_IRWXG | S_IRWXO;
+		if (g_mkdir_with_parents (directory, mode)) {
+			g_critical ("tracker data directory does not exist and "
+			            "could not be created: %s",
+			            g_strerror (errno));
+
+			g_free (directory);
+			g_free (writer.journal_filename);
+			writer.journal_filename = NULL;
+
+			return FALSE;
+		}
+	}
+	g_free (directory);
+
+	writer.do_rotating = do_rotating;
+	writer.chunk_size = chunk_size;
+
+	return tracker_db_journal_init_file (truncate);
+}
+
+gboolean
 tracker_db_journal_shutdown (void)
 {
 	if (writer.journal == 0) {
@@ -606,6 +624,74 @@ tracker_db_journal_truncate (gsize new_size)
 	return (ftruncate (writer.journal, new_size) != -1);
 }
 
+void
+tracker_db_journal_get_rotating (gboolean *do_rotating,
+                                 gsize    *chunk_size)
+{
+	*do_rotating = writer.do_rotating;
+	*chunk_size = writer.chunk_size;
+}
+
+static gboolean
+tracker_db_journal_rotate (void)
+{
+	gchar *fullpath;
+	static guint max = 0;
+
+	if (max == 0) {
+		gchar *directory;
+		GDir *journal_dir;
+		const gchar *f_name;
+
+		directory = g_path_get_dirname (writer.journal_filename);
+
+		journal_dir = g_dir_open (directory, 0, NULL);
+
+		f_name = g_dir_read_name (journal_dir);
+
+		while (f_name) {
+			gchar *ptr;
+			guint cur;
+
+			if (f_name) {
+
+				if (!g_str_has_prefix (f_name, "tracker-store.journal.")) {
+					f_name = g_dir_read_name (journal_dir);
+					continue;
+				}
+
+				ptr = strrchr (f_name, '.');
+				if (ptr) {
+					ptr++;
+					cur = atoi (ptr);
+					max = MAX (cur, max);
+				}
+			}
+
+			f_name = g_dir_read_name (journal_dir);
+		}
+
+		g_dir_close (journal_dir);
+		g_free (directory);
+	}
+
+	tracker_db_journal_fsync ();
+
+	if (close (writer.journal) != 0) {
+		g_warning ("Could not close journal, %s", 
+		           g_strerror (errno));
+		return FALSE;
+	}
+
+	fullpath = g_strdup_printf ("%s.%d", writer.journal_filename, ++max);
+
+	g_rename (writer.journal_filename, fullpath);
+
+	g_free (fullpath);
+
+	return tracker_db_journal_init_file (TRUE);
+}
+
 gboolean
 tracker_db_journal_commit_db_transaction (void)
 {
@@ -651,6 +737,13 @@ tracker_db_journal_commit_db_transaction (void)
 	/* Clean up for next transaction */
 	cur_block_kill ();
 
+	if (writer.do_rotating && (writer.cur_size > writer.chunk_size)) {
+		if (!tracker_db_journal_rotate ()) {
+			g_critical ("Could not rotate journal, %s", g_strerror (errno));
+			return FALSE;
+		}
+	}
+
 	return TRUE;
 }
 
@@ -670,6 +763,8 @@ tracker_db_journal_reader_init (const gchar *filename)
 {
 	GError *error = NULL;
 	gchar *filename_used;
+	gchar *filename_open;
+	gchar *test;
 
 	g_return_val_if_fail (reader.file == NULL, FALSE);
 
@@ -684,16 +779,29 @@ tracker_db_journal_reader_init (const gchar *filename)
 		                                  NULL);
 	}
 
+	test = g_strdup_printf ("%s.1", filename_used);
+
+	if (g_file_test (test, G_FILE_TEST_EXISTS)) {
+		filename_open = test;
+		reader.current_file = 1;
+	} else {
+		g_free (test);
+		filename_open = g_strdup (filename_used);
+		reader.current_file = 0;
+	}
+
 	reader.type = TRACKER_DB_JOURNAL_START;
 	reader.filename = filename_used;
-	reader.file = g_mapped_file_new (reader.filename, FALSE, &error);
+	reader.file = g_mapped_file_new (filename_open, FALSE, &error);
+
+	g_free (filename_open);
 
 	if (error) {
 		if (!g_error_matches (error, G_FILE_ERROR, G_FILE_ERROR_NOENT)) {
 			/* do not warn if the file does not exist, just return FALSE */
 			g_warning ("Could not create TrackerDBJournalReader for file '%s', %s",
-				   reader.filename,
-				   error->message ? error->message : "no error given");
+			           reader.filename,
+			           error->message ? error->message : "no error given");
 		}
 		g_error_free (error);
 		g_free (reader.filename);
@@ -713,7 +821,7 @@ tracker_db_journal_reader_init (const gchar *filename)
 		return FALSE;
 	}
 
-	if (memcmp (reader.current, "trlog\00002", 8)) {
+	if (memcmp (reader.current, "trlog\00003", 8)) {
 		tracker_db_journal_reader_shutdown ();
 		return FALSE;
 	}
@@ -731,6 +839,70 @@ tracker_db_journal_reader_get_size_of_correct (void)
 	return (gsize) (reader.last_success - reader.start);
 }
 
+static gboolean
+reader_next_file (GError **error)
+{
+	gchar *filename_open;
+	gchar *test;
+	GError *new_error = NULL;
+
+	test = g_strdup_printf ("%s.%d", reader.filename, ++reader.current_file);
+
+	if (g_file_test (test, G_FILE_TEST_EXISTS)) {
+		filename_open = test;
+	} else {
+		g_free (test);
+		filename_open = g_strdup (reader.filename);
+		/* Last file is the active journal file */
+		reader.current_file = 0;
+	}
+
+#if GLIB_CHECK_VERSION(2,22,0)
+	g_mapped_file_unref (reader.file);
+#else
+	g_mapped_file_free (reader.file);
+#endif
+
+	reader.file = g_mapped_file_new (filename_open, FALSE, &new_error);
+
+	if (new_error) {
+		g_propagate_error (error, new_error);
+		return FALSE;
+	}
+
+	reader.last_success = reader.start = reader.current = 
+		g_mapped_file_get_contents (reader.file);
+
+	reader.end = reader.current + g_mapped_file_get_length (reader.file);
+
+	/* verify journal file header */
+	if (reader.end - reader.current < 8) {
+		g_set_error (error, TRACKER_DB_JOURNAL_ERROR, 0, 
+		             "Damaged journal entry at begin of journal");
+		tracker_db_journal_reader_shutdown ();
+		return FALSE;
+	}
+
+	if (memcmp (reader.current, "trlog\00003", 8)) {
+		g_set_error (error, TRACKER_DB_JOURNAL_ERROR, 0, 
+		             "Damaged journal entry at begin of journal");
+		tracker_db_journal_reader_shutdown ();
+		return FALSE;
+	}
+
+	reader.current += 8;
+
+	reader.type = TRACKER_DB_JOURNAL_END_TRANSACTION;
+
+	reader.entry_begin = NULL;
+	reader.entry_end = NULL;
+	reader.amount_of_triples = 0;
+
+	g_free (filename_open);
+
+	return TRUE;
+}
+
 gboolean
 tracker_db_journal_reader_shutdown (void)
 {
@@ -840,9 +1012,11 @@ tracker_db_journal_reader_next (GError **error)
 		/* Check the end is not before where we currently are */
 		if (reader.current >= reader.end) {
 			/* Return FALSE as there is no further entry but
-			 * do not set error as it's not an error case.
-			 */
-			return FALSE;
+			 * do not set error as it's not an error case. */
+			if (reader.current_file != 0)
+				return reader_next_file (error);
+			else
+				return FALSE;
 		}
 
 		/* Check the end is not smaller than the first uint32
@@ -1154,6 +1328,7 @@ gdouble
 tracker_db_journal_reader_get_progress (void)
 {
 	gdouble percent = ((gdouble)(reader.end - reader.start));
-	return ((gdouble)(reader.current - reader.start)) / percent;
+	/* TODO: Fix this now that multiple chunks can exist */
+	return (((gdouble)(reader.current - reader.start)) / percent);
 }
 
diff --git a/src/libtracker-db/tracker-db-journal.h b/src/libtracker-db/tracker-db-journal.h
index 254f670..4ef0ca5 100644
--- a/src/libtracker-db/tracker-db-journal.h
+++ b/src/libtracker-db/tracker-db-journal.h
@@ -48,11 +48,15 @@ GQuark       tracker_db_journal_error_quark                  (void);
  * Writer API
  */
 gboolean     tracker_db_journal_init                         (const gchar *filename,
-                                                              gboolean     truncate);
+                                                              gboolean     truncate,
+                                                              gboolean     do_rotating,
+                                                              gsize        chunk_size);
 gboolean     tracker_db_journal_shutdown                     (void);
 
 const gchar* tracker_db_journal_get_filename                 (void);
 gsize        tracker_db_journal_get_size                     (void);
+void         tracker_db_journal_get_rotating                 (gboolean    *do_rotating,
+                                                              gsize       *chunk_size);
 
 gboolean     tracker_db_journal_start_transaction            (time_t       time);
 gboolean     tracker_db_journal_start_ontology_transaction   (time_t       time);
diff --git a/src/libtracker-db/tracker-db-manager.c b/src/libtracker-db/tracker-db-manager.c
index 51aa979..fcc8d82 100644
--- a/src/libtracker-db/tracker-db-manager.c
+++ b/src/libtracker-db/tracker-db-manager.c
@@ -422,11 +422,45 @@ db_manager_remove_all (gboolean rm_journal)
 		if (opath) {
 			GFile *file;
 			gchar *cpath;
+			gchar *directory;
+			GDir *journal_dir;
+			const gchar *f_name;
 
 			cpath = g_strdup (opath);
 			tracker_db_journal_shutdown ();
-			g_message ("  Removing journal:'%s'",
-					   cpath);
+
+			g_message ("  Removing journal:'%s'", cpath);
+
+			directory = g_path_get_dirname (cpath);
+			journal_dir = g_dir_open (directory, 0, NULL);
+			f_name = g_dir_read_name (journal_dir);
+
+			/* Remove rotated chunks */
+			while (f_name) {
+				gchar *fullpath;
+
+				if (f_name) {
+					if (!g_str_has_prefix (f_name, "tracker-store.journal.")) {
+						f_name = g_dir_read_name (journal_dir);
+						continue;
+					}
+					
+				}
+
+				fullpath = g_build_filename (directory, f_name, NULL);
+
+				file = g_file_new_for_path (fullpath);
+				g_file_delete (file, NULL, NULL);
+				g_object_unref (file);
+				g_free (fullpath);
+
+				f_name = g_dir_read_name (journal_dir);
+			}
+
+			g_dir_close (journal_dir);
+			g_free (directory);
+
+			/* Remove active journal */
 			file = g_file_new_for_path (cpath);
 			g_file_delete (file, NULL, NULL);
 			g_object_unref (file);
diff --git a/src/tracker-control/tracker-control.c b/src/tracker-control/tracker-control.c
index ec4ab44..380b121 100644
--- a/src/tracker-control/tracker-control.c
+++ b/src/tracker-control/tracker-control.c
@@ -411,7 +411,7 @@ main (int argc, char **argv)
 		g_log_set_default_handler (log_handler, NULL);
 
 		/* This call is needed to set the journal's filename */
-		tracker_db_journal_init (NULL, FALSE);
+		tracker_db_journal_init (NULL, FALSE, FALSE, G_MAXSIZE);
 
 		/* Clean up */
 		if (!tracker_db_manager_init (TRACKER_DB_MANAGER_REMOVE_ALL, NULL, FALSE)) {
diff --git a/src/tracker-store/tracker-config.c b/src/tracker-store/tracker-config.c
index f18b15e..d8736f0 100644
--- a/src/tracker-store/tracker-config.c
+++ b/src/tracker-store/tracker-config.c
@@ -33,16 +33,21 @@
 
 /* GKeyFile defines */
 #define GROUP_GENERAL     "General"
-#define GROUP_INDEXING    "Indexing"
+#define GROUP_JOURNAL     "Journal"
 
 /* Default values */
-#define DEFAULT_VERBOSITY 2
+#define DEFAULT_VERBOSITY                 2
+#define DEFAULT_JOURNAL_CHUNK_SIZE        500
+#define DEFAULT_ENABLE_JOURNAL_ROTATING   TRUE
 
 /* typedef struct TrackerConfigPrivate TrackerConfigPrivate; */
 
 typedef struct {
 	/* General */
 	gint verbosity;
+	/* Journal */
+	gsize journal_chunk_size;
+	gboolean enable_journal_rotating;
 }  TrackerConfigPrivate;
 
 typedef struct {
@@ -71,11 +76,16 @@ enum {
 	PROP_0,
 
 	/* General */
-	PROP_VERBOSITY
+	PROP_VERBOSITY,
+	/* Journal */
+	PROP_JOURNAL_CHUNK_SIZE,
+	PROP_ENABLE_JOURNAL_ROTATING
 };
 
 static ObjectToKeyFile conversions[] = {
-	{ G_TYPE_INT,     "verbosity",          GROUP_GENERAL,  "Verbosity"       },
+	{ G_TYPE_INT,     "verbosity",               GROUP_GENERAL,  "Verbosity"             },
+	{ G_TYPE_BOOLEAN, "enable-journal-rotating", GROUP_JOURNAL,  "EnableJournalRotating" },
+	{ G_TYPE_UINT64,  "journal-chunk-size",  GROUP_JOURNAL,  "JournalChunkSize"   },
 };
 
 G_DEFINE_TYPE (TrackerConfig, tracker_config, TRACKER_TYPE_CONFIG_FILE);
@@ -101,6 +111,24 @@ tracker_config_class_init (TrackerConfigClass *klass)
 	                                                   DEFAULT_VERBOSITY,
 	                                                   G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
 
+	g_object_class_install_property (object_class,
+	                                 PROP_JOURNAL_CHUNK_SIZE,
+	                                 g_param_spec_uint64 ("journal-chunk-size",
+	                                                      "Journal chunk size",
+	                                                      " Size of a journal-chunk before rotation",
+	                                                      0,
+	                                                      G_MAXSIZE,
+	                                                      DEFAULT_JOURNAL_CHUNK_SIZE,
+	                                                      G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+
+	g_object_class_install_property (object_class,
+	                                 PROP_ENABLE_JOURNAL_ROTATING,
+	                                 g_param_spec_boolean ("enable-journal-rotating",
+	                                                       "Enable journal rotating",
+	                                                       " Enable journal rotating",
+	                                                       DEFAULT_ENABLE_JOURNAL_ROTATING,
+	                                                       G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
+
 	g_type_class_add_private (object_class, sizeof (TrackerConfigPrivate));
 }
 
@@ -121,7 +149,15 @@ config_set_property (GObject      *object,
 		tracker_config_set_verbosity (TRACKER_CONFIG (object),
 		                              g_value_get_int (value));
 		break;
-
+		/* Journal */
+	case PROP_JOURNAL_CHUNK_SIZE:
+		tracker_config_set_journal_chunk_size (TRACKER_CONFIG (object),
+		                                       (gsize) g_value_get_uint64 (value));
+		break;
+	case PROP_ENABLE_JOURNAL_ROTATING:
+		tracker_config_set_enable_journal_rotating (TRACKER_CONFIG (object),
+		                                            g_value_get_boolean (value));
+		break;
 	default:
 		G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
 		break;
@@ -143,7 +179,12 @@ config_get_property (GObject    *object,
 	case PROP_VERBOSITY:
 		g_value_set_int (value, priv->verbosity);
 		break;
-
+	case PROP_JOURNAL_CHUNK_SIZE:
+		g_value_set_uint64 (value, (guint64) priv->journal_chunk_size);
+		break;
+	case PROP_ENABLE_JOURNAL_ROTATING:
+		g_value_set_boolean (value, priv->enable_journal_rotating);
+		break;
 	default:
 		G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec);
 		break;
@@ -179,6 +220,7 @@ config_create_with_defaults (TrackerConfig *config,
 
 	for (i = 0; i < G_N_ELEMENTS (conversions); i++) {
 		gboolean has_key;
+		gchar *str;
 
 		has_key = g_key_file_has_key (key_file,
 		                              conversions[i].group,
@@ -197,6 +239,17 @@ config_create_with_defaults (TrackerConfig *config,
 			                                                            conversions[i].property));
 			break;
 
+		case G_TYPE_UINT64:
+			str = g_strdup_printf ("%" G_GUINT64_FORMAT,
+			                       tracker_keyfile_object_default_uint64 (config,
+			                                                              conversions[i].property));
+			g_key_file_set_string (key_file,
+			                       conversions[i].group,
+			                       conversions[i].key,
+			                       str);
+			g_free (str);
+			break;
+
 		case G_TYPE_BOOLEAN:
 			g_key_file_set_boolean (key_file,
 			                        conversions[i].group,
@@ -249,6 +302,14 @@ config_load (TrackerConfig *config)
 			                                 conversions[i].key);
 			break;
 
+		case G_TYPE_UINT64:
+			tracker_keyfile_object_load_uint64 (G_OBJECT (file),
+			                                    conversions[i].property,
+			                                    file->key_file,
+			                                    conversions[i].group,
+			                                    conversions[i].key);
+			break;
+
 		case G_TYPE_BOOLEAN:
 			tracker_keyfile_object_load_boolean (G_OBJECT (file),
 			                                     conversions[i].property,
@@ -333,6 +394,62 @@ tracker_config_get_verbosity (TrackerConfig *config)
 	return priv->verbosity;
 }
 
+gboolean
+tracker_config_get_enable_journal_rotating (TrackerConfig *config)
+{
+	TrackerConfigPrivate *priv;
+
+	g_return_val_if_fail (TRACKER_IS_CONFIG (config), DEFAULT_ENABLE_JOURNAL_ROTATING);
+
+	priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+	return priv->enable_journal_rotating;
+}
+
+gsize
+tracker_config_get_journal_chunk_size (TrackerConfig *config)
+{
+	TrackerConfigPrivate *priv;
+
+	g_return_val_if_fail (TRACKER_IS_CONFIG (config), DEFAULT_JOURNAL_CHUNK_SIZE);
+
+	priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+	return priv->journal_chunk_size;
+}
+
+void
+tracker_config_set_enable_journal_rotating (TrackerConfig *config,
+                                            gboolean       value)
+{
+	TrackerConfigPrivate *priv;
+
+	g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+	priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+	priv->enable_journal_rotating = value;
+	g_object_notify (G_OBJECT (config), "enable-journal-rotating");
+}
+
+void
+tracker_config_set_journal_chunk_size (TrackerConfig *config,
+                                       gsize          value)
+{
+	TrackerConfigPrivate *priv;
+
+	g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+	if (!tracker_keyfile_object_validate_uint64 (config, "journal-chunk-size", (guint64) value)) {
+		return;
+	}
+
+	priv = TRACKER_CONFIG_GET_PRIVATE (config);
+
+	priv->journal_chunk_size = value;
+	g_object_notify (G_OBJECT (config), "journal-chunk-size");
+}
+
 void
 tracker_config_set_verbosity (TrackerConfig *config,
                               gint           value)
diff --git a/src/tracker-store/tracker-config.h b/src/tracker-store/tracker-config.h
index beb85f3..27f0695 100644
--- a/src/tracker-store/tracker-config.h
+++ b/src/tracker-store/tracker-config.h
@@ -44,13 +44,20 @@ struct TrackerConfigClass {
 	TrackerConfigFileClass parent_class;
 };
 
-GType          tracker_config_get_type               (void) G_GNUC_CONST;
+GType          tracker_config_get_type                             (void) G_GNUC_CONST;
 
-TrackerConfig *tracker_config_new                    (void);
-gboolean       tracker_config_save                   (TrackerConfig *config);
-gint           tracker_config_get_verbosity          (TrackerConfig *config);
-void           tracker_config_set_verbosity          (TrackerConfig *config,
-                                                      gint           value);
+TrackerConfig *tracker_config_new                                  (void);
+gboolean       tracker_config_save                                 (TrackerConfig *config);
+gint           tracker_config_get_verbosity                        (TrackerConfig *config);
+gboolean       tracker_config_get_enable_journal_rotating          (TrackerConfig *config);
+gsize          tracker_config_get_journal_chunk_size               (TrackerConfig *config);
+
+void           tracker_config_set_verbosity                        (TrackerConfig *config,
+                                                                    gint           value);
+void           tracker_config_set_enable_journal_rotating          (TrackerConfig *config,
+                                                                    gboolean       value);
+void           tracker_config_set_journal_chunk_size               (TrackerConfig *config,
+                                                                    gsize          value);
 
 G_END_DECLS
 
diff --git a/src/tracker-store/tracker-main.c b/src/tracker-store/tracker-main.c
index d18b99d..b98d5cb 100644
--- a/src/tracker-store/tracker-main.c
+++ b/src/tracker-store/tracker-main.c
@@ -438,6 +438,8 @@ main (gint argc, gchar *argv[])
 	                                            &busy_user_data);
 
 	if (!tracker_data_manager_init (flags,
+	                                tracker_config_get_enable_journal_rotating (config),
+	                                tracker_config_get_journal_chunk_size (config),
 	                                NULL,
 	                                &is_first_time_index,
 	                                TRUE,
diff --git a/tests/libtracker-data/tracker-backup-test.c b/tests/libtracker-data/tracker-backup-test.c
index d5e3a2a..01e0277 100644
--- a/tests/libtracker-data/tracker-backup-test.c
+++ b/tests/libtracker-data/tracker-backup-test.c
@@ -106,6 +106,7 @@ test_backup_and_restore_helper (gboolean journal)
 	test_schemas[3] = data_prefix;
 
 	tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+	                           FALSE, G_MAXSIZE,
 	                           (const gchar **) test_schemas,
 	                           NULL, FALSE, NULL, NULL, NULL);
 
@@ -158,6 +159,7 @@ test_backup_and_restore_helper (gboolean journal)
 	g_free (meta_db);
 
 	tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+	                           FALSE, G_MAXSIZE,
 	                           (const gchar **) test_schemas,
 	                           NULL, FALSE, NULL, NULL, NULL);
 	check_content_in_db (0, 0);
diff --git a/tests/libtracker-data/tracker-ontology-test.c b/tests/libtracker-data/tracker-ontology-test.c
index 44da5ca..86ddf44 100644
--- a/tests/libtracker-data/tracker-ontology-test.c
+++ b/tests/libtracker-data/tracker-ontology-test.c
@@ -264,7 +264,7 @@ test_ontology_change (void)
 
                 g_chmod (ontology_file, 0666);
 
-		tracker_data_manager_init (0, (const gchar **) test_schemas,
+		tracker_data_manager_init (0, FALSE, G_MAXSIZE, (const gchar **) test_schemas,
 		                           NULL, FALSE, NULL, NULL, NULL);
 
 		if (g_file_get_contents (update, &queries, NULL, NULL)) {
@@ -290,7 +290,7 @@ test_ontology_change (void)
 
 	delete_db (FALSE);
 
-	tracker_data_manager_init (0, (const gchar **) test_schemas,
+	tracker_data_manager_init (0, FALSE, G_MAXSIZE, (const gchar **) test_schemas,
 	                           NULL, TRUE, NULL, NULL, NULL);
 
 	for (i = 0; change_tests[i].test_name != NULL; i++) {
@@ -326,6 +326,7 @@ test_ontology_init (void)
 {
 	/* first-time initialization */
 	tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+	                           FALSE, G_MAXSIZE,
 	                           NULL,
 	                           NULL,
 	                           FALSE,
@@ -336,7 +337,7 @@ test_ontology_init (void)
 	tracker_data_manager_shutdown ();
 
 	/* initialization from existing database */
-	tracker_data_manager_init (0,
+	tracker_data_manager_init (0, FALSE, G_MAXSIZE,
 	                           NULL,
 	                           NULL,
 	                           FALSE,
@@ -368,6 +369,7 @@ test_query (gconstpointer test_data)
 
 	/* initialization */
 	tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+	                           FALSE, G_MAXSIZE,
 	                           NULL,
 	                           NULL, 
 	                           FALSE,
diff --git a/tests/libtracker-data/tracker-sparql-blank-test.c b/tests/libtracker-data/tracker-sparql-blank-test.c
index 5b2f9b7..ecdbf6c 100644
--- a/tests/libtracker-data/tracker-sparql-blank-test.c
+++ b/tests/libtracker-data/tracker-sparql-blank-test.c
@@ -42,6 +42,7 @@ test_blank (void)
 
 	/* initialization */
 	tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+	                           FALSE, G_MAXSIZE,
 	                           NULL,
 	                           NULL,
 	                           FALSE,
diff --git a/tests/libtracker-data/tracker-sparql-test.c b/tests/libtracker-data/tracker-sparql-test.c
index 2ce7805..da9b78b 100644
--- a/tests/libtracker-data/tracker-sparql-test.c
+++ b/tests/libtracker-data/tracker-sparql-test.c
@@ -235,6 +235,7 @@ test_sparql_query (gconstpointer test_data)
 
 	test_schemas[0] = data_prefix;
 	tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+	                           FALSE, G_MAXSIZE,
 	                           test_schemas,
 	                           NULL, FALSE, NULL, NULL, NULL);
 
diff --git a/tests/libtracker-db/tracker-db-journal.c b/tests/libtracker-db/tracker-db-journal.c
index 6614ef9..5a2c623 100644
--- a/tests/libtracker-db/tracker-db-journal.c
+++ b/tests/libtracker-db/tracker-db-journal.c
@@ -29,13 +29,13 @@ test_init_and_shutdown (void)
 	gboolean result;
 
 	/* check double init/shutdown */
-	result = tracker_db_journal_init (NULL, FALSE);
+	result = tracker_db_journal_init (NULL, FALSE, FALSE, G_MAXSIZE);
 	g_assert (result == TRUE);
 
 	result = tracker_db_journal_shutdown ();
 	g_assert (result == TRUE);
 
-	result = tracker_db_journal_init (NULL, FALSE);
+	result = tracker_db_journal_init (NULL, FALSE, FALSE, G_MAXSIZE);
 	g_assert (result == TRUE);
 
 	result = tracker_db_journal_shutdown ();
@@ -53,7 +53,7 @@ test_write_functions (void)
 	path = g_build_filename (TOP_BUILDDIR, "tests", "libtracker-db", "tracker-store.journal", NULL);
 	g_unlink (path);
 
-	tracker_db_journal_init (path, FALSE);
+	tracker_db_journal_init (path, FALSE, FALSE, G_MAXSIZE);
 
         filename = tracker_db_journal_get_filename ();
 	g_assert (filename != NULL);
diff --git a/tests/libtracker-fts/tracker-fts-test.c b/tests/libtracker-fts/tracker-fts-test.c
index 84ec1fa..503dfdb 100644
--- a/tests/libtracker-fts/tracker-fts-test.c
+++ b/tests/libtracker-fts/tracker-fts-test.c
@@ -73,6 +73,7 @@ test_sparql_query (gconstpointer test_data)
 
 	test_schemas[0] = data_prefix;
 	tracker_data_manager_init (TRACKER_DB_MANAGER_FORCE_REINDEX,
+	                           FALSE, G_MAXSIZE,
 	                           test_schemas,
 	                           NULL, FALSE, NULL, NULL, NULL);
 



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]