[tracker/urho-sync] Changed the format of the log, playing it using mmap

[Philip Van Hoof <pvanhoof src gnome org>]

commit 013b277d1f3079d228f20a8fc14b11da8d5f8f6e
Author: Philip Van Hoof <philip codeminded be>
Date:   Fri Aug 21 17:11:44 2009 +0200

    Changed the format of the log, playing it using mmap

 src/libtracker-db/tracker-db-journal.c |   51 +++++++++++++++++++++++++++----
 src/libtracker-db/tracker-db-journal.h |    6 ++-
 src/tracker-store/tracker-store.c      |   41 ++++++++++++++-----------
 3 files changed, 71 insertions(+), 27 deletions(-)
---
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index 07c3c39..91f212c 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -21,6 +21,8 @@
  */
 #include "config.h"
 
+#define _GNU_SOURCE
+
 #include <unistd.h>
 #include <sys/types.h>
 #include <stdio.h>
@@ -30,6 +32,7 @@
 
 static gchar *filename = NULL;
 static FILE *journal = NULL;
+static GMappedFile *mapped = NULL;
 
 #define TRACKER_DB_JOURNAL_LOG_FILENAME		"log.sparql.txt"
 
@@ -64,7 +67,7 @@ tracker_db_journal_log (const gchar *query)
 {
 	if (journal) {
 		write (fileno (journal), query, strlen (query));
-		write (fileno (journal), "\n", 1);
+		write (fileno (journal), "\n\0", 2);
 	}
 }
 
@@ -97,15 +100,49 @@ tracker_db_journal_close (void)
 	filename = NULL;
 }
 
-gchar *
-tracker_db_journal_get_content (void)
+TrackerJournalContents*
+tracker_db_journal_get_contents (guint transaction_size)
 {
-	gchar *lines = NULL;
-	gsize len;
+	GPtrArray *lines;
+	gsize max_pos, next_len;
+	gchar *cur;
 
 	get_filename ();
 
-	g_file_get_contents (filename, &lines, &len, NULL);
+	if (!mapped) {
+		GError *error = NULL;
+
+		mapped = g_mapped_file_new (filename, FALSE, &error);
+
+		if (error) {
+			g_warning ("Journal read: %s", error->message);
+			g_clear_error (&error);
+			mapped = NULL;
+			return NULL;
+		}
+	}
+
+	lines = g_ptr_array_sized_new (transaction_size > 0 ? transaction_size : 2000);
+
+	cur = g_mapped_file_get_contents (mapped);
+	max_pos = (gsize) (cur + g_mapped_file_get_length (mapped));
+
+	while (((gsize)cur) < max_pos) {
+		next_len = strnlen (cur, max_pos - ((gsize)cur)) + 1;
+		g_ptr_array_add (lines, cur);
+		cur += next_len;
+	}
+
+	return (TrackerJournalContents *) lines;
+}
+
+void 
+tracker_db_journal_free_contents (TrackerJournalContents *contents)
+{
+	if (mapped) {
+		g_mapped_file_free (mapped);
+		mapped = NULL;
+	}
 
-	return lines;
+	g_ptr_array_free ((GPtrArray *)contents, TRUE);
 }
diff --git a/src/libtracker-db/tracker-db-journal.h b/src/libtracker-db/tracker-db-journal.h
index 53fa242..3ff6c4c 100644
--- a/src/libtracker-db/tracker-db-journal.h
+++ b/src/libtracker-db/tracker-db-journal.h
@@ -26,15 +26,17 @@
 #include <glib.h>
 #include <gio/gio.h>
 
-
 G_BEGIN_DECLS
 
+typedef GPtrArray TrackerJournalContents;
+
 const gchar* tracker_db_journal_filename (void);
 void tracker_db_journal_open (void);
 void tracker_db_journal_log (const gchar *query);
 void tracker_db_journal_truncate (void);
 void tracker_db_journal_close (void);
-gchar * tracker_db_journal_get_content (void);
+TrackerJournalContents* tracker_db_journal_get_contents (guint transaction_size);
+void tracker_db_journal_free_contents (TrackerJournalContents *contents);
 void tracker_db_journal_fsync (void);
 
 G_END_DECLS
diff --git a/src/tracker-store/tracker-store.c b/src/tracker-store/tracker-store.c
index e99c815..00cc05f 100644
--- a/src/tracker-store/tracker-store.c
+++ b/src/tracker-store/tracker-store.c
@@ -39,6 +39,7 @@
 #include "tracker-store.h"
 
 #define TRACKER_STORE_TRANSACTION_MAX			4000		/* At commit is journal fsynced too */
+#define TRACKER_STORE_TRANSACTION_MAX			4000		/* At commit is journal fsynced too */
 #define TRACKER_STORE_JOURNAL_TIMEOUT_BEFORE_BACKUP	(60 * 60 * 2)	/* Two hours before backup */
 
 typedef struct {
@@ -314,32 +315,42 @@ tracker_store_flush_journal (void)
 	private->journal_log_count = 0;
 }
 
+static void
+interal_play_journal (TrackerStorePrivate *private)
+{
+	TrackerJournalContents *lines;
+
+	lines = tracker_db_journal_get_contents (TRACKER_STORE_TRANSACTION_MAX);
+
+	if (lines) {
+		guint i;
+		private->start_log = FALSE;
+		for (i = 0; i < lines->len; i++) {
+			const gchar *line = g_ptr_array_index (lines, i);
+			tracker_store_sparql_update (line, NULL);
+		}
+		tracker_db_journal_truncate ();
+		private->start_log = TRUE;
+
+		tracker_db_journal_free_contents (lines);
+	}
+}
 
 void
 tracker_store_play_journal (void)
 {
 	TrackerStorePrivate *private;
-	gchar *lines;
 
 	private = g_static_private_get (&private_key);
 	g_return_if_fail (private != NULL);
 
-	lines = tracker_db_journal_get_content ();
-
-	if (lines) {
-		private->start_log = FALSE;
-		tracker_store_sparql_update (lines, NULL);
-		tracker_db_journal_truncate ();
-		private->start_log = TRUE;
-		g_free (lines);
-	}
+	interal_play_journal (private);
 }
 
 void
 tracker_store_init (gboolean load_journal)
 {
 	TrackerStorePrivate *private;
-	gchar *lines = NULL;
 
 	private = g_new0 (TrackerStorePrivate, 1);
 
@@ -351,17 +362,11 @@ tracker_store_init (gboolean load_journal)
 	                      private_free);
 
 	if (load_journal) {
-		lines = tracker_db_journal_get_content ();
+		interal_play_journal (private);
 	}
 
 	tracker_db_journal_open ();
 
-	if (lines) {
-		tracker_store_sparql_update (lines, NULL);
-		tracker_db_journal_truncate ();
-		g_free (lines);
-	}
-
 	private->start_log = TRUE;
 
 	private->sync_handler = g_timeout_add_seconds_full (G_PRIORITY_LOW,