[tracker/binary-log] libtracker-db: Draft journal reading support

[Jürg Billeter <juergbi src gnome org>]

commit 1af2c25ec52e1d7172ac9ad7c924a8ed6b774174
Author: Jürg Billeter <j bitron ch>
Date:   Thu Dec 31 13:21:59 2009 +0100

    libtracker-db: Draft journal reading support

 src/libtracker-db/tracker-db-journal.c |  204 ++++++++++++++++++++++++++++++--
 src/libtracker-db/tracker-db-journal.h |   17 ++-
 2 files changed, 204 insertions(+), 17 deletions(-)
---
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index 0cba52f..37fb380 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -34,12 +34,18 @@
 #include "tracker-db-journal.h"
 
 static struct {
+	GMappedFile *file;
+	const gchar *current;
+	const gchar *end;
+	const gchar *entry_begin;
+	const gchar *entry_end;
+	guint32 amount_of_triples;
 	TrackerDBJournalEntryType type;
-	gchar *uri;
+	const gchar *uri;
 	guint32 s_code;
 	guint32 p_code;
 	guint32 o_code;
-	gchar *object;
+	const gchar *object;
 } journal_reader;
 
 static gchar *filename = NULL;
@@ -346,9 +352,185 @@ tracker_db_journal_close (void)
 	filename = NULL;
 }
 
+void
+tracker_db_journal_reader_init (const gchar *filen)
+{
+	if (!filen) {
+		get_filename ();
+	} else {
+		filename = g_strdup (filen);
+	}
+
+	/* TODO error handling */
+	journal_reader.file = g_mapped_file_new (filename, FALSE, NULL);
+	journal_reader.current = g_mapped_file_get_contents (journal_reader.file);
+	journal_reader.end = journal_reader.current + g_mapped_file_get_length (journal_reader.file);
+
+	/* verify journal file header */
+	g_assert (journal_reader.end - journal_reader.current >= 8);
+	g_assert (memcmp (journal_reader.current, "trlog\001", 8) == 0);
+	journal_reader.current += 8;
+}
+
+static guint32 read_uint32 (const gchar *data)
+{
+	return data[0] << 24 |
+	       data[1] << 16 |
+	       data[2] << 8 |
+	       data[3];
+}
+
 gboolean
 tracker_db_journal_next (void)
 {
+	if (journal_reader.type == TRACKER_DB_JOURNAL_START ||
+	    journal_reader.type == TRACKER_DB_JOURNAL_END_TRANSACTION) {
+		/* expect new transaction or end of file */
+
+		guint32 entry_size;
+		guint32 crc32;
+
+		if (journal_reader.current >= journal_reader.end) {
+			/* end of journal reached */
+			return FALSE;
+		}
+
+		if (journal_reader.end - journal_reader.current < sizeof (guint32)) {
+			/* damaged journal entry */
+			return FALSE;
+		}
+
+		journal_reader.entry_begin = journal_reader.current;
+		entry_size = read_uint32 (journal_reader.current);
+		journal_reader.entry_end = journal_reader.entry_begin + entry_size;
+		if (journal_reader.end < journal_reader.entry_end) {
+			/* damaged journal entry */
+			return FALSE;
+		}
+		journal_reader.current += 4;
+
+		/* compare with entry_size at end */
+		if (entry_size != read_uint32 (journal_reader.entry_end - 4)) {
+			/* damaged journal entry */
+			return FALSE;
+		}
+
+		journal_reader.amount_of_triples = read_uint32 (journal_reader.current);
+		journal_reader.current += 4;
+
+		crc32 = read_uint32 (journal_reader.current);
+		journal_reader.current += 4;
+
+		/* verify checksum */
+		if (crc32 != tracker_crc32 (journal_reader.entry_begin, entry_size)) {
+			/* damaged journal entry */
+			return FALSE;
+		}
+
+		journal_reader.type = TRACKER_DB_JOURNAL_START_TRANSACTION;
+		return TRUE;
+	} else if (journal_reader.amount_of_triples == 0) {
+		/* end of transaction */
+
+		if (journal_reader.current + 4 != journal_reader.entry_end) {
+			/* damaged journal entry */
+			return FALSE;
+		}
+
+		journal_reader.type = TRACKER_DB_JOURNAL_END_TRANSACTION;
+		return TRUE;
+	} else {
+		guint32 data_format;
+		gsize str_length;
+
+		if (journal_reader.end - journal_reader.current < sizeof (guint32)) {
+			/* damaged journal entry */
+			return FALSE;
+		}
+
+		data_format = read_uint32 (journal_reader.current);
+		journal_reader.current += 4;
+
+		if (data_format == 1) {
+			journal_reader.type = TRACKER_DB_JOURNAL_RESOURCE;
+
+			if (journal_reader.end - journal_reader.current < sizeof (guint32) + 1) {
+				/* damaged journal entry */
+				return FALSE;
+			}
+
+			journal_reader.s_code = read_uint32 (journal_reader.current);
+			journal_reader.current += 4;
+
+			str_length = strnlen (journal_reader.current, journal_reader.end - journal_reader.current);
+			if (str_length == journal_reader.end - journal_reader.current) {
+				/* damaged journal entry (no terminating '\0' character) */
+				return FALSE;
+			}
+			if (!g_utf8_validate (journal_reader.current, -1, NULL)) {
+				/* damaged journal entry (invalid UTF-8) */
+				return FALSE;
+			}
+			journal_reader.uri = journal_reader.current;
+			journal_reader.current += str_length + 1;
+		} else {
+			if (data_format & 4) {
+				if (data_format & 2) {
+					journal_reader.type = TRACKER_DB_JOURNAL_DELETE_STATEMENT_CODE;
+				} else {
+					journal_reader.type = TRACKER_DB_JOURNAL_DELETE_STATEMENT;
+				}
+			} else {
+				if (data_format & 2) {
+					journal_reader.type = TRACKER_DB_JOURNAL_INSERT_STATEMENT_CODE;
+				} else {
+					journal_reader.type = TRACKER_DB_JOURNAL_INSERT_STATEMENT;
+				}
+			}
+
+			if (journal_reader.end - journal_reader.current < 2 * sizeof (guint32)) {
+				/* damaged journal entry */
+				return FALSE;
+			}
+
+			journal_reader.s_code = read_uint32 (journal_reader.current);
+			journal_reader.current += 4;
+
+			journal_reader.p_code = read_uint32 (journal_reader.current);
+			journal_reader.current += 4;
+
+			if (data_format & 2) {
+				if (journal_reader.end - journal_reader.current < sizeof (guint32)) {
+					/* damaged journal entry */
+					return FALSE;
+				}
+
+				journal_reader.o_code = read_uint32 (journal_reader.current);
+				journal_reader.current += 4;
+			} else {
+				if (journal_reader.end - journal_reader.current < 1) {
+					/* damaged journal entry */
+					return FALSE;
+				}
+
+				str_length = strnlen (journal_reader.current, journal_reader.end - journal_reader.current);
+				if (str_length == journal_reader.end - journal_reader.current) {
+					/* damaged journal entry (no terminating '\0' character) */
+					return FALSE;
+				}
+				if (!g_utf8_validate (journal_reader.current, -1, NULL)) {
+					/* damaged journal entry (invalid UTF-8) */
+					return FALSE;
+				}
+				journal_reader.object = journal_reader.current;
+				journal_reader.current += str_length + 1;
+			}
+		}
+
+		journal_reader.amount_of_triples--;
+		return TRUE;
+	}
+
 	return FALSE;
 }
 
@@ -358,25 +540,27 @@ tracker_db_journal_get_type (void)
 	return journal_reader.type;
 }
 
-gchar *
-tracker_db_journal_get_resource (void)
+void
+tracker_db_journal_get_resource (guint32      *code,
+                                 const gchar **uri)
 {
-	g_return_val_if_fail (journal_reader.type == TRACKER_DB_JOURNAL_RESOURCE, NULL);
+	g_return_if_fail (journal_reader.type == TRACKER_DB_JOURNAL_RESOURCE);
 
-	return journal_reader.uri;
+	*code = journal_reader.s_code;
+	*uri = journal_reader.uri;
 }
 
 void
-tracker_db_journal_get_statement (guint32    *s_code,
-                                  guint32    *p_code,
-                                  gchar     **object)
+tracker_db_journal_get_statement (guint32      *s_code,
+                                  guint32      *p_code,
+                                  const gchar **object)
 {
 	g_return_if_fail (journal_reader.type == TRACKER_DB_JOURNAL_INSERT_STATEMENT ||
 	                  journal_reader.type == TRACKER_DB_JOURNAL_DELETE_STATEMENT);
 
 	*s_code = journal_reader.s_code;
 	*p_code = journal_reader.p_code;
-	*object = g_strdup (journal_reader.object);
+	*object = journal_reader.object;
 }
 
 void
diff --git a/src/libtracker-db/tracker-db-journal.h b/src/libtracker-db/tracker-db-journal.h
index b05b7c2..2bd946e 100644
--- a/src/libtracker-db/tracker-db-journal.h
+++ b/src/libtracker-db/tracker-db-journal.h
@@ -28,6 +28,7 @@
 G_BEGIN_DECLS
 
 typedef enum {
+	TRACKER_DB_JOURNAL_START,
 	TRACKER_DB_JOURNAL_START_TRANSACTION,
 	TRACKER_DB_JOURNAL_END_TRANSACTION,
 	TRACKER_DB_JOURNAL_RESOURCE,
@@ -63,16 +64,18 @@ void         tracker_db_journal_close                        (void);
 void         tracker_db_journal_fsync                        (void);
 gsize        tracker_db_journal_get_size                     (void);
 
+void         tracker_db_journal_reader_init                  (const gchar  *filen);
 gboolean     tracker_db_journal_next                         (void);
 TrackerDBJournalEntryType
              tracker_db_journal_get_type                     (void);
-gchar *      tracker_db_journal_get_resource                 (void);
-void         tracker_db_journal_get_statement                (guint32    *s_code,
-                                                              guint32    *p_code,
-                                                              gchar     **object);
-void         tracker_db_journal_get_statement_code           (guint32    *s_code,
-                                                              guint32    *p_code,
-                                                              guint32    *o_code);
+void         tracker_db_journal_get_resource                 (guint32      *code,
+                                                              const gchar **uri);
+void         tracker_db_journal_get_statement                (guint32      *s_code,
+                                                              guint32      *p_code,
+                                                              const gchar **object);
+void         tracker_db_journal_get_statement_code           (guint32      *s_code,
+                                                              guint32      *p_code,
+                                                              guint32      *o_code);
 
 G_END_DECLS