[tracker/binary-log-2: 7/63] libtracker-db: Implemented binary log format

[Jürg Billeter <juergbi src gnome org>]

commit 94efdf622776c556b0543f1a5b2d24506ee255fb
Author: Philip Van Hoof <philip codeminded be>
Date:   Tue Dec 29 17:02:20 2009 +0100

    libtracker-db: Implemented binary log format

 src/libtracker-db/tracker-db-journal.c |  244 +++++++++++++++++++++++++++++---
 1 files changed, 224 insertions(+), 20 deletions(-)
---
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index 86100b6..b2cad9e 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -34,8 +34,14 @@
 static gchar *filename = NULL;
 static FILE *journal = NULL;
 static gsize current_size = 0;
+static guint cur_block_len = 0;
+static guint cur_block_alloc = 0;
+static gchar *cur_block = NULL;
+static guint cur_entry_amount = 0;
+static guint cur_pos = 0;
 
-#define TRACKER_DB_JOURNAL_LOG_FILENAME                 "log.sparql.txt"
+#define TRACKER_DB_JOURNAL_LOG_FILENAME  "tracker-store.journal"
+#define MIN_BLOCK_SIZE                   1024
 
 static void
 get_filename (void)
@@ -62,6 +68,40 @@ tracker_db_journal_filename (void)
 	return (const gchar *) filename;
 }
 
+static void
+kill_cur_block (void)
+{
+	cur_block_len = 0;
+	cur_pos = 0;
+	cur_entry_amount = 0;
+	cur_block_alloc = 0;
+	g_free (cur_block);
+	cur_block = NULL;
+}
+
+
+static gint
+nearest_pow (gint num)
+{
+	gint n = 1;
+	while (n < num)
+		n <<= 1;
+	return n;
+}
+
+static void
+cur_block_maybe_expand (guint len)
+{
+	guint want_alloc = cur_block_len + len;
+
+	if (want_alloc > cur_block_alloc) {
+		want_alloc = nearest_pow (want_alloc);
+		want_alloc = MAX (want_alloc, MIN_BLOCK_SIZE);
+		cur_block = g_realloc (cur_block, want_alloc);
+		cur_block_alloc = want_alloc;
+	}
+}
+
 void
 tracker_db_journal_open (void)
 {
@@ -74,23 +114,60 @@ tracker_db_journal_open (void)
 	if (stat (filename, &st) == 0) {
 		current_size = (gsize) st.st_size;
 	}
-}
 
+	if (current_size == 0) {
+		g_assert (cur_block_len == 0);
+		g_assert (cur_block_alloc == 0);
+		g_assert (cur_block == NULL);
+		g_assert (cur_block == NULL);
+
+		cur_block_maybe_expand (8);
+
+		cur_block[0] = 't';
+		cur_block[1] = 'r';
+		cur_block[2] = 'l';
+		cur_block[3] = 'o';
+		cur_block[4] = 'g';
+		cur_block[5] = '\0';
+		cur_block[6] = '0';
+		cur_block[7] = '1';
+
+		write (fileno (journal), cur_block, 8);
 
-//void
-//tracker_db_journal_log (const gchar *query)
-//{
-//	if (journal) {
-//		size_t len = strlen (query);
-//		write (fileno (journal), query, len);
-//		write (fileno (journal), "\n\0", 2);
-//		current_size += (len + 2);
-//	}
-//}
+		current_size += 8;
+
+		kill_cur_block ();
+	}
+}
 
 void
 tracker_db_journal_start_transaction (void)
 {
+	guint size = sizeof (guint32) * 3;
+
+	cur_block_maybe_expand (size);
+
+	/* Leave space for size, amount and crc */
+
+	cur_pos = cur_block_len = size;
+	cur_entry_amount = 0;
+}
+
+static void
+cur_setnum (gchar *dest, guint *pos, guint32 val)
+{
+	memset (dest + (*pos)++, val >> 24 & 0xff, 1);
+	memset (dest + (*pos)++, val >> 16 & 0xff, 1);
+	memset (dest + (*pos)++, val >>  8 & 0xff, 1);
+	memset (dest + (*pos)++, val >>  0 & 0xff, 1);
+}
+
+static void
+cur_setstr (gchar *dest, guint *pos, const gchar *str, gsize len)
+{
+	memcpy (dest + *pos, str, len);
+	(*pos) += len;
+	memset (dest + (*pos)++, 0 & 0xff, 1);
 }
 
 void
@@ -98,6 +175,19 @@ tracker_db_journal_append_delete_statement (guint32      s_code,
                                             guint32      p_code,
                                             const gchar *object)
 {
+	gint o_len = strlen (object);
+	gchar data_format = 0x04;
+	gint size = (sizeof (guint32) * 3) + o_len + 1;
+
+	cur_block_maybe_expand (size);
+
+	cur_setnum (cur_block, &cur_pos, data_format);
+	cur_setnum (cur_block, &cur_pos, s_code);
+	cur_setnum (cur_block, &cur_pos, p_code);
+	cur_setstr (cur_block, &cur_pos, object, o_len);
+
+	cur_entry_amount++;
+	cur_block_len += size;
 }
 
 
@@ -106,6 +196,18 @@ tracker_db_journal_append_delete_statement_code (guint32 s_code,
                                                  guint32 p_code,
                                                  guint32 o_code)
 {
+	gchar data_format = 0x06;
+	gint size = sizeof (guint32) * 4;
+
+	cur_block_maybe_expand (size);
+
+	cur_setnum (cur_block, &cur_pos, data_format);
+	cur_setnum (cur_block, &cur_pos, s_code);
+	cur_setnum (cur_block, &cur_pos, p_code);
+	cur_setnum (cur_block, &cur_pos, o_code);
+
+	cur_entry_amount++;
+	cur_block_len += size;
 }
 
 void
@@ -113,6 +215,19 @@ tracker_db_journal_append_insert_statement (guint32      s_code,
                                             guint32      p_code, 
                                             const gchar *object)
 {
+	gint o_len = strlen (object);
+	gchar data_format = 0x00;
+	gint size = (sizeof (guint32) * 3) + o_len + 1;
+
+	cur_block_maybe_expand (size);
+
+	cur_setnum (cur_block, &cur_pos, data_format);
+	cur_setnum (cur_block, &cur_pos, s_code);
+	cur_setnum (cur_block, &cur_pos, p_code);
+	cur_setstr (cur_block, &cur_pos, object, o_len);
+
+	cur_entry_amount++;
+	cur_block_len += size;
 }
 
 void
@@ -120,41 +235,130 @@ tracker_db_journal_append_insert_statement_code (guint32 s_code,
                                                  guint32 p_code,
                                                  guint32 o_code)
 {
+	gchar data_format = 0x02;
+	gint size = sizeof (guint32) * 4;
+
+	cur_block_maybe_expand (size);
+
+	cur_setnum (cur_block, &cur_pos, data_format);
+	cur_setnum (cur_block, &cur_pos, s_code);
+	cur_setnum (cur_block, &cur_pos, p_code);
+	cur_setnum (cur_block, &cur_pos, o_code);
+
+	cur_entry_amount++;
+	cur_block_len += size;
 }
 
 void
 tracker_db_journal_append_resource (guint32      s_code,
-                                   const gchar *uri)
+                                    const gchar *uri)
 {
+	gint o_len = strlen (uri);
+	gchar data_format = 0x01;
+	gint size = (sizeof (guint32) * 2) + o_len + 1;
+
+	cur_block_maybe_expand (size);
+
+	cur_setnum (cur_block, &cur_pos, data_format);
+	cur_setnum (cur_block, &cur_pos, s_code);
+	cur_setstr (cur_block, &cur_pos, uri, o_len);
+
+	cur_entry_amount++;
+	cur_block_len += size;
 }
 
+
 void
 tracker_db_journal_rollback_transaction (void)
 {
+	kill_cur_block ();
 }
 
+
 void
 tracker_db_journal_commit_transaction (void)
 {
+	guint32 crc = 0xfffffff;
+	guint begin_pos = 0;
+	guint size = sizeof (guint32);
+
+	g_assert (journal);
+
+	/* TODO calculate CRC for entry */
+
+	cur_block_maybe_expand (size);
+
+	cur_setnum (cur_block, &begin_pos, cur_block_len);
+	cur_setnum (cur_block, &begin_pos, cur_entry_amount);
+	cur_setnum (cur_block, &begin_pos, crc);
+
+	cur_setnum (cur_block, &cur_pos, cur_block_len);
+
+	cur_block_len += size;
+
+	write (fileno (journal), cur_block, cur_block_len);
+
+	current_size += cur_block_len;
+
+	kill_cur_block ();
 }
 
 void 
 tracker_db_journal_fsync (void)
 {
-	if (journal) {
-		fsync (fileno (journal));
-	}
+	g_assert (journal);
+
+	fsync (fileno (journal));
 }
 
 
 void
 tracker_db_journal_close (void)
 {
-	if (journal) {
-		fclose (journal);
-		journal = NULL;
-	}
+	g_assert (journal);
+
+	fclose (journal);
+	journal = NULL;
 
 	g_free (filename);
 	filename = NULL;
 }
+
+#ifdef TEST
+int main () {
+	guint i;
+
+	tracker_db_journal_open ();
+
+	tracker_db_journal_start_transaction ();
+	tracker_db_journal_append_resource (10, "http://resource";);
+	tracker_db_journal_append_resource (11, "http://predicate";);
+	tracker_db_journal_append_delete_statement (10, 11, "test");
+	tracker_db_journal_commit_transaction ();
+
+	tracker_db_journal_start_transaction ();
+	tracker_db_journal_append_resource (12, "http://resource";);
+	tracker_db_journal_append_resource (13, "http://predicate";);
+	tracker_db_journal_append_resource (14, "http://resource";);
+	tracker_db_journal_append_delete_statement_code (12, 13, 14);
+	tracker_db_journal_commit_transaction ();
+
+
+	tracker_db_journal_start_transaction ();
+	tracker_db_journal_append_resource (15, "http://resource";);
+	tracker_db_journal_append_resource (16, "http://predicate";);
+	tracker_db_journal_append_insert_statement (15, 16, "test");
+	tracker_db_journal_commit_transaction ();
+
+	tracker_db_journal_start_transaction ();
+	tracker_db_journal_append_resource (17, "http://resource";);
+	tracker_db_journal_append_resource (18, "http://predicate";);
+	tracker_db_journal_append_resource (19, "http://resource";);
+	tracker_db_journal_append_insert_statement_code (17, 18, 19);
+	tracker_db_journal_commit_transaction ();
+
+	tracker_db_journal_fsync ();
+	tracker_db_journal_close ();
+	return 0;
+}
+#endif