[tracker/urho-sync] Make fsync happen less often on the journal

[Philip Van Hoof <pvanhoof src gnome org>]

commit d5ab45435cc1a2e9b56fbafb04a82f150c843e51
Author: Philip Van Hoof <philip codeminded be>
Date:   Fri Aug 21 10:58:04 2009 +0200

    Make fsync happen less often on the journal

 src/libtracker-db/tracker-db-journal.c |    7 +++++
 src/libtracker-db/tracker-db-journal.h |    1 +
 src/tracker-store/tracker-resources.c  |    4 +++
 src/tracker-store/tracker-store.c      |   45 +++++++++++++++++++++++++++----
 src/tracker-store/tracker-store.h      |    1 +
 5 files changed, 52 insertions(+), 6 deletions(-)
---
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index 1c7f79a..07c3c39 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -65,6 +65,13 @@ tracker_db_journal_log (const gchar *query)
 	if (journal) {
 		write (fileno (journal), query, strlen (query));
 		write (fileno (journal), "\n", 1);
+	}
+}
+
+void 
+tracker_db_journal_fsync (void)
+{
+	if (journal) {
 		fsync (fileno (journal));
 	}
 }
diff --git a/src/libtracker-db/tracker-db-journal.h b/src/libtracker-db/tracker-db-journal.h
index 6fe017b..53fa242 100644
--- a/src/libtracker-db/tracker-db-journal.h
+++ b/src/libtracker-db/tracker-db-journal.h
@@ -35,6 +35,7 @@ void tracker_db_journal_log (const gchar *query);
 void tracker_db_journal_truncate (void);
 void tracker_db_journal_close (void);
 gchar * tracker_db_journal_get_content (void);
+void tracker_db_journal_fsync (void);
 
 G_END_DECLS
 
diff --git a/src/tracker-store/tracker-resources.c b/src/tracker-store/tracker-resources.c
index 52bb1ed..ebe985d 100644
--- a/src/tracker-store/tracker-resources.c
+++ b/src/tracker-store/tracker-resources.c
@@ -399,6 +399,10 @@ on_statements_committed (gpointer user_data)
 
 	priv = TRACKER_RESOURCES_GET_PRIVATE (user_data);
 
+	/* For more information about this call, look at the function end_batch
+	 * of tracker-store.c */
+	tracker_store_flush_journal ();
+
 	events = tracker_events_get_pending ();
 
 	if (events) {
diff --git a/src/tracker-store/tracker-store.c b/src/tracker-store/tracker-store.c
index c1b234f..236c2e2 100644
--- a/src/tracker-store/tracker-store.c
+++ b/src/tracker-store/tracker-store.c
@@ -38,12 +38,14 @@
 
 #include "tracker-store.h"
 
-#define TRACKER_STORE_TRANSACTION_MAX	4000
+#define TRACKER_STORE_TRANSACTION_MAX	4000		/* At commit is journal fsynced too */
+#define TRACKER_STORE_JOURNAL_MAX	2000		/* Amount of queries before fsync */
+#define TRACKER_STORE_JOURNAL_TIMEOUT	(60 * 60 * 2)	/* Two hours before fsync */
 
 typedef struct {
 	gboolean  have_handler, have_sync_handler;
 	gboolean  batch_mode, start_log;
-	guint     batch_count;
+	guint     batch_count, journal_log_count;
 	GQueue   *queue;
 	guint     handler, sync_handler;
 } TrackerStorePrivate;
@@ -74,6 +76,8 @@ typedef struct {
 
 static GStaticPrivate private_key = G_STATIC_PRIVATE_INIT;
 
+static void on_backup_done (GError *error, gpointer user_data);
+
 static void
 private_free (gpointer data)
 {
@@ -152,11 +156,29 @@ end_batch (TrackerStorePrivate *private)
 	if (private->batch_mode) {
 		/* commit pending batch items */
 		tracker_data_commit_transaction ();
+
+		/* The on_statements_committed in tracker-resources.c performs
+		 * the flush on the journal, I can only register one callback 
+		 * for this atm, so that's why it's called over there as a 
+		 * tracker_store_flush_journal */
+
 		private->batch_mode = FALSE;
 		private->batch_count = 0;
 	}
 }
 
+static void
+log_to_journal (TrackerStorePrivate *private, const gchar *query)
+{
+	tracker_db_journal_log (query);
+	private->journal_log_count++;
+
+	if (private->journal_log_count > TRACKER_STORE_JOURNAL_MAX) {
+		tracker_db_backup_save (on_backup_done, private, NULL);
+		private->journal_log_count = 0;
+	}
+}
+
 static gboolean
 queue_idle_handler (gpointer user_data)
 {
@@ -174,7 +196,7 @@ queue_idle_handler (gpointer user_data)
 		tracker_data_update_sparql (task->data.query, &error);
 
 		if (private->start_log) {
-			tracker_db_journal_log (task->data.query);
+			log_to_journal (private, task->data.query);
 		}
 
 		if (!error) {
@@ -251,14 +273,18 @@ static void
 on_backup_done (GError *error, gpointer user_data)
 {
 	if (!error) {
+		TrackerStorePrivate *private = user_data;
+
 		tracker_db_journal_truncate ();
+
+		private->journal_log_count = 0;
 	}
 }
 
 static gboolean
 sync_idle_handler (gpointer user_data)
 {
-	tracker_db_backup_save (on_backup_done, NULL, NULL);
+	tracker_db_backup_save (on_backup_done, user_data, NULL);
 
 	return TRUE;
 }
@@ -282,6 +308,12 @@ sync_idle_destroy (gpointer user_data)
 	private->have_sync_handler = FALSE;
 }
 
+void 
+tracker_store_flush_journal (void)
+{
+	tracker_db_journal_fsync ();
+}
+
 void
 tracker_store_play_journal (void)
 {
@@ -311,6 +343,7 @@ tracker_store_init (gboolean load_journal)
 	private = g_new0 (TrackerStorePrivate, 1);
 
 	private->queue = g_queue_new ();
+	private->journal_log_count = 0;
 
 	g_static_private_set (&private_key,
 	                      private,
@@ -331,7 +364,7 @@ tracker_store_init (gboolean load_journal)
 	private->start_log = TRUE;
 
 	private->sync_handler = g_timeout_add_seconds_full (G_PRIORITY_LOW,
-	                                                    60 * 30, /* 30 minutes worth of seconds*/
+	                                                    TRACKER_STORE_JOURNAL_TIMEOUT,
 	                                                    sync_idle_handler,
 	                                                    private,
 	                                                    sync_idle_destroy);
@@ -471,7 +504,7 @@ tracker_store_sparql_update (const gchar *sparql,
 	tracker_data_update_sparql (sparql, error);
 
 	if (private->start_log) {
-		tracker_db_journal_log (sparql);
+		log_to_journal (private, sparql);
 	}
 
 }
diff --git a/src/tracker-store/tracker-store.h b/src/tracker-store/tracker-store.h
index 5affc21..a87320a 100644
--- a/src/tracker-store/tracker-store.h
+++ b/src/tracker-store/tracker-store.h
@@ -64,6 +64,7 @@ TrackerDBResultSet*
 guint        tracker_store_get_queue_size         (void);
 
 void         tracker_store_play_journal           (void);
+void         tracker_store_flush_journal          (void);
 
 G_END_DECLS