[tracker/sqlite-batch-locking] libtracker-db: Add experimental SQLite VFS wrapper

[Jürg Billeter <juergbi src gnome org>]

commit 8173a3c516c405e56cf1f78c558301d427b102c1
Author: Jürg Billeter <j bitron ch>
Date:   Tue May 11 12:15:19 2010 +0200

    libtracker-db: Add experimental SQLite VFS wrapper
    
    The SQLite VFS wrapper tracks database locking to dynamically adjust
    batch transaction size. This only works without shared cache as
    table-level locking does not use locks on the VFS level.

 src/libtracker-db/tracker-db-interface-sqlite.c |   66 ++++++++++++++++++++++-
 src/libtracker-db/tracker-db-interface-sqlite.h |    3 +
 src/libtracker-db/tracker-db-manager.c          |    6 --
 src/tracker-store/tracker-store.c               |   16 ++----
 4 files changed, 74 insertions(+), 17 deletions(-)
---
diff --git a/src/libtracker-db/tracker-db-interface-sqlite.c b/src/libtracker-db/tracker-db-interface-sqlite.c
index c290d16..d209923 100644
--- a/src/libtracker-db/tracker-db-interface-sqlite.c
+++ b/src/libtracker-db/tracker-db-interface-sqlite.c
@@ -137,11 +137,75 @@ G_DEFINE_TYPE_WITH_CODE (TrackerDBCursorSqlite, tracker_db_cursor_sqlite, G_TYPE
                          G_IMPLEMENT_INTERFACE (TRACKER_TYPE_DB_CURSOR,
                                                 tracker_db_cursor_sqlite_iface_init))
 
+/* from GLib */
+#define STRUCT_ALIGNMENT (2 * sizeof (gsize))
+#define ALIGN_STRUCT(offset) \
+      ((offset + (STRUCT_ALIGNMENT - 1)) & -STRUCT_ALIGNMENT)
+
+static sqlite3_vfs *default_vfs = NULL;
+
+#define DEFAULT_METHODS(file) (*((const sqlite3_io_methods **) ((gchar *) file + ALIGN_STRUCT(default_vfs->szOsFile))))
+
+static gboolean db_locked = FALSE;
+
+static int
+my_lock (sqlite3_file* file, int locktype)
+{
+	gint rc;
+	if (locktype >= SQLITE_LOCK_PENDING) {
+		db_locked = TRUE;
+	}
+	rc = DEFAULT_METHODS(file)->xLock (file, locktype);
+	return rc;
+}
+
+static int
+my_open (sqlite3_vfs *vfs, const char *zName, sqlite3_file* file, int flags, int *pOutFlags)
+{
+	int rc = default_vfs->xOpen (vfs, zName, file, flags, pOutFlags);
+	if ((flags & SQLITE_OPEN_MAIN_DB) && file->pMethods) {
+		sqlite3_io_methods *my_methods = g_memdup (file->pMethods, sizeof (sqlite3_io_methods));
+		my_methods->xLock = my_lock;
+
+		DEFAULT_METHODS(file) = file->pMethods;
+		file->pMethods = my_methods;
+	}
+	return rc;
+}
+
+gboolean
+tracker_db_manager_pending_lock (void)
+{
+	if (db_locked) {
+		db_locked = FALSE;
+		return TRUE;
+	} else {
+		return FALSE;
+	}
+}
+
+static gboolean initialized = FALSE;
+
 void
 tracker_db_interface_sqlite_enable_shared_cache (void)
 {
+	sqlite3_vfs *wrapper;
+
+	if (initialized) {
+		return;
+	}
+
+	initialized = TRUE;
+
+	default_vfs = sqlite3_vfs_find (NULL);
+	wrapper = g_memdup (default_vfs, sizeof (sqlite3_vfs));
+	wrapper->szOsFile = ALIGN_STRUCT(default_vfs->szOsFile) + sizeof (gpointer);
+	wrapper->xOpen = my_open;
+
+	sqlite3_vfs_register (wrapper, TRUE);
+
 	sqlite3_config (SQLITE_CONFIG_MULTITHREAD);
-	sqlite3_enable_shared_cache (1);
+	sqlite3_enable_shared_cache (0);
 }
 
 static void
diff --git a/src/libtracker-db/tracker-db-interface-sqlite.h b/src/libtracker-db/tracker-db-interface-sqlite.h
index 042d62c..aaa74e8 100644
--- a/src/libtracker-db/tracker-db-interface-sqlite.h
+++ b/src/libtracker-db/tracker-db-interface-sqlite.h
@@ -65,6 +65,9 @@ void                tracker_db_interface_sqlite_enable_shared_cache    (void);
 void                tracker_db_interface_sqlite_fts_init               (TrackerDBInterfaceSqlite *interface,
                                                                         gboolean                  create);
 
+gboolean
+tracker_db_manager_pending_lock (void);
+
 G_END_DECLS
 
 #endif /* __LIBTRACKER_DB_INTERFACE_SQLITE_H__ */
diff --git a/src/libtracker-db/tracker-db-manager.c b/src/libtracker-db/tracker-db-manager.c
index 66efc03..fb3ea94 100644
--- a/src/libtracker-db/tracker-db-manager.c
+++ b/src/libtracker-db/tracker-db-manager.c
@@ -267,12 +267,6 @@ db_set_params (TrackerDBInterface *iface,
 	tracker_db_interface_execute_query (iface, NULL, "PRAGMA encoding = \"UTF-8\"");
 	tracker_db_interface_execute_query (iface, NULL, "PRAGMA auto_vacuum = 0;");
 
-	/* if we add direct access library (or enable running queries in parallel with updates,
-	   we need to disable read_uncommitted again
-	   however, when read_uncommitted is disabled, we need the custom page cache (wrapper)
-	   to avoid locking issues between long running batch transactions and queries */
-	tracker_db_interface_execute_query (iface, NULL, "PRAGMA read_uncommitted = True;");
-
 	if (page_size != TRACKER_DB_PAGE_SIZE_DONT_SET) {
 		g_message ("  Setting page size to %d", page_size);
 		tracker_db_interface_execute_query (iface, NULL, "PRAGMA page_size = %d", page_size);
diff --git a/src/tracker-store/tracker-store.c b/src/tracker-store/tracker-store.c
index bb55c55..5ec04c7 100644
--- a/src/tracker-store/tracker-store.c
+++ b/src/tracker-store/tracker-store.c
@@ -35,7 +35,6 @@
 
 #include "tracker-store.h"
 
-#define TRACKER_STORE_TRANSACTION_MAX                   4000
 #define TRACKER_STORE_MAX_CONCURRENT_QUERIES               2
 
 #define TRACKER_STORE_QUERY_WATCHDOG_TIMEOUT 10
@@ -44,7 +43,6 @@
 typedef struct {
 	gboolean     have_handler, have_sync_handler;
 	gboolean     batch_mode, start_log;
-	guint        batch_count;
 	GQueue      *queues[TRACKER_STORE_N_PRIORITIES];
 	guint        handler, sync_handler;
 	guint        n_queries_running;
@@ -209,7 +207,6 @@ begin_batch (TrackerStorePrivate *private)
 		   delays database commits to improve performance */
 		tracker_data_begin_db_transaction ();
 		private->batch_mode = TRUE;
-		private->batch_count = 0;
 	}
 }
 
@@ -221,8 +218,10 @@ end_batch (TrackerStorePrivate *private)
 		tracker_data_commit_db_transaction ();
 		tracker_data_notify_db_transaction ();
 
+		/* the above commit will trigger a lock again, reset it */
+		tracker_db_manager_pending_lock ();
+
 		private->batch_mode = FALSE;
-		private->batch_count = 0;
 	}
 }
 
@@ -447,8 +446,7 @@ pool_dispatch_cb (gpointer data,
 
 		if (task->data.update.batch) {
 			if (!task->error) {
-				private->batch_count++;
-				if (private->batch_count >= TRACKER_STORE_TRANSACTION_MAX) {
+				if (tracker_db_manager_pending_lock ()) {
 					end_batch (private);
 				}
 			}
@@ -467,8 +465,7 @@ pool_dispatch_cb (gpointer data,
 
 		if (task->data.update.batch) {
 			if (!task->error) {
-				private->batch_count++;
-				if (private->batch_count >= TRACKER_STORE_TRANSACTION_MAX) {
+				if (tracker_db_manager_pending_lock ()) {
 					end_batch (private);
 				}
 			}
@@ -560,8 +557,7 @@ queue_idle_handler (gpointer user_data)
 
 		if (process_turtle_file_part (turtle_reader, &error)) {
 			/* import still in progress */
-			private->batch_count++;
-			if (private->batch_count >= TRACKER_STORE_TRANSACTION_MAX) {
+			if (tracker_db_manager_pending_lock ()) {
 				end_batch (private);
 			}