[tracker] libtracker-data: Move WAL checkpointing to TrackerDBManager

[Carlos Garnacho <carlosg src gnome org>]

commit 202c73c8d204b2e45923ccdf7889bcff1b8e498c
Author: Carlos Garnacho <carlosg gnome org>
Date:   Sun Sep 30 10:51:20 2018 +0200

    libtracker-data: Move WAL checkpointing to TrackerDBManager
    
    The WAL logging and manual checkpoint handling is defined in the deepmost
    layers, it doesn't make sense to handle WAL checkpoints in the highest ones.
    
    Besides, the GThread for the deferred checkpoint is no longer leaked, and
    we ensure that deferred and blocking checkpoints are mutually exclusive.

 src/libtracker-data/tracker-db-interface-sqlite.c |  7 +-
 src/libtracker-data/tracker-db-interface-sqlite.h |  6 +-
 src/libtracker-data/tracker-db-manager.c          | 81 ++++++++++++++++++++++-
 src/libtracker-direct/tracker-direct.c            | 71 --------------------
 4 files changed, 88 insertions(+), 77 deletions(-)
---
diff --git a/src/libtracker-data/tracker-db-interface-sqlite.c 
b/src/libtracker-data/tracker-db-interface-sqlite.c
index 079be7a1c..e1c249409 100644
--- a/src/libtracker-data/tracker-db-interface-sqlite.c
+++ b/src/libtracker-data/tracker-db-interface-sqlite.c
@@ -95,6 +95,7 @@ struct TrackerDBInterface {
 
        /* Wal */
        TrackerDBWalCallback wal_hook;
+       gpointer wal_hook_data;
 
        /* User data */
        gpointer user_data;
@@ -1868,15 +1869,17 @@ wal_hook (gpointer     user_data,
 {
        TrackerDBInterface *iface = user_data;
 
-       iface->wal_hook (iface, n_pages);
+       iface->wal_hook (iface, n_pages, iface->wal_hook_data);
        return SQLITE_OK;
 }
 
 void
 tracker_db_interface_sqlite_wal_hook (TrackerDBInterface   *interface,
-                                      TrackerDBWalCallback  callback)
+                                      TrackerDBWalCallback  callback,
+                                      gpointer              user_data)
 {
        interface->wal_hook = callback;
+       interface->wal_hook_data = user_data;
        sqlite3_wal_hook (interface->db, wal_hook, interface);
 }
 
diff --git a/src/libtracker-data/tracker-db-interface-sqlite.h 
b/src/libtracker-data/tracker-db-interface-sqlite.h
index 49fab3c17..8ae70fd42 100644
--- a/src/libtracker-data/tracker-db-interface-sqlite.h
+++ b/src/libtracker-data/tracker-db-interface-sqlite.h
@@ -34,7 +34,8 @@ G_BEGIN_DECLS
 #define TRACKER_TITLE_COLLATION_NAME "TRACKER_TITLE"
 
 typedef void (*TrackerDBWalCallback) (TrackerDBInterface *iface,
-                                      gint                n_pages);
+                                      gint                n_pages,
+                                      gpointer            user_data);
 
 typedef enum {
        TRACKER_DB_INTERFACE_READONLY  = 1 << 0,
@@ -52,7 +53,8 @@ void                tracker_db_interface_sqlite_fts_init               (TrackerD
                                                                         gboolean                  create);
 void                tracker_db_interface_sqlite_reset_collator         (TrackerDBInterface       *interface);
 void                tracker_db_interface_sqlite_wal_hook               (TrackerDBInterface       *interface,
-                                                                        TrackerDBWalCallback      callback);
+                                                                        TrackerDBWalCallback      callback,
+                                                                        gpointer                  user_data);
 gboolean            tracker_db_interface_sqlite_wal_checkpoint         (TrackerDBInterface       *interface,
                                                                         gboolean                  blocking,
                                                                         GError                  **error);
diff --git a/src/libtracker-data/tracker-db-manager.c b/src/libtracker-data/tracker-db-manager.c
index 9f30fe0ae..e40273ff3 100644
--- a/src/libtracker-data/tracker-db-manager.c
+++ b/src/libtracker-data/tracker-db-manager.c
@@ -143,6 +143,7 @@ struct _TrackerDBManager {
        GWeakRef iface_data;
 
        GAsyncQueue *interfaces;
+       GThread *wal_thread;
 };
 
 static gboolean            db_exec_no_reply                        (TrackerDBInterface   *iface,
@@ -506,6 +507,7 @@ db_recreate_all (TrackerDBManager  *db_manager,
        }
 
        g_clear_object (&db_manager->db.iface);
+       g_clear_object (&db_manager->db.wal_iface);
 
        locale = tracker_locale_get (TRACKER_LOCALE_COLLATE);
        /* Initialize locale file */
@@ -550,6 +552,7 @@ perform_recreate (TrackerDBManager  *db_manager,
        }
 
        g_clear_object (&db_manager->db.iface);
+       g_clear_object (&db_manager->db.wal_iface);
 
        if (!tracker_file_system_has_enough_space (db_manager->data_dir, TRACKER_DB_MIN_REQUIRED_SPACE, 
TRUE)) {
                g_set_error (error,
@@ -914,15 +917,28 @@ tracker_db_manager_new (TrackerDBManagerFlags   flags,
 void
 tracker_db_manager_free (TrackerDBManager *db_manager)
 {
+       gboolean readonly = (db_manager->flags & TRACKER_DB_MANAGER_READONLY) != 0;
+
        g_async_queue_unref (db_manager->interfaces);
        g_free (db_manager->db.abs_filename);
-       g_clear_object (&db_manager->db.iface);
+
+       if (db_manager->wal_thread)
+               g_thread_join (db_manager->wal_thread);
+
+       g_clear_object (&db_manager->db.wal_iface);
+
+       if (db_manager->db.iface) {
+               if (!readonly)
+                       tracker_db_interface_sqlite_wal_checkpoint (db_manager->db.iface, TRUE, NULL);
+               g_object_unref (db_manager->db.iface);
+       }
+
        g_weak_ref_clear (&db_manager->iface_data);
 
        g_free (db_manager->data_dir);
        g_free (db_manager->user_data_dir);
 
-       if ((db_manager->flags & TRACKER_DB_MANAGER_READONLY) == 0) {
+       if (!readonly) {
                /* do not delete in-use file for read-only mode (direct access) */
                g_unlink (db_manager->in_use_filename);
        }
@@ -1077,6 +1093,61 @@ tracker_db_manager_get_db_interface (TrackerDBManager *db_manager)
        return interface;
 }
 
+static void
+wal_checkpoint (TrackerDBInterface *iface,
+                gboolean            blocking)
+{
+       GError *error = NULL;
+
+       g_debug ("Checkpointing database...");
+
+       tracker_db_interface_sqlite_wal_checkpoint (iface, blocking,
+                                                   blocking ? &error : NULL);
+
+       if (error) {
+               g_warning ("Error in %s WAL checkpoint: %s",
+                          blocking ? "blocking" : "deferred",
+                          error->message);
+               g_error_free (error);
+       }
+
+       g_debug ("Checkpointing complete");
+}
+
+static gpointer
+wal_checkpoint_thread (gpointer data)
+{
+       TrackerDBManager *db_manager = data;
+       TrackerDBInterface *wal_iface = tracker_db_manager_get_wal_db_interface (db_manager);
+
+       wal_checkpoint (wal_iface, FALSE);
+
+       return NULL;
+}
+
+static void
+wal_hook (TrackerDBInterface *iface,
+          gint                n_pages,
+          gpointer            user_data)
+{
+       TrackerDBManager *db_manager = user_data;
+
+       /* Ensure there is only one WAL checkpoint at a time */
+       if (db_manager->wal_thread)
+               g_thread_join (db_manager->wal_thread);
+
+       if (n_pages >= 10000) {
+               /* Do immediate checkpointing (blocking updates) to
+                * prevent excessive WAL file growth.
+                */
+               wal_checkpoint (iface, TRUE);
+       } else {
+               /* Defer non-blocking checkpoint to thread */
+               db_manager->wal_thread = g_thread_try_new ("wal-checkpoint", wal_checkpoint_thread,
+                                                          db_manager, NULL);
+       }
+}
+
 static TrackerDBInterface *
 init_writable_db_interface (TrackerDBManager *db_manager)
 {
@@ -1100,6 +1171,12 @@ tracker_db_manager_get_writable_db_interface (TrackerDBManager *db_manager)
 {
        if (db_manager->db.iface == NULL) {
                db_manager->db.iface = init_writable_db_interface (db_manager);
+
+               if (db_manager->db.iface &&
+                   (db_manager->flags & TRACKER_DB_MANAGER_READONLY) == 0) {
+                       tracker_db_interface_sqlite_wal_hook (db_manager->db.iface,
+                                                             wal_hook, db_manager);
+               }
        }
 
        return db_manager->db.iface;
diff --git a/src/libtracker-direct/tracker-direct.c b/src/libtracker-direct/tracker-direct.c
index 3dee1bc8b..9864e7d01 100644
--- a/src/libtracker-direct/tracker-direct.c
+++ b/src/libtracker-direct/tracker-direct.c
@@ -186,57 +186,6 @@ query_thread_pool_func (gpointer data,
        g_object_unref (task);
 }
 
-static void
-wal_checkpoint (TrackerDBInterface *iface,
-                gboolean            blocking)
-{
-       GError *error = NULL;
-
-       g_debug ("Checkpointing database...");
-
-       tracker_db_interface_sqlite_wal_checkpoint (iface, blocking,
-                                                   blocking ? &error : NULL);
-
-       if (error) {
-               g_warning ("Error in WAL checkpoint: %s", error->message);
-               g_error_free (error);
-       }
-
-       g_debug ("Checkpointing complete");
-}
-
-static gpointer
-wal_checkpoint_thread (gpointer data)
-{
-       TrackerDBInterface *wal_iface = data;
-
-       wal_checkpoint (wal_iface, FALSE);
-       g_object_unref (wal_iface);
-       return NULL;
-}
-
-static void
-wal_hook (TrackerDBInterface *iface,
-          gint                n_pages)
-{
-       TrackerDataManager *data_manager = tracker_db_interface_get_user_data (iface);
-       TrackerDBInterface *wal_iface = tracker_data_manager_get_wal_db_interface (data_manager);
-
-       if (!wal_iface)
-               return;
-
-       if (n_pages >= 10000) {
-               /* Do immediate checkpointing (blocking updates) to
-                * prevent excessive WAL file growth.
-                */
-               wal_checkpoint (wal_iface, TRUE);
-       } else {
-               /* Defer non-blocking checkpoint to thread */
-               g_thread_try_new ("wal-checkpoint", wal_checkpoint_thread,
-                                 g_object_ref (wal_iface), NULL);
-       }
-}
-
 static gint
 task_compare_func (GTask    *a,
                    GTask    *b,
@@ -280,7 +229,6 @@ tracker_direct_connection_initable_init (GInitable     *initable,
        TrackerDirectConnectionPrivate *priv;
        TrackerDirectConnection *conn;
        TrackerDBManagerFlags db_flags = TRACKER_DB_MANAGER_ENABLE_MUTEXES;
-       TrackerDBInterface *iface;
        GHashTable *namespaces;
        GHashTableIter iter;
        gchar *prefix, *ns;
@@ -305,12 +253,6 @@ tracker_direct_connection_initable_init (GInitable     *initable,
                return FALSE;
        }
 
-       if ((priv->flags & TRACKER_SPARQL_CONNECTION_FLAGS_READONLY) == 0) {
-               /* Set up WAL hook on our connection */
-               iface = tracker_data_manager_get_writable_db_interface (priv->data_manager);
-               tracker_db_interface_sqlite_wal_hook (iface, wal_hook);
-       }
-
        /* Initialize namespace manager */
        priv->namespace_manager = tracker_namespace_manager_new ();
        namespaces = tracker_data_manager_get_namespaces (priv->data_manager);
@@ -394,19 +336,11 @@ tracker_direct_connection_finalize (GObject *object)
        if (priv->select_pool)
                g_thread_pool_free (priv->select_pool, TRUE, FALSE);
 
-       if (priv->data_manager) {
-               TrackerDBInterface *wal_iface;
-               wal_iface = tracker_data_manager_get_wal_db_interface (priv->data_manager);
-               if (wal_iface)
-                       tracker_db_interface_sqlite_wal_checkpoint (wal_iface, TRUE, NULL);
-       }
-
        tracker_data_manager_shutdown (priv->data_manager);
 
        g_clear_object (&priv->store);
        g_clear_object (&priv->journal);
        g_clear_object (&priv->ontology);
-       g_clear_object (&priv->data_manager);
        g_clear_object (&priv->namespace_manager);
 
        G_OBJECT_CLASS (tracker_direct_connection_parent_class)->finalize (object);
@@ -881,7 +815,6 @@ void
 tracker_direct_connection_sync (TrackerDirectConnection *conn)
 {
        TrackerDirectConnectionPrivate *priv;
-       TrackerDBInterface *wal_iface;
 
        priv = tracker_direct_connection_get_instance_private (conn);
 
@@ -896,8 +829,4 @@ tracker_direct_connection_sync (TrackerDirectConnection *conn)
                g_thread_pool_free (priv->select_pool, TRUE, FALSE);
 
        set_up_thread_pools (conn, NULL);
-
-       wal_iface = tracker_data_manager_get_wal_db_interface (priv->data_manager);
-       if (wal_iface)
-               tracker_db_interface_sqlite_wal_checkpoint (wal_iface, TRUE, NULL);
 }