[tracker/wip/carlosg/sparql1.1: 163/201] libtracker-data: Use versioning to know when to update union views



commit 6a38f94fe059d739cb4a305281208bcc6eb000fc
Author: Carlos Garnacho <carlosg gnome org>
Date:   Sun Jul 14 21:27:29 2019 +0200

    libtracker-data: Use versioning to know when to update union views
    
    Instead of poking the cache dir mtime, use a version number that is
    bumped when the attached databases change. On the external readonly
    case, there's no way around checking through the database.
    
    Also, add provisioning for only updating a few selected views on
    demand for the query being prepared.

 src/libtracker-data/tracker-data-manager.c | 322 +++++++++++++++++++++--------
 src/libtracker-data/tracker-data-manager.h |   5 +
 src/libtracker-data/tracker-sparql.c       |  10 +-
 src/tracker/tracker-sql.c                  |   2 +
 4 files changed, 244 insertions(+), 95 deletions(-)
---
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index e7681eda1..60e4917f0 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -78,6 +78,7 @@ struct _TrackerDataManager {
 
        gint select_cache_size;
        gint update_cache_size;
+       guint generation;
 
        TrackerDBManager *db_manager;
        TrackerOntologies *ontologies;
@@ -86,7 +87,6 @@ struct _TrackerDataManager {
        GHashTable *graphs;
 
        gchar *status;
-       guint64 cache_mtime;
 };
 
 struct _TrackerDataManagerClass {
@@ -145,6 +145,7 @@ G_DEFINE_TYPE_WITH_CODE (TrackerDataManager, tracker_data_manager, G_TYPE_OBJECT
 static void
 tracker_data_manager_init (TrackerDataManager *manager)
 {
+       manager->generation = 1;
 }
 
 GQuark
@@ -3509,6 +3510,9 @@ tracker_data_ontology_import_into_db (TrackerDataManager  *manager,
        TrackerProperty **properties;
        guint i, n_props, n_classes;
 
+       if (!tracker_data_manager_update_union_views (manager, iface, NULL, error))
+               return;
+
        classes = tracker_ontologies_get_classes (manager->ontologies, &n_classes);
        properties = tracker_ontologies_get_properties (manager->ontologies, &n_props);
 
@@ -3870,9 +3874,10 @@ tracker_data_manager_get_data_location (TrackerDataManager *manager)
        return manager->data_location ? g_object_ref (manager->data_location) : NULL;
 }
 
-static gboolean
+gboolean
 tracker_data_manager_update_union_views (TrackerDataManager  *manager,
                                          TrackerDBInterface  *iface,
+                                         GHashTable          *tables,
                                          GError             **error)
 {
        TrackerOntologies *ontologies = manager->ontologies;
@@ -3885,6 +3890,10 @@ tracker_data_manager_update_union_views (TrackerDataManager  *manager,
        GHashTable *graphs;
        gpointer graph_name, graph_id;
        GString *str;
+       GHashTable *view_generations;
+       gpointer generation;
+
+       generation = GUINT_TO_POINTER (manager->generation);
 
        classes = tracker_ontologies_get_classes (ontologies, &n_classes);
        properties = tracker_ontologies_get_properties (ontologies, &n_properties);
@@ -3893,10 +3902,28 @@ tracker_data_manager_update_union_views (TrackerDataManager  *manager,
        if (!graphs)
                return FALSE;
 
+       view_generations = g_object_get_data (G_OBJECT (iface),
+                                             "tracker-data-view-generations");
+
+       if (!view_generations) {
+               view_generations = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+               g_object_set_data_full (G_OBJECT (iface),
+                                       "tracker-data-view-generations",
+                                       view_generations,
+                                       (GDestroyNotify) g_hash_table_unref);
+       }
+
        for (i = 0; !inner_error && i < n_classes; i++) {
+               const gchar *name;
+
                if (g_str_has_prefix (tracker_class_get_name (classes[i]), "xsd:"))
                        continue;
 
+               name = tracker_class_get_name (classes[i]);
+               if ((tables && !g_hash_table_contains (tables, name)) ||
+                   g_hash_table_lookup (view_generations, name) == generation)
+                       continue;
+
                stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, 
&inner_error,
                                                              "DROP VIEW IF EXISTS temp.\"unionGraph_%s\"",
                                                              tracker_class_get_name (classes[i]));
@@ -3929,21 +3956,41 @@ tracker_data_manager_update_union_views (TrackerDataManager  *manager,
 
                tracker_db_statement_execute (stmt, &inner_error);
                g_object_unref (stmt);
+
+               if (inner_error)
+                       goto error;
+
+               g_hash_table_insert (view_generations,
+                                    g_strdup (tracker_class_get_name (classes[i])),
+                                    generation);
        }
 
        for (i = 0; !inner_error && i < n_properties; i++) {
                TrackerClass *service;
+               gchar *name;
 
                if (!tracker_property_get_multiple_values (properties[i]))
                        continue;
 
                service = tracker_property_get_domain (properties[i]);
+               name = g_strdup_printf ("%s_%s",
+                                       tracker_class_get_name (service),
+                                       tracker_property_get_name (properties[i]));
+
+               if ((tables && !g_hash_table_contains (tables, name)) ||
+                   g_hash_table_lookup (view_generations, name) == generation) {
+                       g_free (name);
+                       continue;
+               }
+
                stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, 
&inner_error,
                                                              "DROP VIEW IF EXISTS temp.\"unionGraph_%s_%s\"",
                                                              tracker_class_get_name (service),
                                                              tracker_property_get_name (properties[i]));
-               if (!stmt)
+               if (!stmt) {
+                       g_free (name);
                        goto error;
+               }
 
                tracker_db_statement_execute (stmt, NULL);
                g_object_unref (stmt);
@@ -3970,43 +4017,59 @@ tracker_data_manager_update_union_views (TrackerDataManager  *manager,
                                                              "%s", str->str);
                g_string_free (str, TRUE);
 
-               if (!stmt)
+               if (!stmt) {
+                       g_free (name);
                        goto error;
+               }
 
                tracker_db_statement_execute (stmt, &inner_error);
                g_object_unref (stmt);
+
+               if (inner_error) {
+                       g_free (name);
+                       goto error;
+               }
+
+               g_hash_table_insert (view_generations, name, generation);
        }
 
        /* Update FTS5 union view */
-       stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, 
&inner_error,
-                                                     "DROP VIEW IF EXISTS temp.\"unionGraph_fts5\"");
-       if (!stmt)
-               goto error;
+       if ((!tables || g_hash_table_contains (tables, "fts5")) &&
+           g_hash_table_lookup (view_generations, "fts5") != generation) {
+               stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, 
&inner_error,
+                                                             "DROP VIEW IF EXISTS temp.\"unionGraph_fts5\"");
+               if (!stmt)
+                       goto error;
 
-       tracker_db_statement_execute (stmt, NULL);
-       g_object_unref (stmt);
+               tracker_db_statement_execute (stmt, NULL);
+               g_object_unref (stmt);
 
-       str = g_string_new (NULL);
-       g_string_append (str,
-                        "CREATE VIEW temp.\"unionGraph_fts5\" AS "
-                        "SELECT 0 AS graph, ROWID, *, fts5, rank, tracker_offsets(fts5) AS offsets FROM 
\"main\".\"fts5\" ");
+               str = g_string_new (NULL);
+               g_string_append (str,
+                                "CREATE VIEW temp.\"unionGraph_fts5\" AS "
+                                "SELECT 0 AS graph, ROWID, *, fts5, rank, tracker_offsets(fts5) AS offsets 
FROM \"main\".\"fts5\" ");
 
-       g_hash_table_iter_init (&iter, graphs);
-       while (g_hash_table_iter_next (&iter, &graph_name, &graph_id)) {
-               g_string_append_printf (str, "UNION ALL SELECT %d AS graph, ROWID, *, fts5, rank, 
tracker_offsets(fts5) AS offsets FROM \"%s\".\"fts5\" ",
-                                       GPOINTER_TO_INT (graph_id),
-                                       (gchar *) graph_name);
-       }
+               g_hash_table_iter_init (&iter, graphs);
+               while (g_hash_table_iter_next (&iter, &graph_name, &graph_id)) {
+                       g_string_append_printf (str, "UNION ALL SELECT %d AS graph, ROWID, *, fts5, rank, 
tracker_offsets(fts5) AS offsets FROM \"%s\".\"fts5\" ",
+                                               GPOINTER_TO_INT (graph_id),
+                                               (gchar *) graph_name);
+               }
 
-       stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, 
&inner_error,
-                                                     "%s", str->str);
-       g_string_free (str, TRUE);
+               stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, 
&inner_error,
+                                                             "%s", str->str);
+               g_string_free (str, TRUE);
 
-       if (!stmt)
-               goto error;
+               if (stmt) {
+                       tracker_db_statement_execute (stmt, &inner_error);
+                       g_object_unref (stmt);
+               }
 
-       tracker_db_statement_execute (stmt, &inner_error);
-       g_object_unref (stmt);
+               if (inner_error)
+                       goto error;
+
+               g_hash_table_insert (view_generations, g_strdup ("fts5"), generation);
+       }
 
 error:
        if (inner_error) {
@@ -4073,93 +4136,164 @@ update_ontology_last_modified (TrackerDataManager  *manager,
        }
 }
 
-static void
-setup_interface_cb (TrackerDBManager   *db_manager,
-                    TrackerDBInterface *iface,
-                    TrackerDataManager *data_manager)
+static gboolean
+tracker_data_manager_initialize_iface (TrackerDataManager  *data_manager,
+                                       TrackerDBInterface  *iface,
+                                       GError             **error)
 {
        GHashTable *graphs;
-       GError *error = NULL;
 
-#if HAVE_TRACKER_FTS
-       tracker_data_manager_init_fts (iface, FALSE);
-#endif
        graphs = tracker_data_manager_ensure_graphs (data_manager, iface, NULL);
 
        if (graphs) {
                GHashTableIter iter;
                gpointer value;
-               GError *error = NULL;
 
                g_hash_table_iter_init (&iter, graphs);
 
                while (g_hash_table_iter_next (&iter, &value, NULL)) {
-                       if (!tracker_db_manager_attach_database (db_manager,
+                       if (!tracker_db_manager_attach_database (data_manager->db_manager,
                                                                 iface, value, FALSE,
-                                                                &error)) {
-                               g_critical ("Could not attach database '%s': %s\n",
-                                           (gchar *) value, error->message);
-                               g_clear_error (&error);
-                               continue;
+                                                                error)) {
+                               return FALSE;
                        }
                }
        }
 
-       if (!tracker_data_manager_update_union_views (data_manager, iface, &error)) {
-               g_critical ("Could not update union views: %s\n", error->message);
-               g_error_free (error);
-       }
+#if HAVE_TRACKER_FTS
+       tracker_data_manager_init_fts (iface, FALSE);
+#endif
+
+       return TRUE;
 }
 
 static void
-update_interface_cb (TrackerDBManager   *db_manager,
-                     TrackerDBInterface *iface,
-                     TrackerDataManager *data_manager)
+setup_interface_cb (TrackerDBManager   *db_manager,
+                    TrackerDBInterface *iface,
+                    TrackerDataManager *data_manager)
 {
        GError *error = NULL;
-       guint64 cache_mtime;
-       gchar *cache_dir;
+       guint flags;
 
-       cache_dir = g_file_get_path (data_manager->cache_location);
-       cache_mtime = tracker_file_get_mtime (cache_dir);
+       if (!tracker_data_manager_initialize_iface (data_manager, iface, &error)) {
+               g_critical ("Could not set up interface %p: %s",
+                           iface, error->message);
+               g_error_free (error);
+       }
 
-       if (cache_mtime > data_manager->cache_mtime) {
-               GHashTable *prev_graphs;
+       g_object_get (iface, "flags", &flags, NULL);
 
-               prev_graphs = data_manager->graphs;
-               data_manager->graphs = NULL;
+       if (flags & TRACKER_DB_INTERFACE_READONLY) {
+               tracker_data_manager_update_union_views (data_manager, iface, NULL, NULL);
+       }
+}
 
-               tracker_data_manager_ensure_graphs (data_manager, iface, NULL);
+static gboolean
+update_attached_databases (TrackerDBInterface  *iface,
+                           TrackerDataManager  *data_manager,
+                           gboolean            *changed,
+                           GError             **error)
+{
+       TrackerDBStatement *stmt;
+       TrackerDBCursor *cursor;
+       gboolean retval = TRUE;
 
-               if (data_manager->graphs) {
-                       GHashTableIter iter;
-                       gpointer value;
-                       GError *error = NULL;
+       *changed = FALSE;
+       stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT, error,
+                                                     "SELECT name, 1, 0, 0 FROM pragma_database_list "
+                                                     "WHERE name NOT IN (SELECT Uri FROM RESOURCE WHERE ID 
IN (SELECT ID FROM Graph))"
+                                                     "UNION "
+                                                     "SELECT Uri, 0, 1, ID FROM Resource "
+                                                     "WHERE ID IN (SELECT ID FROM Graph) "
+                                                     "AND Uri NOT IN (SELECT name FROM 
pragma_database_list)");
 
-                       g_hash_table_iter_init (&iter, data_manager->graphs);
+       if (!stmt)
+               return FALSE;
 
-                       while (g_hash_table_iter_next (&iter, &value, NULL)) {
-                               if (g_hash_table_contains (prev_graphs, value))
-                                       continue;
+       cursor = tracker_db_statement_start_cursor (stmt, error);
+       g_object_unref (stmt);
 
-                               if (!tracker_db_manager_attach_database (db_manager,
-                                                                        iface, value, FALSE,
-                                                                        &error)) {
-                                       g_critical ("Could not attach database '%s': %s\n",
-                                                   (gchar *) value, error->message);
-                                       g_clear_error (&error);
-                                       continue;
-                               }
+       if (!cursor)
+               return FALSE;
+
+       while (retval && tracker_db_cursor_iter_next (cursor, NULL, NULL)) {
+               const gchar *name;
+
+               name = tracker_db_cursor_get_string (cursor, 0, NULL);
+
+               /* Ignore the main and temp databases, they are always attached */
+               if (strcmp (name, "main") == 0 || strcmp (name, "temp") == 0)
+                       continue;
+
+               if (tracker_db_cursor_get_int (cursor, 1)) {
+                       if (!tracker_db_manager_detach_database (data_manager->db_manager,
+                                                                iface, name, error)) {
+                               retval = FALSE;
+                               break;
+                       }
+
+                       g_hash_table_remove (data_manager->graphs, name);
+                       *changed = TRUE;
+               } else if (tracker_db_cursor_get_int (cursor, 2)) {
+                       if (!tracker_db_manager_attach_database (data_manager->db_manager,
+                                                                iface, name, FALSE, error)) {
+                               retval = FALSE;
+                               break;
                        }
+
+                       g_hash_table_insert (data_manager->graphs, g_strdup (name),
+                                            GINT_TO_POINTER (tracker_db_cursor_get_int (cursor, 3)));
+                       *changed = TRUE;
                }
+       }
+
+       g_object_unref (cursor);
+
+       return retval;
+}
 
-               if (!tracker_data_manager_update_union_views (data_manager, iface, &error)) {
-                       g_critical ("Could not update union views: %s\n", error->message);
+static void
+update_interface_cb (TrackerDBManager   *db_manager,
+                     TrackerDBInterface *iface,
+                     TrackerDataManager *data_manager)
+{
+       GError *error = NULL;
+       guint iface_generation;
+       gboolean update = FALSE, changed, readonly;
+
+       readonly = (tracker_db_manager_get_flags (db_manager, NULL, NULL) & TRACKER_DB_MANAGER_READONLY) != 0;
+
+       if (readonly) {
+               update = TRUE;
+       } else {
+               iface_generation = GPOINTER_TO_UINT (g_object_get_data (G_OBJECT (iface),
+                                                                       "tracker-data-iface-generation"));
+               update = (iface_generation != data_manager->generation);
+       }
+
+       if (update) {
+               if (update_attached_databases (iface, data_manager, &changed, &error)) {
+                       /* This is where we bump the generation for the readonly case, in response to
+                        * tables being attached or detached due to graph changes.
+                        */
+                       if (readonly && changed) {
+                               data_manager->generation++;
+                               if (!tracker_data_manager_update_union_views (data_manager, iface,
+                                                                             NULL, &error)) {
+                                       g_critical ("Could not update union views: %s\n",
+                                                   error->message);
+                                       g_error_free (error);
+                               }
+                       }
+               } else {
+                       g_critical ("Could not update attached databases: %s\n",
+                                   error->message);
                        g_error_free (error);
                }
-       }
 
-       data_manager->cache_mtime = cache_mtime;
+               g_object_set_data (G_OBJECT (iface), "tracker-data-iface-generation",
+                                  GUINT_TO_POINTER (data_manager->generation));
+       }
 }
 
 static gboolean
@@ -4334,8 +4468,11 @@ tracker_data_manager_initable_init (GInitable     *initable,
                tracker_data_manager_init_fts (iface, TRUE);
 #endif
 
-               // FIXME
-               setup_interface_cb (manager->db_manager, iface, manager);
+               tracker_data_manager_initialize_iface (manager, iface, &internal_error);
+               if (internal_error) {
+                       g_propagate_error (error, internal_error);
+                       return FALSE;
+               }
 
                /* store ontology in database */
                for (l = sorted; l; l = l->next) {
@@ -4386,12 +4523,11 @@ tracker_data_manager_initable_init (GInitable     *initable,
                        }
                }
 
-#if HAVE_TRACKER_FTS
-               tracker_data_manager_init_fts (iface, FALSE);
-#endif
-
-               // FIXME: a bit hackish
-               setup_interface_cb (manager->db_manager, iface, manager);
+               tracker_data_manager_initialize_iface (manager, iface, &internal_error);
+               if (internal_error) {
+                       g_propagate_error (error, internal_error);
+                       return FALSE;
+               }
        }
 
        if (!read_only) {
@@ -4800,6 +4936,9 @@ tracker_data_manager_initable_init (GInitable     *initable,
                }
        }
 
+       if (!tracker_data_manager_update_union_views (manager, iface, NULL, error))
+               return FALSE;
+
 skip_ontology_check:
        /* If locale changed, re-create indexes */
        if (!read_only && tracker_db_manager_locale_changed (manager->db_manager, NULL)) {
@@ -4826,9 +4965,6 @@ skip_ontology_check:
                tracker_ontologies_sort (manager->ontologies);
        }
 
-       // FIXME: a bit hackish
-       manager->cache_mtime = tracker_file_get_mtime (g_file_get_path (manager->cache_location));
-
        manager->initialized = TRUE;
 
        /* This is the only one which doesn't show the 'OPERATION' part */
@@ -5064,7 +5200,9 @@ tracker_data_manager_create_graph (TrackerDataManager  *manager,
 
        g_hash_table_insert (manager->graphs, g_strdup (name), GINT_TO_POINTER (id));
 
-       if (!tracker_data_manager_update_union_views (manager, iface, error))
+       manager->generation++;
+
+       if (!tracker_data_manager_update_union_views (manager, iface, NULL, error))
                goto detach;
 
        return TRUE;
@@ -5098,7 +5236,9 @@ tracker_data_manager_drop_graph (TrackerDataManager  *manager,
        if (!tracker_data_delete_graph (manager->data_update, name, error))
                return FALSE;
 
-       if (!tracker_data_manager_update_union_views (manager, iface, error))
+       manager->generation++;
+
+       if (!tracker_data_manager_update_union_views (manager, iface, NULL, error))
                return FALSE;
 
        if (manager->graphs)
diff --git a/src/libtracker-data/tracker-data-manager.h b/src/libtracker-data/tracker-data-manager.h
index 2e6d0029b..7170e01eb 100644
--- a/src/libtracker-data/tracker-data-manager.h
+++ b/src/libtracker-data/tracker-data-manager.h
@@ -103,6 +103,11 @@ gint                 tracker_data_manager_find_graph       (TrackerDataManager *
 const gchar *        tracker_data_manager_find_graph_by_id (TrackerDataManager *manager,
                                                             gint                id);
 
+gboolean             tracker_data_manager_update_union_views (TrackerDataManager  *manager,
+                                                              TrackerDBInterface  *iface,
+                                                              GHashTable          *tables,
+                                                              GError             **error);
+
 G_END_DECLS
 
 #endif /* __LIBTRACKER_DATA_MANAGER_H__ */
diff --git a/src/libtracker-data/tracker-sparql.c b/src/libtracker-data/tracker-sparql.c
index af4581e55..59d5d5ab5 100644
--- a/src/libtracker-data/tracker-sparql.c
+++ b/src/libtracker-data/tracker-sparql.c
@@ -65,7 +65,8 @@ static gboolean helper_translate_date (TrackerSparql  *sparql,
 static gboolean helper_translate_time (TrackerSparql  *sparql,
                                        guint           format,
                                        GError        **error);
-static TrackerDBStatement * prepare_query (TrackerDBInterface    *iface,
+static TrackerDBStatement * prepare_query (TrackerSparql         *sparql,
+                                           TrackerDBInterface    *iface,
                                            TrackerStringBuilder  *str,
                                            GPtrArray             *literals,
                                            GHashTable            *parameters,
@@ -3588,7 +3589,7 @@ get_solution_for_pattern (TrackerSparql      *sparql,
        }
 
        iface = tracker_data_manager_get_writable_db_interface (sparql->data_manager);
-       stmt = prepare_query (iface, sparql->sql,
+       stmt = prepare_query (sparql, iface, sparql->sql,
                              TRACKER_SELECT_CONTEXT (sparql->context)->literal_bindings,
                              NULL, FALSE,
                              error);
@@ -8238,7 +8239,8 @@ tracker_sparql_new (TrackerDataManager *manager,
 }
 
 static TrackerDBStatement *
-prepare_query (TrackerDBInterface    *iface,
+prepare_query (TrackerSparql         *sparql,
+               TrackerDBInterface    *iface,
                TrackerStringBuilder  *str,
                GPtrArray             *literals,
               GHashTable            *parameters,
@@ -8356,7 +8358,7 @@ tracker_sparql_execute_cursor (TrackerSparql  *sparql,
                return NULL;
 
        iface = tracker_data_manager_get_db_interface (sparql->data_manager);
-       stmt = prepare_query (iface, sparql->sql,
+       stmt = prepare_query (sparql, iface, sparql->sql,
                              TRACKER_SELECT_CONTEXT (sparql->context)->literal_bindings,
                              parameters,
                              sparql->cacheable,
diff --git a/src/tracker/tracker-sql.c b/src/tracker/tracker-sql.c
index 98e35ec04..ed8b49a12 100644
--- a/src/tracker/tracker-sql.c
+++ b/src/tracker/tracker-sql.c
@@ -128,6 +128,8 @@ sql_by_query (void)
 
        iface = tracker_data_manager_get_db_interface (data_manager);
 
+       tracker_data_manager_update_union_views (data_manager, iface, NULL, NULL);
+
        stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, &error, 
"%s", query);
 
        if (stmt) {


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]