[tracker/binary-log-2: 19/50] libtracker-data: Add experimental journal replay code
- From: Jürg Billeter <juergbi src gnome org>
- To: svn-commits-list gnome org
- Cc:
- Subject: [tracker/binary-log-2: 19/50] libtracker-data: Add experimental journal replay code
- Date: Mon, 11 Jan 2010 11:03:47 +0000 (UTC)
commit 33b2d185433a77028b73bc0fc746816007834c3d
Author: Jürg Billeter <j bitron ch>
Date: Tue Jan 5 17:51:27 2010 +0100
libtracker-data: Add experimental journal replay code
src/libtracker-data/tracker-data-manager.c | 529 +++++++++++++++++++---------
1 files changed, 358 insertions(+), 171 deletions(-)
---
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index 0810d16..7aaad34 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -65,210 +65,219 @@ static gchar *ontologies_dir;
static gboolean initialized;
static void
-load_ontology_file_from_path (const gchar *ontology_file)
+load_ontology_statement (const gchar *ontology_file,
+ const gchar *subject,
+ const gchar *predicate,
+ const gchar *object)
{
- TrackerTurtleReader *reader;
- GError *error = NULL;
+ if (g_strcmp0 (predicate, RDF_TYPE) == 0) {
+ if (g_strcmp0 (object, RDFS_CLASS) == 0) {
+ TrackerClass *class;
- reader = tracker_turtle_reader_new (ontology_file, &error);
- if (error) {
- g_critical ("Turtle parse error: %s", error->message);
- g_error_free (error);
- return;
- }
+ if (tracker_ontology_get_class_by_uri (subject) != NULL) {
+ g_critical ("%s: Duplicate definition of class %s", ontology_file, subject);
+ return;
+ }
- while (error == NULL && tracker_turtle_reader_next (reader, &error)) {
- const gchar *subject, *predicate, *object;
+ class = tracker_class_new ();
+ tracker_class_set_uri (class, subject);
+ tracker_ontology_add_class (class);
+ g_object_unref (class);
+ } else if (g_strcmp0 (object, RDF_PROPERTY) == 0) {
+ TrackerProperty *property;
- subject = tracker_turtle_reader_get_subject (reader);
- predicate = tracker_turtle_reader_get_predicate (reader);
- object = tracker_turtle_reader_get_object (reader);
+ if (tracker_ontology_get_property_by_uri (subject) != NULL) {
+ g_critical ("%s: Duplicate definition of property %s", ontology_file, subject);
+ return;
+ }
- if (g_strcmp0 (predicate, RDF_TYPE) == 0) {
- if (g_strcmp0 (object, RDFS_CLASS) == 0) {
- TrackerClass *class;
+ property = tracker_property_new ();
+ tracker_property_set_uri (property, subject);
+ tracker_ontology_add_property (property);
+ g_object_unref (property);
+ } else if (g_strcmp0 (object, NRL_INVERSE_FUNCTIONAL_PROPERTY) == 0) {
+ TrackerProperty *property;
- if (tracker_ontology_get_class_by_uri (subject) != NULL) {
- g_critical ("%s: Duplicate definition of class %s", ontology_file, subject);
- continue;
- }
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- class = tracker_class_new ();
- tracker_class_set_uri (class, subject);
- tracker_ontology_add_class (class);
- g_object_unref (class);
- } else if (g_strcmp0 (object, RDF_PROPERTY) == 0) {
- TrackerProperty *property;
+ tracker_property_set_is_inverse_functional_property (property, TRUE);
+ } else if (g_strcmp0 (object, TRACKER_PREFIX "Namespace") == 0) {
+ TrackerNamespace *namespace;
- if (tracker_ontology_get_property_by_uri (subject) != NULL) {
- g_critical ("%s: Duplicate definition of property %s", ontology_file, subject);
- continue;
- }
+ if (tracker_ontology_get_namespace_by_uri (subject) != NULL) {
+ g_critical ("%s: Duplicate definition of namespace %s", ontology_file, subject);
+ return;
+ }
- property = tracker_property_new ();
- tracker_property_set_uri (property, subject);
- tracker_ontology_add_property (property);
- g_object_unref (property);
- } else if (g_strcmp0 (object, NRL_INVERSE_FUNCTIONAL_PROPERTY) == 0) {
- TrackerProperty *property;
-
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ namespace = tracker_namespace_new ();
+ tracker_namespace_set_uri (namespace, subject);
+ tracker_ontology_add_namespace (namespace);
+ g_object_unref (namespace);
+ }
+ } else if (g_strcmp0 (predicate, RDFS_SUB_CLASS_OF) == 0) {
+ TrackerClass *class, *super_class;
- tracker_property_set_is_inverse_functional_property (property, TRUE);
- } else if (g_strcmp0 (object, TRACKER_PREFIX "Namespace") == 0) {
- TrackerNamespace *namespace;
+ class = tracker_ontology_get_class_by_uri (subject);
+ if (class == NULL) {
+ g_critical ("%s: Unknown class %s", ontology_file, subject);
+ return;
+ }
- if (tracker_ontology_get_namespace_by_uri (subject) != NULL) {
- g_critical ("%s: Duplicate definition of namespace %s", ontology_file, subject);
- continue;
- }
+ super_class = tracker_ontology_get_class_by_uri (object);
+ if (super_class == NULL) {
+ g_critical ("%s: Unknown class %s", ontology_file, object);
+ return;
+ }
- namespace = tracker_namespace_new ();
- tracker_namespace_set_uri (namespace, subject);
- tracker_ontology_add_namespace (namespace);
- g_object_unref (namespace);
- }
- } else if (g_strcmp0 (predicate, RDFS_SUB_CLASS_OF) == 0) {
- TrackerClass *class, *super_class;
+ tracker_class_add_super_class (class, super_class);
+ } else if (g_strcmp0 (predicate, RDFS_SUB_PROPERTY_OF) == 0) {
+ TrackerProperty *property, *super_property;
- class = tracker_ontology_get_class_by_uri (subject);
- if (class == NULL) {
- g_critical ("%s: Unknown class %s", ontology_file, subject);
- continue;
- }
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- super_class = tracker_ontology_get_class_by_uri (object);
- if (super_class == NULL) {
- g_critical ("%s: Unknown class %s", ontology_file, object);
- continue;
- }
+ super_property = tracker_ontology_get_property_by_uri (object);
+ if (super_property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, object);
+ return;
+ }
- tracker_class_add_super_class (class, super_class);
- } else if (g_strcmp0 (predicate, RDFS_SUB_PROPERTY_OF) == 0) {
- TrackerProperty *property, *super_property;
+ tracker_property_add_super_property (property, super_property);
+ } else if (g_strcmp0 (predicate, RDFS_DOMAIN) == 0) {
+ TrackerProperty *property;
+ TrackerClass *domain;
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- super_property = tracker_ontology_get_property_by_uri (object);
- if (super_property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, object);
- continue;
- }
+ domain = tracker_ontology_get_class_by_uri (object);
+ if (domain == NULL) {
+ g_critical ("%s: Unknown class %s", ontology_file, object);
+ return;
+ }
- tracker_property_add_super_property (property, super_property);
- } else if (g_strcmp0 (predicate, RDFS_DOMAIN) == 0) {
- TrackerProperty *property;
- TrackerClass *domain;
+ tracker_property_set_domain (property, domain);
+ } else if (g_strcmp0 (predicate, RDFS_RANGE) == 0) {
+ TrackerProperty *property;
+ TrackerClass *range;
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- domain = tracker_ontology_get_class_by_uri (object);
- if (domain == NULL) {
- g_critical ("%s: Unknown class %s", ontology_file, object);
- continue;
- }
+ range = tracker_ontology_get_class_by_uri (object);
+ if (range == NULL) {
+ g_critical ("%s: Unknown class %s", ontology_file, object);
+ return;
+ }
- tracker_property_set_domain (property, domain);
- } else if (g_strcmp0 (predicate, RDFS_RANGE) == 0) {
- TrackerProperty *property;
- TrackerClass *range;
+ tracker_property_set_range (property, range);
+ } else if (g_strcmp0 (predicate, NRL_MAX_CARDINALITY) == 0) {
+ TrackerProperty *property;
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- range = tracker_ontology_get_class_by_uri (object);
- if (range == NULL) {
- g_critical ("%s: Unknown class %s", ontology_file, object);
- continue;
- }
+ if (atoi (object) == 1) {
+ tracker_property_set_multiple_values (property, FALSE);
+ }
+ } else if (g_strcmp0 (predicate, TRACKER_PREFIX "indexed") == 0) {
+ TrackerProperty *property;
- tracker_property_set_range (property, range);
- } else if (g_strcmp0 (predicate, NRL_MAX_CARDINALITY) == 0) {
- TrackerProperty *property;
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ if (strcmp (object, "true") == 0) {
+ tracker_property_set_indexed (property, TRUE);
+ }
+ } else if (g_strcmp0 (predicate, TRACKER_PREFIX "transient") == 0) {
+ TrackerProperty *property;
- if (atoi (object) == 1) {
- tracker_property_set_multiple_values (property, FALSE);
- }
- } else if (g_strcmp0 (predicate, TRACKER_PREFIX "indexed") == 0) {
- TrackerProperty *property;
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ if (g_strcmp0 (object, "true") == 0) {
+ tracker_property_set_transient (property, TRUE);
+ }
+ } else if (g_strcmp0 (predicate, TRACKER_PREFIX "isAnnotation") == 0) {
+ TrackerProperty *property;
- if (strcmp (object, "true") == 0) {
- tracker_property_set_indexed (property, TRUE);
- }
- } else if (g_strcmp0 (predicate, TRACKER_PREFIX "transient") == 0) {
- TrackerProperty *property;
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ if (g_strcmp0 (object, "true") == 0) {
+ tracker_property_set_embedded (property, FALSE);
+ }
+ } else if (g_strcmp0 (predicate, TRACKER_PREFIX "fulltextIndexed") == 0) {
+ TrackerProperty *property;
- if (g_strcmp0 (object, "true") == 0) {
- tracker_property_set_transient (property, TRUE);
- }
- } else if (g_strcmp0 (predicate, TRACKER_PREFIX "isAnnotation") == 0) {
- TrackerProperty *property;
+ property = tracker_ontology_get_property_by_uri (subject);
+ if (property == NULL) {
+ g_critical ("%s: Unknown property %s", ontology_file, subject);
+ return;
+ }
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ if (strcmp (object, "true") == 0) {
+ tracker_property_set_fulltext_indexed (property, TRUE);
+ }
+ } else if (g_strcmp0 (predicate, TRACKER_PREFIX "prefix") == 0) {
+ TrackerNamespace *namespace;
- if (g_strcmp0 (object, "true") == 0) {
- tracker_property_set_embedded (property, FALSE);
- }
- } else if (g_strcmp0 (predicate, TRACKER_PREFIX "fulltextIndexed") == 0) {
- TrackerProperty *property;
+ namespace = tracker_ontology_get_namespace_by_uri (subject);
+ if (namespace == NULL) {
+ g_critical ("%s: Unknown namespace %s", ontology_file, subject);
+ return;
+ }
- property = tracker_ontology_get_property_by_uri (subject);
- if (property == NULL) {
- g_critical ("%s: Unknown property %s", ontology_file, subject);
- continue;
- }
+ tracker_namespace_set_prefix (namespace, object);
+ }
+}
- if (strcmp (object, "true") == 0) {
- tracker_property_set_fulltext_indexed (property, TRUE);
- }
- } else if (g_strcmp0 (predicate, TRACKER_PREFIX "prefix") == 0) {
- TrackerNamespace *namespace;
+static void
+load_ontology_file_from_path (const gchar *ontology_file)
+{
+ TrackerTurtleReader *reader;
+ GError *error = NULL;
- namespace = tracker_ontology_get_namespace_by_uri (subject);
- if (namespace == NULL) {
- g_critical ("%s: Unknown namespace %s", ontology_file, subject);
- continue;
- }
+ reader = tracker_turtle_reader_new (ontology_file, &error);
+ if (error) {
+ g_critical ("Turtle parse error: %s", error->message);
+ g_error_free (error);
+ return;
+ }
- tracker_namespace_set_prefix (namespace, object);
- }
+ while (error == NULL && tracker_turtle_reader_next (reader, &error)) {
+ const gchar *subject, *predicate, *object;
+
+ subject = tracker_turtle_reader_get_subject (reader);
+ predicate = tracker_turtle_reader_get_predicate (reader);
+ object = tracker_turtle_reader_get_object (reader);
+
+ load_ontology_statement (ontology_file, subject, predicate, object);
}
g_object_unref (reader);
@@ -289,6 +298,49 @@ load_ontology_file (const gchar *filename)
g_free (ontology_file);
}
+static GHashTable *
+load_ontology_from_journal (void)
+{
+ GHashTable *id_uri_map;
+
+ id_uri_map = g_hash_table_new (g_direct_hash, g_direct_equal);
+
+ while (tracker_db_journal_reader_next (NULL)) {
+ TrackerDBJournalEntryType type;
+
+ type = tracker_db_journal_reader_get_type ();
+ if (type == TRACKER_DB_JOURNAL_RESOURCE) {
+ guint32 id;
+ const gchar *uri;
+
+ tracker_db_journal_reader_get_resource (&id, &uri);
+ g_hash_table_insert (id_uri_map, GUINT_TO_POINTER (id), (gpointer) uri);
+ } else if (type == TRACKER_DB_JOURNAL_END_TRANSACTION) {
+ /* end of initial transaction => end of ontology */
+ break;
+ } else {
+ const gchar *subject, *predicate, *object;
+ guint32 subject_id, predicate_id, object_id;
+
+ if (type == TRACKER_DB_JOURNAL_INSERT_STATEMENT) {
+ tracker_db_journal_reader_get_statement (&subject_id, &predicate_id, &object);
+ } else if (type == TRACKER_DB_JOURNAL_INSERT_STATEMENT_ID) {
+ tracker_db_journal_reader_get_statement_id (&subject_id, &predicate_id, &object_id);
+ object = g_hash_table_lookup (id_uri_map, GUINT_TO_POINTER (object_id));
+ } else {
+ continue;
+ }
+
+ subject = g_hash_table_lookup (id_uri_map, GUINT_TO_POINTER (subject_id));
+ predicate = g_hash_table_lookup (id_uri_map, GUINT_TO_POINTER (predicate_id));
+
+ load_ontology_statement ("journal", subject, predicate, object);
+ }
+ }
+
+ return id_uri_map;
+}
+
static void
import_ontology_file (const gchar *filename)
{
@@ -305,6 +357,89 @@ import_ontology_file (const gchar *filename)
}
}
+static gchar *
+query_resource_by_id (guint32 id)
+{
+ TrackerDBCursor *cursor;
+ TrackerDBInterface *iface;
+ TrackerDBStatement *stmt;
+ gchar *uri;
+
+ g_return_val_if_fail (id > 0, NULL);
+
+ iface = tracker_db_manager_get_db_interface ();
+
+ stmt = tracker_db_interface_create_statement (iface,
+ "SELECT Uri FROM \"rdfs:Resource\" WHERE ID = ?");
+ tracker_db_statement_bind_uint (stmt, 0, id);
+ cursor = tracker_db_statement_start_cursor (stmt, NULL);
+ g_object_unref (stmt);
+
+ tracker_db_cursor_iter_next (cursor);
+ uri = g_strdup (tracker_db_cursor_get_string (cursor, 0));
+ g_object_unref (cursor);
+
+ return uri;
+}
+
+static void
+replay_journal (void)
+{
+ GError *error = NULL;
+
+ tracker_db_journal_reader_init (NULL);
+
+ while (tracker_db_journal_reader_next (NULL)) {
+ TrackerDBJournalEntryType type;
+ const gchar *subject, *predicate, *object;
+ guint32 subject_id, predicate_id, object_id;
+
+ type = tracker_db_journal_reader_get_type ();
+ if (type == TRACKER_DB_JOURNAL_RESOURCE) {
+ TrackerDBInterface *iface;
+ TrackerDBStatement *stmt;
+ guint32 id;
+ const gchar *uri;
+
+ tracker_db_journal_reader_get_resource (&id, &uri);
+
+ iface = tracker_db_manager_get_db_interface ();
+
+ stmt = tracker_db_interface_create_statement (iface,
+ "INSERT "
+ "INTO \"rdfs:Resource\" "
+ "(ID, Uri, \"tracker:added\", "
+ "\"tracker:modified\", Available) "
+ "VALUES (?, ?, ?, 0, 1)");
+ tracker_db_statement_bind_int (stmt, 0, id);
+ tracker_db_statement_bind_text (stmt, 1, uri);
+ tracker_db_statement_bind_int64 (stmt, 2, (gint64) time (NULL));
+ tracker_db_statement_execute (stmt, &error);
+ } else if (type == TRACKER_DB_JOURNAL_START_TRANSACTION) {
+ tracker_data_begin_transaction ();
+ } else if (type == TRACKER_DB_JOURNAL_END_TRANSACTION) {
+ tracker_data_commit_transaction ();
+ } else if (type == TRACKER_DB_JOURNAL_INSERT_STATEMENT) {
+ tracker_db_journal_reader_get_statement (&subject_id, &predicate_id, &object);
+
+ subject = query_resource_by_id (subject_id);
+ predicate = query_resource_by_id (predicate_id);
+
+ tracker_data_insert_statement_with_string (NULL, subject, predicate, object, &error);
+ } else if (type == TRACKER_DB_JOURNAL_INSERT_STATEMENT_ID) {
+ tracker_db_journal_reader_get_statement_id (&subject_id, &predicate_id, &object_id);
+
+ subject = query_resource_by_id (subject_id);
+ predicate = query_resource_by_id (predicate_id);
+ object = query_resource_by_id (object_id);
+
+ tracker_data_insert_statement_with_uri (NULL, subject, predicate, object, &error);
+ }
+ }
+
+ tracker_db_journal_reader_shutdown ();
+}
+
static void
class_add_super_classes_from_db (TrackerDBInterface *iface, TrackerClass *class)
{
@@ -807,7 +942,7 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
gboolean *need_journal)
{
TrackerDBInterface *iface;
- gboolean is_first_time_index;
+ gboolean is_first_time_index, read_journal;
/* First set defaults for return values */
if (first_time) {
@@ -822,6 +957,8 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
return TRUE;
}
+ read_journal = FALSE;
+
tracker_db_manager_init (flags, &is_first_time_index, FALSE, need_journal);
if (first_time != NULL) {
@@ -830,9 +967,55 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
iface = tracker_db_manager_get_db_interface ();
- tracker_db_journal_init (NULL);
+ if (is_first_time_index && !test_schema) {
+ if (tracker_db_journal_reader_init (NULL)) {
+ if (tracker_db_journal_reader_next (NULL)) {
+ /* journal with at least one valid transaction
+ is required to trigger journal replay */
+ read_journal = TRUE;
+ } else {
+ tracker_db_journal_reader_shutdown ();
+ }
+ }
+ }
+
+ if (read_journal) {
+ TrackerClass **classes;
+ TrackerProperty **properties;
+ gint max_id = 0;
+ guint i, n_props, n_classes;
+
+ /* load ontology from journal into memory */
+ /*id_uri_map =*/ load_ontology_from_journal ();
+
+ classes = tracker_ontology_get_classes (&n_classes);
+
+ tracker_data_begin_transaction ();
+
+ /* create tables */
+ for (i = 0; i < n_classes; i++) {
+ create_decomposed_metadata_tables (iface, classes[i], &max_id);
+ }
+
+ /* insert properties into rdfs:Resource table */
+ properties = tracker_ontology_get_properties (&n_props);
+
+ for (i = 0; i < n_props; i++) {
+ insert_uri_in_resource_table (iface, tracker_property_get_uri (properties[i]),
+ &max_id, NULL, properties[i]);
+ }
+
+ create_fts_table (iface);
+
+ tracker_data_commit_transaction ();
+
+ tracker_db_journal_reader_shutdown ();
- if (is_first_time_index) {
+ replay_journal ();
+
+ /* open journal for writing */
+ tracker_db_journal_init (NULL);
+ } else if (is_first_time_index) {
TrackerClass **classes;
TrackerProperty **properties;
gint max_id = 0;
@@ -881,6 +1064,8 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
g_dir_close (ontologies);
}
+ tracker_db_journal_init (NULL);
+
/* load ontology from files into memory */
for (l = sorted; l; l = l->next) {
g_debug ("Loading ontology %s", (char *) l->data);
@@ -942,6 +1127,8 @@ tracker_data_manager_init (TrackerDBManagerFlags flags,
g_free (ontologies_dir);
ontologies_dir = NULL;
} else {
+ tracker_db_journal_init (NULL);
+
/* load ontology from database into memory */
db_get_static_data (iface);
create_decomposed_transient_metadata_tables (iface);
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]