GNOME.org
 

[tracker/wal: 2/18] libtracker-data: Do not use overly large transactions anymore

commit 89a75182a2bc52104e997c2446c9c20f351aab7c
Author: Jürg Billeter <j bitron ch>
Date:   Tue Jun 29 14:41:40 2010 +0200

    libtracker-data: Do not use overly large transactions anymore
    
    This is not necessary anymore with WAL.

 src/libtracker-data/tracker-data-manager.c         |   17 +--
 src/libtracker-data/tracker-data-update.c          |  118 ++++++++-----------
 src/libtracker-data/tracker-data-update.h          |    7 +-
 src/libtracker-data/tracker-db-manager.c           |    2 +
 src/tracker-store/tracker-store.c                  |   18 +---
 tests/libtracker-data/tracker-backup-test.c        |    2 -
 .../libtracker-data/tracker-ontology-change-test.c |    2 -
 tests/libtracker-data/tracker-ontology-test.c      |    2 -
 tests/libtracker-data/tracker-sparql-blank-test.c  |    2 -
 tests/libtracker-data/tracker-sparql-test.c        |    4 -
 tests/libtracker-fts/tracker-fts-test.c            |    2 -
 11 files changed, 65 insertions(+), 111 deletions(-)
---
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index 53c47a1..8a0618d 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -2751,13 +2751,13 @@ tracker_data_manager_init (TrackerDBManagerFlags  flags,
 		load_ontology_from_journal (&classes, &properties, &id_uri_map);
 
 		/* Read first ontology and commit it into the DB */
-		tracker_data_begin_db_transaction_for_replay (tracker_db_journal_reader_get_time ());
+		tracker_data_begin_transaction_for_replay (tracker_db_journal_reader_get_time (), NULL);
 
 		/* This is a no-op when FTS is disabled */
 		tracker_db_interface_sqlite_fts_init (iface, TRUE);
 
 		tracker_data_ontology_import_into_db (FALSE);
-		tracker_data_commit_db_transaction ();
+		tracker_data_commit_transaction (NULL);
 		tracker_db_journal_reader_shutdown ();
 
 		/* Start replay. Ontology changes might happen during replay of the journal. */
@@ -2806,10 +2806,7 @@ tracker_data_manager_init (TrackerDBManagerFlags  flags,
 			}
 		}
 
-		tracker_data_begin_db_transaction ();
-
-		/* Not an ontology transaction: this is the first ontology */
-		tracker_db_journal_start_transaction (time (NULL));
+		tracker_data_begin_transaction (NULL);
 
 		/* This is a no-op when FTS is disabled */
 		tracker_db_interface_sqlite_fts_init (iface, TRUE);
@@ -2834,8 +2831,7 @@ tracker_data_manager_init (TrackerDBManagerFlags  flags,
 			}
 		}
 
-		tracker_db_journal_commit_db_transaction ();
-		tracker_data_commit_db_transaction ();
+		tracker_data_commit_transaction (NULL);
 
 		g_list_foreach (sorted, (GFunc) g_free, NULL);
 		g_list_free (sorted);
@@ -2888,7 +2884,7 @@ tracker_data_manager_init (TrackerDBManagerFlags  flags,
 
 		/* check ontology against database */
 
-		tracker_data_begin_db_transaction ();
+		tracker_data_begin_transaction (NULL);
 
 		/* This _is_ an ontology transaction, it represents a change to the
 		 * ontology. We mark it up as such in the journal, so that replay_journal
@@ -3045,8 +3041,7 @@ tracker_data_manager_init (TrackerDBManagerFlags  flags,
 		/* Reset the is_new flag for all classes and properties */
 		tracker_data_ontology_import_finished ();
 
-		tracker_db_journal_commit_db_transaction ();
-		tracker_data_commit_db_transaction ();
+		tracker_data_commit_transaction (NULL);
 
 		g_hash_table_unref (ontos_table);
 
diff --git a/src/libtracker-data/tracker-data-update.c b/src/libtracker-data/tracker-data-update.c
index ab2dd08..b90c4cd 100644
--- a/src/libtracker-data/tracker-data-update.c
+++ b/src/libtracker-data/tracker-data-update.c
@@ -2109,12 +2109,18 @@ tracker_data_insert_statement_with_string (const gchar            *graph,
 }
 
 void
-tracker_data_begin_db_transaction (void)
+tracker_data_begin_transaction (GError **error)
 {
 	TrackerDBInterface *iface;
 
 	g_return_if_fail (!in_transaction);
 
+	if (!tracker_db_manager_has_enough_space ()) {
+		g_set_error (error, TRACKER_DATA_ERROR, TRACKER_DATA_ERROR_NO_SPACE,
+			"There is not enough space on the file system for update operations");
+		return;
+	}
+
 	resource_time = time (NULL);
 
 	if (update_buffer.resource_cache == NULL) {
@@ -2134,34 +2140,45 @@ tracker_data_begin_db_transaction (void)
 
 	tracker_db_interface_start_transaction (iface);
 
+	if (!in_journal_replay) {
+		tracker_db_journal_start_transaction (resource_time);
+	}
+
+	iface = tracker_db_manager_get_db_interface ();
+
 	in_transaction = TRUE;
 }
 
 void
-tracker_data_begin_db_transaction_for_replay (time_t time)
+tracker_data_begin_transaction_for_replay (time_t time, GError **error)
 {
 	in_journal_replay = TRUE;
-	tracker_data_begin_db_transaction ();
+	tracker_data_begin_transaction (error);
 	resource_time = time;
 }
 
 void
-tracker_data_commit_db_transaction (void)
+tracker_data_commit_transaction (GError **error)
 {
 	TrackerDBInterface *iface;
+	GError *actual_error = NULL;
 
 	g_return_if_fail (in_transaction);
 
-	in_transaction = FALSE;
+	iface = tracker_db_manager_get_db_interface ();
 
-	tracker_data_update_buffer_flush (NULL);
+	tracker_data_update_buffer_flush (&actual_error);
+	if (actual_error) {
+		g_propagate_error (error, actual_error);
+		return;
+	}
 
-#if HAVE_TRACKER_FTS
-	if (update_buffer.fts_ever_updated) {
-		tracker_fts_update_commit ();
-		update_buffer.fts_ever_updated = FALSE;
+	in_transaction = FALSE;
+
+	if (!in_journal_replay) {
+		tracker_db_journal_commit_db_transaction ();
 	}
-#endif
+	resource_time = 0;
 
 	if (update_buffer.class_counts) {
 		/* successful transaction, no need to rollback class counts,
@@ -2169,7 +2186,12 @@ tracker_data_commit_db_transaction (void)
 		g_hash_table_remove_all (update_buffer.class_counts);
 	}
 
-	iface = tracker_db_manager_get_db_interface ();
+#if HAVE_TRACKER_FTS
+	if (update_buffer.fts_ever_updated) {
+		tracker_fts_update_commit ();
+		update_buffer.fts_ever_updated = FALSE;
+	}
+#endif
 
 	tracker_db_interface_end_db_transaction (iface);
 
@@ -2181,7 +2203,7 @@ tracker_data_commit_db_transaction (void)
 }
 
 void
-tracker_data_notify_db_transaction (void)
+tracker_data_notify_transaction (void)
 {
 	if (commit_callbacks) {
 		guint n;
@@ -2451,57 +2473,18 @@ tracker_data_delete_resource_description (const gchar *graph,
 }
 
 void
-tracker_data_begin_transaction (GError **error)
-{
-	TrackerDBInterface *iface;
-
-	if (!tracker_db_manager_has_enough_space ()) {
-		g_set_error (error, TRACKER_DATA_ERROR, TRACKER_DATA_ERROR_NO_SPACE,
-			"There is not enough space on the file system for update operations");
-		return;
-	}
-
-	iface = tracker_db_manager_get_db_interface ();
-
-	resource_time = time (NULL);
-	tracker_db_interface_execute_query (iface, NULL, "SAVEPOINT sparql");
-	tracker_db_journal_start_transaction (resource_time);
-}
-
-void
-tracker_data_commit_transaction (GError **error)
+tracker_data_rollback_transaction (void)
 {
 	TrackerDBInterface *iface;
-	GError *actual_error = NULL;
 
-	iface = tracker_db_manager_get_db_interface ();
-
-	tracker_data_update_buffer_flush (&actual_error);
-	if (actual_error) {
-		g_propagate_error (error, actual_error);
-		return;
-	}
-
-	tracker_db_journal_commit_db_transaction ();
-	resource_time = 0;
-	tracker_db_interface_execute_query (iface, NULL, "RELEASE sparql");
-
-	if (update_buffer.class_counts) {
-		/* successful transaction, no need to rollback class counts,
-		   so remove them */
-		g_hash_table_remove_all (update_buffer.class_counts);
-	}
-}
+	g_return_if_fail (in_transaction);
 
-void
-tracker_data_rollback_transaction (void)
-{
-	TrackerDBInterface *iface;
+	in_transaction = FALSE;
 
 	iface = tracker_db_manager_get_db_interface ();
 
 	tracker_data_update_buffer_clear ();
-	tracker_db_interface_execute_query (iface, NULL, "ROLLBACK TO sparql");
+	tracker_db_interface_execute_query (iface, NULL, "ROLLBACK");
 	tracker_db_journal_rollback_transaction ();
 
 	if (rollback_callbacks) {
@@ -2727,8 +2710,6 @@ tracker_data_replay_journal (GHashTable          *classes,
 	GPtrArray *seen_classes = NULL;
 	GPtrArray *seen_properties = NULL;
 
-	tracker_data_begin_db_transaction_for_replay (0);
-
 	rdf_type = tracker_ontologies_get_property_by_uri (RDF_PREFIX "type");
 
 	tracker_db_journal_reader_init (NULL);
@@ -2783,10 +2764,14 @@ tracker_data_replay_journal (GHashTable          *classes,
 			}
 
 		} else if (type == TRACKER_DB_JOURNAL_START_ONTOLOGY_TRANSACTION) {
+			tracker_data_begin_transaction_for_replay (tracker_db_journal_reader_get_time (), NULL);
 			in_ontology = TRUE;
 		} else if (type == TRACKER_DB_JOURNAL_START_TRANSACTION) {
-			resource_time = tracker_db_journal_reader_get_time ();
+			tracker_data_begin_transaction_for_replay (tracker_db_journal_reader_get_time (), NULL);
 		} else if (type == TRACKER_DB_JOURNAL_END_TRANSACTION) {
+			GError *new_error = NULL;
+			tracker_data_update_buffer_might_flush (&new_error);
+
 			if (in_ontology) {
 				ontology_transaction_end (ontology_queue, seen_classes, seen_properties);
 				g_list_foreach (ontology_queue, (GFunc) free_queued_statement, NULL);
@@ -2797,13 +2782,12 @@ tracker_data_replay_journal (GHashTable          *classes,
 				seen_classes = NULL;
 				tracker_data_ontology_free_seen (seen_properties);
 				seen_properties = NULL;
-			} else {
-				GError *new_error = NULL;
-				tracker_data_update_buffer_might_flush (&new_error);
-				if (new_error) {
-					g_warning ("Journal replay error: '%s'", new_error->message);
-					g_clear_error (&new_error);
-				}
+			}
+
+			tracker_data_commit_transaction (&new_error);
+			if (new_error) {
+				g_warning ("Journal replay error: '%s'", new_error->message);
+				g_clear_error (&new_error);
 			}
 		} else if (type == TRACKER_DB_JOURNAL_INSERT_STATEMENT) {
 			GError *new_error = NULL;
@@ -3044,6 +3028,4 @@ tracker_data_replay_journal (GHashTable          *classes,
 	} else {
 		tracker_db_journal_reader_shutdown ();
 	}
-
-	tracker_data_commit_db_transaction ();
 }
diff --git a/src/libtracker-data/tracker-data-update.h b/src/libtracker-data/tracker-data-update.h
index 981a702..0b82f5f 100644
--- a/src/libtracker-data/tracker-data-update.h
+++ b/src/libtracker-data/tracker-data-update.h
@@ -80,12 +80,11 @@ void     tracker_data_insert_statement_with_string  (const gchar               *
                                                      const gchar               *predicate,
                                                      const gchar               *object,
                                                      GError                   **error);
-void     tracker_data_begin_db_transaction          (void);
-void     tracker_data_begin_db_transaction_for_replay (time_t                   time);
-void     tracker_data_commit_db_transaction         (void);
-void     tracker_data_notify_db_transaction         (void);
 void     tracker_data_begin_transaction             (GError                   **error);
+void     tracker_data_begin_transaction_for_replay  (time_t                     time,
+                                                     GError                   **error);
 void     tracker_data_commit_transaction            (GError                   **error);
+void     tracker_data_notify_transaction            (void);
 void     tracker_data_rollback_transaction          (void);
 void     tracker_data_update_sparql                 (const gchar               *update,
                                                      GError                   **error);
diff --git a/src/libtracker-data/tracker-db-manager.c b/src/libtracker-data/tracker-db-manager.c
index 05904a9..b0d0d66 100644
--- a/src/libtracker-data/tracker-db-manager.c
+++ b/src/libtracker-data/tracker-db-manager.c
@@ -210,12 +210,14 @@ db_set_params (TrackerDBInterface *iface,
                gint                page_size)
 {
 	tracker_db_interface_execute_query (iface, NULL, "PRAGMA synchronous = OFF;");
+	//tracker_db_interface_execute_query (iface, NULL, "PRAGMA synchronous = NORMAL;");
 	tracker_db_interface_execute_query (iface, NULL, "PRAGMA count_changes = 0;");
 	tracker_db_interface_execute_query (iface, NULL, "PRAGMA temp_store = FILE;");
 	tracker_db_interface_execute_query (iface, NULL, "PRAGMA encoding = \"UTF-8\"");
 	tracker_db_interface_execute_query (iface, NULL, "PRAGMA auto_vacuum = 0;");
 
 	tracker_db_interface_execute_query (iface, NULL, "PRAGMA journal_mode = WAL;");
+	//tracker_db_interface_execute_query (iface, NULL, "PRAGMA wal_autocheckpoint = 0;");
 
 	if (page_size != TRACKER_DB_PAGE_SIZE_DONT_SET) {
 		g_message ("  Setting page size to %d", page_size);
diff --git a/src/tracker-store/tracker-store.c b/src/tracker-store/tracker-store.c
index 4b4b619..2d16896 100644
--- a/src/tracker-store/tracker-store.c
+++ b/src/tracker-store/tracker-store.c
@@ -223,9 +223,6 @@ static void
 begin_batch (TrackerStorePrivate *private)
 {
 	if (!private->batch_mode) {
-		/* switch to batch mode
-		   delays database commits to improve performance */
-		tracker_data_begin_db_transaction ();
 		private->batch_mode = TRUE;
 		private->batch_count = 0;
 	}
@@ -236,8 +233,7 @@ end_batch (TrackerStorePrivate *private)
 {
 	if (private->batch_mode) {
 		/* commit pending batch items */
-		tracker_data_commit_db_transaction ();
-		tracker_data_notify_db_transaction ();
+		tracker_data_notify_transaction ();
 
 		private->batch_mode = FALSE;
 		private->batch_count = 0;
@@ -376,7 +372,7 @@ task_finish_cb (gpointer data)
 		private->n_queries_running--;
 	} else if (task->type == TRACKER_STORE_TASK_TYPE_UPDATE) {
 		if (!task->data.update.batch && !task->error) {
-			tracker_data_notify_db_transaction ();
+			tracker_data_notify_transaction ();
 		}
 
 		if (task->callback.update_callback) {
@@ -390,7 +386,7 @@ task_finish_cb (gpointer data)
 		private->update_running = FALSE;
 	} else if (task->type == TRACKER_STORE_TASK_TYPE_UPDATE_BLANK) {
 		if (!task->data.update.batch && !task->error) {
-			tracker_data_notify_db_transaction ();
+			tracker_data_notify_transaction ();
 		}
 
 		if (task->callback.update_blank_callback) {
@@ -437,7 +433,7 @@ task_finish_cb (gpointer data)
 			g_queue_pop_head (private->queues[TRACKER_STORE_PRIORITY_TURTLE]);
 		}
 	} else if (task->type == TRACKER_STORE_TASK_TYPE_COMMIT) {
-		tracker_data_notify_db_transaction ();
+		tracker_data_notify_transaction ();
 
 		if (task->callback.commit_callback) {
 			task->callback.commit_callback (task->user_data);
@@ -495,7 +491,6 @@ pool_dispatch_cb (gpointer data,
 			begin_batch (private);
 		} else {
 			end_batch (private);
-			tracker_data_begin_db_transaction ();
 		}
 
 		tracker_data_update_sparql (task->data.update.query, &task->error);
@@ -507,15 +502,12 @@ pool_dispatch_cb (gpointer data,
 					end_batch (private);
 				}
 			}
-		} else {
-			tracker_data_commit_db_transaction ();
 		}
 	} else if (task->type == TRACKER_STORE_TASK_TYPE_UPDATE_BLANK) {
 		if (task->data.update.batch) {
 			begin_batch (private);
 		} else {
 			end_batch (private);
-			tracker_data_begin_db_transaction ();
 		}
 
 		task->data.update.blank_nodes = tracker_data_update_sparql_blank (task->data.update.query, &task->error);
@@ -527,8 +519,6 @@ pool_dispatch_cb (gpointer data,
 					end_batch (private);
 				}
 			}
-		} else {
-			tracker_data_commit_db_transaction ();
 		}
 	} else if (task->type == TRACKER_STORE_TASK_TYPE_TURTLE) {
 		if (!task->data.turtle.in_progress) {
diff --git a/tests/libtracker-data/tracker-backup-test.c b/tests/libtracker-data/tracker-backup-test.c
index cca86a0..a060f29 100644
--- a/tests/libtracker-data/tracker-backup-test.c
+++ b/tests/libtracker-data/tracker-backup-test.c
@@ -114,9 +114,7 @@ test_backup_and_restore_helper (gboolean journal)
 	/* load data set */
 	data_filename = g_strconcat (data_prefix, ".data", NULL);
 	if (g_file_test (data_filename, G_FILE_TEST_IS_REGULAR)) {
-		tracker_data_begin_db_transaction ();
 		tracker_turtle_reader_load (data_filename, &error);
-		tracker_data_commit_db_transaction ();
 		g_assert_no_error (error);
 	} else {
 		g_assert_not_reached ();
diff --git a/tests/libtracker-data/tracker-ontology-change-test.c b/tests/libtracker-data/tracker-ontology-change-test.c
index 5893589..9ae8beb 100644
--- a/tests/libtracker-data/tracker-ontology-change-test.c
+++ b/tests/libtracker-data/tracker-ontology-change-test.c
@@ -252,9 +252,7 @@ test_ontology_change (void)
 			gchar *query = strtok (queries, "\n");
 			while (query) {
 
-				tracker_data_begin_db_transaction ();
 				tracker_data_update_sparql (query, &error);
-				tracker_data_commit_db_transaction ();
 
 				g_assert_no_error (error);
 				query = strtok (NULL, "\n");
diff --git a/tests/libtracker-data/tracker-ontology-test.c b/tests/libtracker-data/tracker-ontology-test.c
index f7e658c..6913e19 100644
--- a/tests/libtracker-data/tracker-ontology-test.c
+++ b/tests/libtracker-data/tracker-ontology-test.c
@@ -248,9 +248,7 @@ test_query (gconstpointer test_data)
 
 	/* load data set */
 	data_filename = g_strconcat (data_prefix, ".ttl", NULL);
-	tracker_data_begin_db_transaction ();
 	tracker_turtle_reader_load (data_filename, &error);
-	tracker_data_commit_db_transaction ();
 	g_assert_no_error (error);
 
 	query_filename = g_strconcat (test_prefix, ".rq", NULL);
diff --git a/tests/libtracker-data/tracker-sparql-blank-test.c b/tests/libtracker-data/tracker-sparql-blank-test.c
index 8d614a2..e4850f0 100644
--- a/tests/libtracker-data/tracker-sparql-blank-test.c
+++ b/tests/libtracker-data/tracker-sparql-blank-test.c
@@ -52,12 +52,10 @@ test_blank (void)
 
 	/* perform update in transaction */
 
-	tracker_data_begin_db_transaction ();
 	updates = tracker_data_update_sparql_blank (
 	                                            "INSERT { _:foo a rdfs:Resource } "
 	                                            "INSERT { _:foo a rdfs:Resource . _:bar a rdfs:Resource } ",
 	                                            &error);
-	tracker_data_commit_db_transaction ();
 	g_assert_no_error (error);
 
 	g_assert_cmpint (updates->len, ==, 2);
diff --git a/tests/libtracker-data/tracker-sparql-test.c b/tests/libtracker-data/tracker-sparql-test.c
index ccd688c..3aea0a9 100644
--- a/tests/libtracker-data/tracker-sparql-test.c
+++ b/tests/libtracker-data/tracker-sparql-test.c
@@ -256,9 +256,7 @@ test_sparql_query (gconstpointer test_data)
 	/* load data set */
 	data_filename = g_strconcat (data_prefix, ".ttl", NULL);
 	if (g_file_test (data_filename, G_FILE_TEST_IS_REGULAR)) {
-		tracker_data_begin_db_transaction ();
 		tracker_turtle_reader_load (data_filename, &error);
-		tracker_data_commit_db_transaction ();
 		g_assert_no_error (error);
 	} else {
 		/* no .ttl available, assume .rq with SPARQL Update */
@@ -270,9 +268,7 @@ test_sparql_query (gconstpointer test_data)
 		g_file_get_contents (data_filename, &data, NULL, &error);
 		g_assert_no_error (error);
 
-		tracker_data_begin_db_transaction ();
 		tracker_data_update_sparql (data, &error);
-		tracker_data_commit_db_transaction ();
 		if (test_info->expect_update_error) {
 			g_assert (error != NULL);
 			g_clear_error (&error);
diff --git a/tests/libtracker-fts/tracker-fts-test.c b/tests/libtracker-fts/tracker-fts-test.c
index 6058822..333041d 100644
--- a/tests/libtracker-fts/tracker-fts-test.c
+++ b/tests/libtracker-fts/tracker-fts-test.c
@@ -82,9 +82,7 @@ test_sparql_query (gconstpointer test_data)
 	g_file_get_contents (update_filename, &update, NULL, &error);
 	g_assert_no_error (error);
 
-	tracker_data_begin_db_transaction ();
 	tracker_data_update_sparql (update, &error);
-	tracker_data_commit_db_transaction ();
 	g_assert_no_error (error);
 
 	g_free (update_filename);
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]