[tracker/wip/fts4: 4/21] libtracker-data: Create several columns for FTS data



commit 1b788ea12829709f0999d2c76f0d0733153eab61
Author: Carlos Garnacho <carlos lanedo com>
Date:   Fri Nov 4 12:47:27 2011 +0100

    libtracker-data: Create several columns for FTS data
    
    there is now one column per ontology property with FTS enabled,
    the column name is that of the property.

 src/libtracker-data/tracker-data-manager.c        |   51 ++++++++-
 src/libtracker-data/tracker-data-update.c         |   21 ++--
 src/libtracker-data/tracker-db-interface-sqlite.c |  120 +++++++-------------
 src/libtracker-data/tracker-db-interface-sqlite.h |    9 +-
 src/libtracker-data/tracker-db-manager.c          |    2 +-
 src/libtracker-fts/tracker-fts.c                  |   38 +++++---
 src/libtracker-fts/tracker-fts.h                  |    6 +-
 7 files changed, 136 insertions(+), 111 deletions(-)
---
diff --git a/src/libtracker-data/tracker-data-manager.c b/src/libtracker-data/tracker-data-manager.c
index e700da6..81f4adb 100644
--- a/src/libtracker-data/tracker-data-manager.c
+++ b/src/libtracker-data/tracker-data-manager.c
@@ -2487,7 +2487,6 @@ db_get_static_data (TrackerDBInterface  *iface,
 	}
 }
 
-
 static void
 insert_uri_in_resource_table (TrackerDBInterface  *iface,
                               const gchar         *uri,
@@ -3583,6 +3582,48 @@ load_ontologies_gvdb (GError **error)
 	g_free (filename);
 }
 
+static gint
+compare_fts_property_ids (gconstpointer a,
+                          gconstpointer b)
+{
+	TrackerProperty *pa, *pb;
+
+	pa = (TrackerProperty *) a;
+	pb = (TrackerProperty *) b;
+
+	return tracker_property_get_id (pa) - tracker_property_get_id (pb);
+}
+
+static const gchar **
+ontology_get_fts_properties (void)
+{
+	TrackerProperty **properties;
+	GList *fts_props = NULL, *l;
+	const gchar **prop_names;
+	guint i, len;
+
+	properties = tracker_ontologies_get_properties (&len);
+
+	for (i = 0; i < len; i++) {
+		if (tracker_property_get_fulltext_indexed (properties[i])) {
+			/* Sort them by ID */
+			fts_props =
+				g_list_insert_sorted (fts_props, properties[i],
+						      (GCompareFunc) compare_fts_property_ids);
+		}
+	}
+
+	prop_names = g_new0 (const gchar *, g_list_length (fts_props) + 1);
+
+	for (l = fts_props, i = 0; l; l = l->next, i++) {
+		prop_names[i] = tracker_property_get_name (l->data);
+	}
+
+	g_list_free (fts_props);
+
+	return prop_names;
+}
+
 gboolean
 tracker_data_manager_init (TrackerDBManagerFlags   flags,
                            const gchar           **test_schemas,
@@ -3760,6 +3801,8 @@ tracker_data_manager_init (TrackerDBManagerFlags   flags,
 #endif /* DISABLE_JOURNAL */
 
 	if (is_first_time_index && !read_only) {
+		const gchar **fts_props;
+
 		sorted = get_ontologies (test_schemas != NULL, ontologies_dir);
 
 #ifndef DISABLE_JOURNAL
@@ -3848,8 +3891,10 @@ tracker_data_manager_init (TrackerDBManagerFlags   flags,
 			return FALSE;
 		}
 
+		fts_props = ontology_get_fts_properties ();
 		/* This is a no-op when FTS is disabled */
-		tracker_db_interface_sqlite_fts_init (iface, TRUE);
+		tracker_db_interface_sqlite_fts_init (iface, fts_props, TRUE);
+		g_free (fts_props);
 
 		tracker_data_ontology_import_into_db (FALSE,
 		                                      &internal_error);
@@ -3997,7 +4042,7 @@ tracker_data_manager_init (TrackerDBManagerFlags   flags,
 		}
 
 		/* This is a no-op when FTS is disabled */
-		tracker_db_interface_sqlite_fts_init (iface, FALSE);
+		tracker_db_interface_sqlite_fts_init (iface, NULL, FALSE);
 	}
 
 	if (check_ontology) {
diff --git a/src/libtracker-data/tracker-data-update.c b/src/libtracker-data/tracker-data-update.c
index c09c648..e4accfa 100644
--- a/src/libtracker-data/tracker-data-update.c
+++ b/src/libtracker-data/tracker-data-update.c
@@ -961,8 +961,7 @@ tracker_data_resource_buffer_flush (GError **error)
 	if (resource_buffer->fts_updated) {
 		TrackerProperty *prop;
 		GValueArray *values;
-
-		tracker_db_interface_sqlite_fts_update_init (iface, resource_buffer->id);
+		gboolean create = resource_buffer->create;
 
 		g_hash_table_iter_init (&iter, resource_buffer->predicates);
 		while (g_hash_table_iter_next (&iter, (gpointer*) &prop, (gpointer*) &values)) {
@@ -975,15 +974,17 @@ tracker_data_resource_buffer_flush (GError **error)
 					g_string_append_c (fts, ' ');
 				}
 				tracker_db_interface_sqlite_fts_update_text (iface,
-					resource_buffer->id,
-					tracker_data_query_resource_id (tracker_property_get_uri (prop)),
-					fts->str,
-					!tracker_property_get_fulltext_no_limit (prop));
+				                                             resource_buffer->id,
+				                                             tracker_property_get_name (prop),
+				                                             fts->str,
+				                                             !tracker_property_get_fulltext_no_limit (prop),
+				                                             create);
 				g_string_free (fts, TRUE);
 
 				/* Set that we ever updated FTS, so that tracker_db_interface_sqlite_fts_update_commit()
 				 * gets called */
 				update_buffer.fts_ever_updated = TRUE;
+				create = FALSE;
 			}
 		}
 	}
@@ -1450,8 +1451,6 @@ get_old_property_values (TrackerProperty  *property,
 				/* first fulltext indexed property to be modified
 				 * retrieve values of all fulltext indexed properties
 				 */
-				tracker_db_interface_sqlite_fts_update_init (iface, resource_buffer->id);
-
 				properties = tracker_ontologies_get_properties (&n_props);
 
 				for (i = 0; i < n_props; i++) {
@@ -1459,17 +1458,17 @@ get_old_property_values (TrackerProperty  *property,
 
 					if (tracker_property_get_fulltext_indexed (prop)
 					    && check_property_domain (prop)) {
-						gint prop_id;
+						const gchar *property_name;
 						gint i;
 
 						old_values = get_property_values (prop);
-						prop_id = tracker_data_query_resource_id (tracker_property_get_uri (prop));
+						property_name = tracker_property_get_name (prop);
 
 						/* delete old fts entries */
 						for (i = 0; i < old_values->n_values; i++) {
 							tracker_db_interface_sqlite_fts_delete_text (iface,
 							                                             resource_buffer->id,
-							                                             prop_id);
+							                                             property_name);
 						}
 					}
 				}
diff --git a/src/libtracker-data/tracker-db-interface-sqlite.c b/src/libtracker-data/tracker-db-interface-sqlite.c
index 4888e96..165bba6 100644
--- a/src/libtracker-data/tracker-db-interface-sqlite.c
+++ b/src/libtracker-data/tracker-db-interface-sqlite.c
@@ -992,11 +992,15 @@ close_database (TrackerDBInterface *db_interface)
 }
 
 void
-tracker_db_interface_sqlite_fts_init (TrackerDBInterface *db_interface,
-                                      gboolean            create)
+tracker_db_interface_sqlite_fts_init (TrackerDBInterface  *db_interface,
+                                      const gchar        **columns,
+                                      gboolean             create)
 {
 #if HAVE_TRACKER_FTS
-	if (!tracker_fts_init_db (db_interface->db, create)) {
+	tracker_fts_init_db (db_interface->db);
+
+	if (create &&
+	    !tracker_fts_create_table (db_interface->db, "fts", columns)) {
 		g_warning ("FTS tables creation failed");
 	}
 #endif
@@ -1013,63 +1017,48 @@ tracker_db_interface_sqlite_fts_update_init (TrackerDBInterface *db_interface,
 gboolean
 tracker_db_interface_sqlite_fts_update_text (TrackerDBInterface *db_interface,
                                              int                 id,
-                                             int                 column_id,
+                                             const gchar        *property,
                                              const char         *text,
-                                             gboolean            limit_word_length)
+                                             gboolean            limit_word_length,
+                                             gboolean            create)
 {
 	TrackerDBStatement *stmt;
 	GError *error = NULL;
+	gchar *query;
 
-	if (!text || !*text || column_id < 0) {
-		return FALSE;
+	if (create) {
+		query = g_strdup_printf ("INSERT INTO fts (docid, \"%s\") "
+		                         "VALUES (?, ?) ",
+		                         property);
+	} else {
+		query = g_strdup_printf ("UPDATE fts "
+		                         "SET \"%s\" = ? "
+		                         "WHERE docid = ? ",
+		                         property);
 	}
 
-	/* Insert docid/property into property map */
 	stmt = tracker_db_interface_create_statement (db_interface,
 	                                              TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE,
-	                                              &error,
-	                                              "INSERT OR REPLACE "
-						      "INTO fts_prop_map "
-						      "  (resourceid, propid) "
-						      "VALUES (?, ?)");
+	                                              &error, query);
+	g_free (query);
 
 	if (!stmt || error) {
 		if (error) {
-			g_warning ("Could not create FTS property map insert statement: %s\n",
+			g_warning ("Could not create FTS insert statement: %s\n",
 			           error->message);
 			g_error_free (error);
 		}
 		return FALSE;
 	}
 
-	tracker_db_statement_bind_int (stmt, 0, id);
-	tracker_db_statement_bind_int (stmt, 1, column_id);
-
-	tracker_db_statement_execute (stmt, &error);
-	g_object_unref (stmt);
-
-	if (error) {
-		g_warning ("Could not insert FTS property map: %s", error->message);
-		g_error_free (error);
-		return FALSE;
-	}
-
-	stmt = tracker_db_interface_create_statement (db_interface,
-	                                              TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE,
-	                                              &error,
-	                                              "INSERT OR REPLACE "
-						      "INTO fts (docid,content) "
-						      "VALUES (last_insert_rowid(), ?)");
-	if (!stmt || error) {
-		if (error) {
-			g_warning ("Could not create FTS insert statement: %s\n",
-			           error->message);
-			g_error_free (error);
-		}
-		return FALSE;
+	if (create) {
+		tracker_db_statement_bind_int (stmt, 0, id);
+		tracker_db_statement_bind_text (stmt, 1, text);
+	} else {
+		tracker_db_statement_bind_text (stmt, 0, text);
+		tracker_db_statement_bind_int (stmt, 1, id);
 	}
 
-	tracker_db_statement_bind_text (stmt, 0, text);
 	tracker_db_statement_execute (stmt, &error);
 	g_object_unref (stmt);
 
@@ -1085,62 +1074,37 @@ tracker_db_interface_sqlite_fts_update_text (TrackerDBInterface *db_interface,
 gboolean
 tracker_db_interface_sqlite_fts_delete_text (TrackerDBInterface *db_interface,
                                              int                 id,
-                                             int                 column_id)
+					     const gchar        *property)
 {
 	TrackerDBStatement *stmt;
 	GError *error = NULL;
-	int doc_id;
-
-	stmt = tracker_db_interface_create_statement (db_interface,
-	                                              TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE,
-	                                              &error,
-	                                              "DELETE FROM fts "
-	                                              "WHERE docid IN ("
-	                                              "  SELECT docid "
-	                                              "  FROM fts_prop_map "
-	                                              "  WHERE resourceid = ? AND propid = ?"
-	                                              ")");
-	if (!stmt) {
-		if (error) {
-			g_warning ("Could not delete FTS text: %s\n",
-			           error->message);
-			g_error_free (error);
-		}
-		return FALSE;
-	}
+	gchar *query;
 
-	tracker_db_statement_execute (stmt, &error);
-	g_object_unref (stmt);
-
-	if (error) {
-		g_warning ("Could not execute FTS text deletion: %s", error->message);
-		g_error_free (error);
-		return FALSE;
-	}
+	query = g_strdup_printf ("UPDATE fts "
+	                         "SET \"%s\" = '' "
+	                         "WHERE docid = ?",
+	                         property);
 
 	stmt = tracker_db_interface_create_statement (db_interface,
 	                                              TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE,
-	                                              &error,
-	                                              "DELETE FROM fts "
-	                                              "WHERE docid IN ("
-	                                              "  SELECT docid "
-	                                              "  FROM fts_prop_map "
-	                                              "  WHERE resourceid = ? AND propid = ?"
-	                                              ")");
-	if (!stmt) {
+	                                              &error, query);
+	g_free (query);
+
+	if (!stmt || error) {
 		if (error) {
-			g_warning ("Could not delete FTS property map: %s\n",
+			g_warning ("Could not create FTS update statement: %s\n",
 			           error->message);
 			g_error_free (error);
 		}
 		return FALSE;
 	}
 
+	tracker_db_statement_bind_int (stmt, 0, id);
 	tracker_db_statement_execute (stmt, &error);
 	g_object_unref (stmt);
 
 	if (error) {
-		g_warning ("Could not execute FTS properties deletion: %s", error->message);
+		g_warning ("Could not execute FTS update: %s", error->message);
 		g_error_free (error);
 		return FALSE;
 	}
diff --git a/src/libtracker-data/tracker-db-interface-sqlite.h b/src/libtracker-data/tracker-db-interface-sqlite.h
index f31d943..44c4232 100644
--- a/src/libtracker-data/tracker-db-interface-sqlite.h
+++ b/src/libtracker-data/tracker-db-interface-sqlite.h
@@ -41,6 +41,7 @@ TrackerDBInterface *tracker_db_interface_sqlite_new_ro                 (const gc
 gint64              tracker_db_interface_sqlite_get_last_insert_id     (TrackerDBInterface       *interface);
 void                tracker_db_interface_sqlite_enable_shared_cache    (void);
 void                tracker_db_interface_sqlite_fts_init               (TrackerDBInterface       *interface,
+                                                                        const gchar             **columns,
                                                                         gboolean                  create);
 void                tracker_db_interface_sqlite_reset_collator         (TrackerDBInterface       *interface);
 void                tracker_db_interface_sqlite_wal_hook               (TrackerDBInterface       *interface,
@@ -51,12 +52,14 @@ int                 tracker_db_interface_sqlite_fts_update_init        (TrackerD
                                                                         int                       id);
 int                 tracker_db_interface_sqlite_fts_update_text        (TrackerDBInterface       *interface,
                                                                         int                       id,
-                                                                        int                       column_id,
+									const gchar              *property,
                                                                         const char               *text,
-                                                                        gboolean                  limit_word_length);
+                                                                        gboolean                  limit_word_length,
+                                                                        gboolean                  create);
+
 gboolean            tracker_db_interface_sqlite_fts_delete_text        (TrackerDBInterface       *db_interface,
 									int                       id,
-									int                       column_id);
+									const gchar              *property);
 void                tracker_db_interface_sqlite_fts_update_commit      (TrackerDBInterface       *interface);
 void                tracker_db_interface_sqlite_fts_update_rollback    (TrackerDBInterface       *interface);
 #endif
diff --git a/src/libtracker-data/tracker-db-manager.c b/src/libtracker-data/tracker-db-manager.c
index 1deca39..ab59148 100644
--- a/src/libtracker-data/tracker-db-manager.c
+++ b/src/libtracker-data/tracker-db-manager.c
@@ -1522,7 +1522,7 @@ tracker_db_manager_get_db_interface (void)
 			return NULL;
 		}
 
-		tracker_db_interface_sqlite_fts_init (interface, FALSE);
+		tracker_db_interface_sqlite_fts_init (interface, NULL, FALSE);
 
 
 		tracker_db_interface_set_max_stmt_cache_size (interface,
diff --git a/src/libtracker-fts/tracker-fts.c b/src/libtracker-fts/tracker-fts.c
index 57b8ef0..23c6f46 100644
--- a/src/libtracker-fts/tracker-fts.c
+++ b/src/libtracker-fts/tracker-fts.c
@@ -24,7 +24,8 @@
 #include "tracker-fts.h"
 #include "fts3.h"
 
-gboolean tracker_fts_init (void) {
+gboolean
+tracker_fts_init (void) {
 	static gsize module_initialized = 0;
 	int rc = SQLITE_OK;
 
@@ -36,23 +37,34 @@ gboolean tracker_fts_init (void) {
 	return (module_initialized != 0);
 }
 
-gboolean tracker_fts_init_db (sqlite3 *db, int create){
-	int rc = SQLITE_OK;
-
+gboolean
+tracker_fts_init_db (sqlite3 *db) {
 	if (!tracker_tokenizer_initialize (db)) {
 		return FALSE;
 	}
 
-	if (create){
-		rc = sqlite3_exec(db,
-		                  "CREATE VIRTUAL TABLE fts "
-		                  "USING fts4(tokenize=TrackerTokenizer)",
-		                  NULL, 0, NULL);
-	}
+	return TRUE;
+}
 
-	if (SQLITE_OK != rc){
-		return FALSE;
+gboolean
+tracker_fts_create_table (sqlite3  *db,
+			  gchar    *table_name,
+			  gchar   **column_names)
+{
+	GString *str;
+	gint i, rc;
+
+	str = g_string_new ("CREATE VIRTUAL TABLE ");
+	g_string_append_printf (str, "%s USING fts4(", table_name);
+
+	for (i = 0; column_names[i]; i++) {
+		g_string_append_printf (str, "\"%s\", ", column_names[i]);
 	}
 
-	return TRUE;
+	g_string_append (str, " tokenize=TrackerTokenizer)");
+
+	rc = sqlite3_exec(db, str->str, NULL, 0, NULL);
+	g_string_free (str, TRUE);
+
+	return (rc == SQLITE_OK);
 }
diff --git a/src/libtracker-fts/tracker-fts.h b/src/libtracker-fts/tracker-fts.h
index a1beb66..9284885 100644
--- a/src/libtracker-fts/tracker-fts.h
+++ b/src/libtracker-fts/tracker-fts.h
@@ -28,8 +28,10 @@
 G_BEGIN_DECLS
 
 gboolean    tracker_fts_init             (void);
-gboolean    tracker_fts_init_db          (sqlite3    *db,
-                                          int         create);
+gboolean    tracker_fts_init_db          (sqlite3    *db);
+gboolean    tracker_fts_create_table     (sqlite3    *db,
+					  gchar      *table_name,
+					  gchar     **column_names);
 
 G_END_DECLS
 



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]