Index: ChangeLog =================================================================== --- ChangeLog (revision 2710) +++ ChangeLog (working copy) @@ -1,3 +1,14 @@ +2008-12-16 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/tracker-indexer/tracker-indexer.c + * src/libtracker-data/tracker-turtle.h + * src/libtracker-data/tracker-data-metadata.c + * src/libtracker-data/tracker-data-update.c + * src/libtracker-data/tracker-data-update.h + * src/trackerd/tracker-main.c + * src/libtracker-common/tracker-hal.c: Fixed comments from martyn + 2008-12-16 Martyn Russell * src/libtracker-common/tracker-config.c: @@ -21,6 +32,37 @@ for embedded album-art * src/libtracker-common/tracker-thumbnailer.c: coping with paths +2008-12-11 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/tracker-indexer/tracker-removable-device.h + * src/tracker-indexer/tracker-main.c + * src/tracker-indexer/tracker-indexer.c + * src/tracker-indexer/Makefile.am + * src/tracker-indexer/tracker-indexer.h + * src/tracker-indexer/tracker-module-metadata.c + * src/libtracker-data/tracker-data-query.c + * src/libtracker-data/tracker-turtle.c + * src/libtracker-data/tracker-data-query.h + * src/libtracker-data/tracker-turtle.h + * src/libtracker-data/tracker-data-metadata.c + * src/libtracker-data/tracker-data-update.c + * src/libtracker-data/tracker-data-metadata.h + * src/libtracker-data/tracker-data-update.h + * src/libtracker-data/Makefile.am + * src/trackerd/tracker-main.c + * src/trackerd/tracker-backup.c + * src/trackerd/tracker-backup.h + * src/trackerd/Makefile.am + * src/libtracker-db/tracker-db-manager.c + * src/libtracker-common/tracker-hal.c + * src/libtracker-common/tracker-hal.h + * tests/tracker-indexer/Makefile.am + * configure.ac + * data/db/sqlite-stored-procs.sql + * data/dbus/tracker-indexer.xml: Recreated the Turtle branch, and + applied merge-to-trunk patch on it + 2008-12-10 Mikael Ottela * src/tracker-utils/tracker-query.c: Index: src/tracker-indexer/tracker-main.c =================================================================== --- src/tracker-indexer/tracker-main.c (revision 2710) +++ src/tracker-indexer/tracker-main.c (working copy) @@ -43,6 +43,7 @@ #include #include +#include #include "tracker-dbus.h" #include "tracker-indexer.h" @@ -370,13 +371,18 @@ tracker_indexer_process_modules (indexer, modules); } + tracker_turtle_init (); + g_message ("Starting..."); + main_loop = g_main_loop_new (NULL, FALSE); g_main_loop_run (main_loop); g_message ("Shutdown started"); + tracker_turtle_shutdown (); + if (quit_timeout_id) { g_source_remove (quit_timeout_id); } Index: src/tracker-indexer/tracker-indexer.c =================================================================== --- src/tracker-indexer/tracker-indexer.c (revision 2710) +++ src/tracker-indexer/tracker-indexer.c (working copy) @@ -71,11 +71,13 @@ #include #include +#include #include "tracker-indexer.h" #include "tracker-indexer-module.h" #include "tracker-marshal.h" #include "tracker-module-metadata-private.h" +#include "tracker-removable-device.h" #define TRACKER_INDEXER_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_INDEXER, TrackerIndexerPrivate)) @@ -85,9 +87,6 @@ #define LOW_DISK_CHECK_FREQUENCY 10 #define SIGNAL_STATUS_FREQUENCY 10 -/* Transaction every 'x' items */ -#define TRANSACTION_MAX 2000 - /* Throttle defaults */ #define THROTTLE_DEFAULT 0 #define THROTTLE_DEFAULT_ON_BATTERY 5 @@ -385,6 +384,22 @@ indexer); } + +void +tracker_indexer_transaction_commit (TrackerIndexer *indexer) +{ + stop_transaction (indexer); + tracker_indexer_set_running (indexer, TRUE); + +} + +void +tracker_indexer_transaction_open (TrackerIndexer *indexer) +{ + tracker_indexer_set_running (indexer, FALSE); + start_transaction (indexer); +} + #ifdef HAVE_HAL static void @@ -1365,6 +1380,7 @@ } } + static void item_add_or_update (TrackerIndexer *indexer, PathInfo *info, @@ -1375,6 +1391,8 @@ TrackerService *service; gchar *text; guint32 id; + gchar *mount_point = NULL; + gchar *service_path; service = get_service_for_file (info->module_file, info->module); @@ -1419,11 +1437,11 @@ new_text = tracker_module_file_get_text (info->module_file); item_update_content (indexer, service, id, old_text, new_text); - g_free (old_text); g_free (new_text); tracker_data_metadata_free (old_metadata_emb); tracker_data_metadata_free (old_metadata_non_emb); + } else { GHashTable *data; @@ -1463,8 +1481,32 @@ g_hash_table_destroy (data); } + + /* TODO: URI branch path -> uri */ + + service_path = g_build_path (G_DIR_SEPARATOR_S, + dirname, + basename, + NULL); + + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + service_path, + &mount_point, + NULL)) { + + tracker_removable_device_add_metadata (indexer, + mount_point, + service_path, + tracker_service_get_name (service), + metadata); + } + + g_free (mount_point); + g_free (service_path); + } + static gboolean filter_invalid_after_move_properties (TrackerField *field, gpointer value, @@ -1491,6 +1533,7 @@ gchar *path, *other_path; gchar *uri, *other_uri, *mime_type; guint32 service_id; + gchar *mount_point = NULL; service = get_service_for_file (info->other_module_file, info->module); @@ -1541,6 +1584,32 @@ tracker_data_update_move_service (service, path, other_path); + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + path, + &mount_point, + NULL) ) { + + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + other_path, + NULL, + NULL) ) { + + tracker_removable_device_add_move (indexer, + mount_point, + path, + other_path, + tracker_service_get_name (service)); + + } else { + tracker_removable_device_add_removal (indexer, + mount_point, + path, + tracker_service_get_name (service)); + } + } + + g_free (mount_point); + /* * Updating what changes in move event (Path related properties) */ @@ -1576,6 +1645,7 @@ g_free (other_path); } + static void item_remove (TrackerIndexer *indexer, PathInfo *info, @@ -1587,6 +1657,7 @@ gchar *content; gchar *metadata; gchar *path; + gchar *mount_point = NULL; const gchar *service_type; guint service_id, service_type_id; @@ -1693,8 +1764,19 @@ tracker_data_update_delete_service_recursively (service, path); } + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + path, + &mount_point, + NULL)) { + + tracker_removable_device_add_removal (indexer, mount_point, + path, + tracker_service_get_name (service)); + } + tracker_data_update_decrement_stats (indexer->private->common, service); + g_free (mount_point); g_free (path); } @@ -2392,7 +2474,7 @@ /* Signal stopped and clean up */ check_stopped (indexer, FALSE); check_disk_space_stop (indexer); - + return FALSE; } @@ -2400,7 +2482,7 @@ g_free (module_name); } - if (indexer->private->items_processed > TRANSACTION_MAX) { + if (indexer->private->items_processed > TRACKER_INDEXER_TRANSACTION_MAX) { schedule_flush (indexer, TRUE); } @@ -2848,6 +2930,13 @@ dbus_g_method_return (context); tracker_dbus_request_success (request_id); + + /* tracker_turtle_process_ttl will be spinning the mainloop, therefore + we can already return the DBus method */ + + if (enabled) { + tracker_removable_device_load (indexer, path); + } } void @@ -2941,7 +3030,64 @@ tracker_dbus_request_success (request_id); } +static void +restore_backup_triple (void *user_data, const TrackerRaptorStatement *triple) { + + const gchar *values[2]; + TrackerIndexer *indexer = user_data; + GError *error = NULL; + + g_return_if_fail (TRACKER_IS_INDEXER (indexer)); + + g_debug ("Turtle loading <%s, %s, %s>", + (gchar *)triple->subject, + (gchar *)triple->predicate, + (gchar *)triple->object); + + values[0] = triple->object; + values[1] = NULL; + + handle_metadata_add (indexer, + "Files", + triple->subject, + triple->predicate, + (GStrv) values, + &error); + + if (error) { + g_warning ("Restoring backup: %s", error->message); + g_error_free (error); + } + +} + void +tracker_indexer_restore_backup (TrackerIndexer *indexer, + const gchar *backup_file, + DBusGMethodInvocation *context, + GError **error) +{ + guint request_id; + + request_id = tracker_dbus_get_next_request_id (); + + tracker_dbus_async_return_if_fail (TRACKER_IS_INDEXER (indexer), context); + + tracker_dbus_request_new (request_id, + "DBus request to restore backup data from '%s'", + backup_file); + + tracker_turtle_process (backup_file, + "/", + (TurtleTripleCallback) restore_backup_triple, + indexer); + + dbus_g_method_return (context); + tracker_dbus_request_success (request_id); +} + + +void tracker_indexer_shutdown (TrackerIndexer *indexer, DBusGMethodInvocation *context, GError **error) Index: src/tracker-indexer/Makefile.am =================================================================== --- src/tracker-indexer/Makefile.am (revision 2710) +++ src/tracker-indexer/Makefile.am (working copy) @@ -13,6 +13,7 @@ -I$(top_srcdir)/src \ $(DBUS_CFLAGS) \ $(PANGO_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GMODULE_CFLAGS) libtracker_moduledir = $(libdir)/tracker @@ -42,8 +43,9 @@ tracker-indexer-module.c \ tracker-indexer-module.h \ tracker-main.c \ + tracker-module-metadata-private.h \ tracker-marshal-main.c \ - tracker-module-metadata-private.h + tracker-removable-device.c tracker_indexer_LDADD = \ libtracker-module.la \ @@ -58,6 +60,7 @@ $(PANGO_LIBS) \ $(GIO_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ -lz \ -lm Index: src/tracker-indexer/tracker-indexer.h =================================================================== --- src/tracker-indexer/tracker-indexer.h (revision 2710) +++ src/tracker-indexer/tracker-indexer.h (working copy) @@ -30,6 +30,9 @@ #define TRACKER_INDEXER_PATH "/org/freedesktop/Tracker/Indexer" #define TRACKER_INDEXER_INTERFACE "org.freedesktop.Tracker.Indexer" +/* Transaction every 'x' items */ +#define TRACKER_INDEXER_TRANSACTION_MAX 2000 + G_BEGIN_DECLS #define TRACKER_TYPE_INDEXER (tracker_indexer_get_type()) @@ -80,6 +83,8 @@ void tracker_indexer_process_all (TrackerIndexer *indexer); void tracker_indexer_process_modules (TrackerIndexer *indexer, gchar **modules); +void tracker_indexer_transaction_commit (TrackerIndexer *indexer); +void tracker_indexer_transaction_open (TrackerIndexer *indexer); /* DBus methods */ void tracker_indexer_pause (TrackerIndexer *indexer, @@ -136,6 +141,10 @@ GStrv values, DBusGMethodInvocation *context, GError **error); +void tracker_indexer_restore_backup (TrackerIndexer *indexer, + const gchar *backup_file, + DBusGMethodInvocation *context, + GError **error); void tracker_indexer_shutdown (TrackerIndexer *indexer, DBusGMethodInvocation *context, GError **error); Index: src/tracker-indexer/tracker-module-metadata.c =================================================================== --- src/tracker-indexer/tracker-module-metadata.c (revision 2710) +++ src/tracker-indexer/tracker-module-metadata.c (working copy) @@ -188,13 +188,15 @@ list = g_list_prepend (list, value); data = list; } else { - /* FIXME: warn if data already exists */ + data = g_hash_table_lookup (metadata->table, field); + g_free (data); data = value; } - g_hash_table_insert (metadata->table, - g_object_ref (field), - data); + g_hash_table_replace (metadata->table, + g_object_ref (field), + data); + return TRUE; } Index: src/libtracker-data/tracker-data-query.c =================================================================== --- src/libtracker-data/tracker-data-query.c (revision 2710) +++ src/libtracker-data/tracker-data-query.c (working copy) @@ -386,6 +386,9 @@ return service_type_id; } +/* + * Result set with (metadataID, value) per row + */ static void result_set_to_metadata (TrackerDBResultSet *result_set, TrackerDataMetadata *metadata, @@ -400,7 +403,7 @@ GValue transform = {0, }; GValue value = {0, }; gchar *str; - + g_value_init (&transform, G_TYPE_STRING); tracker_db_result_set_get (result_set, 0, &metadata_id, -1); _tracker_db_result_set_get_value (result_set, 1, &value); @@ -492,6 +495,23 @@ return metadata; } +TrackerDBResultSet * +tracker_data_query_backup_metadata (TrackerService *service) +{ + TrackerDBInterface *iface; + TrackerDBResultSet *result_set; + GHashTable *results; + + g_return_val_if_fail (TRACKER_IS_SERVICE (service), NULL); + + iface = tracker_db_manager_get_db_interface_by_service (tracker_service_get_name (service)); + + result_set = tracker_data_manager_exec_proc (iface, + "GetUserMetadataBackup", + NULL); + return result_set; +} + static gchar * db_get_metadata (TrackerService *service, guint service_id, Index: src/libtracker-data/tracker-data-query.h =================================================================== --- src/libtracker-data/tracker-data-query.h (revision 2710) +++ src/libtracker-data/tracker-data-query.h (working copy) @@ -53,6 +53,7 @@ TrackerDataMetadata *tracker_data_query_metadata (TrackerService *service, guint32 service_id, gboolean embedded); +TrackerDBResultSet *tracker_data_query_backup_metadata (TrackerService *service); gchar * tracker_data_query_parsed_metadata (TrackerService *service, guint32 service_id); gchar * tracker_data_query_unparsed_metadata (TrackerService *service, Index: src/libtracker-data/tracker-data-metadata.c =================================================================== --- src/libtracker-data/tracker-data-metadata.c (revision 2710) +++ src/libtracker-data/tracker-data-metadata.c (working copy) @@ -106,6 +106,7 @@ const gchar *value) { TrackerField *field; + gchar *old_value; g_return_if_fail (metadata != NULL); g_return_if_fail (field_name != NULL); @@ -114,13 +115,17 @@ field = tracker_ontology_get_field_by_name (field_name); g_return_if_fail (TRACKER_IS_FIELD (field)); - g_return_if_fail (tracker_field_get_multiple_values (field) == FALSE); + g_return_if_fail (tracker_field_get_multiple_values (field) == FALSE); - g_hash_table_insert (metadata->table, - g_object_ref (field), - g_strdup (value)); + old_value = g_hash_table_lookup (metadata->table, field); + g_free (old_value); + + g_hash_table_replace (metadata->table, + g_object_ref (field), + g_strdup (value)); } + /** * tracker_data_metadata_insert_values: * @metadata: A #TrackerDataMetadata @@ -137,9 +142,10 @@ void tracker_data_metadata_insert_values (TrackerDataMetadata *metadata, const gchar *field_name, - GList *list) + const GList *list) { TrackerField *field; + GList *old_values; g_return_if_fail (metadata != NULL); g_return_if_fail (field_name != NULL); @@ -150,12 +156,24 @@ field = tracker_ontology_get_field_by_name (field_name); + if (!field) { + g_warning ("Field name '%s' has isn't described in the ontology", field_name); + return; + } + g_return_if_fail (TRACKER_IS_FIELD (field)); g_return_if_fail (tracker_field_get_multiple_values (field) == TRUE); - g_hash_table_insert (metadata->table, - g_object_ref (field), - tracker_glist_copy_with_string_data (list)); + old_values = g_hash_table_lookup (metadata->table, field); + + if (old_values) { + g_list_foreach (old_values, (GFunc) g_free, NULL); + g_list_free (old_values); + } + + g_hash_table_replace (metadata->table, + g_object_ref (field), + tracker_glist_copy_with_string_data ((GList *)list)); } /** Index: src/libtracker-data/tracker-data-update.c =================================================================== --- src/libtracker-data/tracker-data-update.c (revision 2710) +++ src/libtracker-data/tracker-data-update.c (working copy) @@ -26,12 +26,21 @@ #include #include +#include #include #include #include "tracker-data-manager.h" #include "tracker-data-update.h" +typedef struct { + TrackerService *service; + guint32 iid_value; + TrackerLanguage *language; + TrackerConfig *config; +} ForeachInMetadataInfo; + + guint32 tracker_data_update_get_new_service_id (TrackerDBInterface *iface) { @@ -566,7 +575,240 @@ tracker_data_manager_exec (iface, "DELETE FROM Events WHERE BeingHandled = 1"); } +/* TODO: URI branch path -> uri */ + void +tracker_data_update_delete_service_by_path (const gchar *path, + const gchar *rdf_type) +{ + TrackerService *service; + const gchar *service_type; + guint32 service_id; + + g_return_if_fail (path != NULL); + + if (!rdf_type) + return; + + service = tracker_ontology_get_service_by_name (rdf_type); + service_type = tracker_service_get_name (service); + service_id = tracker_data_query_file_id (service_type, path); + + /* When merging from the decomposed branch to trunk then this function + * wont exist in the decomposed branch. Create it based on this one. */ + + if (service_id != 0) { + tracker_data_update_delete_service (service, service_id); + if (strcmp (service_type, "Folders") == 0) { + tracker_data_update_delete_service_recursively (service, (gchar *) path); + } + tracker_data_update_delete_all_metadata (service, service_id); + } +} + +/* TODO: URI branch path -> uri */ + +static void +set_metadata (TrackerField *field, + gpointer value, + ForeachInMetadataInfo *info) +{ + TrackerDBIndex *index; + gchar *parsed_value; + gchar **arr; + gint service_id; + gint i; + gint score; + + /* TODO untested and unfinished port that came from the decomposed + * branch of Jürg. When merging from the decomposed branch to trunk + * then pick the version in the decomposed branch for this function */ + + parsed_value = tracker_parser_text_to_string (value, + info->language, + tracker_config_get_max_word_length (info->config), + tracker_config_get_min_word_length (info->config), + tracker_field_get_filtered (field), + tracker_field_get_filtered (field), + tracker_field_get_delimited (field)); + + if (!parsed_value) { + return; + } + + score = tracker_field_get_weight (field); + + arr = g_strsplit (parsed_value, " ", -1); + service_id = tracker_service_get_id (info->service); + index = tracker_db_index_manager_get_index_by_service_id (service_id); + + for (i = 0; arr[i]; i++) { + tracker_db_index_add_word (index, + arr[i], + info->iid_value, + tracker_service_get_id (info->service), + score); + } + + tracker_data_update_set_metadata (info->service, info->iid_value, field, value, parsed_value); + + g_free (parsed_value); + g_strfreev (arr); + +} + +static void +foreach_in_metadata_set_metadata (gpointer predicate, + gpointer value, + gpointer user_data) +{ + ForeachInMetadataInfo *info = user_data; + gchar *parsed_value; + gint throttle; + TrackerField *field; + + field = tracker_ontology_get_field_by_name (predicate); + + if (!field) + return; + + /* Throttle indexer, value 9 is from older code, why 9? */ + throttle = tracker_config_get_throttle (info->config); + if (throttle > 9) { + tracker_throttle (info->config, throttle * 100); + } + + if (!tracker_field_get_multiple_values (field)) { + set_metadata (field, value, user_data); + } else { + GList *list; + for (list = value; list; list = list->next) + set_metadata (field, list->data, user_data); + } +} + +/* TODO: URI branch path -> uri */ + +void +tracker_data_update_replace_service (const gchar *path, + const gchar *rdf_type, + GHashTable *metadata) +{ + TrackerDBInterface *iface; + TrackerDBResultSet *result_set; + const gchar *modified; + GError *error = NULL; + TrackerService *service; + gchar *escaped_path; + gchar *dirname; + const gchar *basename; + + g_return_if_fail (path != NULL); + g_return_if_fail (metadata != NULL); + + /* When merging from the decomposed branch to trunk then pick the version + * in the decomposed branch for this function. However, carefully + * compare the features, as this version is more recent and has + * implemented a few significant items, whereas the version in the + * decomposed branch was a proof of concept implementation, and might + * not have these needed features. */ + + if (!rdf_type) + return; + + service = tracker_ontology_get_service_by_name (rdf_type); + + iface = tracker_db_manager_get_db_interface_by_type (tracker_service_get_name (service), + TRACKER_DB_CONTENT_TYPE_METADATA); + + modified = g_hash_table_lookup (metadata, "File:Modified"); + + if (!modified) { + return; + } + + escaped_path = tracker_escape_string (path); + + basename = g_basename (escaped_path); + dirname = g_dirname (escaped_path); + + /* TODO Warning: comparing Modified against Accessed. Do we have a + * better field for this? */ + + result_set = tracker_db_interface_execute_query (iface, &error, + "SELECT ID, Accessed < '%s' FROM Services " + "WHERE Path = '%s' AND " + "Name = '%s'", + modified, + dirname, basename); + + if (error) { + g_error_free (error); + } + + if (result_set) { + GValue id_value = { 0, }; + GValue is_value = { 0, }; + gint iid_value, iis_value; + + _tracker_db_result_set_get_value (result_set, 0, &id_value); + iid_value = g_value_get_int (&id_value); + + _tracker_db_result_set_get_value (result_set, 1, &is_value); + iis_value = g_value_get_int (&is_value); + + if (iis_value) { + ForeachInMetadataInfo *info = g_slice_new (ForeachInMetadataInfo); + info->service = service; + info->iid_value = iid_value; + + info->config = tracker_data_manager_get_config (); + info->language = tracker_data_manager_get_language (); + + g_hash_table_foreach (metadata, + foreach_in_metadata_set_metadata, + info); + + g_slice_free (ForeachInMetadataInfo, info); + } + + g_value_unset (&id_value); + g_value_unset (&is_value); + + g_object_unref (result_set); + + } else { + guint32 id; + + id = tracker_data_update_get_new_service_id (iface); + + if (tracker_data_update_create_service (service, id, + dirname, basename, + metadata)) { + ForeachInMetadataInfo *info; + + info = g_slice_new (ForeachInMetadataInfo); + + info->service = service; + info->iid_value = id; + + info->config = tracker_data_manager_get_config (); + info->language = tracker_data_manager_get_language (); + + g_hash_table_foreach (metadata, + foreach_in_metadata_set_metadata, + info); + + g_slice_free (ForeachInMetadataInfo, info); + } + + } + + g_free (dirname); + g_free (escaped_path); +} + +void tracker_data_update_enable_volume (const gchar *udi, const gchar *mount_path) { Index: src/libtracker-data/tracker-data-metadata.h =================================================================== --- src/libtracker-data/tracker-data-metadata.h (revision 2710) +++ src/libtracker-data/tracker-data-metadata.h (working copy) @@ -42,7 +42,7 @@ const gchar *value); void tracker_data_metadata_insert_values (TrackerDataMetadata *metadata, const gchar *field_name, - GList *list); + const GList *list); G_CONST_RETURN gchar *tracker_data_metadata_lookup (TrackerDataMetadata *metadata, const gchar *field_name); G_CONST_RETURN GList *tracker_data_metadata_lookup_values (TrackerDataMetadata *metadata, Index: src/libtracker-data/tracker-data-update.h =================================================================== --- src/libtracker-data/tracker-data-update.h (revision 2710) +++ src/libtracker-data/tracker-data-update.h (working copy) @@ -52,6 +52,13 @@ const gchar *from, const gchar *to); +/* Turtle importing */ +void tracker_data_update_replace_service (const gchar *path, + const gchar *rdf_type, + GHashTable *metadata); +void tracker_data_update_delete_service_by_path (const gchar *path, + const gchar *rdf_type); + /* Metadata */ void tracker_data_update_set_metadata (TrackerService *service, guint32 service_id, Index: src/libtracker-data/Makefile.am =================================================================== --- src/libtracker-data/Makefile.am (revision 2710) +++ src/libtracker-data/Makefile.am (working copy) @@ -6,6 +6,7 @@ -DTRACKER_COMPILATION \ -I$(top_srcdir)/src \ $(DBUS_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) libtracker_datadir = $(libdir)/tracker @@ -21,6 +22,7 @@ tracker-data-update.c \ tracker-query-tree.c \ tracker-rdf-query.c \ + tracker-turtle.c \ tracker-xesam-query.c noinst_HEADERS = \ @@ -33,6 +35,7 @@ tracker-data-update.h \ tracker-query-tree.h \ tracker-rdf-query.h \ + tracker-turtle.h \ tracker-xesam-query.h libtracker_data_la_LDFLAGS = -version-info 0:0:0 @@ -41,4 +44,5 @@ $(top_builddir)/src/libtracker-db/libtracker-db.la \ $(DBUS_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ -lz Index: src/trackerd/tracker-main.c =================================================================== --- src/trackerd/tracker-main.c (revision 2710) +++ src/trackerd/tracker-main.c (working copy) @@ -61,6 +61,7 @@ #include "tracker-status.h" #include "tracker-xesam-manager.h" #include "tracker-cleanup.h" +#include "tracker-backup.h" #ifdef G_OS_WIN32 #include @@ -96,7 +97,8 @@ gchar *data_dir; gchar *user_data_dir; gchar *sys_tmp_dir; - + gchar *ttl_backup_file; + gboolean reindex_on_shutdown; TrackerProcessor *processor; @@ -182,6 +184,7 @@ g_free (private->user_data_dir); g_free (private->data_dir); + g_free (private->ttl_backup_file); g_free (private->log_filename); g_main_loop_unref (private->main_loop); @@ -501,6 +504,11 @@ NULL); g_free (filename); + private->ttl_backup_file = + g_build_filename (private->user_data_dir, + "tracker-userdata-backup.ttl", + NULL); + /* Private locations */ private->log_filename = g_build_filename (g_get_user_data_dir (), @@ -601,6 +609,14 @@ static void shutdown_databases (void) { + TrackerMainPrivate *private; + + private = g_static_private_get (&private_key); + + /* If we are reindexing, save the user metadata */ + if (private->reindex_on_shutdown) { + tracker_backup_save (private->ttl_backup_file); + } /* Reset integrity status as threads have closed cleanly */ tracker_data_manager_set_db_option_int ("IntegrityCheck", 0); } @@ -624,6 +640,102 @@ } } +static const gchar * +get_ttl_backup_filename (void) +{ + + TrackerMainPrivate *private; + + private = g_static_private_get (&private_key); + + return private->ttl_backup_file; +} + +static void +backup_user_metadata (TrackerConfig *config, TrackerLanguage *language) +{ + TrackerDBIndex *file_index; + TrackerDBIndex *email_index; + gboolean is_first_time_index; + + g_message ("Saving metadata in %s", get_ttl_backup_filename ()); + + /* + * Init the DB stack to get the user metadata + */ + tracker_db_manager_init (0, &is_first_time_index, TRUE); + + /* + * If some database is missing or the dbs dont exists, we dont need + * to backup anything. + */ + if (is_first_time_index) { + tracker_db_manager_shutdown (); + return; + } + + tracker_db_index_manager_init (0, + tracker_config_get_min_bucket_count (config), + tracker_config_get_max_bucket_count (config)); + + file_index = tracker_db_index_manager_get_index (TRACKER_DB_INDEX_FILE); + email_index = tracker_db_index_manager_get_index (TRACKER_DB_INDEX_EMAIL); + + tracker_data_manager_init (config, language, file_index, email_index); + + /* Actual save of the metadata */ + tracker_backup_save (get_ttl_backup_filename ()); + + /* Shutdown the DB stack */ + tracker_data_manager_shutdown (); + tracker_db_index_manager_shutdown (); + tracker_db_manager_shutdown (); +} + +/* + * TODO: Ugly hack counting signals because the indexer is sending two "Finished" signals + * and only the second really mean "finished processing modules". + * + * Saving the last backup file to help with debugging. + */ +static void +crawling_finished_cb (TrackerProcessor *processor, gpointer user_data) +{ + gulong *callback_id = user_data; + GError *error; + static gint counter = 0; + + counter += 1; + + if (counter >= 2) { + gchar *rebackup; + + g_debug ("Uninstalling initial crawling callback"); + g_signal_handler_disconnect (processor, *callback_id); + + org_freedesktop_Tracker_Indexer_restore_backup (tracker_dbus_indexer_get_proxy (), + get_ttl_backup_filename (), + &error); + rebackup = g_strdup_printf ("%s.old", + get_ttl_backup_filename ()); + g_rename (get_ttl_backup_filename (), rebackup); + g_free (rebackup); + } else { + g_debug ("%d finished signal", counter); + } +} + +static void +backup_restore_on_crawling_finished (TrackerProcessor *processor) +{ + gulong restore_cb_id; + + g_debug ("Setting callback for crawling finish detection"); + restore_cb_id = g_signal_connect (processor, "finished", + G_CALLBACK (crawling_finished_cb), + &restore_cb_id); +} + #ifdef HAVE_HAL static void @@ -786,7 +898,7 @@ /* Set timezone info */ tzset (); - + /* Translators: this messagge will apper immediately after the * usage string - Usage: COMMAND */ @@ -913,12 +1025,16 @@ return EXIT_FAILURE; } + tracker_turtle_init (); + tracker_module_config_init (); flags |= TRACKER_DB_MANAGER_REMOVE_CACHE; index_flags |= TRACKER_DB_INDEX_MANAGER_READONLY; if (force_reindex) { + backup_user_metadata (config, language); + flags |= TRACKER_DB_MANAGER_FORCE_REINDEX; index_flags |= TRACKER_DB_INDEX_MANAGER_FORCE_REINDEX; } @@ -1035,6 +1151,11 @@ tracker_status_set_and_signal (TRACKER_STATUS_IDLE); } + if (flags & TRACKER_DB_MANAGER_FORCE_REINDEX || + g_file_test (get_ttl_backup_filename (), G_FILE_TEST_EXISTS)) { + backup_restore_on_crawling_finished (private->processor); + } + if (tracker_status_get_is_running ()) { private->main_loop = g_main_loop_new (NULL, FALSE); g_main_loop_run (private->main_loop); @@ -1082,6 +1203,7 @@ tracker_module_config_shutdown (); tracker_nfs_lock_shutdown (); tracker_status_shutdown (); + tracker_turtle_shutdown (); tracker_log_shutdown (); #ifdef HAVE_HAL @@ -1141,3 +1263,4 @@ private->reindex_on_shutdown = value; } + Index: src/trackerd/Makefile.am =================================================================== --- src/trackerd/Makefile.am (revision 2710) +++ src/trackerd/Makefile.am (working copy) @@ -17,6 +17,7 @@ $(GIO_CFLAGS) \ $(GMODULE_CFLAGS) \ $(GTHREAD_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) if HAVE_INOTIFY @@ -29,6 +30,8 @@ libexec_PROGRAMS = trackerd trackerd_SOURCES = \ + tracker-backup.h \ + tracker-backup.c \ tracker-crawler.c \ tracker-crawler.h \ tracker-daemon.c \ @@ -85,6 +88,7 @@ $(GMODULE_LIBS) \ $(GTHREAD_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ $(trackerd_win_libs) \ -lz \ -lm Index: src/libtracker-db/tracker-db-manager.c =================================================================== --- src/libtracker-db/tracker-db-manager.c (revision 2710) +++ src/libtracker-db/tracker-db-manager.c (working copy) @@ -2726,12 +2726,20 @@ g_free (services_dir); g_free (sql_dir); - if (file_iface) + if (file_iface) { g_object_unref (file_iface); - if (email_iface) + file_iface = NULL; + } + + if (email_iface) { g_object_unref (email_iface); - if (xesam_iface) + email_iface = NULL; + } + + if (xesam_iface) { g_object_unref (xesam_iface); + xesam_iface = NULL; + } /* Since we don't reference this enum anywhere, we do Index: src/libtracker-common/tracker-hal.c =================================================================== --- src/libtracker-common/tracker-hal.c (revision 2710) +++ src/libtracker-common/tracker-hal.c (working copy) @@ -1057,7 +1057,7 @@ gboolean is_mounted; gr = (GetRoots*) user_data; - udi = (const gchar*) key; + udi = key; volume = libhal_volume_from_udi (gr->context, udi); if (!volume) { @@ -1136,6 +1136,79 @@ } /** + * tracker_hal_path_is_on_removable_device: + * @hal: A #TrackerHal + * @path: a path + * @mount_mount: if @path is on a removable device, the mount point will + * be filled in here. You must free the returned result + * @available: if @path is on a removable device, this will be set to + * TRUE in case the file is available right now + * + * Returns Whether or not @path is on a known removable device + * + * Returns: TRUE if @path on a known removable device, FALSE otherwise + **/ +gboolean +tracker_hal_path_is_on_removable_device (TrackerHal *hal, + const gchar *path, + gchar **mount_point, + gboolean *available) +{ + TrackerHalPriv *priv; + GHashTableIter iter; + gboolean found = FALSE; + gpointer key, value; + + g_return_val_if_fail (TRACKER_IS_HAL (hal), FALSE); + + if (!path) + return FALSE; + + priv = GET_PRIV (hal); + + g_hash_table_iter_init (&iter, priv->removable_devices); + + while (g_hash_table_iter_next (&iter, &key, &value)) { + LibHalVolume *volume; + const gchar *udi; + const gchar *mp; + + udi = key; + + volume = libhal_volume_from_udi (priv->context, udi); + + if (!volume) { + g_message ("HAL device with udi:'%s' has no volume, " + "should we delete?", + udi); + continue; + } + + mp = libhal_volume_get_mount_point (volume); + + if (g_strcmp0 (mp, path) != 0) { + if (g_strrstr (path, mp)) { + found = TRUE; + + if (mount_point) + *mount_point = g_strdup (mp); + + if (available) + *available = libhal_volume_is_mounted (volume); + + libhal_volume_free (volume); + break; + } + } + + libhal_volume_free (volume); + } + + return found; +} + + +/** * tracker_hal_get_removable_device_udis: * @hal: A #TrackerHal * Index: src/libtracker-common/tracker-hal.h =================================================================== --- src/libtracker-common/tracker-hal.h (revision 2710) +++ src/libtracker-common/tracker-hal.h (working copy) @@ -66,6 +66,10 @@ const gchar *udi); gboolean tracker_hal_udi_get_is_mounted (TrackerHal *hal, const gchar *udi); +gboolean tracker_hal_path_is_on_removable_device (TrackerHal *hal, + const gchar *path, + gchar **mount_point, + gboolean *available); #endif /* HAVE_HAL */ Index: tests/tracker-indexer/Makefile.am =================================================================== --- tests/tracker-indexer/Makefile.am (revision 2710) +++ tests/tracker-indexer/Makefile.am (working copy) @@ -18,6 +18,7 @@ $(PANGO_CFLAGS) \ $(GMODULE_CFLAGS) \ $(GTHREAD_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) tracker_metadata_utils_SOURCES = \ @@ -37,7 +38,9 @@ tracker-module-file.c \ tracker-module-file.h \ tracker-module-iteratable.c \ - tracker-module-iteratable.h + tracker-module-iteratable.h \ + tracker-removable-device.c \ + tracker-removable-device.h tracker_metadata_utils_LDADD = \ $(top_builddir)/src/libtracker-data/libtracker-data.la \ @@ -49,6 +52,7 @@ $(GMODULE_LIBS) \ $(GTHREAD_LIBS) \ $(GIO_LIBS) \ + $(RAPTOR_LIBS) \ $(GLIB2_LIBS) # Index: configure.ac =================================================================== --- configure.ac (revision 2710) +++ configure.ac (working copy) @@ -170,6 +170,15 @@ AM_CONDITIONAL(HAVE_GCONF, test "$have_gconf" = "yes") +# Check for Raptor +PKG_CHECK_MODULES(RAPTOR, [raptor], have_raptor=yes, have_raptor=no) +AC_SUBST(RAPTOR_CFLAGS) +AC_SUBST(RAPTOR_LIBS) + +if test x$have_raptor == "xyes"; then + AC_DEFINE(HAVE_RAPTOR, 1, [Raptor RDF parsers]) +fi + # Check we have the DBUS binding tool we need AC_PATH_PROG(DBUSBINDINGTOOL, dbus-binding-tool) if test -z $DBUSBINDINGTOOL; then @@ -1113,6 +1122,8 @@ tests/tracker-indexer/tracker-module-file.h:src/tracker-indexer/tracker-module-file.h tests/tracker-indexer/tracker-module-iteratable.c:src/tracker-indexer/tracker-module-iteratable.c tests/tracker-indexer/tracker-module-iteratable.h:src/tracker-indexer/tracker-module-iteratable.h + tests/tracker-indexer/tracker-removable-device.c:src/tracker-indexer/tracker-removable-device.c + tests/tracker-indexer/tracker-removable-device.h:src/tracker-indexer/tracker-removable-device.h ) ################################################################## Index: data/db/sqlite-stored-procs.sql =================================================================== --- data/db/sqlite-stored-procs.sql (revision 2710) +++ data/db/sqlite-stored-procs.sql (working copy) @@ -63,6 +63,7 @@ /* * Metadata/MIME queries */ +GetUserMetadataBackup SELECT S.Path || '/' || S.Name, T.TypeName, M.MetadataID, M.MetadataDisplay From Services S, ServiceMetadata M, ServiceTypes T WHERE (S.ID == M.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (M.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)) UNION SELECT S.Path || '/' || S.Name, T.TypeName, M.MetadataID, upper(M.MetadataValue) From Services S, ServiceNumericMetadata M, ServiceTypes T WHERE (S.ID == M.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (M.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)) UNION SELECT S.Path || '/' || S.Name, T.Typename, M.MetadataID, M.MetadataValue From Services S, ServiceKeywordMetadata M, ServiceTypes T WHERE (S.ID == M.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (M.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)); GetAllMetadata SELECT MetadataID, MetadataDisplay FROM ServiceMetadata WHERE ServiceID = ? UNION SELECT MetadataID, MetadataValue FROM ServiceKeywordMetadata WHERE ServiceID = ? UNION SELECT MetadataID, upper(MetadataValue) FROM ServiceNumericMetadata WHERE ServiceID = ?; GetMetadata SELECT MetaDataDisplay FROM ServiceMetaData WHERE ServiceID = ? AND MetaDataID = ?; GetMetadataAliases SELECT DISTINCT M.MetaName, M.ID FROM MetaDataTypes AS M, MetaDataChildren AS C WHERE M.ID = C.ChildID AND C.MetaDataID = ?; Index: data/dbus/tracker-indexer.xml =================================================================== --- data/dbus/tracker-indexer.xml (revision 2710) +++ data/dbus/tracker-indexer.xml (working copy) @@ -74,6 +74,11 @@ + + + + +