Index: src/tracker-indexer/tracker-main.c =================================================================== --- src/tracker-indexer/tracker-main.c (revision 2649) +++ src/tracker-indexer/tracker-main.c (working copy) @@ -43,7 +43,9 @@ #include #include +#include + #include "tracker-dbus.h" #include "tracker-indexer.h" @@ -370,13 +372,18 @@ tracker_indexer_process_modules (indexer, modules); } + tracker_turtle_init (); + g_message ("Starting..."); + main_loop = g_main_loop_new (NULL, FALSE); g_main_loop_run (main_loop); g_message ("Shutdown started"); + tracker_turtle_shutdown (); + if (quit_timeout_id) { g_source_remove (quit_timeout_id); } Index: src/tracker-indexer/tracker-indexer.c =================================================================== --- src/tracker-indexer/tracker-indexer.c (revision 2649) +++ src/tracker-indexer/tracker-indexer.c (working copy) @@ -71,10 +71,12 @@ #include #include +#include #include "tracker-indexer.h" #include "tracker-indexer-module.h" #include "tracker-marshal.h" +#include "tracker-removable-device.h" #define TRACKER_INDEXER_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_INDEXER, TrackerIndexerPrivate)) @@ -84,9 +86,6 @@ #define LOW_DISK_CHECK_FREQUENCY 10 #define SIGNAL_STATUS_FREQUENCY 10 -/* Transaction every 'x' items */ -#define TRANSACTION_MAX 2000 - /* Throttle defaults */ #define THROTTLE_DEFAULT 0 #define THROTTLE_DEFAULT_ON_BATTERY 5 @@ -384,6 +383,22 @@ indexer); } + +void +tracker_indexer_commit_transaction (TrackerIndexer *indexer) +{ + stop_transaction (indexer); + tracker_indexer_set_running (indexer, TRUE); + +} + +void +tracker_indexer_open_transaction (TrackerIndexer *indexer) +{ + tracker_indexer_set_running (indexer, FALSE); + start_transaction (indexer); +} + #ifdef HAVE_HAL static void @@ -1376,6 +1391,8 @@ TrackerService *service; gchar *text; guint32 id; + gchar *mount_point = NULL; + gchar *service_path; service = get_service_for_file (info->module_file, info->module); @@ -1413,6 +1430,29 @@ index_metadata (indexer, id, service, metadata); + /* TODO: URI branch path -> uri */ + + service_path = g_build_path (G_DIR_SEPARATOR_S, + dirname, + basename, + NULL); + + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + service_path, + &mount_point, + NULL)) { + + tracker_removable_device_add_metadata (indexer, + mount_point, + service_path, + tracker_service_get_name (service), + metadata); + } + + g_free (mount_point); + mount_point = NULL; + g_free (service_path); + /* Take the old text -> the new one, calculate * difference and add the words. */ @@ -1446,6 +1486,29 @@ index_metadata (indexer, id, service, metadata); + /* TODO: URI branch path -> uri */ + + service_path = g_build_path (G_DIR_SEPARATOR_S, + dirname, + basename, + NULL); + + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + service_path, + &mount_point, + NULL)) { + + tracker_removable_device_add_metadata (indexer, + mount_point, + service_path, + tracker_service_get_name (service), + metadata); + } + + g_free (mount_point); + mount_point = NULL; + g_free (service_path); + text = tracker_module_file_get_text (info->module_file); if (text) { @@ -1486,6 +1549,7 @@ GFile *file, *other_file; gchar *path, *other_path; gchar *uri, *other_uri, *mime_type; + gchar *mount_point = NULL; guint32 id; service = get_service_for_file (info->other_module_file, info->module); @@ -1528,6 +1592,32 @@ tracker_data_update_move_service (service, path, other_path); + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + path, + &mount_point, + NULL) ) { + + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + other_path, + NULL, + NULL) ) { + + tracker_removable_device_add_move (indexer, + mount_point, + path, + other_path, + tracker_service_get_name (service)); + + } else { + tracker_removable_device_add_removal (indexer, + mount_point, + path, + tracker_service_get_name (service)); + } + } + + g_free (mount_point); + /* * Updating what changes in move event (Path related properties) */ @@ -1581,6 +1671,7 @@ gchar *service_path; const gchar *service_type; guint service_id, service_type_id; + gchar *mount_point = NULL; service_type = tracker_module_config_get_index_service (info->module->name); @@ -1675,6 +1766,9 @@ g_free (metadata); /* Delete service */ + + /* TODO: URI branch path -> uri */ + service_path = g_build_path (G_DIR_SEPARATOR_S, dirname, basename, @@ -1686,6 +1780,18 @@ tracker_data_update_delete_service_recursively (service, service_path); } + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + service_path, + &mount_point, + NULL)) { + + tracker_removable_device_add_removal (indexer, mount_point, + service_path, + tracker_service_get_name (service)); + } + + g_free (mount_point); + tracker_data_update_decrement_stats (indexer->private->common, service); g_free (service_path); @@ -2385,7 +2491,7 @@ /* Signal stopped and clean up */ check_stopped (indexer, FALSE); check_disk_space_stop (indexer); - + return FALSE; } @@ -2840,6 +2946,13 @@ dbus_g_method_return (context); tracker_dbus_request_success (request_id); + + /* tracker_turtle_process_ttl will be spinning the mainloop, therefore + we can already return the DBus method */ + + if (enabled) { + tracker_removable_device_load (indexer, path); + } } void @@ -2933,7 +3046,59 @@ tracker_dbus_request_success (request_id); } +static void +restore_backup_triple (void *user_data, const stmt *triple) { + + gchar *values[2]; + TrackerIndexer *indexer = user_data; + GError *error = NULL; + + g_return_if_fail (TRACKER_IS_INDEXER (indexer)); + + g_debug ("Turtle loading <%s, %s, %s>", + (gchar *)triple->subject, + (gchar *)triple->predicate, + (gchar *)triple->object); + + values[0] = g_strdup (triple->object); + values[1] = NULL; + + handle_metadata_add (indexer, "Files", triple->subject, triple->predicate, values, &error); + + if (error) { + g_warning ("Restoring backup: %s", error->message); + g_error_free (error); + } + + g_free (values[0]); +} + void +tracker_indexer_restore_backup (TrackerIndexer *indexer, + const gchar *backup_file, + DBusGMethodInvocation *context, + GError **error) +{ + guint request_id; + + tracker_dbus_async_return_if_fail (TRACKER_IS_INDEXER (indexer), context); + + request_id = tracker_dbus_get_next_request_id (); + tracker_dbus_request_new (request_id, + "DBus request to restore backup data from '%s'", + backup_file); + + tracker_turtle_process (backup_file, + "/", + (TurtleTripleCallback) restore_backup_triple, + indexer); + + dbus_g_method_return (context); + tracker_dbus_request_success (request_id); +} + + +void tracker_indexer_shutdown (TrackerIndexer *indexer, DBusGMethodInvocation *context, GError **error) Index: src/tracker-indexer/Makefile.am =================================================================== --- src/tracker-indexer/Makefile.am (revision 2649) +++ src/tracker-indexer/Makefile.am (working copy) @@ -13,6 +13,7 @@ -I$(top_srcdir)/src \ $(DBUS_CFLAGS) \ $(PANGO_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GMODULE_CFLAGS) libtracker_indexerdir = $(libdir)/tracker @@ -39,7 +40,8 @@ tracker-indexer-module.c \ tracker-indexer-module.h \ tracker-main.c \ - tracker-marshal-main.c + tracker-marshal-main.c \ + tracker-removable-device.c tracker_indexer_LDADD = \ libtracker-indexer.la \ @@ -54,6 +56,7 @@ $(PANGO_LIBS) \ $(GIO_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ -lz \ -lm Index: src/tracker-indexer/tracker-indexer.h =================================================================== --- src/tracker-indexer/tracker-indexer.h (revision 2649) +++ src/tracker-indexer/tracker-indexer.h (working copy) @@ -30,6 +30,9 @@ #define TRACKER_INDEXER_PATH "/org/freedesktop/Tracker/Indexer" #define TRACKER_INDEXER_INTERFACE "org.freedesktop.Tracker.Indexer" +/* Transaction every 'x' items */ +#define TRANSACTION_MAX 2000 + G_BEGIN_DECLS #define TRACKER_TYPE_INDEXER (tracker_indexer_get_type()) @@ -80,6 +83,8 @@ void tracker_indexer_process_all (TrackerIndexer *indexer); void tracker_indexer_process_modules (TrackerIndexer *indexer, gchar **modules); +void tracker_indexer_commit_transaction (TrackerIndexer *indexer); +void tracker_indexer_open_transaction (TrackerIndexer *indexer); /* DBus methods */ void tracker_indexer_pause (TrackerIndexer *indexer, @@ -136,6 +141,10 @@ GStrv values, DBusGMethodInvocation *context, GError **error); +void tracker_indexer_restore_backup (TrackerIndexer *indexer, + const gchar *backup_file, + DBusGMethodInvocation *context, + GError **error); void tracker_indexer_shutdown (TrackerIndexer *indexer, DBusGMethodInvocation *context, GError **error); Index: src/libtracker-data/tracker-data-query.c =================================================================== --- src/libtracker-data/tracker-data-query.c (revision 2649) +++ src/libtracker-data/tracker-data-query.c (working copy) @@ -386,6 +386,9 @@ return service_type_id; } +/* + * Result set with (metadataID, value) per row + */ static void result_set_to_metadata (TrackerDBResultSet *result_set, TrackerDataMetadata *metadata, @@ -400,7 +403,7 @@ GValue transform = {0, }; GValue value = {0, }; gchar *str; - + g_value_init (&transform, G_TYPE_STRING); tracker_db_result_set_get (result_set, 0, &metadata_id, -1); _tracker_db_result_set_get_value (result_set, 1, &value); @@ -492,6 +495,23 @@ return metadata; } +TrackerDBResultSet * +tracker_data_query_backup_metadata (TrackerService *service) +{ + TrackerDBInterface *iface; + TrackerDBResultSet *result_set; + GHashTable *results; + + g_return_val_if_fail (TRACKER_IS_SERVICE (service), NULL); + + iface = tracker_db_manager_get_db_interface_by_service (tracker_service_get_name (service)); + + result_set = tracker_data_manager_exec_proc (iface, + "GetEmbeddedMetadataBackup", + NULL); + return result_set; +} + static gchar * db_get_metadata (TrackerService *service, guint service_id, Index: src/libtracker-data/tracker-data-query.h =================================================================== --- src/libtracker-data/tracker-data-query.h (revision 2649) +++ src/libtracker-data/tracker-data-query.h (working copy) @@ -53,6 +53,7 @@ TrackerDataMetadata *tracker_data_query_metadata (TrackerService *service, guint32 service_id, gboolean embedded); +TrackerDBResultSet *tracker_data_query_backup_metadata (TrackerService *service); gchar * tracker_data_query_parsed_metadata (TrackerService *service, guint32 service_id); gchar * tracker_data_query_unparsed_metadata (TrackerService *service, Index: src/libtracker-data/tracker-data-metadata.c =================================================================== --- src/libtracker-data/tracker-data-metadata.c (revision 2649) +++ src/libtracker-data/tracker-data-metadata.c (working copy) @@ -106,6 +106,7 @@ const gchar *value) { TrackerField *field; + gchar *old_value; g_return_if_fail (metadata != NULL); g_return_if_fail (field_name != NULL); @@ -114,14 +115,54 @@ field = tracker_ontology_get_field_by_name (field_name); g_return_if_fail (TRACKER_IS_FIELD (field)); - g_return_if_fail (tracker_field_get_multiple_values (field) == FALSE); + g_return_if_fail (tracker_field_get_multiple_values (field) == FALSE); - g_hash_table_insert (metadata->table, - g_object_ref (field), - g_strdup (value)); + old_value = g_hash_table_lookup (metadata->table, field); + g_free (old_value); + + g_hash_table_replace (metadata->table, + g_object_ref (field), + g_strdup (value)); + } + + /** + * tracker_data_metadata_append_to_list: + * @metadata: A #TrackerDataMetadata + * @field_name: Field name for the metadata to insert. + * @value: Value for the metadata to insert. + * + * Inserts a new metadata element into @metadata. + **/ +void +tracker_data_metadata_append_to_list (TrackerDataMetadata *metadata, + const gchar *field_name, + gchar *value) +{ + TrackerField *field; + GList *values; + + field = tracker_ontology_get_field_by_name (field_name); + + if (!field) { + g_warning ("Field name '%s' has isn't described in the ontology", field_name); + g_free (value); + return; + } + + g_return_if_fail (tracker_field_get_multiple_values (field) == TRUE); + + values = g_hash_table_lookup (metadata->table, field); + + values = g_list_append (values, value); + + g_hash_table_replace (metadata->table, + g_object_ref (field), + values); +} +/** * tracker_data_metadata_insert_values: * @metadata: A #TrackerDataMetadata * @field_name: Field name for the metadata to insert @@ -140,6 +181,7 @@ GList *list) { TrackerField *field; + GList *old_values; g_return_if_fail (metadata != NULL); g_return_if_fail (field_name != NULL); @@ -150,7 +192,13 @@ field = tracker_ontology_get_field_by_name (field_name); - g_return_if_fail (TRACKER_IS_FIELD (field)); + if (!field) { + g_warning ("Field name '%s' has isn't described in the ontology", field_name); + g_list_foreach (list, (GFunc) g_free, NULL); + g_list_free (list); + return; + } + g_return_if_fail (tracker_field_get_multiple_values (field) == TRUE); g_hash_table_insert (metadata->table, Index: src/libtracker-data/tracker-data-update.c =================================================================== --- src/libtracker-data/tracker-data-update.c (revision 2649) +++ src/libtracker-data/tracker-data-update.c (working copy) @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -571,7 +572,271 @@ tracker_data_manager_exec (iface, "DELETE FROM Events WHERE BeingHandled = 1"); } +/* TODO: URI branch path -> uri */ + void +tracker_data_delete_service (const gchar *path, + const gchar *rdf_type) +{ + TrackerService *service; + const gchar *service_type; + guint32 service_id; + + service = tracker_ontology_get_service_by_name (rdf_type); + service_type = tracker_service_get_name (service); + service_id = tracker_data_query_file_id (service_type, path); + + /* When merging from the decomposed branch to trunk then this function + * wont exist in the decomposed branch. Create it based on this one. */ + + if (service_id != 0) { + tracker_data_update_delete_service (service, service_id); + tracker_data_update_delete_service_recursively (service, (gchar *) path); + tracker_data_update_delete_all_metadata (service, service_id); + } +} + +/* TODO: URI branch path -> uri */ + +typedef struct { + TrackerService *service; + guint32 iid_value; + TrackerLanguage *language; + TrackerConfig *config; +} ForeachInMetadataInfo; + + +static void +set_metadata (TrackerField *field, gpointer value, ForeachInMetadataInfo *info) +{ + TrackerDBIndex *index; + gchar *parsed_value; + gchar **arr; + gint service_id; + gint i; + gint score; + + /* TODO untested and unfinished port that came from the decomposed + * branch of Jürg. When merging from the decomposed branch to trunk + * then pick the version in the decomposed branch for this function */ + + parsed_value = tracker_parser_text_to_string (value, + info->language, + tracker_config_get_max_word_length (info->config), + tracker_config_get_min_word_length (info->config), + tracker_field_get_filtered (field), + tracker_field_get_filtered (field), + tracker_field_get_delimited (field)); + + if (!parsed_value) { + return; + } + + score = tracker_field_get_weight (field); + + arr = g_strsplit (parsed_value, " ", -1); + service_id = tracker_service_get_id (info->service); + index = tracker_db_index_manager_get_index_by_service_id (service_id); + + for (i = 0; arr[i]; i++) { + tracker_db_index_add_word (index, + arr[i], + info->iid_value, + tracker_service_get_id (info->service), + score); + } + + tracker_data_update_set_metadata (info->service, info->iid_value, field, value, parsed_value); + + g_free (parsed_value); + g_strfreev (arr); + +} + +static void +foreach_in_metadata_set_metadata (TrackerField *field, + gpointer value, + gpointer user_data) +{ + ForeachInMetadataInfo *info = user_data; + gchar *parsed_value; + gint throttle; + + /* Throttle indexer, value 9 is from older code, why 9? */ + throttle = tracker_config_get_throttle (info->config); + if (throttle > 9) { + tracker_throttle (info->config, throttle * 100); + } + + if (!tracker_field_get_multiple_values (field)) { + set_metadata (field, value, user_data); + } else { + GList *list; + + list = value; + + while (list) { + set_metadata (field, list->data, user_data); + list = list->next; + } + } + +} + +static TrackerConfig *config = NULL; +static TrackerLanguage *language = NULL; + + +void +tracker_data_start_turtle_import (void) +{ + if (!config) + config = tracker_config_new (); + if (!language) + language = tracker_language_new (config); +} + +void +tracker_data_stop_turtle_import (void) +{ + g_object_unref (language); + g_object_unref (config); + language = NULL; + config = NULL; +} + + +/* TODO: URI branch path -> uri */ + +void +tracker_data_replace_service (const gchar *path, + const gchar *rdf_type, + TrackerDataMetadata *metadata) +{ + TrackerDBInterface *iface; + TrackerDBResultSet *result_set; + const gchar *modified; + GError *error = NULL; + TrackerService *service; + gchar *escaped_path; + gchar *dirname; + const gchar *basename; + + /* When merging from the decomposed branch to trunk then pick the version + * in the decomposed branch for this function. However, carefully + * compare the features, as this version is more recent and has + * implemented a few significant items, whereas the version in the + * decomposed branch was a proof of concept implementation, and might + * not have these needed features. */ + + if (!rdf_type) + return; + + service = tracker_ontology_get_service_by_name (rdf_type); + + iface = tracker_db_manager_get_db_interface_by_type (tracker_service_get_name (service), + TRACKER_DB_CONTENT_TYPE_METADATA); + + modified = tracker_data_metadata_lookup (metadata, "File:Modified"); + + if (!modified) { + return; + } + + escaped_path = tracker_escape_string (path); + + basename = g_basename (escaped_path); + dirname = g_dirname (escaped_path); + + /* TODO Warning: comparing Modified against Accessed. Do we have a + * better field for this? */ + + result_set = tracker_db_interface_execute_query (iface, &error, + "SELECT ID, Accessed < '%s' FROM Services " + "WHERE Path = '%s' AND " + "Name = '%s'", + modified, + dirname, basename); + + if (error) { + #ifdef TURTLE_DEBUG + g_print ("Q ERROR: %s\n", error->message); + #endif /* TURTLE_DEBUG */ + g_error_free (error); + } + + if (result_set) { + GValue id_value = { 0, }; + GValue is_value = { 0, }; + gint iid_value, iis_value; + + _tracker_db_result_set_get_value (result_set, 0, &id_value); + iid_value = g_value_get_int (&id_value); + + _tracker_db_result_set_get_value (result_set, 1, &is_value); + iis_value = g_value_get_int (&is_value); + + if (iis_value) { + ForeachInMetadataInfo *info = g_slice_new (ForeachInMetadataInfo); + info->service = service; + info->iid_value = iid_value; + + info->config = config?g_object_ref (config):NULL; + info->language = language?g_object_ref (language):NULL; + + tracker_data_metadata_foreach (metadata, + foreach_in_metadata_set_metadata, + info); + + if (info->language) + g_object_unref (info->language); + if (info->config) + g_object_unref (info->config); + + g_slice_free (ForeachInMetadataInfo, info); + } + + g_value_unset (&id_value); + g_value_unset (&is_value); + + g_object_unref (result_set); + + } else { + guint32 id; + + id = tracker_data_update_get_new_service_id (iface); + + if (tracker_data_update_create_service (service, id, + dirname, basename, + metadata)) { + ForeachInMetadataInfo *info; + + info = g_slice_new (ForeachInMetadataInfo); + + info->service = service; + info->iid_value = id; + + info->config = config?g_object_ref (config):NULL; + info->language = language?g_object_ref (language):NULL; + + tracker_data_metadata_foreach (metadata, + foreach_in_metadata_set_metadata, + info); + + if (info->language) + g_object_unref (info->language); + if (info->config) + g_object_unref (info->config); + + g_slice_free (ForeachInMetadataInfo, info); + } + } + + g_free (dirname); + g_free (escaped_path); +} + +void tracker_data_update_enable_volume (const gchar *udi, const gchar *mount_path) { Index: src/libtracker-data/tracker-data-metadata.h =================================================================== --- src/libtracker-data/tracker-data-metadata.h (revision 2649) +++ src/libtracker-data/tracker-data-metadata.h (working copy) @@ -43,6 +43,9 @@ void tracker_data_metadata_insert_values (TrackerDataMetadata *metadata, const gchar *field_name, GList *list); +void tracker_data_metadata_append_to_list (TrackerDataMetadata *metadata, + const gchar *field_name, + gchar *value); G_CONST_RETURN gchar *tracker_data_metadata_lookup (TrackerDataMetadata *metadata, const gchar *field_name); G_CONST_RETURN GList *tracker_data_metadata_lookup_values (TrackerDataMetadata *metadata, Index: src/libtracker-data/tracker-data-update.h =================================================================== --- src/libtracker-data/tracker-data-update.h (revision 2649) +++ src/libtracker-data/tracker-data-update.h (working copy) @@ -52,6 +52,15 @@ const gchar *from, const gchar *to); +/* Turtle importing */ +void tracker_data_replace_service (const gchar *path, + const gchar *rdf_type, + TrackerDataMetadata *metadata); +void tracker_data_delete_service (const gchar *path, + const gchar *rdf_type); +void tracker_data_start_turtle_import (void); +void tracker_data_stop_turtle_import (void); + /* Metadata */ void tracker_data_update_set_metadata (TrackerService *service, guint32 service_id, Index: src/libtracker-data/Makefile.am =================================================================== --- src/libtracker-data/Makefile.am (revision 2649) +++ src/libtracker-data/Makefile.am (working copy) @@ -6,6 +6,7 @@ -DTRACKER_COMPILATION \ -I$(top_srcdir)/src \ $(DBUS_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) libtracker_datadir = $(libdir)/tracker @@ -21,6 +22,7 @@ tracker-data-update.c \ tracker-query-tree.c \ tracker-rdf-query.c \ + tracker-turtle.c \ tracker-xesam-query.c noinst_HEADERS = \ @@ -33,6 +35,7 @@ tracker-data-update.h \ tracker-query-tree.h \ tracker-rdf-query.h \ + tracker-turtle.h \ tracker-xesam-query.h libtracker_data_la_LDFLAGS = -version-info 0:0:0 @@ -41,4 +44,5 @@ $(top_builddir)/src/libtracker-db/libtracker-db.la \ $(DBUS_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ -lz Index: src/trackerd/tracker-main.c =================================================================== --- src/trackerd/tracker-main.c (revision 2649) +++ src/trackerd/tracker-main.c (working copy) @@ -61,6 +61,7 @@ #include "tracker-status.h" #include "tracker-xesam-manager.h" #include "tracker-cleanup.h" +#include "tracker-backup.h" #ifdef G_OS_WIN32 #include @@ -747,6 +748,52 @@ return FALSE; } +static gchar * +get_turtle_userdata_backup_filename () +{ + TrackerMainPrivate *private; + + private = g_static_private_get (&private_key); + + if (private) { + return g_build_filename (private->user_data_dir, + "tracker-userdata-backup.ttl", + NULL); + } else { + g_critical ("Directories not initialized"); + return NULL; + } +} + + +/* + * TODO: Ugly hack counting signals because the indexer is sending two "Finished" signals + * and only the second really mean "finished processing modules". + */ +static void +crawling_finished_cb (TrackerProcessor *processor, gpointer user_data) +{ + gulong *callback_id = (gulong *)user_data; + GError *error; + gchar *turtle_file; + static gint counter = 0; + + counter += 1; + + if (counter >= 2) { + g_debug ("Uninstalling initial crawling callback"); + g_signal_handler_disconnect (processor, *callback_id); + + turtle_file = get_turtle_userdata_backup_filename (); + org_freedesktop_Tracker_Indexer_restore_backup (tracker_dbus_indexer_get_proxy (), + turtle_file, + &error); + g_free (turtle_file); + } else { + g_debug ("%d finished signal", counter); + } +} + gint main (gint argc, gchar *argv[]) { @@ -764,6 +811,7 @@ TrackerDBManagerFlags flags = 0; TrackerDBIndexManagerFlags index_flags = 0; gboolean is_first_time_index; + gulong callback_id; g_type_init (); @@ -786,7 +834,7 @@ /* Set timezone info */ tzset (); - + /* Translators: this messagge will apper immediately after the * usage string - Usage: COMMAND */ @@ -913,12 +961,43 @@ return EXIT_FAILURE; } + tracker_turtle_init (); + tracker_module_config_init (); flags |= TRACKER_DB_MANAGER_REMOVE_CACHE; index_flags |= TRACKER_DB_INDEX_MANAGER_READONLY; if (force_reindex) { + + gchar *turtle_file; + + turtle_file = get_turtle_userdata_backup_filename (); + + g_message ("Saving metadata in %s", turtle_file); + + /* Init the DB stack */ + tracker_db_manager_init (0, &is_first_time_index, TRUE); + + tracker_db_index_manager_init (0, + tracker_config_get_min_bucket_count (config), + tracker_config_get_max_bucket_count (config)); + + file_index = tracker_db_index_manager_get_index (TRACKER_DB_INDEX_FILE); + email_index = tracker_db_index_manager_get_index (TRACKER_DB_INDEX_EMAIL); + + tracker_data_manager_init (config, language, file_index, email_index); + + tracker_backup_save (turtle_file); + + /* Shutdown the DB stack */ + tracker_data_manager_shutdown (); + + tracker_db_index_manager_shutdown (); + tracker_db_manager_shutdown (); + + g_free (turtle_file); + flags |= TRACKER_DB_MANAGER_FORCE_REINDEX; index_flags |= TRACKER_DB_INDEX_MANAGER_FORCE_REINDEX; } @@ -1035,6 +1114,13 @@ tracker_status_set_and_signal (TRACKER_STATUS_IDLE); } + if (flags & TRACKER_DB_MANAGER_FORCE_REINDEX) { + g_debug ("Setting callback for crawling finish detection"); + callback_id = g_signal_connect (private->processor, "finished", + G_CALLBACK (crawling_finished_cb), + &callback_id); + } + if (tracker_status_get_is_running ()) { private->main_loop = g_main_loop_new (NULL, FALSE); g_main_loop_run (private->main_loop); @@ -1082,6 +1168,7 @@ tracker_module_config_shutdown (); tracker_nfs_lock_shutdown (); tracker_status_shutdown (); + tracker_turtle_shutdown (); tracker_log_shutdown (); #ifdef HAVE_HAL Index: src/trackerd/Makefile.am =================================================================== --- src/trackerd/Makefile.am (revision 2649) +++ src/trackerd/Makefile.am (working copy) @@ -17,6 +17,7 @@ $(GIO_CFLAGS) \ $(GMODULE_CFLAGS) \ $(GTHREAD_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) if HAVE_INOTIFY @@ -29,6 +30,8 @@ libexec_PROGRAMS = trackerd trackerd_SOURCES = \ + tracker-backup.h \ + tracker-backup.c \ tracker-crawler.c \ tracker-crawler.h \ tracker-daemon.c \ @@ -85,6 +88,7 @@ $(GMODULE_LIBS) \ $(GTHREAD_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ $(trackerd_win_libs) \ -lz \ -lm Index: src/libtracker-db/tracker-db-manager.c =================================================================== --- src/libtracker-db/tracker-db-manager.c (revision 2649) +++ src/libtracker-db/tracker-db-manager.c (working copy) @@ -2726,12 +2726,20 @@ g_free (services_dir); g_free (sql_dir); - if (file_iface) + if (file_iface) { g_object_unref (file_iface); - if (email_iface) + file_iface = NULL; + } + + if (email_iface) { g_object_unref (email_iface); - if (xesam_iface) + email_iface = NULL; + } + + if (xesam_iface) { g_object_unref (xesam_iface); + xesam_iface = NULL; + } /* Since we don't reference this enum anywhere, we do Index: src/libtracker-common/tracker-hal.c =================================================================== --- src/libtracker-common/tracker-hal.c (revision 2649) +++ src/libtracker-common/tracker-hal.c (working copy) @@ -1136,6 +1136,77 @@ } /** + * tracker_hal_path_is_on_removable_device: + * @hal: A #TrackerHal + * @path: a path + * @mount_mount: (out): if @path is on a removable device, the mount point will + * be filled in here + * @available: (out): if @path is on a removable device, this will be set to + * TRUE in case the file is available right now + * + * Returns Whether or not @path is on a known removable device + * + * Returns: Whether or not @path is on a known removable device + **/ +gboolean +tracker_hal_path_is_on_removable_device (TrackerHal *hal, + const gchar *path, + gchar **mount_point, + gboolean *available) +{ + TrackerHalPriv *priv; + GHashTableIter iter; + gboolean found = FALSE; + gpointer key, value; + + g_return_val_if_fail (TRACKER_IS_HAL (hal), FALSE); + + if (!path) + return FALSE; + + priv = GET_PRIV (hal); + + g_hash_table_iter_init (&iter, priv->removable_devices); + + while (g_hash_table_iter_next (&iter, &key, &value)) { + LibHalVolume *volume; + const gchar *udi; + const gchar *mp; + + udi = (const gchar*) key; + + volume = libhal_volume_from_udi (priv->context, udi); + + if (!volume) { + g_message ("HAL device with udi:'%s' has no volume, " + "should we delete?", + udi); + continue; + } + + mp = libhal_volume_get_mount_point (volume); + + if (mp && strcmp (mp, path) != 0) { + if (g_strrstr (path, mp)) { + found = TRUE; + + if (mount_point) + *mount_point = g_strdup (mp); + + if (available) + *available = libhal_volume_is_mounted (volume); + break; + } + } + + libhal_volume_free (volume); + } + + return found; +} + + +/** * tracker_hal_get_removable_device_udis: * @hal: A #TrackerHal * Index: src/libtracker-common/tracker-hal.h =================================================================== --- src/libtracker-common/tracker-hal.h (revision 2649) +++ src/libtracker-common/tracker-hal.h (working copy) @@ -66,6 +66,10 @@ const gchar *udi); gboolean tracker_hal_udi_get_is_mounted (TrackerHal *hal, const gchar *udi); +gboolean tracker_hal_path_is_on_removable_device (TrackerHal *hal, + const gchar *path, + gchar **mount_point, + gboolean *available); #endif /* HAVE_HAL */ Index: ChangeLog =================================================================== --- ChangeLog (revision 2649) +++ ChangeLog (working copy) @@ -1,5 +1,19 @@ 2008-12-04 Ivan Frade + * src/tracker-indexer/tracker-removable-device.c: + Fixed compilation without libraptor. + +2008-12-02 Ivan Frade + + * src/trackerd/tracker-main.c + Init and shutdown of tracker_turtle in the main secuence. + Dynamic creation of backup turtle file. + + * src/trackerd/tracker-backup.[ch]: + Turtle filename as parameter to the module. + +2008-12-04 Ivan Frade + * src/tracker-indexer/tracker-indexer.c (item_remove): Call recursively delete only when the uri is a Folder. This speed up a lot the delete operation. @@ -133,6 +147,60 @@ Fixes NB #93935. +2008-12-01 Ivan Frade + + * src/trackerd/tracker-main.c: Readded static counter for + "Finished" signals and call to the "restore_backup" method in the + indexer. + + * src/trackerd/tracker-backup.[ch]: Removed "restore" method. + +2008-12-01 Ivan Frade + + * src/tracker-indexer/tracker-indexer.c: + Implemented restore metadata method. First version with "Files" + hardcoded. + +2008-12-01 Philip Van Hoof + + * src/tracker-indexer/tracker-indexer.c: The if-then-else here was just + wrong. We were always writing something, even if the removed file was + not on a removable device. + +2008-12-01 Ivan Frade + + * data/db/sqlite-stored-procs.sql + * src/trackerd/tracker-backup.c + Retrieve also the service type of the file to backup. Needed to + set the rdf:type. + +2008-12-01 Ivan Frade + + * src/tracker-indexer/tracker-indexer.[ch] + * data/dbus/tracker-indexer.xml: + Added new method "RestoreBackup" in the dbus API of the indexer. + +2008-12-01 Ivan Frade + + * src/libtracker-data/tracker-turtle.c: + (tracker_turtle_open): Check the module is initialized + +2008-12-01 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/tracker-indexer/tracker-removable-device.h + * src/tracker-indexer/tracker-indexer.c: Added detecting the rdf-type + while writing out Turtle files to the metadata.ttl of a removable + device + +2008-12-01 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/libtracker-data/tracker-turtle.c + * src/libtracker-data/tracker-turtle.h + * src/trackerd/tracker-backup.c: Cleaning up code, fixed an #ifdef/#endif + placement in case you don't have raptor + 2008-12-01 Martyn Russell * src/tracker-indexer/modules/dummy.c: Small code clean up. @@ -158,11 +226,117 @@ directories to ignore and "*.rcore.lzo" to the list of files to ignore. +2008-11-28 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/libtracker-data/tracker-turtle.c + * src/libtracker-data/tracker-data-update.c: Making removals work + +2008-11-28 Philip Van Hoof + + * src/libtracker-data/tracker-data-update.c + * src/libtracker-data/tracker-data-update.h + * src/tracker-indexer/tracker-removable-device.c: Making it actually work + * src/trackerd/tracker-backup.c: Fixed API of tracker_turtle_process, + base_uri + * src/libtracker-data/tracker-turtle.c + * src/libtracker-data/tracker-turtle.h + * src/libtracker-common/tracker-hal.c: Fixed that the dir of the mount- + point itself is not detected as "on the device" + +2008-11-28 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/libtracker-common/tracker-hal.c: Various bugfixes for the Turtle + support + +2008-11-28 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/tracker-indexer/tracker-indexer.c + * src/libtracker-data/tracker-turtle.c: Implementing add, remove, move + events for Turtle (untested) + 2008-11-27 Philip Van Hoof * src/libtracker-common/tracker-thumbnailer.c: Replaced a non-critical warning with a message +2008-11-27 Philip Van Hoof + + * src/tracker-indexer/tracker-main.c: Added turtle-support's init + + * src/libtracker-data/tracker-turtle.c: + * src/tracker-indexer/tracker-removable-device.c + * src/libtracker-data/tracker-turtle.h: Migrated the optimizer to + tracker-turtle.c, migrated the 'storer' to use tracker-turtle.c/h's + tracker_turtle_processor + +2008-11-27 Ivan Frade + + * src/trackerd/tracker-backup.c: + Remove the previous file when saving. + Cleaning a little bit the crime scene. + +2008-11-27 Ivan Frade + + * src/libtracker-data/tracker-turtle.[ch] + (tracker_turtle_process): Added a wrapper to raptor parse method. + + * src/trackerd/tracker-backup.[ch] + Using the new wrapper method to print parsed triplets. + + * src/trackerd/Makefile.am: Added compilation flags for raptor. + +2008-11-27 Philip Van Hoof + + * src/libtracker-data/tracker-data-update.c: Mimicking what(ever) + index_metadata_item is doing to store metadata (also to cope with + list values) + +2008-11-27 Philip Van Hoof + + * src/libtracker-data/tracker-data-update.c: Coping with list-value + values while storing predicates coming from Turtle + +2008-11-27 Philip Van Hoof + + * turtle/src/libtracker-data/tracker-data-metadata.c: Memory leaks + when overwriting predicates in TrackerDataMetadata + +2008-11-27 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/libtracker-data/tracker-data-metadata.c + * src/libtracker-data/tracker-data-metadata.h: Coping with list-value + values in the Turtle file + +2008-11-27 Philip Van Hoof + + * src/tracker-indexer/tracker-indexer.c + * src/libtracker-data/tracker-data-update.c + * src/libtracker-common/tracker-hal.c + * src/libtracker-common/tracker-hal.h: Calling the (stub) functions + added in previous commit at the right moments (at item_remove and + item_add_or_update) + +2008-11-27 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/tracker-indexer/tracker-removable-device.h: Created stubs for + adding 'adding', 'moving', 'removing' triples about resources to the + Turtle file + + * src/libtracker-data/tracker-data-update.c: Added some comments + +2008-11-27 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/libtracker-data/tracker-data-update.c + * src/libtracker-data/tracker-data-update.h: Implemented removing + a resource. Backported storing a resource from the decomposed + tables -branch. + 2008-11-27 Martyn Russell * src/libinotify/inotify-handle.c: Disable warning causing breaks @@ -193,6 +367,51 @@ * src/tracker-indexer/tracker-metadata-utils.c: Added support for multivalued field extraction using '|' as the separator. +2008-11-26 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/tracker-indexer/tracker-indexer.c + * src/tracker-indexer/tracker-indexer.h: Respecting TRANSACTION_MAX + during the Turtle import + +2008-11-26 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c: Copying which + predicates that we don't know about in our own ontology. + +2008-11-26 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/libtracker-data/tracker-turtle.c: Implemented Turtle optimizer + +2008-11-26 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/tracker-indexer/tracker-removable-device.h + * src/tracker-indexer/tracker-indexer.c + * src/tracker-indexer/Makefile.am + * src/tracker-indexer/tracker-indexer.h + * src/libtracker-data/tracker-data-update.c + * src/libtracker-data/tracker-data-update.h + * tests/tracker-indexer/Makefile.am + * configure.ac: Further merging with Jürg's git's Turtle branch. + Note for the person who'll merg Jürg's git with this work: + tracker-indexer/tracker-turtle.c is replaced with + tracker-indexer/tracker-removable-device.c and with + libtracker-data/tracker-turtle.c (the code is split over the two + files). The function tracker_data_replace_service in + libtracker-data/tracker-data-update.c was replaced with an empty + body implementation. Of course should the version in Jürg's git + branch be taken for the decomposed data model (instead of the empty + body implementation). + +2008-11-26 Philip Van Hoof + + * src/tracker-indexer/tracker-removable-device.c + * src/libtracker-data/tracker-turtle.c + * src/libtracker-data/tracker-turtle.h: Various fixes for the Turtle + writer. Added turtle writers for metadata fields + 2008-11-26 Martyn Russell * src/trackerd/tracker-cleanup.[ch]: Committed the changes I made @@ -250,6 +469,27 @@ This patch was written by Philip and reviewed/updated by Martyn. +2008-11-25 Ivan Frade + + * src/libtracker-data/tracker-data-query.[ch] + * data/db/sqlite-stored-procs.sql: + Added code and procedures to retrieve metadata for backup. + + * src/trackerd/tracker-main.c + * src/trackerd/Makefile.am + * src/trackerd/tracker-backup.[ch] + Logic in main to save data on --reindex + + * src/libtracker-db/tracker-db-manager.c: + Set pointers to NULL after unref to shutdown db manager cleanly. + + * configure.ac + * src/tracker-indexer/Makefile.am + * src/libtracker-data/Makefile.am + * src/tracker-indexer/tracker-removable-device.[ch] + * src/libtracker-data/tracker-turtle.[ch] + Import and split of turtle code from Juergs branch. + 2008-11-24 Philip Van Hoof * src/tracker-extract/tracker-albumart.c: Requesting thumbnails of Index: tests/tracker-indexer/Makefile.am =================================================================== --- tests/tracker-indexer/Makefile.am (revision 2649) +++ tests/tracker-indexer/Makefile.am (working copy) @@ -18,6 +18,7 @@ $(PANGO_CFLAGS) \ $(GMODULE_CFLAGS) \ $(GTHREAD_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) tracker_metadata_utils_SOURCES = \ @@ -34,7 +35,9 @@ tracker-module-file.c \ tracker-module-file.h \ tracker-module-iteratable.c \ - tracker-module-iteratable.h + tracker-module-iteratable.h \ + tracker-removable-device.c \ + tracker-removable-device.h tracker_metadata_utils_LDADD = \ $(top_builddir)/src/libtracker-data/libtracker-data.la \ @@ -46,6 +49,7 @@ $(GMODULE_LIBS) \ $(GTHREAD_LIBS) \ $(GIO_LIBS) \ + $(RAPTOR_LIBS) \ $(GLIB2_LIBS) # Index: configure.ac =================================================================== --- configure.ac (revision 2649) +++ configure.ac (working copy) @@ -170,6 +170,15 @@ AM_CONDITIONAL(HAVE_GCONF, test "$have_gconf" = "yes") +# Check for Raptor +PKG_CHECK_MODULES(RAPTOR, [raptor], have_raptor=yes, have_raptor=no) +AC_SUBST(RAPTOR_CFLAGS) +AC_SUBST(RAPTOR_LIBS) + +if test x$have_raptor == "xyes"; then + AC_DEFINE(HAVE_RAPTOR, 1, [Raptor RDF parsers]) +fi + # Check we have the DBUS binding tool we need AC_PATH_PROG(DBUSBINDINGTOOL, dbus-binding-tool) if test -z $DBUSBINDINGTOOL; then @@ -1110,6 +1119,8 @@ tests/tracker-indexer/tracker-module-file.h:src/tracker-indexer/tracker-module-file.h tests/tracker-indexer/tracker-module-iteratable.c:src/tracker-indexer/tracker-module-iteratable.c tests/tracker-indexer/tracker-module-iteratable.h:src/tracker-indexer/tracker-module-iteratable.h + tests/tracker-indexer/tracker-removable-device.c:src/tracker-indexer/tracker-removable-device.c + tests/tracker-indexer/tracker-removable-device.h:src/tracker-indexer/tracker-removable-device.h ) ################################################################## Index: data/db/sqlite-stored-procs.sql =================================================================== --- data/db/sqlite-stored-procs.sql (revision 2649) +++ data/db/sqlite-stored-procs.sql (working copy) @@ -63,6 +63,7 @@ /* * Metadata/MIME queries */ +GetEmbeddedMetadataBackup SELECT S.Path || '/' || S.Name, T.TypeName, F.MetadataID, F.MetadataDisplay From Services S, ServiceMetadata F, ServiceTypes T WHERE (S.ID == F.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (F.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)) UNION SELECT S.Path || '/' || S.Name, T.TypeName, F.MetadataID, upper(F.MetadataValue) From Services S, ServiceNumericMetadata F, ServiceTypes T WHERE (S.ID == F.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (F.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)) UNION SELECT S.Path || '/' || S.Name, T.Typename, F.MetadataID, F.MetadataValue From Services S, ServiceKeywordMetadata F, ServiceTypes T WHERE (S.ID == F.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (F.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)); GetAllMetadata SELECT MetadataID, MetadataDisplay FROM ServiceMetadata WHERE ServiceID = ? UNION SELECT MetadataID, MetadataValue FROM ServiceKeywordMetadata WHERE ServiceID = ? UNION SELECT MetadataID, upper(MetadataValue) FROM ServiceNumericMetadata WHERE ServiceID = ?; GetMetadata SELECT MetaDataDisplay FROM ServiceMetaData WHERE ServiceID = ? AND MetaDataID = ?; GetMetadataAliases SELECT DISTINCT M.MetaName, M.ID FROM MetaDataTypes AS M, MetaDataChildren AS C WHERE M.ID = C.ChildID AND C.MetaDataID = ?; Index: data/dbus/tracker-indexer.xml =================================================================== --- data/dbus/tracker-indexer.xml (revision 2649) +++ data/dbus/tracker-indexer.xml (working copy) @@ -74,6 +74,11 @@ + + + + +