Index: src/tracker-indexer/tracker-removable-device.c =================================================================== --- src/tracker-indexer/tracker-removable-device.c (revision 0) +++ src/tracker-indexer/tracker-removable-device.c (revision 0) @@ -0,0 +1,564 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* + * Copyright (C) 2008, Nokia + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * Author: Philip Van Hoof + */ + +/* When merging the decomposed branch to trunk: this filed used to be called + * tracker-indexer/tracker-turtle.c. What has happened in trunk is that this + * file got renamed to tracker-indexer/tracker-removable-device.c and that + * we created a new file libtracker-data/tracker-turtle.c which contains some + * of the functions that used to be available in this file. + * + * The reason for that is that Ivan's backup support, which runs in trackerd, + * needed access to the same Turtle related routines. So we moved it to one of + * our internally shared libraries (libtracker-data got elected for this). + * + * When merging the decomposed branch, simply pick this file over the file + * tracker-indexer/tracker-turtle.c (in the decomposed branch). */ + +#include "config.h" + +#ifdef HAVE_RAPTOR + +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +#include "tracker-removable-device.h" +#include "tracker-module-metadata-private.h" + +typedef struct { + const gchar *ttl_file; + gchar *last_subject; + gchar *base; + guint amount; + TrackerIndexer *indexer; + TrackerModuleMetadata *metadata; + gchar *rdf_type; +} TurtleStorerInfo; + +typedef enum { + REMOVAL, + REPLACE, + MOVE +} StorerTask; + +static void +commit_turtle_parse_info_storer (TurtleStorerInfo *info, gboolean may_flush, StorerTask task, gchar *destination) +{ + if (info->last_subject) { + + GHashTable *data; + + /* We have it as a URI, database api wants Paths. Update this when + * the database api becomes sane and uses URIs everywhere, the + * '+ 7' means that we cut away the 'file://' prefix. */ + + if (info->rdf_type) { + + /* We ignore records that didn't have an + * predicate. Those are just wrong anyway. */ + + switch (task) { + case REMOVAL: + tracker_data_update_delete_service_by_path (info->last_subject + 7, + info->rdf_type); + break; + case MOVE: + data = tracker_module_metadata_get_hash_table (info->metadata); + tracker_data_update_delete_service_by_path (info->last_subject + 7, + info->rdf_type); + tracker_data_update_replace_service (destination + 7, + info->rdf_type, + data); + g_hash_table_destroy (data); + break; + default: + case REPLACE: + data = tracker_module_metadata_get_hash_table (info->metadata); + tracker_data_update_replace_service (info->last_subject + 7, + info->rdf_type, + data); + g_hash_table_destroy (data); + break; + } + } + + info->amount++; + + g_object_unref (info->metadata); + + g_free (info->last_subject); + g_free (info->rdf_type); + info->last_subject = NULL; + info->metadata = NULL; + info->rdf_type = NULL; + } + + /* We commit per transaction of 100 here, and then we also iterate the + * mainloop so that the state-machine gets the opportunity to run for a + * moment */ + + if (may_flush && info->amount > TRACKER_INDEXER_TRANSACTION_MAX) { + tracker_indexer_transaction_commit (info->indexer); + g_main_context_iteration (NULL, FALSE); + tracker_indexer_transaction_open (info->indexer); + info->amount = 0; + } +} + +static void +consume_triple_storer (void* user_data, const raptor_statement* triple) +{ + TurtleStorerInfo *info = user_data; + gchar *subject; + + /* TODO: cope with multi-value values like User:Keywords */ + + subject = (gchar *) raptor_uri_as_string ((raptor_uri *) triple->subject); + + if (!info->last_subject || strcmp (subject, info->last_subject) != 0) { + + /* Commit previous subject */ + commit_turtle_parse_info_storer (info, TRUE, REPLACE, NULL); + + /* Install next subject */ + info->last_subject = g_strdup (subject); + info->metadata = tracker_module_metadata_new (); + } + + if (triple->object_type == RAPTOR_IDENTIFIER_TYPE_LITERAL) { + gchar *predicate; + + predicate = g_strdup ((const gchar *) raptor_uri_as_string ((raptor_uri *) triple->predicate)); + + if (strcmp (predicate, "rdf:type") == 0) { + g_free (info->rdf_type); + + /* TODO: ontology */ + /* Change this when Files and Emails becomes File and Email */ + + info->rdf_type = g_strdup_printf ("%ss", triple->object); + } else { + tracker_module_metadata_add_string (info->metadata, + predicate, + triple->object); + } + + } else if (triple->object_type == RAPTOR_IDENTIFIER_TYPE_RESOURCE) { + gchar *key = g_strdup_printf ("file://%s/", info->base); + + if (triple->object && strcmp (key, triple->object) == 0 && + triple->predicate && strcmp (key, triple->predicate) == 0) + { + /* "Type" ; * + * <> <> - is a removal of the resource */ + + /* We commit this subject as a removal, the last_subject + * field will be cleared for the next subject to be set + * ready first next process loop. */ + + commit_turtle_parse_info_storer (info, FALSE, REMOVAL, NULL); + } + else + if (triple->object && strcmp (key, triple->object) == 0 && + triple->predicate && strcmp (key, triple->predicate) != 0) + { + gchar *predicate; + + /* "Type" ; * + * <> - is a removal of the * + * resource's Pfx:Predicate */ + + predicate = (gchar *) raptor_uri_as_string ((raptor_uri *) triple->predicate); + + /* We put NULL here, so that a null value goes into + * SQLite. Perhaps we should change this? If so, Why? */ + + tracker_module_metadata_add_string (info->metadata, + predicate, + NULL); + + } + else + if (triple->object && strcmp (key, triple->object) != 0 && + triple->predicate && strcmp (key, triple->predicate) == 0) + { + gchar *object; + /* <> - is a move of the subject */ + object = (gchar *) raptor_uri_as_string ((raptor_uri *) triple->object); + commit_turtle_parse_info_storer (info, FALSE, MOVE, object); + + } + g_free (key); + } + +} + +#endif /* HAVE_RAPTOR */ + +void +tracker_removable_device_optimize (TrackerIndexer *indexer, const gchar *mount_point) +{ + gchar *file = g_build_filename (mount_point, ".cache", + "metadata", "metadata.ttl", NULL); + + if (g_file_test (file, G_FILE_TEST_EXISTS)) { + tracker_turtle_optimize (file); + } + + g_free (file); +} + +void +tracker_removable_device_load (TrackerIndexer *indexer, const gchar *mount_point) +{ +#ifdef HAVE_RAPTOR + gchar *file; + + file = g_build_filename (mount_point, ".cache", + "metadata", "metadata.ttl", NULL); + + if (g_file_test (file, G_FILE_TEST_EXISTS)) { + static gboolean has_init = FALSE; + TurtleStorerInfo *info; + gchar *copy_file, *ptr, *base_uri; + + info = g_slice_new0 (TurtleStorerInfo); + + info->ttl_file = file; + info->indexer = g_object_ref (indexer); + info->amount = 0; + + info->base = g_strdup (mount_point); + + /* We need to open the transaction, during the parsing will the + * transaction be committed and reopened */ + + tracker_indexer_transaction_open (info->indexer); + + tracker_data_update_turtle_import_start (); + + base_uri = g_strdup_printf ("file://%s/", mount_point); + tracker_turtle_process (file, base_uri, consume_triple_storer, info); + g_free (base_uri); + + /* Commit final subject (our loop doesn't handle the very last) + * It can't be a REMOVAL nor MOVE as those happen immediately. */ + + commit_turtle_parse_info_storer (info, FALSE, REPLACE, NULL); + + + /* We will (always) be left in open state, so we commit the + * last opened transaction */ + + tracker_data_update_turtle_import_stop (); + + tracker_indexer_transaction_commit (info->indexer); + + g_free (info->base); + g_object_unref (info->indexer); + g_slice_free (TurtleStorerInfo, info); + } + + g_free (file); + +#endif /* HAVE_RAPTOR */ + +} + +#ifdef HAVE_RAPTOR +typedef struct { + raptor_serializer *serializer; + gchar *about_uri; +} AddMetadataInfo; +#endif + +static void +set_metadata (const gchar *key, const gchar *value, gpointer user_data) +{ +#ifdef HAVE_RAPTOR + raptor_statement *statement; + AddMetadataInfo *item = user_data; + const gchar *about_uri = item->about_uri; + raptor_serializer *serializer = item->serializer; + + statement = g_new0 (raptor_statement, 1); + + statement->subject = (void *) raptor_new_uri (about_uri); + statement->subject_type = RAPTOR_IDENTIFIER_TYPE_RESOURCE; + + statement->predicate = (void *) raptor_new_uri (key); + statement->predicate_type = RAPTOR_IDENTIFIER_TYPE_RESOURCE; + + statement->object = (unsigned char *) g_strdup (value); + statement->object_type = RAPTOR_IDENTIFIER_TYPE_LITERAL; + + raptor_serialize_statement (serializer, + statement); + + raptor_free_uri ((raptor_uri *) statement->subject); + raptor_free_uri ((raptor_uri *) statement->predicate); + g_free ((unsigned char *) statement->object); + + g_free (statement); +#endif +} + + +/* TODO URI branch: path -> uri */ +#ifdef HAVE_RAPTOR +static void +foreach_in_metadata_set_metadata (TrackerField *field, + gpointer value, + gpointer user_data) +{ + AddMetadataInfo *info = user_data; + gchar *parsed_value; + gint throttle; + + + if (!tracker_field_get_multiple_values (field)) { + set_metadata (tracker_field_get_name (field), value, user_data); + } else { + GList *list; + + list = value; + + while (list) { + set_metadata (tracker_field_get_name (field), list->data, user_data); + list = list->next; + } + } + +} +#endif + +void +tracker_removable_device_add_metadata (TrackerIndexer *indexer, + const gchar *mount_point, + const gchar *path, + const gchar *rdf_type, + TrackerModuleMetadata *metadata) +{ +#ifdef HAVE_RAPTOR + AddMetadataInfo *info = g_slice_new (AddMetadataInfo); + gchar *file, *muri; + FILE *target_file; + raptor_uri *suri; + + file = g_build_filename (mount_point, ".cache", + "metadata", NULL); + g_mkdir_with_parents (file, 0700); + g_free (file); + + file = g_build_filename (mount_point, ".cache", + "metadata", "metadata.ttl", NULL); + + target_file = fopen (file, "a"); + /* Similar to a+ */ + if (!target_file) + target_file = fopen (file, "w"); + + if (!target_file) { + g_free (target_file); + g_free (file); + return; + } + + info->serializer = raptor_new_serializer ("turtle"); + info->about_uri = g_strdup (path+strlen (mount_point)+1); + + raptor_serializer_set_feature (info->serializer, + RAPTOR_FEATURE_WRITE_BASE_URI, 0); + + raptor_serializer_set_feature (info->serializer, + RAPTOR_FEATURE_ASSUME_IS_RDF, 1); + + raptor_serializer_set_feature (info->serializer, + RAPTOR_FEATURE_ALLOW_NON_NS_ATTRIBUTES, 1); + + muri = g_strdup_printf ("file://%s/base", mount_point); + suri = raptor_new_uri (muri); + g_free (muri); + + raptor_serialize_start_to_file_handle (info->serializer, + suri, target_file); + + set_metadata ("rdf:type", rdf_type, info); + + tracker_module_metadata_foreach (metadata, + foreach_in_metadata_set_metadata, + info); + + g_free (info->about_uri); + raptor_serialize_end (info->serializer); + raptor_free_serializer (info->serializer); + fclose (target_file); + raptor_free_uri (suri); + + g_slice_free (AddMetadataInfo, info); +#endif /* HAVE_RAPTOR */ +} + +/* TODO URI branch: path -> uri */ + +void +tracker_removable_device_add_removal (TrackerIndexer *indexer, + const gchar *mount_point, + const gchar *path, + const gchar *rdf_type) +{ +#ifdef HAVE_RAPTOR + gchar *file, *about_uri, *dirname, *muri; + FILE *target_file; + raptor_uri *suri; + raptor_statement *statement; + raptor_serializer *serializer; + AddMetadataInfo *info; + + file = g_build_filename (mount_point, ".cache", + "metadata", NULL); + g_mkdir_with_parents (file, 0700); + g_free (file); + + file = g_build_filename (mount_point, ".cache", + "metadata", "metadata.ttl", NULL); + + target_file = fopen (file, "a"); + /* Similar to a+ */ + if (!target_file) + target_file = fopen (file, "w"); + + if (!target_file) { + g_free (target_file); + g_free (file); + return; + } + + serializer = raptor_new_serializer ("turtle"); + about_uri = g_strdup (path+strlen (mount_point)+1); + + raptor_serializer_set_feature (serializer, + RAPTOR_FEATURE_WRITE_BASE_URI, 0); + + raptor_serialize_start_to_file_handle (serializer, + suri, target_file); + + info = g_slice_new (AddMetadataInfo); + + info->serializer = serializer; + info->about_uri = about_uri; + + set_metadata ("rdf:type", rdf_type, info); + set_metadata (NULL, NULL, info); + + g_slice_free (AddMetadataInfo, info); + g_free (about_uri); + raptor_serialize_end (serializer); + raptor_free_serializer (serializer); + fclose (target_file); + +#endif /* HAVE_RAPTOR */ +} + +/* TODO URI branch: path -> uri */ + +void +tracker_removable_device_add_move (TrackerIndexer *indexer, + const gchar *mount_point, + const gchar *from_path, + const gchar *to_path, + const gchar *rdf_type) +{ +#ifdef HAVE_RAPTOR + gchar *file, *about_uri, *to_uri, *muri; + FILE *target_file; + raptor_uri *suri; + raptor_statement *statement; + raptor_serializer *serializer; + AddMetadataInfo *info; + + file = g_build_filename (mount_point, ".cache", + "metadata", NULL); + g_mkdir_with_parents (file, 0700); + g_free (file); + + file = g_build_filename (mount_point, ".cache", + "metadata", "metadata.ttl", NULL); + + target_file = fopen (file, "a"); + /* Similar to a+ */ + if (!target_file) + target_file = fopen (file, "w"); + + if (!target_file) { + g_free (target_file); + g_free (file); + return; + } + + serializer = raptor_new_serializer ("turtle"); + + raptor_serializer_set_feature (serializer, + RAPTOR_FEATURE_WRITE_BASE_URI, 0); + + about_uri = g_strdup (from_path+strlen (mount_point)+1); + to_uri = g_strdup (to_path+strlen (mount_point)+1); + + muri = g_strdup_printf ("file://%s/", mount_point); + suri = raptor_new_uri (muri); + g_free (muri); + + raptor_serialize_start_to_file_handle (serializer, + suri, target_file); + + info = g_slice_new (AddMetadataInfo); + + info->serializer = serializer; + info->about_uri = about_uri; + + set_metadata ("rdf:type", rdf_type, info); + set_metadata (NULL, to_uri, info); + + g_slice_free (AddMetadataInfo, info); + + + g_free (about_uri); + g_free (to_uri); + raptor_serialize_end (serializer); + raptor_free_serializer (serializer); + fclose (target_file); + raptor_free_uri (suri); + + +#endif /* HAVE_RAPTOR */ +} + Index: src/tracker-indexer/tracker-removable-device.h =================================================================== --- src/tracker-indexer/tracker-removable-device.h (revision 0) +++ src/tracker-indexer/tracker-removable-device.h (revision 0) @@ -0,0 +1,57 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* + * Copyright (C) 2008, Nokia + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * Author: Philip Van Hoof + */ + +#ifndef __TRACKER_REMOVABLE_DEVICE_H__ +#define __TRACKER_REMOVABLE_DEVICE_H__ + +#include + +#include "tracker-module-metadata.h" +#include "tracker-indexer.h" + +G_BEGIN_DECLS + +void tracker_removable_device_load (TrackerIndexer *indexer, + const gchar *mount_point); +void tracker_removable_device_optimize (TrackerIndexer *indexer, + const gchar *mount_point); +void tracker_removable_device_add_metadata (TrackerIndexer *indexer, + const gchar *mount_point, + const gchar *path, + const gchar *rdf_type, + TrackerModuleMetadata *metadata); +void tracker_removable_device_add_removal (TrackerIndexer *indexer, + const gchar *mount_point, + const gchar *path, + const gchar *rdf_type); +void tracker_removable_device_add_move (TrackerIndexer *indexer, + const gchar *mount_point, + const gchar *from_path, + const gchar *to_path, + const gchar *rdf_type); + +G_END_DECLS + +#endif /* __TRACKER_REMOVABLE_DEVICE_H__ */ + + + Index: src/tracker-indexer/tracker-main.c =================================================================== --- src/tracker-indexer/tracker-main.c (revision 2683) +++ src/tracker-indexer/tracker-main.c (working copy) @@ -43,6 +43,7 @@ #include #include +#include #include "tracker-dbus.h" #include "tracker-indexer.h" @@ -370,13 +371,18 @@ tracker_indexer_process_modules (indexer, modules); } + tracker_turtle_init (); + g_message ("Starting..."); + main_loop = g_main_loop_new (NULL, FALSE); g_main_loop_run (main_loop); g_message ("Shutdown started"); + tracker_turtle_shutdown (); + if (quit_timeout_id) { g_source_remove (quit_timeout_id); } Index: src/tracker-indexer/tracker-indexer.c =================================================================== --- src/tracker-indexer/tracker-indexer.c (revision 2683) +++ src/tracker-indexer/tracker-indexer.c (working copy) @@ -71,11 +71,13 @@ #include #include +#include #include "tracker-indexer.h" #include "tracker-indexer-module.h" #include "tracker-marshal.h" #include "tracker-module-metadata-private.h" +#include "tracker-removable-device.h" #define TRACKER_INDEXER_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_INDEXER, TrackerIndexerPrivate)) @@ -85,9 +87,6 @@ #define LOW_DISK_CHECK_FREQUENCY 10 #define SIGNAL_STATUS_FREQUENCY 10 -/* Transaction every 'x' items */ -#define TRANSACTION_MAX 2000 - /* Throttle defaults */ #define THROTTLE_DEFAULT 0 #define THROTTLE_DEFAULT_ON_BATTERY 5 @@ -385,6 +384,22 @@ indexer); } + +void +tracker_indexer_transaction_commit (TrackerIndexer *indexer) +{ + stop_transaction (indexer); + tracker_indexer_set_running (indexer, TRUE); + +} + +void +tracker_indexer_transaction_open (TrackerIndexer *indexer) +{ + tracker_indexer_set_running (indexer, FALSE); + start_transaction (indexer); +} + #ifdef HAVE_HAL static void @@ -1365,6 +1380,7 @@ } } + static void item_add_or_update (TrackerIndexer *indexer, PathInfo *info, @@ -1375,6 +1391,8 @@ TrackerService *service; gchar *text; guint32 id; + gchar *mount_point = NULL; + gchar *service_path; service = get_service_for_file (info->module_file, info->module); @@ -1419,11 +1437,11 @@ new_text = tracker_module_file_get_text (info->module_file); item_update_content (indexer, service, id, old_text, new_text); - g_free (old_text); g_free (new_text); tracker_data_metadata_free (old_metadata_emb); tracker_data_metadata_free (old_metadata_non_emb); + } else { GHashTable *data; @@ -1463,8 +1481,32 @@ g_hash_table_destroy (data); } + + /* TODO: URI branch path -> uri */ + + service_path = g_build_path (G_DIR_SEPARATOR_S, + dirname, + basename, + NULL); + + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + service_path, + &mount_point, + NULL)) { + + tracker_removable_device_add_metadata (indexer, + mount_point, + service_path, + tracker_service_get_name (service), + metadata); + } + + g_free (mount_point); + g_free (service_path); + } + static gboolean filter_invalid_after_move_properties (TrackerField *field, gpointer value, @@ -1491,6 +1533,7 @@ gchar *path, *other_path; gchar *uri, *other_uri, *mime_type; guint32 service_id; + gchar *mount_point = NULL; service = get_service_for_file (info->other_module_file, info->module); @@ -1541,6 +1584,32 @@ tracker_data_update_move_service (service, path, other_path); + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + path, + &mount_point, + NULL) ) { + + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + other_path, + NULL, + NULL) ) { + + tracker_removable_device_add_move (indexer, + mount_point, + path, + other_path, + tracker_service_get_name (service)); + + } else { + tracker_removable_device_add_removal (indexer, + mount_point, + path, + tracker_service_get_name (service)); + } + } + + g_free (mount_point); + /* * Updating what changes in move event (Path related properties) */ @@ -1576,6 +1645,7 @@ g_free (other_path); } + static void item_remove (TrackerIndexer *indexer, PathInfo *info, @@ -1587,6 +1657,7 @@ gchar *content; gchar *metadata; gchar *path; + gchar *mount_point = NULL; const gchar *service_type; guint service_id, service_type_id; @@ -1693,8 +1764,19 @@ tracker_data_update_delete_service_recursively (service, path); } + if (tracker_hal_path_is_on_removable_device (indexer->private->hal, + path, + &mount_point, + NULL)) { + + tracker_removable_device_add_removal (indexer, mount_point, + path, + tracker_service_get_name (service)); + } + tracker_data_update_decrement_stats (indexer->private->common, service); + g_free (mount_point); g_free (path); } @@ -2392,7 +2474,7 @@ /* Signal stopped and clean up */ check_stopped (indexer, FALSE); check_disk_space_stop (indexer); - + return FALSE; } @@ -2400,7 +2482,7 @@ g_free (module_name); } - if (indexer->private->items_processed > TRANSACTION_MAX) { + if (indexer->private->items_processed > TRACKER_INDEXER_TRANSACTION_MAX) { schedule_flush (indexer, TRUE); } @@ -2848,6 +2930,13 @@ dbus_g_method_return (context); tracker_dbus_request_success (request_id); + + /* tracker_turtle_process_ttl will be spinning the mainloop, therefore + we can already return the DBus method */ + + if (enabled) { + tracker_removable_device_load (indexer, path); + } } void @@ -2941,7 +3030,64 @@ tracker_dbus_request_success (request_id); } +static void +restore_backup_triple (void *user_data, const stmt *triple) { + + const gchar *values[2]; + TrackerIndexer *indexer = user_data; + GError *error = NULL; + + g_return_if_fail (TRACKER_IS_INDEXER (indexer)); + + g_debug ("Turtle loading <%s, %s, %s>", + (gchar *)triple->subject, + (gchar *)triple->predicate, + (gchar *)triple->object); + + values[0] = triple->object; + values[1] = NULL; + + handle_metadata_add (indexer, + "Files", + triple->subject, + triple->predicate, + (GStrv) values, + &error); + + if (error) { + g_warning ("Restoring backup: %s", error->message); + g_error_free (error); + } + +} + void +tracker_indexer_restore_backup (TrackerIndexer *indexer, + const gchar *backup_file, + DBusGMethodInvocation *context, + GError **error) +{ + guint request_id; + + request_id = tracker_dbus_get_next_request_id (); + + tracker_dbus_async_return_if_fail (TRACKER_IS_INDEXER (indexer), context); + + tracker_dbus_request_new (request_id, + "DBus request to restore backup data from '%s'", + backup_file); + + tracker_turtle_process (backup_file, + "/", + (TurtleTripleCallback) restore_backup_triple, + indexer); + + dbus_g_method_return (context); + tracker_dbus_request_success (request_id); +} + + +void tracker_indexer_shutdown (TrackerIndexer *indexer, DBusGMethodInvocation *context, GError **error) Index: src/tracker-indexer/Makefile.am =================================================================== --- src/tracker-indexer/Makefile.am (revision 2683) +++ src/tracker-indexer/Makefile.am (working copy) @@ -13,6 +13,7 @@ -I$(top_srcdir)/src \ $(DBUS_CFLAGS) \ $(PANGO_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GMODULE_CFLAGS) libtracker_moduledir = $(libdir)/tracker @@ -42,8 +43,9 @@ tracker-indexer-module.c \ tracker-indexer-module.h \ tracker-main.c \ + tracker-module-metadata-private.h \ tracker-marshal-main.c \ - tracker-module-metadata-private.h + tracker-removable-device.c tracker_indexer_LDADD = \ libtracker-module.la \ @@ -58,6 +60,7 @@ $(PANGO_LIBS) \ $(GIO_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ -lz \ -lm Index: src/tracker-indexer/tracker-indexer.h =================================================================== --- src/tracker-indexer/tracker-indexer.h (revision 2683) +++ src/tracker-indexer/tracker-indexer.h (working copy) @@ -30,6 +30,9 @@ #define TRACKER_INDEXER_PATH "/org/freedesktop/Tracker/Indexer" #define TRACKER_INDEXER_INTERFACE "org.freedesktop.Tracker.Indexer" +/* Transaction every 'x' items */ +#define TRACKER_INDEXER_TRANSACTION_MAX 2000 + G_BEGIN_DECLS #define TRACKER_TYPE_INDEXER (tracker_indexer_get_type()) @@ -80,6 +83,8 @@ void tracker_indexer_process_all (TrackerIndexer *indexer); void tracker_indexer_process_modules (TrackerIndexer *indexer, gchar **modules); +void tracker_indexer_transaction_commit (TrackerIndexer *indexer); +void tracker_indexer_transaction_open (TrackerIndexer *indexer); /* DBus methods */ void tracker_indexer_pause (TrackerIndexer *indexer, @@ -136,6 +141,10 @@ GStrv values, DBusGMethodInvocation *context, GError **error); +void tracker_indexer_restore_backup (TrackerIndexer *indexer, + const gchar *backup_file, + DBusGMethodInvocation *context, + GError **error); void tracker_indexer_shutdown (TrackerIndexer *indexer, DBusGMethodInvocation *context, GError **error); Index: src/tracker-indexer/tracker-module-metadata.c =================================================================== --- src/tracker-indexer/tracker-module-metadata.c (revision 2683) +++ src/tracker-indexer/tracker-module-metadata.c (working copy) @@ -188,13 +188,15 @@ list = g_list_prepend (list, value); data = list; } else { - /* FIXME: warn if data already exists */ + data = g_hash_table_lookup (metadata->table, field); + g_free (data); data = value; } - g_hash_table_insert (metadata->table, - g_object_ref (field), - data); + g_hash_table_replace (metadata->table, + g_object_ref (field), + data); + return TRUE; } Index: src/libtracker-data/tracker-data-query.c =================================================================== --- src/libtracker-data/tracker-data-query.c (revision 2683) +++ src/libtracker-data/tracker-data-query.c (working copy) @@ -386,6 +386,9 @@ return service_type_id; } +/* + * Result set with (metadataID, value) per row + */ static void result_set_to_metadata (TrackerDBResultSet *result_set, TrackerDataMetadata *metadata, @@ -400,7 +403,7 @@ GValue transform = {0, }; GValue value = {0, }; gchar *str; - + g_value_init (&transform, G_TYPE_STRING); tracker_db_result_set_get (result_set, 0, &metadata_id, -1); _tracker_db_result_set_get_value (result_set, 1, &value); @@ -492,6 +495,23 @@ return metadata; } +TrackerDBResultSet * +tracker_data_query_backup_metadata (TrackerService *service) +{ + TrackerDBInterface *iface; + TrackerDBResultSet *result_set; + GHashTable *results; + + g_return_val_if_fail (TRACKER_IS_SERVICE (service), NULL); + + iface = tracker_db_manager_get_db_interface_by_service (tracker_service_get_name (service)); + + result_set = tracker_data_manager_exec_proc (iface, + "GetEmbeddedMetadataBackup", + NULL); + return result_set; +} + static gchar * db_get_metadata (TrackerService *service, guint service_id, Index: src/libtracker-data/tracker-turtle.c =================================================================== --- src/libtracker-data/tracker-turtle.c (revision 0) +++ src/libtracker-data/tracker-turtle.c (revision 0) @@ -0,0 +1,417 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* + * Copyright (C) 2008, Nokia + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * Author: Philip Van Hoof + */ + +#include "config.h" + +#include +#include +#include + +#include +#include + +#include "tracker-turtle.h" + +static gboolean initialized = FALSE; + +struct TurtleFile { + FILE *file; +#ifdef HAVE_RAPTOR + raptor_uri *uri; + raptor_serializer *serializer; +#endif /* HAVE_RAPTOR */ +}; + +#ifdef HAVE_RAPTOR +typedef struct { + gchar *last_subject; + raptor_serializer *serializer; + GHashTable *hash; +} TurtleOptimizerInfo; +#endif /* HAVE_RAPTOR */ + +void +tracker_turtle_init (void) +{ + if (!initialized) { +#ifdef HAVE_RAPTOR + raptor_init (); +#endif /* HAVE_RAPTOR */ + initialized = TRUE; + } +} + +void +tracker_turtle_shutdown (void) +{ + if (initialized) { +#ifdef HAVE_RAPTOR + raptor_finish (); +#endif /* HAVE_RAPTOR */ + initialized = FALSE; + } +} + + +#ifdef HAVE_RAPTOR + + +static void +foreach_in_hash (gpointer key, gpointer value, gpointer user_data) +{ + raptor_statement *statement; + TurtleOptimizerInfo *item = user_data; + const gchar *about_uri = item->last_subject; + raptor_serializer *serializer = item->serializer; + + statement = g_new0 (raptor_statement, 1); + + statement->subject = (void *) raptor_new_uri (about_uri); + statement->subject_type = RAPTOR_IDENTIFIER_TYPE_RESOURCE; + + statement->predicate = (void *) raptor_new_uri (key); + statement->predicate_type = RAPTOR_IDENTIFIER_TYPE_RESOURCE; + + statement->object = (unsigned char *) g_strdup (value); + statement->object_type = RAPTOR_IDENTIFIER_TYPE_LITERAL; + + raptor_serialize_statement (serializer, + statement); + + raptor_free_uri ((raptor_uri *) statement->subject); + raptor_free_uri ((raptor_uri *) statement->predicate); + g_free ((unsigned char *) statement->object); + + g_free (statement); +} + + +static void +commit_turtle_parse_info_optimizer (TurtleOptimizerInfo *info) +{ + if (info->last_subject) { + + g_hash_table_foreach (info->hash, + foreach_in_hash, + info); + + g_hash_table_destroy (info->hash); + g_free (info->last_subject); + info->last_subject = NULL; + info->hash = NULL; + } +} + + +static void +consume_triple_optimizer (void* user_data, const raptor_statement* triple) +{ + TurtleOptimizerInfo *info = user_data; + gchar *subject; + gchar *predicate; + + subject = (gchar *) raptor_uri_as_string ((raptor_uri *) triple->subject); + predicate = (gchar *) raptor_uri_as_string ((raptor_uri *) triple->predicate); + + if (!info->last_subject || strcmp (subject, info->last_subject) != 0) { + /* Commit previous subject */ + commit_turtle_parse_info_optimizer (info); + info->last_subject = g_strdup (subject); + info->hash = g_hash_table_new_full (g_str_hash, g_str_equal, + (GDestroyNotify) g_free, + (GDestroyNotify) g_free); + } + + if (triple->object_type == RAPTOR_IDENTIFIER_TYPE_RESOURCE) { + + /* TODO: these checks for removals of resources and predicates + * are incorrect atm */ + + if (g_str_has_suffix (predicate, ":")) { + /* <> <> */ + g_hash_table_destroy (info->hash); + g_free (info->last_subject); + info->last_subject = NULL; + info->hash = NULL; + } else { + /* <> */ + g_hash_table_remove (info->hash, predicate); + } + } else { + + /* TODO: Add conflict resolution here (if any is needed) */ + + g_hash_table_replace (info->hash, + g_strdup (predicate), + g_strdup (triple->object)); + } + +} + +static void +foreach_in_metadata (TrackerField *field, gpointer value, gpointer user_data) +{ + raptor_statement *statement; + TrackerTurtleMetadataItem *item = user_data; + const gchar *about_uri = item->about_uri; + TurtleFile *turtle = item->turtle; + raptor_serializer *serializer = turtle->serializer; + + /* TODO: cope with group values + * + * If you want to reuse the importer of tracker-indexer (for the remov- + * able devices), then you'll need to ensure that the predicates + * File:Modified and rdf:type are added per record (you separate triples + * using a ; and you end a record using a . (a dot). + * + * Also look at tracker-indexer/tracker-removable-device.c */ + + statement = g_new0 (raptor_statement, 1); + + statement->subject = (void *) raptor_new_uri (about_uri); + statement->subject_type = RAPTOR_IDENTIFIER_TYPE_RESOURCE; + + statement->predicate = (void *) raptor_new_uri (tracker_field_get_name (field)); + statement->predicate_type = RAPTOR_IDENTIFIER_TYPE_RESOURCE; + + statement->object = (unsigned char *) g_strdup (value); + statement->object_type = RAPTOR_IDENTIFIER_TYPE_LITERAL; + + raptor_serialize_statement (serializer, + statement); + + raptor_free_uri ((raptor_uri *) statement->subject); + raptor_free_uri ((raptor_uri *) statement->predicate); + g_free ((unsigned char *) statement->object); + + g_free (statement); +} +#endif /* HAVE_RAPTOR */ + + + + +TurtleFile * +tracker_turtle_open (const gchar *turtle_file) +{ + g_return_val_if_fail (initialized, NULL); + +#ifdef HAVE_RAPTOR + TurtleFile *turtle; + + turtle = g_new0 (TurtleFile, 1); + + turtle->file = g_fopen (turtle_file, "a"); + /* Similar to a+ */ + if (!turtle->file) + turtle->file = g_fopen (turtle_file, "w"); + + turtle->serializer = raptor_new_serializer ("turtle"); + turtle->uri = raptor_new_uri ("/"); + raptor_serialize_start_to_file_handle (turtle->serializer, + turtle->uri, turtle->file); + + return turtle; +#else + return NULL; +#endif +} + +void +tracker_turtle_add_metadata (TurtleFile *turtle, + const gchar *uri, + TrackerDataMetadata *metadata) +{ +#ifdef HAVE_RAPTOR + TrackerTurtleMetadataItem *info = g_slice_new (TrackerTurtleMetadataItem); + + info->about_uri = (gchar *) uri; + info->metadata = metadata; + info->turtle = turtle; + + tracker_data_metadata_foreach (metadata, + foreach_in_metadata, + info); + + g_slice_free (TrackerTurtleMetadataItem, info); +#endif /* HAVE_RAPTOR */ +} + +void +tracker_turtle_add_metadatas (TurtleFile *turtle, + GPtrArray *metadata_items) +{ +#ifdef HAVE_RAPTOR + guint count; + + for (count = 0; count < metadata_items->len; count++) { + TrackerTurtleMetadataItem *item = g_ptr_array_index (metadata_items, count); + + item->turtle = turtle; + + tracker_data_metadata_foreach (item->metadata, + foreach_in_metadata, + item); + } +#endif /* HAVE_RAPTOR */ +} + + +void +tracker_turtle_add_triple (TurtleFile *turtle, + const gchar *uri, + TrackerField *property, + const gchar *value) +{ +#ifdef HAVE_RAPTOR + TrackerTurtleMetadataItem *item = g_slice_new (TrackerTurtleMetadataItem); + item->about_uri = (gchar *) uri; + item->turtle = turtle; + foreach_in_metadata (property, (gpointer) value, item); + g_slice_free (TrackerTurtleMetadataItem, item); +#endif +} + + +void +tracker_turtle_close (TurtleFile *turtle) +{ +#ifdef HAVE_RAPTOR + raptor_free_uri (turtle->uri); + raptor_serialize_end (turtle->serializer); + raptor_free_serializer(turtle->serializer); + fclose (turtle->file); + g_free (turtle); +#endif +} + +#ifdef HAVE_RAPTOR +static void +raptor_error (void *user_data, raptor_locator* locator, const char *message) +{ + g_message ("RAPTOR parse error: %s for %s\n", + message, + (gchar *) user_data); +} +#endif + +void +tracker_turtle_process (const gchar *turtle_file, + const gchar *base_uri, + TurtleTripleCallback callback, + void *user_data) +{ +#ifdef HAVE_RAPTOR + unsigned char *uri_string; + raptor_uri *uri, *buri; + raptor_parser *parser; +#endif + if (!initialized) { + g_critical ("Using tracker_turtle module without initialization"); + } + +#ifdef HAVE_RAPTOR + + parser = raptor_new_parser ("turtle"); + + raptor_set_statement_handler (parser, user_data, (raptor_statement_handler) callback); + raptor_set_fatal_error_handler (parser, (void *)turtle_file, raptor_error); + raptor_set_error_handler (parser, (void *)turtle_file, raptor_error); + raptor_set_warning_handler (parser, (void *)turtle_file, raptor_error); + + uri_string = raptor_uri_filename_to_uri_string (turtle_file); + uri = raptor_new_uri (uri_string); + buri = raptor_new_uri (base_uri); + + raptor_parse_file (parser, uri, buri); + + raptor_free_uri (uri); + raptor_free_memory (uri_string); + raptor_free_uri (buri); + + raptor_free_parser (parser); + +#endif +} + + +void +tracker_turtle_optimize (const gchar *turtle_file) +{ +#ifdef HAVE_RAPTOR + raptor_uri *suri; + TurtleOptimizerInfo *info; + gchar *tmp_file, *base_uri; + FILE *target_file; +#endif + + if (!initialized) { + g_critical ("Using tracker_turtle module without initialization"); + } + +#ifdef HAVE_RAPTOR + tmp_file = g_strdup_printf ("%s.tmp", turtle_file); + + target_file = g_fopen (tmp_file, "a"); + /* Similar to a+ */ + if (!target_file) + target_file = g_fopen (tmp_file, "w"); + + if (!target_file) { + g_free (target_file); + g_free (tmp_file); + return; + } + + info = g_slice_new0 (TurtleOptimizerInfo); + info->serializer = raptor_new_serializer ("turtle"); + suri = raptor_new_uri ("/"); + + base_uri = g_strdup_printf ("file://%s", turtle_file); + + raptor_serialize_start_to_file_handle (info->serializer, + suri, target_file); + + tracker_turtle_process (turtle_file, base_uri, consume_triple_optimizer, info); + + g_free (base_uri); + + /* Commit final subject (or loop doesn't handle the very last) */ + commit_turtle_parse_info_optimizer (info); + + raptor_serialize_end (info->serializer); + raptor_free_serializer (info->serializer); + fclose (target_file); + + g_slice_free (TurtleOptimizerInfo, info); + + raptor_free_uri (suri); + + /* When we are finished we atomicly overwrite the original with + * our newly created .tmp file */ + + g_rename (tmp_file, turtle_file); + + g_free (tmp_file); + +#endif /* HAVE_RAPTOR */ +} Index: src/libtracker-data/tracker-data-query.h =================================================================== --- src/libtracker-data/tracker-data-query.h (revision 2683) +++ src/libtracker-data/tracker-data-query.h (working copy) @@ -53,6 +53,7 @@ TrackerDataMetadata *tracker_data_query_metadata (TrackerService *service, guint32 service_id, gboolean embedded); +TrackerDBResultSet *tracker_data_query_backup_metadata (TrackerService *service); gchar * tracker_data_query_parsed_metadata (TrackerService *service, guint32 service_id); gchar * tracker_data_query_unparsed_metadata (TrackerService *service, Index: src/libtracker-data/tracker-turtle.h =================================================================== --- src/libtracker-data/tracker-turtle.h (revision 0) +++ src/libtracker-data/tracker-turtle.h (revision 0) @@ -0,0 +1,95 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* + * Copyright (C) 2008, Nokia + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * Author: Philip Van Hoof + */ + +#ifndef __TRACKER_TURTLE_H__ +#define __TRACKER_TURTLE_H__ + +#include +#include + +#ifdef HAVE_RAPTOR +#include +#endif + +/* + * TODO: Is it possible to do this in the .c file? Dont expose raptor here. +*/ + +#ifdef HAVE_RAPTOR +typedef raptor_statement stmt; +#else +typedef struct { + gconstpointer subject; + gint subject_type; + gconstpointer predicate; + gint predicate_type; + gconstpointer object; + gint object_type; + gpointer object_literal_datatype; + const guchar *object_literal_language; +} stmt; +#endif + +G_BEGIN_DECLS + +typedef void (* TurtleTripleCallback) (void *user_data, const stmt *triple); + +typedef struct TurtleFile TurtleFile; + + + +typedef struct { + gchar *about_uri; + TrackerDataMetadata *metadata; + TurtleFile *turtle; /* For internal use only */ +} TrackerTurtleMetadataItem; + +/* Initialization (use in both cases) */ +void tracker_turtle_init (void); +void tracker_turtle_shutdown (void); + +/* Transactions style */ +TurtleFile *tracker_turtle_open (const gchar *turtle_file); +void tracker_turtle_add_triple (TurtleFile *turtle, + const gchar *uri, + TrackerField *property, + const gchar *value); +void tracker_turtle_add_metadata (TurtleFile *turtle, + const gchar *uri, + TrackerDataMetadata *metadata); +void tracker_turtle_add_metadatas (TurtleFile *turtle, + GPtrArray /* */ *metadata_items); +void tracker_turtle_close (TurtleFile *turtle); + +/* Reading functions */ +void tracker_turtle_process (const gchar *turtle_file, + const gchar *base_uri, + TurtleTripleCallback callback, + void *user_data); + + +/* Optimizer, reparser */ +void tracker_turtle_optimize (const gchar *turtle_file); + +G_END_DECLS + +#endif /* __TRACKER_TURTLE_H__ */ Index: src/libtracker-data/tracker-data-metadata.c =================================================================== --- src/libtracker-data/tracker-data-metadata.c (revision 2683) +++ src/libtracker-data/tracker-data-metadata.c (working copy) @@ -106,6 +106,7 @@ const gchar *value) { TrackerField *field; + gchar *old_value; g_return_if_fail (metadata != NULL); g_return_if_fail (field_name != NULL); @@ -114,13 +115,17 @@ field = tracker_ontology_get_field_by_name (field_name); g_return_if_fail (TRACKER_IS_FIELD (field)); - g_return_if_fail (tracker_field_get_multiple_values (field) == FALSE); + g_return_if_fail (tracker_field_get_multiple_values (field) == FALSE); - g_hash_table_insert (metadata->table, - g_object_ref (field), - g_strdup (value)); + old_value = g_hash_table_lookup (metadata->table, field); + g_free (old_value); + + g_hash_table_replace (metadata->table, + g_object_ref (field), + g_strdup (value)); } + /** * tracker_data_metadata_insert_values: * @metadata: A #TrackerDataMetadata @@ -137,9 +142,10 @@ void tracker_data_metadata_insert_values (TrackerDataMetadata *metadata, const gchar *field_name, - GList *list) + const GList *list) { TrackerField *field; + GList *old_values; g_return_if_fail (metadata != NULL); g_return_if_fail (field_name != NULL); @@ -150,12 +156,26 @@ field = tracker_ontology_get_field_by_name (field_name); + if (!field) { + g_warning ("Field name '%s' has isn't described in the ontology", field_name); + return; + } + + field = tracker_ontology_get_field_by_name (field_name); + g_return_if_fail (TRACKER_IS_FIELD (field)); g_return_if_fail (tracker_field_get_multiple_values (field) == TRUE); - g_hash_table_insert (metadata->table, - g_object_ref (field), - tracker_glist_copy_with_string_data (list)); + old_values = g_hash_table_lookup (metadata->table, field); + + if (old_values) { + g_list_foreach (old_values, (GFunc) g_free, NULL); + g_list_free (old_values); + } + + g_hash_table_replace (metadata->table, + g_object_ref (field), + tracker_glist_copy_with_string_data ((GList *)list)); } /** Index: src/libtracker-data/tracker-data-update.c =================================================================== --- src/libtracker-data/tracker-data-update.c (revision 2683) +++ src/libtracker-data/tracker-data-update.c (working copy) @@ -26,12 +26,23 @@ #include #include +#include #include #include #include "tracker-data-manager.h" #include "tracker-data-update.h" +typedef struct { + TrackerService *service; + guint32 iid_value; + TrackerLanguage *language; + TrackerConfig *config; +} ForeachInMetadataInfo; + +static TrackerConfig *config = NULL; +static TrackerLanguage *language = NULL; + guint32 tracker_data_update_get_new_service_id (TrackerDBInterface *iface) { @@ -566,7 +577,273 @@ tracker_data_manager_exec (iface, "DELETE FROM Events WHERE BeingHandled = 1"); } +/* TODO: URI branch path -> uri */ + void +tracker_data_update_delete_service_by_path (const gchar *path, + const gchar *rdf_type) +{ + TrackerService *service; + const gchar *service_type; + guint32 service_id; + + g_return_if_fail (path != NULL); + + if (!rdf_type) + return; + + service = tracker_ontology_get_service_by_name (rdf_type); + service_type = tracker_service_get_name (service); + service_id = tracker_data_query_file_id (service_type, path); + + /* When merging from the decomposed branch to trunk then this function + * wont exist in the decomposed branch. Create it based on this one. */ + + if (service_id != 0) { + tracker_data_update_delete_service (service, service_id); + tracker_data_update_delete_service_recursively (service, (gchar *) path); + tracker_data_update_delete_all_metadata (service, service_id); + } +} + +/* TODO: URI branch path -> uri */ + +static void +set_metadata (TrackerField *field, + gpointer value, + ForeachInMetadataInfo *info) +{ + TrackerDBIndex *index; + gchar *parsed_value; + gchar **arr; + gint service_id; + gint i; + gint score; + + /* TODO untested and unfinished port that came from the decomposed + * branch of Jürg. When merging from the decomposed branch to trunk + * then pick the version in the decomposed branch for this function */ + + parsed_value = tracker_parser_text_to_string (value, + info->language, + tracker_config_get_max_word_length (info->config), + tracker_config_get_min_word_length (info->config), + tracker_field_get_filtered (field), + tracker_field_get_filtered (field), + tracker_field_get_delimited (field)); + + if (!parsed_value) { + return; + } + + score = tracker_field_get_weight (field); + + arr = g_strsplit (parsed_value, " ", -1); + service_id = tracker_service_get_id (info->service); + index = tracker_db_index_manager_get_index_by_service_id (service_id); + + for (i = 0; arr[i]; i++) { + tracker_db_index_add_word (index, + arr[i], + info->iid_value, + tracker_service_get_id (info->service), + score); + } + + tracker_data_update_set_metadata (info->service, info->iid_value, field, value, parsed_value); + + g_free (parsed_value); + g_strfreev (arr); + +} + +static void +foreach_in_metadata_set_metadata (gpointer predicate, + gpointer value, + gpointer user_data) +{ + ForeachInMetadataInfo *info = user_data; + gchar *parsed_value; + gint throttle; + TrackerField *field; + + field = tracker_ontology_get_field_by_name (predicate); + + if (!field) + return; + + /* Throttle indexer, value 9 is from older code, why 9? */ + throttle = tracker_config_get_throttle (info->config); + if (throttle > 9) { + tracker_throttle (info->config, throttle * 100); + } + + if (!tracker_field_get_multiple_values (field)) { + set_metadata (field, value, user_data); + } else { + GList *list; + + for (list = value; list; list = list->next) + set_metadata (field, list->data, user_data); + + } +} + + +void +tracker_data_update_turtle_import_start (void) +{ + if (!config) + config = tracker_config_new (); + if (!language) + language = tracker_language_new (config); +} + +void +tracker_data_update_turtle_import_stop (void) +{ + g_object_unref (language); + g_object_unref (config); + language = NULL; + config = NULL; +} + + +/* TODO: URI branch path -> uri */ + +void +tracker_data_update_replace_service (const gchar *path, + const gchar *rdf_type, + GHashTable *metadata) +{ + TrackerDBInterface *iface; + TrackerDBResultSet *result_set; + const gchar *modified; + GError *error = NULL; + TrackerService *service; + gchar *escaped_path; + gchar *dirname; + const gchar *basename; + + g_return_if_fail (path != NULL); + g_return_if_fail (metadata != NULL); + + /* When merging from the decomposed branch to trunk then pick the version + * in the decomposed branch for this function. However, carefully + * compare the features, as this version is more recent and has + * implemented a few significant items, whereas the version in the + * decomposed branch was a proof of concept implementation, and might + * not have these needed features. */ + + if (!rdf_type) + return; + + service = tracker_ontology_get_service_by_name (rdf_type); + + iface = tracker_db_manager_get_db_interface_by_type (tracker_service_get_name (service), + TRACKER_DB_CONTENT_TYPE_METADATA); + + modified = g_hash_table_lookup (metadata, "File:Modified"); + + if (!modified) { + return; + } + + escaped_path = tracker_escape_string (path); + + basename = g_basename (escaped_path); + dirname = g_dirname (escaped_path); + + /* TODO Warning: comparing Modified against Accessed. Do we have a + * better field for this? */ + + result_set = tracker_db_interface_execute_query (iface, &error, + "SELECT ID, Accessed < '%s' FROM Services " + "WHERE Path = '%s' AND " + "Name = '%s'", + modified, + dirname, basename); + + if (error) { +#ifdef TURTLE_DEBUG + g_debug ("Q ERROR: %s\n", error->message); +#endif /* TURTLE_DEBUG */ + g_error_free (error); + } + + if (result_set) { + GValue id_value = { 0, }; + GValue is_value = { 0, }; + gint iid_value, iis_value; + + _tracker_db_result_set_get_value (result_set, 0, &id_value); + iid_value = g_value_get_int (&id_value); + + _tracker_db_result_set_get_value (result_set, 1, &is_value); + iis_value = g_value_get_int (&is_value); + + if (iis_value) { + ForeachInMetadataInfo *info = g_slice_new (ForeachInMetadataInfo); + info->service = service; + info->iid_value = iid_value; + + info->config = config ? g_object_ref (config) : NULL; + info->language = language ? g_object_ref (language) : NULL; + + g_hash_table_foreach (metadata, + foreach_in_metadata_set_metadata, + info); + + if (info->language) + g_object_unref (info->language); + if (info->config) + g_object_unref (info->config); + + g_slice_free (ForeachInMetadataInfo, info); + } + + g_value_unset (&id_value); + g_value_unset (&is_value); + + g_object_unref (result_set); + + } else { + guint32 id; + + id = tracker_data_update_get_new_service_id (iface); + + if (tracker_data_update_create_service (service, id, + dirname, basename, + metadata)) { + ForeachInMetadataInfo *info; + + info = g_slice_new (ForeachInMetadataInfo); + + info->service = service; + info->iid_value = id; + + info->config = config ? g_object_ref (config) : NULL; + info->language = language ? g_object_ref (language) : NULL; + + g_hash_table_foreach (metadata, + foreach_in_metadata_set_metadata, + info); + + if (info->language) + g_object_unref (info->language); + if (info->config) + g_object_unref (info->config); + + g_slice_free (ForeachInMetadataInfo, info); + } + + } + + g_free (dirname); + g_free (escaped_path); +} + +void tracker_data_update_enable_volume (const gchar *udi, const gchar *mount_path) { Index: src/libtracker-data/tracker-data-metadata.h =================================================================== --- src/libtracker-data/tracker-data-metadata.h (revision 2683) +++ src/libtracker-data/tracker-data-metadata.h (working copy) @@ -42,7 +42,7 @@ const gchar *value); void tracker_data_metadata_insert_values (TrackerDataMetadata *metadata, const gchar *field_name, - GList *list); + const GList *list); G_CONST_RETURN gchar *tracker_data_metadata_lookup (TrackerDataMetadata *metadata, const gchar *field_name); G_CONST_RETURN GList *tracker_data_metadata_lookup_values (TrackerDataMetadata *metadata, Index: src/libtracker-data/tracker-data-update.h =================================================================== --- src/libtracker-data/tracker-data-update.h (revision 2683) +++ src/libtracker-data/tracker-data-update.h (working copy) @@ -52,6 +52,15 @@ const gchar *from, const gchar *to); +/* Turtle importing */ +void tracker_data_update_replace_service (const gchar *path, + const gchar *rdf_type, + GHashTable *metadata); +void tracker_data_update_delete_service_by_path (const gchar *path, + const gchar *rdf_type); +void tracker_data_update_turtle_import_start (void); +void tracker_data_update_turtle_import_stop (void); + /* Metadata */ void tracker_data_update_set_metadata (TrackerService *service, guint32 service_id, Index: src/libtracker-data/Makefile.am =================================================================== --- src/libtracker-data/Makefile.am (revision 2683) +++ src/libtracker-data/Makefile.am (working copy) @@ -6,6 +6,7 @@ -DTRACKER_COMPILATION \ -I$(top_srcdir)/src \ $(DBUS_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) libtracker_datadir = $(libdir)/tracker @@ -21,6 +22,7 @@ tracker-data-update.c \ tracker-query-tree.c \ tracker-rdf-query.c \ + tracker-turtle.c \ tracker-xesam-query.c noinst_HEADERS = \ @@ -33,6 +35,7 @@ tracker-data-update.h \ tracker-query-tree.h \ tracker-rdf-query.h \ + tracker-turtle.h \ tracker-xesam-query.h libtracker_data_la_LDFLAGS = -version-info 0:0:0 @@ -41,4 +44,5 @@ $(top_builddir)/src/libtracker-db/libtracker-db.la \ $(DBUS_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ -lz Index: src/trackerd/tracker-main.c =================================================================== --- src/trackerd/tracker-main.c (revision 2683) +++ src/trackerd/tracker-main.c (working copy) @@ -61,6 +61,7 @@ #include "tracker-status.h" #include "tracker-xesam-manager.h" #include "tracker-cleanup.h" +#include "tracker-backup.h" #ifdef G_OS_WIN32 #include @@ -747,6 +748,52 @@ return FALSE; } +static gchar * +get_turtle_userdata_backup_filename (void) +{ + TrackerMainPrivate *private; + + private = g_static_private_get (&private_key); + + if (private) { + return g_build_filename (private->user_data_dir, + "tracker-userdata-backup.ttl", + NULL); + } else { + g_critical ("Directories not initialized"); + return NULL; + } +} + + +/* + * TODO: Ugly hack counting signals because the indexer is sending two "Finished" signals + * and only the second really mean "finished processing modules". + */ +static void +crawling_finished_cb (TrackerProcessor *processor, gpointer user_data) +{ + gulong *callback_id = (gulong *)user_data; + GError *error; + gchar *turtle_file; + static gint counter = 0; + + counter += 1; + + if (counter >= 2) { + g_debug ("Uninstalling initial crawling callback"); + g_signal_handler_disconnect (processor, *callback_id); + + turtle_file = get_turtle_userdata_backup_filename (); + org_freedesktop_Tracker_Indexer_restore_backup (tracker_dbus_indexer_get_proxy (), + turtle_file, + &error); + g_free (turtle_file); + } else { + g_debug ("%d finished signal", counter); + } +} + gint main (gint argc, gchar *argv[]) { @@ -764,6 +811,7 @@ TrackerDBManagerFlags flags = 0; TrackerDBIndexManagerFlags index_flags = 0; gboolean is_first_time_index; + gulong callback_id; g_type_init (); @@ -786,7 +834,7 @@ /* Set timezone info */ tzset (); - + /* Translators: this messagge will apper immediately after the * usage string - Usage: COMMAND */ @@ -913,12 +961,43 @@ return EXIT_FAILURE; } + tracker_turtle_init (); + tracker_module_config_init (); flags |= TRACKER_DB_MANAGER_REMOVE_CACHE; index_flags |= TRACKER_DB_INDEX_MANAGER_READONLY; if (force_reindex) { + + gchar *turtle_file; + + turtle_file = get_turtle_userdata_backup_filename (); + + g_message ("Saving metadata in %s", turtle_file); + + /* Init the DB stack */ + tracker_db_manager_init (0, &is_first_time_index, TRUE); + + tracker_db_index_manager_init (0, + tracker_config_get_min_bucket_count (config), + tracker_config_get_max_bucket_count (config)); + + file_index = tracker_db_index_manager_get_index (TRACKER_DB_INDEX_FILE); + email_index = tracker_db_index_manager_get_index (TRACKER_DB_INDEX_EMAIL); + + tracker_data_manager_init (config, language, file_index, email_index); + + tracker_backup_save (turtle_file); + + /* Shutdown the DB stack */ + tracker_data_manager_shutdown (); + + tracker_db_index_manager_shutdown (); + tracker_db_manager_shutdown (); + + g_free (turtle_file); + flags |= TRACKER_DB_MANAGER_FORCE_REINDEX; index_flags |= TRACKER_DB_INDEX_MANAGER_FORCE_REINDEX; } @@ -1035,6 +1114,13 @@ tracker_status_set_and_signal (TRACKER_STATUS_IDLE); } + if (flags & TRACKER_DB_MANAGER_FORCE_REINDEX) { + g_debug ("Setting callback for crawling finish detection"); + callback_id = g_signal_connect (private->processor, "finished", + G_CALLBACK (crawling_finished_cb), + &callback_id); + } + if (tracker_status_get_is_running ()) { private->main_loop = g_main_loop_new (NULL, FALSE); g_main_loop_run (private->main_loop); @@ -1082,6 +1168,7 @@ tracker_module_config_shutdown (); tracker_nfs_lock_shutdown (); tracker_status_shutdown (); + tracker_turtle_shutdown (); tracker_log_shutdown (); #ifdef HAVE_HAL Index: src/trackerd/tracker-backup.c =================================================================== --- src/trackerd/tracker-backup.c (revision 0) +++ src/trackerd/tracker-backup.c (revision 0) @@ -0,0 +1,105 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* + * Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org) + * Copyright (C) 2008, Nokia + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#include "config.h" + +#include "tracker-backup.h" + +#include +#include +#include + +#ifdef HAVE_RAPTOR +#include +#endif + +/* + * (uri, metadataid, value) + */ +static void +extended_result_set_to_turtle (TrackerDBResultSet *result_set, TurtleFile *turtle_file) +{ + TrackerField *field; + gint metadata_id; + gboolean valid = TRUE; + TrackerDataMetadata *metadata; + + while (valid) { + GValue transform = {0, }; + GValue value = {0, }; + gchar *str = NULL; + gchar *uri; + gchar *service_type; + + g_value_init (&transform, G_TYPE_STRING); + + tracker_db_result_set_get (result_set, 0, &uri, -1); + tracker_db_result_set_get (result_set, 1, &service_type, -1); + tracker_db_result_set_get (result_set, 2, &metadata_id, -1); + tracker_db_result_set_get (result_set, 3, &str, -1); + + field = tracker_ontology_get_field_by_id (metadata_id); + if (!field) { + g_critical ("Field id %d in database but not in tracker-ontology", + metadata_id); + g_free (str); + return; + } + + g_debug ("Insertion in turtle <%s, %s, %s>", + uri, tracker_field_get_name (field), str); + tracker_turtle_add_triple (turtle_file, uri, field, str); + + g_free (str); + g_free (service_type); + + valid = tracker_db_result_set_iter_next (result_set); + } + +} + + +void +tracker_backup_save (const gchar *turtle_filename) +{ + TrackerDBResultSet *data; + TrackerService *service; + TurtleFile *turtle_file; + + /* TODO: temporary location */ + if (g_file_test (turtle_filename, G_FILE_TEST_EXISTS)) { + g_unlink (turtle_filename); + } + + turtle_file = tracker_turtle_open (turtle_filename); + + g_message ("***** tracker_backup: Saving metadata in turtle file *****"); + + service = tracker_ontology_get_service_by_name ("Files"); + data = tracker_data_query_backup_metadata (service); + + if (data) { + extended_result_set_to_turtle (data, turtle_file); + g_object_unref (data); + } + + tracker_turtle_close (turtle_file); +} + Index: src/trackerd/tracker-backup.h =================================================================== --- src/trackerd/tracker-backup.h (revision 0) +++ src/trackerd/tracker-backup.h (revision 0) @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* + * Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org) + * Copyright (C) 2008, Nokia + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef __TRACKER_BACKUP_H__ +#define __TRACKER_BACKUP_H__ + +#include + +void tracker_backup_save (const gchar *turtle_filename); + +#endif /* __TRACKER_BACKUP_H__ */ Index: src/trackerd/Makefile.am =================================================================== --- src/trackerd/Makefile.am (revision 2683) +++ src/trackerd/Makefile.am (working copy) @@ -17,6 +17,7 @@ $(GIO_CFLAGS) \ $(GMODULE_CFLAGS) \ $(GTHREAD_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) if HAVE_INOTIFY @@ -29,6 +30,8 @@ libexec_PROGRAMS = trackerd trackerd_SOURCES = \ + tracker-backup.h \ + tracker-backup.c \ tracker-crawler.c \ tracker-crawler.h \ tracker-daemon.c \ @@ -85,6 +88,7 @@ $(GMODULE_LIBS) \ $(GTHREAD_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ $(trackerd_win_libs) \ -lz \ -lm Index: src/libtracker-db/tracker-db-manager.c =================================================================== --- src/libtracker-db/tracker-db-manager.c (revision 2683) +++ src/libtracker-db/tracker-db-manager.c (working copy) @@ -2726,12 +2726,20 @@ g_free (services_dir); g_free (sql_dir); - if (file_iface) + if (file_iface) { g_object_unref (file_iface); - if (email_iface) + file_iface = NULL; + } + + if (email_iface) { g_object_unref (email_iface); - if (xesam_iface) + email_iface = NULL; + } + + if (xesam_iface) { g_object_unref (xesam_iface); + xesam_iface = NULL; + } /* Since we don't reference this enum anywhere, we do Index: src/libtracker-common/tracker-hal.c =================================================================== --- src/libtracker-common/tracker-hal.c (revision 2683) +++ src/libtracker-common/tracker-hal.c (working copy) @@ -1057,7 +1057,7 @@ gboolean is_mounted; gr = (GetRoots*) user_data; - udi = (const gchar*) key; + udi = key; volume = libhal_volume_from_udi (gr->context, udi); if (!volume) { @@ -1136,6 +1136,77 @@ } /** + * tracker_hal_path_is_on_removable_device: + * @hal: A #TrackerHal + * @path: a path + * @mount_mount: if @path is on a removable device, the mount point will + * be filled in here. You must free the returned result + * @available: if @path is on a removable device, this will be set to + * TRUE in case the file is available right now + * + * Returns Whether or not @path is on a known removable device + * + * Returns: TRUE if @path on a known removable device, FALSE otherwise + **/ +gboolean +tracker_hal_path_is_on_removable_device (TrackerHal *hal, + const gchar *path, + gchar **mount_point, + gboolean *available) +{ + TrackerHalPriv *priv; + GHashTableIter iter; + gboolean found = FALSE; + gpointer key, value; + + g_return_val_if_fail (TRACKER_IS_HAL (hal), FALSE); + + if (!path) + return FALSE; + + priv = GET_PRIV (hal); + + g_hash_table_iter_init (&iter, priv->removable_devices); + + while (g_hash_table_iter_next (&iter, &key, &value)) { + LibHalVolume *volume; + const gchar *udi; + const gchar *mp; + + udi = key; + + volume = libhal_volume_from_udi (priv->context, udi); + + if (!volume) { + g_message ("HAL device with udi:'%s' has no volume, " + "should we delete?", + udi); + continue; + } + + mp = libhal_volume_get_mount_point (volume); + + if (g_strcmp0 (mp, path) != 0) { + if (g_strrstr (path, mp)) { + found = TRUE; + + if (mount_point) + *mount_point = g_strdup (mp); + + if (available) + *available = libhal_volume_is_mounted (volume); + break; + } + } + + libhal_volume_free (volume); + } + + return found; +} + + +/** * tracker_hal_get_removable_device_udis: * @hal: A #TrackerHal * Index: src/libtracker-common/tracker-hal.h =================================================================== --- src/libtracker-common/tracker-hal.h (revision 2683) +++ src/libtracker-common/tracker-hal.h (working copy) @@ -66,6 +66,10 @@ const gchar *udi); gboolean tracker_hal_udi_get_is_mounted (TrackerHal *hal, const gchar *udi); +gboolean tracker_hal_path_is_on_removable_device (TrackerHal *hal, + const gchar *path, + gchar **mount_point, + gboolean *available); #endif /* HAVE_HAL */ Index: tests/tracker-indexer/Makefile.am =================================================================== --- tests/tracker-indexer/Makefile.am (revision 2683) +++ tests/tracker-indexer/Makefile.am (working copy) @@ -18,6 +18,7 @@ $(PANGO_CFLAGS) \ $(GMODULE_CFLAGS) \ $(GTHREAD_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GLIB2_CFLAGS) tracker_metadata_utils_SOURCES = \ @@ -37,7 +38,9 @@ tracker-module-file.c \ tracker-module-file.h \ tracker-module-iteratable.c \ - tracker-module-iteratable.h + tracker-module-iteratable.h \ + tracker-removable-device.c \ + tracker-removable-device.h tracker_metadata_utils_LDADD = \ $(top_builddir)/src/libtracker-data/libtracker-data.la \ @@ -49,6 +52,7 @@ $(GMODULE_LIBS) \ $(GTHREAD_LIBS) \ $(GIO_LIBS) \ + $(RAPTOR_LIBS) \ $(GLIB2_LIBS) # Index: configure.ac =================================================================== --- configure.ac (revision 2683) +++ configure.ac (working copy) @@ -170,6 +170,15 @@ AM_CONDITIONAL(HAVE_GCONF, test "$have_gconf" = "yes") +# Check for Raptor +PKG_CHECK_MODULES(RAPTOR, [raptor], have_raptor=yes, have_raptor=no) +AC_SUBST(RAPTOR_CFLAGS) +AC_SUBST(RAPTOR_LIBS) + +if test x$have_raptor == "xyes"; then + AC_DEFINE(HAVE_RAPTOR, 1, [Raptor RDF parsers]) +fi + # Check we have the DBUS binding tool we need AC_PATH_PROG(DBUSBINDINGTOOL, dbus-binding-tool) if test -z $DBUSBINDINGTOOL; then @@ -1113,6 +1122,8 @@ tests/tracker-indexer/tracker-module-file.h:src/tracker-indexer/tracker-module-file.h tests/tracker-indexer/tracker-module-iteratable.c:src/tracker-indexer/tracker-module-iteratable.c tests/tracker-indexer/tracker-module-iteratable.h:src/tracker-indexer/tracker-module-iteratable.h + tests/tracker-indexer/tracker-removable-device.c:src/tracker-indexer/tracker-removable-device.c + tests/tracker-indexer/tracker-removable-device.h:src/tracker-indexer/tracker-removable-device.h ) ################################################################## Index: data/db/sqlite-stored-procs.sql =================================================================== --- data/db/sqlite-stored-procs.sql (revision 2683) +++ data/db/sqlite-stored-procs.sql (working copy) @@ -63,6 +63,7 @@ /* * Metadata/MIME queries */ +GetEmbeddedMetadataBackup SELECT S.Path || '/' || S.Name, T.TypeName, M.MetadataID, M.MetadataDisplay From Services S, ServiceMetadata M, ServiceTypes T WHERE (S.ID == M.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (M.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)) UNION SELECT S.Path || '/' || S.Name, T.TypeName, M.MetadataID, upper(M.MetadataValue) From Services S, ServiceNumericMetadata M, ServiceTypes T WHERE (S.ID == M.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (M.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)) UNION SELECT S.Path || '/' || S.Name, T.Typename, M.MetadataID, M.MetadataValue From Services S, ServiceKeywordMetadata M, ServiceTypes T WHERE (S.ID == M.ServiceID) AND (S.ServiceTypeID == T.TypeID) AND (M.MetadataID IN (SELECT ID From MetadataTypes WHERE Embedded = 0)); GetAllMetadata SELECT MetadataID, MetadataDisplay FROM ServiceMetadata WHERE ServiceID = ? UNION SELECT MetadataID, MetadataValue FROM ServiceKeywordMetadata WHERE ServiceID = ? UNION SELECT MetadataID, upper(MetadataValue) FROM ServiceNumericMetadata WHERE ServiceID = ?; GetMetadata SELECT MetaDataDisplay FROM ServiceMetaData WHERE ServiceID = ? AND MetaDataID = ?; GetMetadataAliases SELECT DISTINCT M.MetaName, M.ID FROM MetaDataTypes AS M, MetaDataChildren AS C WHERE M.ID = C.ChildID AND C.MetaDataID = ?; Index: data/dbus/tracker-indexer.xml =================================================================== --- data/dbus/tracker-indexer.xml (revision 2683) +++ data/dbus/tracker-indexer.xml (working copy) @@ -74,6 +74,11 @@ + + + + +