diff --git a/configure.ac b/configure.ac index e7d06e4..f3e5115 100644 --- a/configure.ac +++ b/configure.ac @@ -142,6 +142,15 @@ AC_SUBST(GCONF_LIBS) AM_CONDITIONAL(HAVE_GCONF, test "$have_gconf" = "yes") +# Check for Raptor +PKG_CHECK_MODULES(RAPTOR, [raptor], have_raptor=yes, have_raptor=no) +AC_SUBST(RAPTOR_CFLAGS) +AC_SUBST(RAPTOR_LIBS) + +if test x$have_raptor == "xyes"; then + AC_DEFINE(HAVE_RAPTOR, 1, [Raptor RDF parsers]) +fi + # Check we have the DBUS binding tool we need AC_PATH_PROG(DBUSBINDINGTOOL, dbus-binding-tool) if test -z $DBUSBINDINGTOOL; then @@ -1022,6 +1031,8 @@ AC_CONFIG_LINKS(tests/tracker-indexer/tracker-dbus.c:src/tracker-indexer/tracker tests/tracker-indexer/tracker-metadata-utils.c:src/tracker-indexer/tracker-metadata-utils.c tests/tracker-indexer/tracker-metadata-utils.h:src/tracker-indexer/tracker-metadata-utils.h tests/tracker-indexer/tracker-module.h:src/tracker-indexer/tracker-module.h + tests/tracker-indexer/tracker-turtle.h:src/tracker-indexer/tracker-turtle.h + tests/tracker-indexer/tracker-turtle.c:src/tracker-indexer/tracker-turtle.c ) ################################################################## diff --git a/data/dbus/tracker-indexer.xml b/data/dbus/tracker-indexer.xml index 11f3723..1462f7f 100644 --- a/data/dbus/tracker-indexer.xml +++ b/data/dbus/tracker-indexer.xml @@ -11,6 +11,11 @@ + + + + + diff --git a/src/libtracker-common/tracker-utils.c b/src/libtracker-common/tracker-utils.c index 3d030bd..a7d64c5 100644 --- a/src/libtracker-common/tracker-utils.c +++ b/src/libtracker-common/tracker-utils.c @@ -106,6 +106,10 @@ tracker_escape_string (const gchar *in) { gchar **array, *out; + if (!in) { + return g_strdup (""); + } + if (!strchr (in, '\'')) { return g_strdup (in); } diff --git a/src/libtracker-data/tracker-data-metadata.c b/src/libtracker-data/tracker-data-metadata.c index f23f393..dee563a 100644 --- a/src/libtracker-data/tracker-data-metadata.c +++ b/src/libtracker-data/tracker-data-metadata.c @@ -111,9 +111,9 @@ tracker_data_metadata_insert (TrackerDataMetadata *metadata, g_return_if_fail (tracker_field_get_multiple_values (field) == FALSE); - g_hash_table_insert (metadata->table, - g_object_ref (field), - value); + g_hash_table_replace (metadata->table, + g_object_ref (field), + value); } /** diff --git a/src/libtracker-data/tracker-data-update.c b/src/libtracker-data/tracker-data-update.c index ec21573..69d5ac5 100644 --- a/src/libtracker-data/tracker-data-update.c +++ b/src/libtracker-data/tracker-data-update.c @@ -76,6 +76,22 @@ tracker_data_update_get_new_service_id (TrackerDBInterface *iface) g_object_unref (result_set); } + temp_iface = tracker_db_manager_get_db_interface (TRACKER_DB_METADATA_DECOMPOSED); + + result_set = tracker_db_interface_execute_query (temp_iface, NULL, + "SELECT MAX(ID) AS A FROM Resource"); + + if (result_set) { + GValue val = {0, }; + _tracker_db_result_set_get_value (result_set, 0, &val); + if (G_VALUE_TYPE (&val) == G_TYPE_INT) { + files_max = g_value_get_int (&val); + max = MAX (files_max, max); + } + g_value_unset (&val); + g_object_unref (result_set); + } + return ++max; } @@ -666,3 +682,120 @@ tracker_data_update_delete_handled_events (TrackerDBInterface *iface) tracker_data_manager_exec (iface, "DELETE FROM Events WHERE BeingHandled = 1"); } + +typedef struct { + TrackerService *service; + guint32 iid_value; +} ForeachInMetadataInfo; + +static void +foreach_in_metadata_set_metadata_decomposed (TrackerField *field, + gpointer value, + gpointer user_data) +{ + ForeachInMetadataInfo *info = user_data; + set_metadata_decomposed (info->service, info->iid_value, field, value); +} + +void +tracker_data_replace_service (const gchar *path, + const gchar *rdf_type, + TrackerDataMetadata *metadata) +{ + TrackerDBInterface *iface; + const gchar *service_type = "Files"; + const gchar *predicate = "File:"; + TrackerDBResultSet *result_set; + const gchar *modified; + GError *error = NULL; + TrackerService *service; + gchar *dirname; + const gchar *basename; + + if (!rdf_type) + return; + + service = tracker_ontology_get_service_by_name (rdf_type); + + iface = tracker_db_manager_get_db_interface_by_type (tracker_service_get_name (service), + TRACKER_DB_CONTENT_TYPE_METADATA_DECOMPOSED); + + modified = tracker_data_metadata_lookup (metadata, "File:Modified"); + + if (!modified) + return; + + dirname = g_dirname (path); + basename = g_basename (path); + + result_set = tracker_db_interface_execute_query (iface, &error, + "SELECT ID, \"File:Modified\" < '%s' FROM Resource " + "WHERE Path = '%s' AND Name = '%s'", + modified, + dirname, basename); + + if (error) { + #ifdef TURTLE_DEBUG + g_print ("Q ERROR: %s\n", error->message); + #endif /* TURTLE_DEBUG */ + g_error_free (error); + } + + if (result_set) { + GValue id_value = { 0, }; + GValue is_value = { 0, }; + gint iid_value, iis_value; + + _tracker_db_result_set_get_value (result_set, 0, &id_value); + iid_value = g_value_get_int (&id_value); + + _tracker_db_result_set_get_value (result_set, 1, &is_value); + iis_value = g_value_get_int (&is_value); + + if (iis_value) { + ForeachInMetadataInfo *info = g_slice_new (ForeachInMetadataInfo); + info->service = service; + info->iid_value = iid_value; + + tracker_data_metadata_foreach (metadata, + foreach_in_metadata_set_metadata_decomposed, + info); + + g_slice_free (ForeachInMetadataInfo, info); + } + + g_value_unset (&id_value); + g_value_unset (&is_value); + + g_object_unref (result_set); + + } else { + ForeachInMetadataInfo *info; + guint32 id; + + info = g_slice_new (ForeachInMetadataInfo); + + /* This is BS, it's using the wrong table to create a new ID, + * but appearently that's how to do it ... (this will make + * the decomposed tables out of sync with the old ones) */ + + id = tracker_data_update_get_new_service_id (iface); + + create_service_decomposed (service, id, + dirname, + basename, + metadata); + + info->service = service; + info->iid_value = id; + + tracker_data_metadata_foreach (metadata, + foreach_in_metadata_set_metadata_decomposed, + info); + + g_slice_free (ForeachInMetadataInfo, info); + } + + g_free (dirname); + +} diff --git a/src/libtracker-data/tracker-data-update.h b/src/libtracker-data/tracker-data-update.h index 0cff9df..b909af6 100644 --- a/src/libtracker-data/tracker-data-update.h +++ b/src/libtracker-data/tracker-data-update.h @@ -51,6 +51,9 @@ void tracker_data_update_delete_service_recursively (TrackerService *se void tracker_data_update_move_service (TrackerService *service, const gchar *from, const gchar *to); +void tracker_data_replace_service (const gchar *path, + const gchar *rdf_type, + TrackerDataMetadata *metadata); /* Metadata */ void tracker_data_update_set_metadata (TrackerService *service, diff --git a/src/tracker-indexer/Makefile.am b/src/tracker-indexer/Makefile.am index 3cd2beb..b30eaae 100644 --- a/src/tracker-indexer/Makefile.am +++ b/src/tracker-indexer/Makefile.am @@ -13,6 +13,7 @@ INCLUDES = \ -I$(top_srcdir)/src \ $(DBUS_CFLAGS) \ $(PANGO_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GMODULE_CFLAGS) libtracker_indexerdir = $(libdir)/tracker @@ -37,7 +38,9 @@ tracker_indexer_SOURCES = \ tracker-indexer-module.c \ tracker-indexer-module.h \ tracker-main.c \ - tracker-marshal-main.c + tracker-marshal-main.c \ + tracker-turtle.c \ + tracker-turtle.h tracker_indexer_LDADD = \ libtracker-indexer.la \ @@ -52,6 +55,7 @@ tracker_indexer_LDADD = \ $(PANGO_LIBS) \ $(GIO_LIBS) \ $(GLIB2_LIBS) \ + $(RAPTOR_LIBS) \ -lz \ -lm diff --git a/src/tracker-indexer/tracker-indexer.c b/src/tracker-indexer/tracker-indexer.c index 8f5a9c6..543f878 100644 --- a/src/tracker-indexer/tracker-indexer.c +++ b/src/tracker-indexer/tracker-indexer.c @@ -75,6 +75,7 @@ #include "tracker-indexer-module.h" #include "tracker-marshal.h" #include "tracker-module.h" +#include "tracker-turtle.h" #define TRACKER_INDEXER_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_INDEXER, TrackerIndexerPrivate)) @@ -122,6 +123,7 @@ struct TrackerIndexerPrivate { TrackerDBInterface *email_contents; TrackerDBInterface *common; TrackerDBInterface *cache; + TrackerDBInterface *meta; TrackerConfig *config; TrackerLanguage *language; @@ -243,12 +245,15 @@ start_transaction (TrackerIndexer *indexer) indexer->private->in_transaction = TRUE; + tracker_db_interface_start_transaction (indexer->private->meta); + tracker_db_interface_start_transaction (indexer->private->cache); tracker_db_interface_start_transaction (indexer->private->file_contents); tracker_db_interface_start_transaction (indexer->private->email_contents); tracker_db_interface_start_transaction (indexer->private->file_metadata); tracker_db_interface_start_transaction (indexer->private->email_metadata); tracker_db_interface_start_transaction (indexer->private->common); + } static void @@ -265,6 +270,8 @@ stop_transaction (TrackerIndexer *indexer) tracker_db_interface_end_transaction (indexer->private->file_contents); tracker_db_interface_end_transaction (indexer->private->cache); + tracker_db_interface_end_transaction (indexer->private->meta); + indexer->private->files_indexed += indexer->private->files_processed; indexer->private->files_processed = 0; indexer->private->in_transaction = FALSE; @@ -368,6 +375,21 @@ schedule_flush (TrackerIndexer *indexer, indexer); } +void +tracker_indexer_commit_transaction (TrackerIndexer *indexer) +{ + stop_transaction (indexer); + tracker_indexer_set_running (indexer, TRUE); + +} + +void +tracker_indexer_open_transaction (TrackerIndexer *indexer) +{ + tracker_indexer_set_running (indexer, FALSE); + start_transaction (indexer); +} + #ifdef HAVE_HAL static void @@ -887,6 +909,9 @@ tracker_indexer_init (TrackerIndexer *indexer) * interfaces as singletons, it's safe to just ask it * again for an interface. */ + + priv->meta = tracker_db_manager_get_db_interface (TRACKER_DB_METADATA_DECOMPOSED); + priv->cache = tracker_db_manager_get_db_interface (TRACKER_DB_CACHE); priv->common = tracker_db_manager_get_db_interface (TRACKER_DB_COMMON); priv->file_metadata = tracker_db_manager_get_db_interface (TRACKER_DB_FILE_METADATA); @@ -2479,6 +2504,33 @@ tracker_indexer_process_modules (TrackerIndexer *indexer, } } +// For martyn: this is that { A01 } code + +void +tracker_indexer_turtle_add (TrackerIndexer *indexer, + gchar *file, + DBusGMethodInvocation *context, + GError **error) +{ + guint request_id; + + request_id = tracker_dbus_get_next_request_id (); + + tracker_dbus_async_return_if_fail (TRACKER_IS_INDEXER (indexer), context); + tracker_dbus_async_return_if_fail (file != NULL, context); + + tracker_dbus_request_new (request_id, + "DBus request to check TTL file %s", + file); + dbus_g_method_return (context); + + tracker_dbus_request_success (request_id); + + tracker_turtle_process_ttl (indexer, file); + +} + + void tracker_indexer_files_check (TrackerIndexer *indexer, const gchar *module_name, diff --git a/src/tracker-indexer/tracker-indexer.h b/src/tracker-indexer/tracker-indexer.h index 8401607..616373d 100644 --- a/src/tracker-indexer/tracker-indexer.h +++ b/src/tracker-indexer/tracker-indexer.h @@ -78,8 +78,10 @@ void tracker_indexer_set_running (TrackerIndexer *indexer, gboolean running); void tracker_indexer_stop (TrackerIndexer *indexer); void tracker_indexer_process_all (TrackerIndexer *indexer); -void tracker_indexer_process_modules (TrackerIndexer *indexer, +void tracker_indexer_process_modules (TrackerIndexer *indexer, gchar **modules); +void tracker_indexer_commit_transaction (TrackerIndexer *indexer); +void tracker_indexer_open_transaction (TrackerIndexer *indexer); /* DBus methods */ void tracker_indexer_pause (TrackerIndexer *indexer, @@ -92,6 +94,10 @@ void tracker_indexer_pause_for_duration (TrackerIndexer *indexer, void tracker_indexer_continue (TrackerIndexer *indexer, DBusGMethodInvocation *context, GError **error); +void tracker_indexer_turtle_add (TrackerIndexer *indexer, + gchar *file, + DBusGMethodInvocation *context, + GError **error); void tracker_indexer_files_check (TrackerIndexer *indexer, const gchar *module, GStrv files, diff --git a/src/tracker-indexer/tracker-turtle.c b/src/tracker-indexer/tracker-turtle.c new file mode 100644 index 0000000..96068e4 --- /dev/null +++ b/src/tracker-indexer/tracker-turtle.c @@ -0,0 +1,248 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* + * Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org) + * Copyright (C) 2008, Nokia + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * Author: Philip Van Hoof + */ + +#include "config.h" + +#ifdef HAVE_RAPTOR + +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "tracker-turtle.h" + +typedef struct { + gchar *last_subject; + TrackerDataMetadata *metadata; + gchar *base; + guint amount; + TrackerIndexer *indexer; + gchar *rdf_type; +} TurtleParseInfo; + +static void +commit_turtle_parse_info_data (TurtleParseInfo *info, gboolean may_flush) +{ + + if (info->last_subject) { + + /* We have it as a URI, database api wants Paths. Update this when + * the database api becomes sane and uses URIs everywhere */ + + g_print ("."); + + tracker_data_replace_service (info->last_subject + 7, + info->rdf_type, + info->metadata); + + info->amount++; + + tracker_data_metadata_free (info->metadata); + g_free (info->last_subject); + g_free (info->rdf_type); + info->last_subject = NULL; + info->metadata = NULL; + info->rdf_type = NULL; + } + + /* We commit per transaction of 100 here, and then we also iterate the + * mainloop so that the state-machine gets the opportunity to run for a + * moment */ + + if (may_flush && info->amount > 100) { + tracker_indexer_commit_transaction (info->indexer); + g_main_context_iteration (NULL, FALSE); + tracker_indexer_open_transaction (info->indexer); + info->amount = 0; + } +} + +static void +consume_triple (void* user_data, const raptor_statement* triple) +{ + TurtleParseInfo *info = user_data; + gchar *subject; + + subject = (gchar *) raptor_uri_as_string ((raptor_uri *) triple->subject); + + if (!info->last_subject || strcmp (subject, info->last_subject) != 0) { + + /* Commit previous subject */ + + commit_turtle_parse_info_data (info, TRUE); + info->last_subject = g_strdup (subject); + info->metadata = tracker_data_metadata_new (); + } + + if (triple->object_type == RAPTOR_IDENTIFIER_TYPE_LITERAL) { + gchar *predicate; + + predicate = g_strdup ((const gchar *) raptor_uri_as_string ((raptor_uri *) triple->predicate)); + + if (strcmp (predicate, "rdf:type") == 0) { + g_free (info->rdf_type); + + /* TODO: ontology */ + /* Change this when Files and Emails becomes File and Email */ + + info->rdf_type = g_strdup_printf ("%ss", triple->object); + } else + tracker_data_metadata_insert (info->metadata, + predicate, + g_strdup (triple->object)); + + } else if (triple->object_type == RAPTOR_IDENTIFIER_TYPE_RESOURCE) { + gchar *key = g_strdup_printf ("file://%s/:", info->base); + + if (strcmp (key, triple->object) == 0) { + gchar *predicate; + + predicate = (gchar *) raptor_uri_as_string ((raptor_uri *) triple->predicate); + + tracker_data_metadata_insert (info->metadata, + predicate, + g_strdup ("")); + + } + + g_free (key); + } + +} + +static void +raptor_error (void *user_data, raptor_locator* locator, const char *message) +{ + g_print ("RAPTOR_ERROR: %s\n", message); +} + +#endif /* HAVE_RAPTOR */ + +void +tracker_turtle_process_ttl (TrackerIndexer *indexer, const gchar *file) +{ +#ifdef HAVE_RAPTOR + + unsigned char *uri_stringa, *uri_stringb; + raptor_uri *uri, *base_uri; + static gboolean has_init = FALSE; + raptor_parser *parser; + TurtleParseInfo *info; + gchar *copy_file, *ptr; + + if (!has_init) + raptor_init(); + + parser = raptor_new_parser ("turtle"); + + info = g_slice_new0 (TurtleParseInfo); + + info->indexer = g_object_ref (indexer); + info->amount = 0; + + raptor_set_statement_handler (parser, info, consume_triple); + raptor_set_fatal_error_handler (parser, info, raptor_error); + raptor_set_error_handler (parser, info, raptor_error); + raptor_set_warning_handler (parser, info, raptor_error); + + copy_file = g_strdup (file); + + ptr = strstr (copy_file, "/metadata/metadata.ttl"); + if (ptr) { + /* .cache remains, and will be cut later, just like dummy_file is */ + *ptr = '\0'; + } else { + g_free (copy_file); + copy_file = g_strdup ("/home/pvanhoof/dummy_file"); + } + + + uri_stringa = raptor_uri_filename_to_uri_string (file); + uri_stringb = raptor_uri_filename_to_uri_string (copy_file); + + uri = raptor_new_uri (uri_stringa); + base_uri = raptor_new_uri (uri_stringb); + + /* Take the file (dummy_file or .cache) from base */ + ptr = strrchr (copy_file, '/'); + if (ptr) + *ptr = '\0'; + + info->base = copy_file; + + tracker_indexer_open_transaction (info->indexer); + + GTimer *timer = g_timer_new (); + g_timer_start (timer); + + raptor_parse_file (parser, uri, base_uri); + /* Commit final subject */ + + commit_turtle_parse_info_data (info, FALSE); + + g_timer_stop (timer); + g_print ("\nTIME: %f\n", g_timer_elapsed (timer, NULL)); + + tracker_indexer_commit_transaction (info->indexer); + + g_free (copy_file); + g_object_unref (info->indexer); + g_slice_free (TurtleParseInfo, info); + + raptor_free_parser (parser); + + raptor_free_uri (base_uri); + raptor_free_uri (uri); + raptor_free_memory (uri_stringa); + raptor_free_memory (uri_stringb); + + + /* raptor_finish(); */ + +#endif /* HAVE_RAPTOR */ + +} + diff --git a/src/tracker-indexer/tracker-turtle.h b/src/tracker-indexer/tracker-turtle.h new file mode 100644 index 0000000..a3a132c --- /dev/null +++ b/src/tracker-indexer/tracker-turtle.h @@ -0,0 +1,31 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* + * Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org) + * Copyright (C) 2008, Nokia + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * Author: Philip Van Hoof + */ + +#ifndef __TRACKER_TURTLE_H__ +#define __TRACKER_TURTLE_H__ + +#include "tracker-indexer.h" + +void tracker_turtle_process_ttl (TrackerIndexer *indexer, const gchar *file); + +#endif /* __TRACKER_TURTLE_H__ */ diff --git a/src/trackerd/tracker-processor.c b/src/trackerd/tracker-processor.c index c36bf7a..67b66a1 100644 --- a/src/trackerd/tracker-processor.c +++ b/src/trackerd/tracker-processor.c @@ -1416,11 +1416,26 @@ mount_point_added_cb (TrackerHal *hal, { TrackerProcessor *processor; TrackerStatus status; + gchar *metadata_file; processor = user_data; g_message ("** TRAWLING THROUGH NEW MOUNT POINT:'%s'", mount_point); + metadata_file = g_build_filename (mount_point, ".cache", + "metadata", + "metadata.ttl", NULL); + + if (g_file_test (metadata_file, G_FILE_TEST_EXISTS)) { + g_message ("** FOUND TURTLE FILE:'%s'", metadata_file); + + // Process a TTL file + // This is where the code in the DBus API handler + // tracker_indexer_turtle_add must come { A01 } + } + + g_free (metadata_file); + status = tracker_status_get (); processor->private->iterated_removable_media = FALSE; diff --git a/tests/tracker-indexer/Makefile.am b/tests/tracker-indexer/Makefile.am index 00b9cfb..76ed0d9 100644 --- a/tests/tracker-indexer/Makefile.am +++ b/tests/tracker-indexer/Makefile.am @@ -17,6 +17,7 @@ INCLUDES = \ $(DBUS_CFLAGS) \ $(PANGO_CFLAGS) \ $(GMODULE_CFLAGS) \ + $(RAPTOR_CFLAGS) \ $(GTHREAD_CFLAGS) \ $(GLIB2_CFLAGS) @@ -31,7 +32,9 @@ tracker_metadata_utils_SOURCES = \ tracker-metadata-utils.c \ tracker-metadata-utils.h \ tracker-metadata-utils-test.c \ - tracker-module.h + tracker-module.h \ + tracker-turtle.c \ + tracker-turtle.h tracker_metadata_utils_LDADD = \ $(top_builddir)/src/libtracker-data/libtracker-data.la \ @@ -43,6 +46,7 @@ tracker_metadata_utils_LDADD = \ $(GMODULE_LIBS) \ $(GTHREAD_LIBS) \ $(GIO_LIBS) \ + $(RAPTOR_LIBS) $(GLIB2_LIBS) #