tracker r2101 - in branches/indexer-split: . src/libtracker-common src/tracker-extract src/tracker-indexer

[carlosg svn gnome org]

Author: carlosg
Date: Tue Aug 19 11:49:42 2008
New Revision: 2101
URL: http://svn.gnome.org/viewvc/tracker?rev=2101&view=rev

Log:
2008-08-19  Carlos Garnacho  <carlos imendio com>

        * src/libtracker-common/tracker-os-dependant-unix.c:
        * src/libtracker-common/tracker-os-dependant-win.c:
        * src/libtracker-common/tracker-os-dependant.h
        (tracker_spawn_async_with_channels): Added function.

        * src/tracker-extract/tracker-extract.c: Modified to read queried
        files from stdin and stay alive until it has stopped being asked for
        metadata. According to sysprof, tracker-metadata spent a long time in
        initialization due to being respawned once and again.

        * src/tracker-indexer/tracker-metadata-utils.c: Modified to send
        filename/mimetype pairs to the extractor through a pipe and keep track
        of the extractor state.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/libtracker-common/tracker-os-dependant-unix.c
   branches/indexer-split/src/libtracker-common/tracker-os-dependant-win.c
   branches/indexer-split/src/libtracker-common/tracker-os-dependant.h
   branches/indexer-split/src/tracker-extract/tracker-extract.c
   branches/indexer-split/src/tracker-indexer/tracker-metadata-utils.c

Modified: branches/indexer-split/src/libtracker-common/tracker-os-dependant-unix.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-os-dependant-unix.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-os-dependant-unix.c	Tue Aug 19 11:49:42 2008
@@ -58,6 +58,50 @@
                              NULL);
 }
 
+gboolean
+tracker_spawn_async_with_channels (const gchar **argv,
+                                   gint          timeout,
+                                   GPid         *pid,
+                                   GIOChannel  **stdin_channel,
+                                   GIOChannel  **stdout_channel,
+                                   GIOChannel  **stderr_channel)
+{
+        gint stdin, stdout, stderr;
+        gboolean result;
+        GError *error = NULL;
+
+        result = g_spawn_async_with_pipes (NULL,
+                                           (gchar **) argv,
+                                           NULL,
+                                           G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD,
+                                           tracker_child_cb,
+                                           GINT_TO_POINTER (timeout),
+                                           pid,
+                                           (stdin_channel) ? &stdin : NULL,
+                                           (stdout_channel) ? &stdout : NULL,
+                                           (stderr_channel) ? &stderr : NULL,
+                                           &error);
+
+        if (error) {
+                g_warning ("Could not spawn command: %s", error->message);
+                g_error_free (error);
+        }
+
+        if (stdin_channel) {
+                *stdin_channel = (result) ? g_io_channel_unix_new (stdin) : NULL;
+        }
+
+        if (stdout_channel) {
+                *stdout_channel = (result) ? g_io_channel_unix_new (stdout) : NULL;
+        }
+
+        if (stderr_channel) {
+                *stderr_channel = (result) ? g_io_channel_unix_new (stderr) : NULL;
+        }
+
+        return result;
+}
+
 gchar *
 tracker_create_permission_string (struct stat finfo)
 {

Modified: branches/indexer-split/src/libtracker-common/tracker-os-dependant-win.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-os-dependant-win.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-os-dependant-win.c	Tue Aug 19 11:49:42 2008
@@ -77,6 +77,48 @@
 	return status;
 }
 
+gboolean
+tracker_spawn_async_with_channels (const gchar **argv,
+				   gint          timeout,
+				   GPid         *pid,
+				   GIOChannel  **stdin_channel,
+				   GIOChannel  **stdout_channel,
+				   GIOChannel  **strerr_channel)
+{
+	gint stdin, stdout, stderr;
+	gboolean result;
+	GError *error = NULL;
+
+	result = g_spawn_async_with_pipes (NULL,
+					   (gchar **) argv,
+					   NULL,
+					   G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD,
+					   tracker_child_cb, GINT_TO_POINTER (timeout), pid,
+					   (stdin_channel) ? &stdin : NULL,
+					   (stdout_channel) ? &stdout : NULL,
+					   (stderr_channel) ? &stderr : NULL,
+					   &error);
+
+	if (error) {
+                g_warning ("Could not spawn command: %s", error->message);
+                g_error_free (error);
+	}
+
+	if (stdin_channel) {
+		*stdin_channel = (result) ? g_io_channel_win32_new_fd (stdin) : NULL;
+	}
+
+	if (stdout_channel) {
+		*stdout_channel = (result) ? g_io_channel_win32_new_fd (stdout) : NULL;
+	}
+
+	if (stderr_channel) {
+		*stderr_channel = (result) ? g_io_channel_win32_new_fd (stderr) : NULL;
+	}
+
+	return result;
+}
+
 void
 tracker_child_cb (gpointer user_data)
 {

Modified: branches/indexer-split/src/libtracker-common/tracker-os-dependant.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-os-dependant.h	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-os-dependant.h	Tue Aug 19 11:49:42 2008
@@ -25,11 +25,18 @@
 #include <glib.h>
 #include <glib/gstdio.h>
 
-gboolean tracker_spawn                    (gchar       **argv,
-                                           gint          timeout,
-                                           gchar       **tmp_stdout,
-                                           gint         *exit_status);
-void     tracker_child_cb                 (gpointer      user_data);
-gchar *  tracker_create_permission_string (struct stat   finfo);
+gboolean tracker_spawn                     (gchar       **argv,
+					    gint          timeout,
+					    gchar       **tmp_stdout,
+					    gint         *exit_status);
+gboolean tracker_spawn_async_with_channels (const gchar **argv,
+					    gint          timeout,
+					    GPid         *pid,
+					    GIOChannel  **stdin_channel,
+					    GIOChannel  **stdout_channel,
+					    GIOChannel  **stderr_channel);
+
+void     tracker_child_cb                  (gpointer      user_data);
+gchar *  tracker_create_permission_string  (struct stat   finfo);
 
 #endif /* __LIBTRACKER_COMMON_OS_DEPENDANT_H__ */

Modified: branches/indexer-split/src/tracker-extract/tracker-extract.c
==============================================================================
--- branches/indexer-split/src/tracker-extract/tracker-extract.c	(original)
+++ branches/indexer-split/src/tracker-extract/tracker-extract.c	Tue Aug 19 11:49:42 2008
@@ -23,6 +23,7 @@
 #define _XOPEN_SOURCE
 #include <time.h>
 
+#include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <errno.h>
@@ -47,6 +48,7 @@
 #define ISO8601_FORMAT "%Y-%m-%dT%H:%M:%S%z"
 
 GArray *extractors = NULL;
+guint   shutdown_timeout_id = 0;
 
 gchar *
 tracker_generic_date_to_iso8601 (const gchar *date, const gchar *format)
@@ -295,9 +297,10 @@
 	return NULL;
 }
 
-
 static void
-get_meta_table_data (gpointer pkey, gpointer pvalue, gpointer user_data)
+print_meta_table_data (gpointer pkey,
+                       gpointer pvalue,
+                       gpointer user_data)
 {
 	char *value;
 
@@ -319,66 +322,88 @@
 	}
 }
 
+static gboolean
+shutdown_app_timeout (gpointer user_data)
+{
+	GMainLoop *main_loop;
+
+	main_loop = (GMainLoop *) user_data;
+	g_main_loop_quit (main_loop);
+
+	return FALSE;
+}
 
 static void
-kill_app_timeout (void)
+reset_shutdown_timeout (GMainLoop *main_loop)
 {
-	g_usleep (1000 * 1000 * 10);
-	exit (EXIT_FAILURE);
+	if (shutdown_timeout_id != 0) {
+		g_source_remove (shutdown_timeout_id);
+	}
+
+	shutdown_timeout_id = g_timeout_add (30000, shutdown_app_timeout, main_loop);
 }
 
+static gboolean
+process_input_cb (GIOChannel   *channel,
+		  GIOCondition  condition,
+		  gpointer      user_data)
+{
+	GHashTable *meta;
+	gchar *filename, *mimetype;
+
+	reset_shutdown_timeout ((GMainLoop *) user_data);
+
+	g_io_channel_read_line (channel, &filename, NULL, NULL, NULL);
+	g_io_channel_read_line (channel, &mimetype, NULL, NULL, NULL);
+
+	g_strstrip (filename);
+	g_strstrip (mimetype);
+
+	if (mimetype && *mimetype) {
+		meta = tracker_get_file_metadata (filename, mimetype);
+	} else {
+		meta = tracker_get_file_metadata (filename, NULL);
+	}
+
+	if (meta) {
+		g_hash_table_foreach (meta, print_meta_table_data, NULL);
+		g_hash_table_destroy (meta);
+	}
+
+	/* Add an empty line so the indexer
+	 * knows when to stop reading
+	 */
+	g_print ("\n");
+
+	g_free (filename);
+	g_free (mimetype);
+
+	return TRUE;
+}
 
 gint
 main (gint argc, gchar *argv[])
 {
-	GHashTable *meta;
-	gchar	   *filename;
+	GMainLoop  *main_loop;
+	GIOChannel *input;
 
 	set_memory_rlimits ();
 
 	if (!g_thread_supported ())
 		g_thread_init (NULL);
 
-	g_thread_create ((GThreadFunc) kill_app_timeout, NULL, FALSE, NULL);
-
 	g_set_application_name ("tracker-extract");
 
 	setlocale (LC_ALL, "");
 
 	initialize_extractors ();
+	main_loop = g_main_loop_new (NULL, FALSE);
 
-	if (argc == 1 || argc > 3) {
-		g_print ("usage: tracker-extract file [mimetype]\n");
-		return EXIT_FAILURE;
-	}
-
-	filename = g_filename_to_utf8 (argv[1], -1, NULL, NULL, NULL);
-
-	if (!filename) {
-		g_warning ("locale to UTF8 failed for filename!");
-		return 1;
-	}
-
-	if (argc == 3) {
-		gchar *mime = g_locale_to_utf8 (argv[2], -1, NULL, NULL, NULL);
+	input = g_io_channel_unix_new (0);
+	g_io_add_watch (input, G_IO_IN, process_input_cb, main_loop);
 
-		if (!mime) {
-			g_warning ("locale to UTF8 failed for mime!");
-			return EXIT_FAILURE;
-		}
-
-		meta = tracker_get_file_metadata (filename, mime);
-		g_free (mime);
-	} else {
-		meta = tracker_get_file_metadata (filename, NULL);
-	}
-
-	g_free (filename);
-
-	if (meta) {
-		g_hash_table_foreach (meta, get_meta_table_data, NULL);
-		g_hash_table_destroy (meta);
-	}
+	reset_shutdown_timeout (main_loop);
+	g_main_loop_run (main_loop);
 
 	return EXIT_SUCCESS;
 }

Modified: branches/indexer-split/src/tracker-indexer/tracker-metadata-utils.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-metadata-utils.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-metadata-utils.c	Tue Aug 19 11:49:42 2008
@@ -36,13 +36,163 @@
 #define METADATA_FILE_MODIFIED       "File:Modified"
 #define METADATA_FILE_ACCESSED       "File:Accessed"
 
+typedef struct {
+	GPid pid;
+	GIOChannel *stdin_channel;
+	GIOChannel *stdout_channel;
+	GMainLoop  *data_incoming_loop;
+} MetadataContext;
+
+static MetadataContext *context = NULL;
+
+static void
+tracker_extract_watch_cb (GPid     pid,
+			  gint     status,
+			  gpointer data)
+{
+	g_debug ("Metadata extractor exited with code: %d\n", status);
+
+	if (!context) {
+		return;
+	}
+
+	g_io_channel_shutdown (context->stdin_channel, FALSE, NULL);
+	g_io_channel_unref (context->stdin_channel);
+
+	g_io_channel_shutdown (context->stdout_channel, FALSE, NULL);
+	g_io_channel_unref (context->stdout_channel);
+
+	if (g_main_loop_is_running (context->data_incoming_loop))
+		g_main_loop_quit (context->data_incoming_loop);
+
+	g_main_loop_unref (context->data_incoming_loop);
+
+	g_spawn_close_pid (context->pid);
+
+	g_free (context);
+	context = NULL;
+}
+
+static gboolean
+tracker_metadata_read (GIOChannel   *channel,
+		       GIOCondition  condition,
+		       gpointer      user_data)
+{
+	GPtrArray *array;
+	GIOStatus status = G_IO_STATUS_NORMAL;
+	gchar *line;
+
+	array = (GPtrArray *) user_data;
+
+	if (!context) {
+		return FALSE;
+	}
+
+	if (condition & G_IO_IN || condition & G_IO_PRI) {
+		do {
+			status = g_io_channel_read_line (context->stdout_channel, &line, NULL, NULL, NULL);
+
+			if (line && *line) {
+				g_strstrip (line);
+				g_ptr_array_add (array, line);
+			}
+		} while (status == G_IO_STATUS_NORMAL && line && *line);
+
+		if (status == G_IO_STATUS_NORMAL && !*line) {
+			/* Empty line, all extractor output has been processed */
+			g_main_loop_quit (context->data_incoming_loop);
+			return FALSE;
+		}
+	}
+
+	if (condition & G_IO_HUP || condition & G_IO_ERR) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static gboolean
+create_metadata_context (void)
+{
+	GIOChannel *stdin_channel, *stdout_channel;
+	const gchar *argv[2] = { "tracker-extract", NULL };
+	GIOFlags flags;
+	GPid pid;
+
+	if (!tracker_spawn_async_with_channels (argv, 10, &pid, &stdin_channel, &stdout_channel, NULL))
+		return FALSE;
+
+	g_child_watch_add (pid, tracker_extract_watch_cb, NULL);
+
+	context = g_new0 (MetadataContext, 1);
+	context->pid = pid;
+	context->stdin_channel = stdin_channel;
+	context->stdout_channel = stdout_channel;
+	context->data_incoming_loop = g_main_loop_new (NULL, FALSE);
+
+	flags = g_io_channel_get_flags (context->stdout_channel);
+	flags |= G_IO_FLAG_NONBLOCK;
+
+	g_io_channel_set_flags (context->stdout_channel, flags, NULL);
+
+	return TRUE;
+}
+
+static gchar **
+tracker_metadata_query_file (const gchar *path,
+			     const gchar *mimetype)
+{
+	gchar *utf_path, *str;
+	GPtrArray *array;
+	GIOStatus status;
+
+	if (!path || !mimetype) {
+		return NULL;
+	}
+
+	if (!context && !create_metadata_context ()) {
+		return NULL;
+	}
+
+	utf_path = g_filename_from_utf8 (path, -1, NULL, NULL, NULL);
+
+	if (!utf_path) {
+		g_free (utf_path);
+		return NULL;
+	}
+
+	array = g_ptr_array_sized_new (10);
+
+	g_io_add_watch (context->stdout_channel,
+			G_IO_IN | G_IO_PRI | G_IO_ERR | G_IO_HUP,
+			tracker_metadata_read,
+			array);
+
+	/* write path and mimetype */
+	str = g_strdup_printf ("%s\n%s\n", utf_path, mimetype);
+	status = g_io_channel_write_chars (context->stdin_channel, str, -1, NULL, NULL);
+	g_io_channel_flush (context->stdin_channel, NULL);
+
+	/* It will block here until all incoming
+	 * metadata has been processed
+	 */
+	g_main_loop_run (context->data_incoming_loop);
+
+	g_ptr_array_add (array, NULL);
+
+	g_free (utf_path);
+	g_free (str);
+
+	return (gchar **) g_ptr_array_free (array, FALSE);
+}
+
 static void
 tracker_metadata_utils_get_embedded (const char      *path,
 				     const char      *mimetype,
 				     TrackerMetadata *metadata)
 {
-	gboolean success;
-	gchar **argv, *output, **values, *service_type;
+	gchar **values, *service_type;
 	gint i;
 
 	service_type = tracker_ontology_get_service_type_for_mime (mimetype);
@@ -58,34 +208,19 @@
 
         g_free (service_type);
 
-	/* we extract metadata out of process using pipes */
-	argv = g_new0 (gchar *, 4);
-	argv[0] = g_strdup ("tracker-extract");
-	argv[1] = g_filename_from_utf8 (path, -1, NULL, NULL, NULL);
-	argv[2] = g_strdup (mimetype);
-
-	if (!argv[1] || !argv[2]) {
-		g_critical ("path or mime could not be converted to locale format");
-		g_strfreev (argv);
-		return;
-	}
-
-	success = tracker_spawn (argv, 10, &output, NULL);
-	g_strfreev (argv);
+	values = tracker_metadata_query_file (path, mimetype);
 
-	if (!success || !output)
+	if (!values) {
 		return;
+	}
 
-	/* parse returned stdout and extract keys and associated metadata values */
-
-	values = g_strsplit_set (output, ";", -1);
-
+	/* parse returned values and extract keys and associated metadata */
 	for (i = 0; values[i]; i++) {
 		char *meta_data, *sep;
 		const char *name, *value;
 		char *utf_value;
 
-		meta_data = g_strstrip (values[i]);
+		meta_data = values[i];
 		sep = strchr (meta_data, '=');
 
 		if (!sep)
@@ -114,7 +249,6 @@
 	}
 
 	g_strfreev (values);
-	g_free (output);
 }
 
 TrackerMetadata *