GNOME.org
 

[tracker/miner-extractor-ipc: 2/4] libtracker-miner, miners/fs: Queue requests to the extractor

commit e01bcc30c6aaf71ff625cdcde7f8c11148e0f66a
Author: Philip Van Hoof <philip codeminded be>
Date:   Thu Feb 24 11:32:46 2011 +0100

    libtracker-miner, miners/fs: Queue requests to the extractor

 src/libtracker-miner/tracker-miner-fs.c    |   24 ++++
 src/libtracker-miner/tracker-miner-fs.h    |    3 +
 src/miners/fs/tracker-miner-applications.c |    1 -
 src/miners/fs/tracker-miner-files.c        |  162 +++++++++++++++++++++++-----
 src/miners/fs/tracker-miner-files.h        |   32 +++---
 5 files changed, 180 insertions(+), 42 deletions(-)
---
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index 6fd9e8d..917da1b 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -2656,6 +2656,8 @@ item_queue_handlers_cb (gpointer user_data)
 		tracker_processing_pool_buffer_flush (fs->private->processing_pool,
 		                                      "Queue handlers WAIT");
 
+		tracker_miner_fs_flush_wait (fs);
+
 		return FALSE;
 	}
 
@@ -2760,6 +2762,8 @@ item_queue_handlers_cb (gpointer user_data)
 		tracker_processing_pool_buffer_flush (fs->private->processing_pool,
 		                                      "Queue handlers NONE");
 
+		tracker_miner_fs_flush_wait (fs);
+
 		tracker_thumbnailer_send ();
 		/* No more files left to process */
 		keep_processing = FALSE;
@@ -4184,6 +4188,26 @@ check_file_parents (TrackerMinerFS *fs,
 }
 
 /**
+ * tracker_miner_fs_flush_wait:
+ * @fs: a #TrackerMinerFS
+ *
+ * Flushes the items that are queued and that we are waiting on
+ *
+ * Since: 0.10.1
+ **/
+void
+tracker_miner_fs_flush_wait (TrackerMinerFS *fs)
+{
+	void (*m_flush) (TrackerMinerFS *fs);
+
+	m_flush = TRACKER_MINER_FS_GET_CLASS (fs)->flush_wait;
+
+	if (m_flush) {
+		m_flush (fs);
+	}
+}
+
+/**
  * tracker_miner_fs_check_file:
  * @fs: a #TrackerMinerFS
  * @file: #GFile for the file to check
diff --git a/src/libtracker-miner/tracker-miner-fs.h b/src/libtracker-miner/tracker-miner-fs.h
index aa0803e..f840fa9 100644
--- a/src/libtracker-miner/tracker-miner-fs.h
+++ b/src/libtracker-miner/tracker-miner-fs.h
@@ -104,6 +104,8 @@ typedef struct {
 	                                       GFile                *file,
 	                                       TrackerSparqlBuilder *builder,
 	                                       GCancellable         *cancellable);
+	void     (* flush_wait)               (TrackerMinerFS       *fs);
+
 } TrackerMinerFSClass;
 
 GType                 tracker_miner_fs_get_type             (void) G_GNUC_CONST;
@@ -133,6 +135,7 @@ G_CONST_RETURN gchar *tracker_miner_fs_get_parent_urn       (TrackerMinerFS *fs,
 gchar                *tracker_miner_fs_query_urn            (TrackerMinerFS *fs,
                                                              GFile          *file);
 void                  tracker_miner_fs_force_recheck        (TrackerMinerFS *fs);
+void                  tracker_miner_fs_flush_wait           (TrackerMinerFS *fs);
 
 void                  tracker_miner_fs_set_mtime_checking   (TrackerMinerFS *fs,
                                                              gboolean        check_mtimes);
diff --git a/src/miners/fs/tracker-miner-applications.c b/src/miners/fs/tracker-miner-applications.c
index e3e015e..96f9a8e 100644
--- a/src/miners/fs/tracker-miner-applications.c
+++ b/src/miners/fs/tracker-miner-applications.c
@@ -57,7 +57,6 @@ static gboolean miner_applications_monitor_directory       (TrackerMinerFS
                                                             GFile                *file);
 static void     miner_applications_finalize                (GObject              *object);
 
-
 static GQuark miner_applications_error_quark = 0;
 
 typedef struct ProcessApplicationData ProcessApplicationData;
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index cd61a06..bac1e9b 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -90,6 +90,14 @@ typedef struct {
 	fast_async_cb callback;
 } SendAndSpliceData;
 
+typedef struct {
+	gchar *uri;
+	gchar *mime_type;
+	GCancellable *cancellable;
+	fast_async_cb callback;
+	ProcessFileData *user_data;
+} ExtractQueueItem;
+
 struct TrackerMinerFilesPrivate {
 	TrackerConfig *config;
 	TrackerStorage *storage;
@@ -123,6 +131,9 @@ struct TrackerMinerFilesPrivate {
 	gboolean mount_points_initialized;
 
 	guint stale_volumes_check_id;
+
+	GPtrArray *extract_queue;
+	guint extract_queue_id;
 };
 
 enum {
@@ -234,6 +245,8 @@ static void        miner_files_add_removable_or_optical_directory (TrackerMinerF
                                                                    const gchar       *mount_path,
                                                                    const gchar       *uuid);
 
+static void        miner_files_flush_wait                         (TrackerMinerFS *miner);
+
 static GInitableIface* miner_files_initable_parent_iface;
 
 G_DEFINE_TYPE_WITH_CODE (TrackerMinerFiles, tracker_miner_files, TRACKER_TYPE_MINER_FS,
@@ -258,6 +271,7 @@ tracker_miner_files_class_init (TrackerMinerFilesClass *klass)
 	miner_fs_class->process_file_attributes = miner_files_process_file_attributes;
 	miner_fs_class->ignore_next_update_file = miner_files_ignore_next_update_file;
 	miner_fs_class->finished = miner_files_finished;
+	miner_fs_class->flush_wait = miner_files_flush_wait;
 
 	g_object_class_install_property (object_class,
 	                                 PROP_CONFIG,
@@ -2050,20 +2064,24 @@ get_metadata_fast_async_callback (SendAndSpliceData *data)
 		buffer = g_memory_output_stream_get_data (G_MEMORY_OUTPUT_STREAM (data->output_stream));
 		buffer_size = g_memory_output_stream_get_data_size (G_MEMORY_OUTPUT_STREAM (data->output_stream));
 
-		preupdate = buffer;
-		preupdate_length = strnlen (preupdate, buffer_size);
-
-		if (preupdate_length < buffer_size && preupdate[buffer_size - 1] == '\0') {
-			/* sparql is stored just after preupdate in the original buffer */
-			sparql = preupdate + preupdate_length + 1;
+		if (buffer_size == 0) {
+			(* data->callback) (NULL, NULL, NULL, data->user_data);
 		} else {
-			preupdate = NULL;
-			error = g_error_new_literal (miner_files_error_quark,
-			                             0,
-			                             "Invalid data received from GetMetadataFast");
-		}
+			preupdate = buffer;
+			preupdate_length = strnlen (preupdate, buffer_size);
 
-		(* data->callback) (preupdate, sparql, error, data->user_data);
+			if (preupdate_length < buffer_size && preupdate[buffer_size - 1] == '\0') {
+				/* sparql is stored just after preupdate in the original buffer */
+				sparql = preupdate + preupdate_length + 1;
+			} else {
+				preupdate = NULL;
+				error = g_error_new_literal (miner_files_error_quark,
+				                             0,
+				                             "Invalid data received from GetMetadataFast");
+			}
+
+			(* data->callback) (preupdate, sparql, error, data->user_data);
+		}
 
 		g_clear_error (&error);
 
@@ -2226,19 +2244,113 @@ get_metadata_fast_async (GDBusConnection *connection,
 }
 
 static void
-extractor_get_embedded_metadata (ProcessFileData *data,
-                                 const gchar     *uri,
-                                 const gchar     *mime_type)
-{
-	get_metadata_fast_async (data->miner->private->connection,
-	                         uri,
-	                         mime_type,
-	                         data->cancellable,
-	                         extractor_get_embedded_metadata_cb,
-	                         data);
+free_queue_item (ExtractQueueItem *item)
+{
+	g_free (item->uri);
+	g_free (item->mime_type);
+	g_object_unref (item->cancellable);
+	g_slice_free (ExtractQueueItem, item);
+}
+
+static void
+flush_extract_queue_shared (TrackerMinerFiles *miner)
+{
+	guint i;
+	GPtrArray *queue = miner->private->extract_queue;
+	GDBusConnection *connection = miner->private->connection;
+
+	if (!queue || queue->len == 0) {
+		return;
+	}
+
+	for (i = 0; i < queue->len; i++) {
+		ExtractQueueItem *item = g_ptr_array_index (queue, i);
+
+		get_metadata_fast_async (connection,
+		                         item->uri,
+		                         item->mime_type,
+		                         item->cancellable,
+		                         item->callback,
+		                         item->user_data);
+	}
+
+	g_ptr_array_remove_range (queue, 0, queue->len);
+}
+
+static gboolean
+flush_extract_queue_idle (gpointer user_data)
+{
+	flush_extract_queue_shared (user_data);
+	return FALSE;
+}
+
+static void
+flush_extract_queue_destroy (gpointer user_data)
+{
+	TrackerMinerFiles *miner = user_data;
+	miner->private->extract_queue_id = 0;
+	g_object_unref (miner);
+}
+
+static void
+miner_files_flush_wait (TrackerMinerFS *miner)
+{
+	flush_extract_queue_shared (TRACKER_MINER_FILES (miner));
+}
+
+static void
+get_metadata_fast_queue_async (TrackerMinerFiles *miner,
+                               const gchar       *uri,
+                               const gchar       *mime_type,
+                               GCancellable      *cancellable,
+                               fast_async_cb      callback,
+                               ProcessFileData   *user_data)
+{
+	ExtractQueueItem *item;
+
+	if (!miner->private->extract_queue) {
+		miner->private->extract_queue = g_ptr_array_new_with_free_func ((GDestroyNotify) free_queue_item);
+	}
+
+	item = g_slice_new (ExtractQueueItem);
+	item->uri = g_strdup (uri);
+	item->mime_type = g_strdup (mime_type);
+	item->cancellable = g_object_ref (cancellable);
+	item->callback = callback;
+	item->user_data = user_data;
+
+	g_ptr_array_add (miner->private->extract_queue, item);
+
+	if (miner->private->extract_queue->len > 5) {
+		flush_extract_queue_shared (miner);
+	} else {
+		if (!miner->private->extract_queue_id) {
+			/* Automatic flush each second */
+			miner->private->extract_queue_id = g_timeout_add_seconds_full (G_PRIORITY_DEFAULT, 10,
+			                                                               flush_extract_queue_idle,
+			                                                               g_object_ref (miner),
+			                                                               flush_extract_queue_destroy);
+		}
+	}
+}
+
+static void
+extractor_get_embedded_metadata (TrackerMinerFiles *miner,
+                                 GCancellable      *cancellable,
+                                 const gchar       *uri,
+                                 const gchar       *mime_type,
+                                 gpointer           user_data)
+{
+	get_metadata_fast_queue_async (miner,
+	                               uri,
+	                               mime_type,
+	                               cancellable,
+	                               extractor_get_embedded_metadata_cb,
+	                               user_data);
 
-	g_signal_connect (data->cancellable, "cancelled",
-	                  G_CALLBACK (extractor_get_embedded_metadata_cancel), data);
+	g_signal_connect (cancellable, "cancelled",
+	                  G_CALLBACK (extractor_get_embedded_metadata_cancel),
+	                  user_data);
 }
 
 static void
@@ -2332,7 +2444,7 @@ process_file_cb (GObject      *object,
 
 	if (!is_directory) {
 		/* Next step, if NOT a directory, get embedded metadata */
-		extractor_get_embedded_metadata (data, uri, mime_type);
+		extractor_get_embedded_metadata (data->miner, data->cancellable, uri, mime_type, data);
 	} else {
 		/* For directories, don't request embedded metadata extraction.
 		 * We setup an idle so that we keep the previous behavior. */
diff --git a/src/miners/fs/tracker-miner-files.h b/src/miners/fs/tracker-miner-files.h
index fc651fb..da95384 100644
--- a/src/miners/fs/tracker-miner-files.h
+++ b/src/miners/fs/tracker-miner-files.h
@@ -48,24 +48,24 @@ struct TrackerMinerFilesClass {
 
 GType         tracker_miner_files_get_type                 (void) G_GNUC_CONST;
 
-TrackerMiner *tracker_miner_files_new                      (TrackerConfig  *config,
-                                                            GError        **error);
+TrackerMiner *tracker_miner_files_new                      (TrackerConfig     *config,
+                                                            GError           **error);
 
 /* Convenience functions for --eligible tracker-miner-fs cmdline */
-gboolean      tracker_miner_files_check_file               (GFile          *file,
-                                                            GSList         *ignored_file_paths,
-                                                            GSList         *ignored_file_patterns);
-gboolean      tracker_miner_files_check_directory          (GFile          *file,
-                                                            GSList         *index_recursive_directories,
-                                                            GSList         *index_single_directories,
-                                                            GSList         *ignored_directory_paths,
-                                                            GSList         *ignored_directory_patterns);
-gboolean      tracker_miner_files_check_directory_contents (GFile          *parent,
-                                                            GList          *children,
-                                                            GSList         *ignored_content);
-gboolean      tracker_miner_files_monitor_directory        (GFile          *file,
-                                                            gboolean        enable_monitors,
-                                                            GSList         *directories_to_check);
+gboolean      tracker_miner_files_check_file               (GFile             *file,
+                                                            GSList            *ignored_file_paths,
+                                                            GSList            *ignored_file_patterns);
+gboolean      tracker_miner_files_check_directory          (GFile             *file,
+                                                            GSList            *index_recursive_directories,
+                                                            GSList            *index_single_directories,
+                                                            GSList            *ignored_directory_paths,
+                                                            GSList            *ignored_directory_patterns);
+gboolean      tracker_miner_files_check_directory_contents (GFile             *parent,
+                                                            GList             *children,
+                                                            GSList            *ignored_content);
+gboolean      tracker_miner_files_monitor_directory        (GFile             *file,
+                                                            gboolean           enable_monitors,
+                                                            GSList            *directories_to_check);
 
 G_END_DECLS
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]