[tracker/failsafe-extraction-0.10: 2/3] tracker-miner-fs: Implement failsafe metadata extraction

[Carlos Garnacho <carlosg src gnome org>]

commit 52e75cebdea4330287f96f7723f3fc886c41fbf3
Author: Carlos Garnacho <carlosg gnome org>
Date:   Fri May 6 15:02:19 2011 +0200

    tracker-miner-fs: Implement failsafe metadata extraction
    
    This method is independent of how the extractor works, unlike the
    previous method. Now on a extractor failure, the miner does:
    
    1) Pause itself
    2) Wait for all pending extractor requests to finish
    3) Accumulate all failed extractions on a list
    4) Run through that list items, extracting again one file at a time.
    5) Resume itself
    
    Conflicts:
    
    	src/miners/fs/tracker-miner-files.c

 src/miners/fs/tracker-miner-files.c |  152 ++++++++++++++++++++++++++++------
 1 files changed, 125 insertions(+), 27 deletions(-)
---
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index a33f034..9cb54ce 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -72,7 +72,6 @@ struct ProcessFileData {
 	GCancellable *cancellable;
 	GFile *file;
 	gchar *mime_type;
-	guint retried : 1;
 };
 
 typedef void (*fast_async_cb) (const gchar *preupdate,
@@ -135,6 +134,10 @@ struct TrackerMinerFilesPrivate {
 	gboolean mount_points_initialized;
 
 	guint stale_volumes_check_id;
+
+	guint failed_extraction_pause_cookie;
+	GList *extraction_queue;
+	GList *failed_extraction_queue;
 };
 
 enum {
@@ -243,12 +246,18 @@ static void        miner_files_in_removable_media_remove_by_date  (TrackerMinerF
 static void        miner_files_add_removable_or_optical_directory (TrackerMinerFiles *mf,
                                                                    const gchar       *mount_path,
                                                                    const gchar       *uuid);
-static void        extractor_get_embedded_metadata                (ProcessFileData *data,
+static void        get_metadata_fast_async                        (GDBusConnection *connection,
                                                                    const gchar     *uri,
-                                                                   const gchar     *mime_type);
+                                                                   const gchar     *mime_type,
+                                                                   GCancellable    *cancellable,
+                                                                   fast_async_cb    callback,
+                                                                   ProcessFileData *user_data);
 static void        extractor_cancel_tasks                         (GDBusConnection   *connection,
                                                                    GFile             *prefix);
 
+static void        extractor_process_failsafe                     (TrackerMinerFiles *miner);
+
+
 static GInitableIface* miner_files_initable_parent_iface;
 
 G_DEFINE_TYPE_WITH_CODE (TrackerMinerFiles, tracker_miner_files, TRACKER_TYPE_MINER_FS,
@@ -674,6 +683,9 @@ miner_files_finalize (GObject *object)
 		priv->stale_volumes_check_id = 0;
 	}
 
+	g_list_free (priv->extraction_queue);
+	g_list_free (priv->failed_extraction_queue);
+
 	G_OBJECT_CLASS (tracker_miner_files_parent_class)->finalize (object);
 }
 
@@ -2036,13 +2048,96 @@ sparql_builder_finish (ProcessFileData *data,
 }
 
 static void
+extractor_get_failsafe_metadata_cb (const gchar *preupdate,
+                                    const gchar *sparql,
+                                    GError      *error,
+                                    gpointer     user_data)
+{
+	ProcessFileData *data = user_data;
+	TrackerMinerFiles *miner = data->miner;
+	gchar *uri;
+
+	if (error) {
+		uri = g_file_get_uri (data->file);
+		g_warning ("  Got second extraction DBus error on '%s'. "
+			   "Adding only non-embedded metadata to the SparQL, "
+			   "the error was: %s",
+			   uri, error->message);
+
+		sparql_builder_finish (data, NULL, NULL);
+
+		g_error_free (error);
+		g_free (uri);
+	} else {
+		g_debug ("  Extraction succeeded the second time");
+		sparql_builder_finish (data, preupdate, sparql);
+	}
+
+	/* Notify success even if the extraction failed
+	 * again, so we get the essential data in the store.
+	 */
+	tracker_miner_fs_file_notify (TRACKER_MINER_FS (miner), data->file, NULL);
+	process_file_data_free (data);
+
+	/* Get on to the next failed extraction, or resume miner */
+	extractor_process_failsafe (miner);
+}
+
+/* This function processes failed files one by one,
+ * the function will be called after each operation
+ * is finished, so elements are processed linearly.
+ */
+static void
+extractor_process_failsafe (TrackerMinerFiles *miner)
+{
+	TrackerMinerFilesPrivate *priv;
+	ProcessFileData *data;
+
+	priv = miner->private;
+
+	if (priv->failed_extraction_queue) {
+		gchar *uri;
+
+		data = priv->failed_extraction_queue->data;
+		priv->failed_extraction_queue = g_list_remove (priv->failed_extraction_queue, data);
+
+		uri = g_file_get_uri (data->file);
+		g_message ("Performing failsafe extraction on '%s'", uri);
+		g_free (uri);
+
+		get_metadata_fast_async (data->miner->private->connection,
+		                         uri,
+		                         data->mime_type,
+		                         data->cancellable,
+		                         extractor_get_failsafe_metadata_cb,
+		                         data);
+	} else {
+		g_debug ("Failsafe extraction finished. Resuming miner...");
+
+		if (priv->failed_extraction_pause_cookie != 0) {
+			tracker_miner_resume (TRACKER_MINER (miner),
+					      priv->failed_extraction_pause_cookie,
+					      NULL);
+
+			priv->failed_extraction_pause_cookie = 0;
+		}
+	}
+}
+
+static void
 extractor_get_embedded_metadata_cb (const gchar *preupdate,
                                     const gchar *sparql,
                                     GError      *error,
                                     gpointer     user_data)
 {
+	TrackerMinerFilesPrivate *priv;
+	TrackerMinerFiles *miner;
 	ProcessFileData *data = user_data;
 
+	miner = data->miner;
+	priv = miner->private;
+	priv->extraction_queue = g_list_remove (priv->extraction_queue, data);
+
 	if (error) {
 		if (error->code == G_DBUS_ERROR_NO_REPLY ||
 		    error->code == G_DBUS_ERROR_TIMEOUT ||
@@ -2050,25 +2145,18 @@ extractor_get_embedded_metadata_cb (const gchar *preupdate,
 			gchar *uri;
 
 			uri = g_file_get_uri (data->file);
-
-			if (!data->retried) {
-				data->retried = TRUE;
-
-				g_debug ("  Got extraction DBus error on '%s'. Retrying file.", uri);
-
-				/* Try again extraction */
-				extractor_get_embedded_metadata (data, uri, data->mime_type);
-			} else {
-				g_warning ("  Got second extraction DBus error on '%s'. "
-				           "Adding only non-embedded metadata to the SparQL, "
-				           "the error was: %s",
-				           uri, error->message);
-
-				sparql_builder_finish (data, NULL, NULL);
-				tracker_miner_fs_file_notify (TRACKER_MINER_FS (data->miner), data->file, NULL);
-				process_file_data_free (data);
+			g_warning ("  Got extraction DBus error on '%s': %s", uri, error->message);
+
+			/* Pause the miner until we've finished failsafe extraction retry */
+			if (priv->failed_extraction_pause_cookie != 0) {
+				priv->failed_extraction_pause_cookie =
+					tracker_miner_pause (TRACKER_MINER (data->miner),
+							     _("Extractor error, performing "
+							       "failsafe embedded metadata extraction"),
+							     NULL);
 			}
 
+			priv->failed_extraction_queue = g_list_prepend (priv->failed_extraction_queue, data);
 			g_free (uri);
 		} else {
 			/* Something bad happened, notify about the error */
@@ -2077,15 +2165,21 @@ extractor_get_embedded_metadata_cb (const gchar *preupdate,
 		}
 
 		g_error_free (error);
-		return;
-	}
-
-	sparql_builder_finish (data, preupdate, sparql);
+	} else {
+		sparql_builder_finish (data, preupdate, sparql);
 
-	/* Notify about the success */
-	tracker_miner_fs_file_notify (TRACKER_MINER_FS (data->miner), data->file, NULL);
+		/* Notify about the success */
+		tracker_miner_fs_file_notify (TRACKER_MINER_FS (data->miner), data->file, NULL);
+		process_file_data_free (data);
+	}
 
-	process_file_data_free (data);
+	/* Wait until there are no pending extraction requests
+	 * before starting failsafe extraction process.
+	 */
+	if (!priv->extraction_queue &&
+	    priv->failed_extraction_queue) {
+		extractor_process_failsafe (miner);
+	}
 }
 
 static SendAndSpliceData *
@@ -2414,6 +2508,7 @@ process_file_cb (GObject      *object,
                  GAsyncResult *result,
                  gpointer      user_data)
 {
+	TrackerMinerFilesPrivate *priv;
 	TrackerSparqlBuilder *sparql;
 	ProcessFileData *data;
 	const gchar *mime_type, *urn, *parent_urn;
@@ -2429,6 +2524,7 @@ process_file_cb (GObject      *object,
 	file = G_FILE (object);
 	sparql = data->sparql;
 	file_info = g_file_query_info_finish (file, result, &error);
+	priv = data->miner->private;
 
 	if (error) {
 		/* Something bad happened, notify about the error */
@@ -2502,6 +2598,8 @@ process_file_cb (GObject      *object,
 	miner_files_add_to_datasource (data->miner, file, sparql);
 
 	if (!is_directory) {
+		priv->extraction_queue = g_list_prepend (priv->extraction_queue, data);
+
 		/* Next step, if NOT a directory, get embedded metadata */
 		extractor_get_embedded_metadata (data, uri, mime_type);
 	} else {