[tracker-miners/sam/file-processed-signal: 13/18] libtracker-miner: Allow a decorator to insert SPARQL on errors

[Sam Thursfield <sthursfield src gnome org>]

commit 1265bc0d307b7b093117c933112f5797d7236c56
Author: Sam Thursfield <sam afuera me uk>
Date:   Sun Mar 22 00:35:03 2020 +0100

    libtracker-miner: Allow a decorator to insert SPARQL on errors
    
    The tracker_decorator_complete_error() function was unused, because
    the extractor wants to insert some SPARQL in case of error to mark
    the file as failed and that function didn't allow it.
    
    We now allow you to return an error and still insert some data into
    the store, which will allow the extract modules to properly report
    errors.

 src/libtracker-miner/tracker-decorator.c | 44 +++++++++++++++++++++++++++-----
 src/libtracker-miner/tracker-decorator.h |  3 ++-
 2 files changed, 39 insertions(+), 8 deletions(-)
---
diff --git a/src/libtracker-miner/tracker-decorator.c b/src/libtracker-miner/tracker-decorator.c
index 099a82b18..7494cd635 100644
--- a/src/libtracker-miner/tracker-decorator.c
+++ b/src/libtracker-miner/tracker-decorator.c
@@ -54,6 +54,7 @@ struct _TrackerDecoratorInfo {
        gchar *mimetype;
        gint id;
        gint ref_count;
+       gchar *failsafe_sparql;
 };
 
 struct _ClassInfo {
@@ -65,6 +66,7 @@ struct _SparqlUpdate {
        gchar *sparql;
        gint id;
        GFile *file;
+       gboolean is_failsafe;
 };
 
 struct _TrackerDecoratorPrivate {
@@ -199,6 +201,7 @@ tracker_decorator_info_unref (TrackerDecoratorInfo *info)
 
        if (info->task)
                g_object_unref (info->task);
+       g_clear_pointer (&info->failsafe_sparql, g_free);
        g_free (info->urn);
        g_free (info->url);
        g_free (info->mimetype);
@@ -356,9 +359,11 @@ decorator_commit_cb (GObject      *object,
 
                        update = &g_array_index (priv->commit_buffer, SparqlUpdate, i);
 
-                       tracker_miner_file_processed (TRACKER_MINER (decorator), update->file, FALSE, 
error->message);
-                       decorator_blacklist_add (decorator, update->id);
-                       item_warn (conn, update->id, update->sparql, error);
+                       if (!update->is_failsafe) {
+                               tracker_miner_file_processed (TRACKER_MINER (decorator), update->file, FALSE, 
error->message);
+                               decorator_blacklist_add (decorator, update->id);
+                               item_warn (conn, update->id, update->sparql, error);
+                       }
                }
        } else {
                /* Notify success */
@@ -367,7 +372,9 @@ decorator_commit_cb (GObject      *object,
 
                        update = &g_array_index (priv->commit_buffer, SparqlUpdate, i);
 
-                       tracker_miner_file_processed (TRACKER_MINER (decorator), update->file, TRUE, "");
+                       if (!update->is_failsafe) {
+                               tracker_miner_file_processed (TRACKER_MINER (decorator), update->file, TRUE, 
"");
+                       }
                }
        }
 
@@ -532,11 +539,13 @@ decorator_task_done (GObject      *object,
        GError *error = NULL;
        GFile *file;
        gchar *sparql;
+       gboolean failed = FALSE;
 
        priv = decorator->priv;
        sparql = g_task_propagate_pointer (G_TASK (result), &error);
 
        if (!sparql) {
+               failed = TRUE;
                file = g_file_new_for_uri (info->url);
 
                /* Blacklist item */
@@ -553,7 +562,11 @@ decorator_task_done (GObject      *object,
                }
 
                g_object_unref (file);
-       } else {
+
+               sparql = g_strdup (info->failsafe_sparql);
+       }
+
+       if (sparql) {
                SparqlUpdate update;
 
                /* Add resulting sparql to buffer and check whether flushing */
@@ -561,6 +574,8 @@ decorator_task_done (GObject      *object,
                update.id = info->id;
                update.file = g_file_new_for_uri (info->url);
 
+               update.is_failsafe = failed;
+
                if (!priv->sparql_buffer)
                        priv->sparql_buffer = sparql_buffer_new ();
 
@@ -1671,17 +1686,32 @@ tracker_decorator_info_complete (TrackerDecoratorInfo *info,
 /**
  * tracker_decorator_info_complete_error:
  * @info: a #TrackerDecoratorInfo
- * @error: (transfer full): An error occurred during SPARQL generation
+ * @error: (transfer full): Error which occurred during processing
+ * @failsafe_sparql: (transfer full): (allow-none): Data to insert to mark the failure
  *
  * Completes the task associated to this #TrackerDecoratorInfo,
  * returning the given @error happened during SPARQL generation.
  *
+ * It's usually a good idea to record failed tasks, to avoid repeatedly
+ * processing the same task. You can do this using the @failsafe_sparql
+ * parameter. You might pass something like this:
+ *
+ * |[<!-- language="SPARQL" -->
+ *     INSERT DATA { GRAPH <urn:graph:my-graph> {
+ *          <file://bad-file> nie:dataSource <urn:data-source:my-decorator>;
+ *              nie:dataSource <urn:data-source:my-decorator-failures>.
+ *     } }
+ * ]|
+ *
  * Since: 2.0
  **/
 void
 tracker_decorator_info_complete_error (TrackerDecoratorInfo *info,
-                                       GError               *error)
+                                       GError               *error,
+                                       gchar                *failsafe_sparql)
 {
+       info->failsafe_sparql = failsafe_sparql;
+
        g_task_return_error (info->task, error);
 }
 
diff --git a/src/libtracker-miner/tracker-decorator.h b/src/libtracker-miner/tracker-decorator.h
index af00d566f..77aa793d0 100644
--- a/src/libtracker-miner/tracker-decorator.h
+++ b/src/libtracker-miner/tracker-decorator.h
@@ -131,7 +131,8 @@ GTask       * tracker_decorator_info_get_task     (TrackerDecoratorInfo *info);
 void          tracker_decorator_info_complete     (TrackerDecoratorInfo *info,
                                                    gchar                *sparql);
 void          tracker_decorator_info_complete_error (TrackerDecoratorInfo *info,
-                                                     GError               *error);
+                                                     GError               *error,
+                                                     gchar                *failsafe_sparql);
 
 G_END_DECLS