[latexila/wip/build-tools-revamp] PostProcessor and PostProcessorAllOutput (not finished)

[Sébastien Wilmet <swilmet src gnome org>]

commit 867bc66977f8bc6408ee6c56320f25333be70f34
Author: Sébastien Wilmet <swilmet gnome org>
Date:   Fri May 23 12:25:30 2014 +0200

    PostProcessor and PostProcessorAllOutput (not finished)
    
    The post processors work on a GInputStream obtained by the GSubprocess.
    The GInputStream is read asynchronously.
    
    Before, the Vala code worked on the full command output, as one big
    string. This big string was split into lines, and then processed.
    
    One exception: the latexmk post-processor worked on the full output with
    a big regex, without splitting the output into lines (splitting the
    lines was delegated to the all-output and latex post-processors, called
    by the latexmk post-processor after extracting the relevant contents
    from the big string).
    
    Next steps:
    - write the latex post-processor (the Vala code works already line by
    line, so working on the stream will use the same algorithm). One
    difference though: the latex post-processor should read the log file,
    not the command output.
    
    - write the latexmk post-processor. Working on the stream (line by line)
    will be more complicated, the big regex cannot be reused (unless the
    command output is constructed to obtain one big string, but I don't want
    this solution, it uses more memory and is thus most probably slower).
    
    - write unit tests!

 src/liblatexila/latexila-build-job.c               |   37 ++++++-
 .../latexila-post-processor-all-output.c           |   41 +++-----
 src/liblatexila/latexila-post-processor.c          |  114 +++++++++++++++++---
 src/liblatexila/latexila-post-processor.h          |   21 ++--
 4 files changed, 161 insertions(+), 52 deletions(-)
---
diff --git a/src/liblatexila/latexila-build-job.c b/src/liblatexila/latexila-build-job.c
index bd35801..01b3e02 100644
--- a/src/liblatexila/latexila-build-job.c
+++ b/src/liblatexila/latexila-build-job.c
@@ -29,6 +29,7 @@
 #include <string.h>
 #include <glib/gi18n.h>
 #include "latexila-build-view.h"
+#include "latexila-post-processor-all-output.h"
 #include "latexila-utils.h"
 #include "latexila-enum-types.h"
 
@@ -43,6 +44,7 @@ struct _LatexilaBuildJobPrivate
   LatexilaBuildView *build_view;
   GtkTreeIter job_title;
   GNode *build_messages;
+  LatexilaPostProcessor *post_processor;
 };
 
 enum
@@ -408,6 +410,16 @@ display_command_line (LatexilaBuildJob *build_job)
 }
 
 static void
+post_processor_cb (LatexilaPostProcessor *pp,
+                   GAsyncResult          *result,
+                   LatexilaBuildJob      *build_job)
+{
+  latexila_post_processor_process_finish (pp, result);
+
+  /* TODO get messages */
+}
+
+static void
 subprocess_wait_cb (GSubprocess      *subprocess,
                     GAsyncResult     *result,
                     LatexilaBuildJob *build_job)
@@ -461,9 +473,16 @@ launch_subprocess (LatexilaBuildJob *build_job)
   gchar **argv;
   GError *error = NULL;
 
-  /* No output for the moment */
-  launcher = g_subprocess_launcher_new (G_SUBPROCESS_FLAGS_STDOUT_SILENCE |
-                                        G_SUBPROCESS_FLAGS_STDERR_SILENCE);
+  if (build_job->priv->post_processor_type == LATEXILA_POST_PROCESSOR_TYPE_NO_OUTPUT)
+    {
+      launcher = g_subprocess_launcher_new (G_SUBPROCESS_FLAGS_STDOUT_SILENCE |
+                                            G_SUBPROCESS_FLAGS_STDERR_SILENCE);
+    }
+  else
+    {
+      launcher = g_subprocess_launcher_new (G_SUBPROCESS_FLAGS_STDOUT_PIPE |
+                                            G_SUBPROCESS_FLAGS_STDERR_MERGE);
+    }
 
   parent_dir = g_file_get_parent (build_job->priv->file);
   working_directory = g_file_get_path (parent_dir);
@@ -485,6 +504,18 @@ launch_subprocess (LatexilaBuildJob *build_job)
       return;
     }
 
+  if (build_job->priv->post_processor_type != LATEXILA_POST_PROCESSOR_TYPE_NO_OUTPUT)
+    {
+      g_clear_object (&build_job->priv->post_processor);
+      build_job->priv->post_processor = latexila_post_processor_all_output_new ();
+
+      latexila_post_processor_process_async (build_job->priv->post_processor,
+                                             g_subprocess_get_stdout_pipe (subprocess),
+                                             g_task_get_cancellable (build_job->priv->task),
+                                             (GAsyncReadyCallback) post_processor_cb,
+                                             build_job);
+    }
+
   g_subprocess_wait_async (subprocess,
                            g_task_get_cancellable (build_job->priv->task),
                            (GAsyncReadyCallback) subprocess_wait_cb,
diff --git a/src/liblatexila/latexila-post-processor-all-output.c 
b/src/liblatexila/latexila-post-processor-all-output.c
index f4bf671..c58ceb6 100644
--- a/src/liblatexila/latexila-post-processor-all-output.c
+++ b/src/liblatexila/latexila-post-processor-all-output.c
@@ -21,25 +21,18 @@
 
 struct _LatexilaPostProcessorAllOutputPrivate
 {
-  GSList *messages;
+  gint something;
 };
 
 G_DEFINE_TYPE_WITH_PRIVATE (LatexilaPostProcessorAllOutput, latexila_post_processor_all_output, 
LATEXILA_TYPE_POST_PROCESSOR)
 
 static void
-latexila_post_processor_all_output_process (LatexilaPostProcessor *post_processor,
-                                            const gchar           *output)
+latexila_post_processor_all_output_process_lines (LatexilaPostProcessor  *post_processor,
+                                                  gchar                 **lines)
 {
+  /*
   LatexilaPostProcessorAllOutput *pp = LATEXILA_POST_PROCESSOR_ALL_OUTPUT (post_processor);
-  gchar **lines;
-  gchar **l;
-
-  lines = g_strsplit (output, "\n", 0);
-
-  for (l = lines; l != NULL && *l != NULL; l++)
-    {
-      pp->priv->messages = g_slist_prepend (pp->priv->messages, *l);
-    }
+  */
 
   /* Generally a single \n is present at the end of the output, so an empty line
    * is added to the list. But we don't want to display it.
@@ -61,30 +54,21 @@ latexila_post_processor_all_output_process (LatexilaPostProcessor *post_processo
         }
     }
 #endif
-
-  pp->priv->messages = g_slist_reverse (pp->priv->messages);
-
-  /* Do not use g_strfreev() because the strings are reused in the list. */
-  g_free (lines);
 }
 
-static GSList *
+static const GNode *
 latexila_post_processor_all_output_get_messages (LatexilaPostProcessor *post_processor)
 {
+  /*
   LatexilaPostProcessorAllOutput *pp = LATEXILA_POST_PROCESSOR_ALL_OUTPUT (post_processor);
+  */
 
-  return pp->priv->messages;
+  return NULL;
 }
 
 static void
 latexila_post_processor_all_output_finalize (GObject *object)
 {
-  LatexilaPostProcessorAllOutputPrivate *priv;
-
-  priv = latexila_post_processor_all_output_get_instance_private (LATEXILA_POST_PROCESSOR_ALL_OUTPUT 
(object));
-
-  g_slist_free_full (priv->messages, g_free);
-  priv->messages = NULL;
 
   G_OBJECT_CLASS (latexila_post_processor_all_output_parent_class)->finalize (object);
 }
@@ -97,7 +81,7 @@ latexila_post_processor_all_output_class_init (LatexilaPostProcessorAllOutputCla
 
   object_class->finalize = latexila_post_processor_all_output_finalize;
 
-  post_processor_class->process = latexila_post_processor_all_output_process;
+  post_processor_class->process_lines = latexila_post_processor_all_output_process_lines;
   post_processor_class->get_messages = latexila_post_processor_all_output_get_messages;
 }
 
@@ -107,6 +91,11 @@ latexila_post_processor_all_output_init (LatexilaPostProcessorAllOutput *pp)
   pp->priv = latexila_post_processor_all_output_get_instance_private (pp);
 }
 
+/**
+ * latexila_post_processor_all_output_new:
+ *
+ * Returns: a new #LatexilaPostProcessorAllOutput object.
+ */
 LatexilaPostProcessor *
 latexila_post_processor_all_output_new (void)
 {
diff --git a/src/liblatexila/latexila-post-processor.c b/src/liblatexila/latexila-post-processor.c
index 19e2b40..777b1e7 100644
--- a/src/liblatexila/latexila-post-processor.c
+++ b/src/liblatexila/latexila-post-processor.c
@@ -21,6 +21,9 @@
 
 struct _LatexilaPostProcessorPrivate
 {
+  GTask *task;
+  GInputStream *stream;
+
   guint has_details : 1;
   guint show_details : 1;
 };
@@ -151,17 +154,26 @@ latexila_post_processor_set_property (GObject      *object,
 }
 
 static void
-latexila_post_processor_process_default (LatexilaPostProcessor *pp,
-                                         const gchar           *output)
+latexila_post_processor_dispose (GObject *object)
 {
-  g_return_if_fail (LATEXILA_IS_POST_PROCESSOR (pp));
+  LatexilaPostProcessor *pp = LATEXILA_POST_PROCESSOR (object);
+
+  g_clear_object (&pp->priv->task);
+  g_clear_object (&pp->priv->stream);
+
+  G_OBJECT_CLASS (latexila_post_processor_parent_class)->dispose (object);
 }
 
-static GSList *
-latexila_post_processor_get_messages_default (LatexilaPostProcessor *pp)
+static void
+latexila_post_processor_process_lines_default (LatexilaPostProcessor  *pp,
+                                               gchar                 **lines)
 {
-  g_return_val_if_fail (LATEXILA_IS_POST_PROCESSOR (pp), NULL);
+  g_strfreev (lines);
+}
 
+static const GNode *
+latexila_post_processor_get_messages_default (LatexilaPostProcessor *pp)
+{
   return NULL;
 }
 
@@ -172,8 +184,9 @@ latexila_post_processor_class_init (LatexilaPostProcessorClass *klass)
 
   object_class->get_property = latexila_post_processor_get_property;
   object_class->set_property = latexila_post_processor_set_property;
+  object_class->dispose = latexila_post_processor_dispose;
 
-  klass->process = latexila_post_processor_process_default;
+  klass->process_lines = latexila_post_processor_process_lines_default;
   klass->get_messages = latexila_post_processor_get_messages_default;
 
   g_object_class_install_property (object_class,
@@ -203,22 +216,95 @@ latexila_post_processor_init (LatexilaPostProcessor *pp)
   pp->priv = latexila_post_processor_get_instance_private (pp);
 }
 
-LatexilaPostProcessor *
-latexila_post_processor_new (void)
+static void
+read_stream (LatexilaPostProcessor *pp)
 {
-  return g_object_new (LATEXILA_TYPE_POST_PROCESSOR, NULL);
+  g_task_return_boolean (pp->priv->task, TRUE);
 }
 
+/**
+ * latexila_post_processor_process_async:
+ * @pp: a post-processor.
+ * @stream: the input stream to process.
+ * @cancellable: a #GCancellable.
+ * @callback: the callback to call when the operation is finished.
+ * @user_data: the data to pass to the callback.
+ *
+ * Asynchronously process an input stream. The input stream can for example come
+ * from the output of a command launched with #GSubprocess, or it can be the
+ * input stream of a file (e.g. the LaTeX log file), etc.
+ *
+ * @callback will be called when the operation is finished. You can then call
+ * latexila_post_processor_process_finish().
+ */
 void
-latexila_post_processor_process (LatexilaPostProcessor *pp,
-                                 const gchar           *output)
+latexila_post_processor_process_async (LatexilaPostProcessor *pp,
+                                       GInputStream          *stream,
+                                       GCancellable          *cancellable,
+                                       GAsyncReadyCallback    callback,
+                                       gpointer               user_data)
 {
   g_return_if_fail (LATEXILA_IS_POST_PROCESSOR (pp));
+  g_return_if_fail (G_IS_INPUT_STREAM (stream));
+  g_return_if_fail (G_IS_CANCELLABLE (cancellable));
+  g_return_if_fail (pp->priv->task == NULL);
+
+  pp->priv->task = g_task_new (pp, cancellable, callback, user_data);
+  pp->priv->stream = g_object_ref (stream);
 
-  LATEXILA_POST_PROCESSOR_GET_CLASS (pp)->process (pp, output);
+  read_stream (pp);
 }
 
-GSList *
+/**
+ * latexila_post_processor_process_finish:
+ * @pp: a post-processor.
+ * @result: a #GAsyncResult.
+ *
+ * Finishes an operation started with latexila_post_processor_process_async().
+ * After calling this function, you can get the filtered messages with
+ * latexila_post_processor_get_messages().
+ */
+void
+latexila_post_processor_process_finish (LatexilaPostProcessor *pp,
+                                        GAsyncResult          *result)
+{
+  g_return_if_fail (g_task_is_valid (result, pp));
+
+  g_task_propagate_boolean (G_TASK (result), NULL);
+
+  g_clear_object (&pp->priv->task);
+  g_clear_object (&pp->priv->stream);
+}
+
+/**
+ * latexila_post_processor_get_messages:
+ * @pp: a post-processor.
+ *
+ * Gets the filtered messages. Call this function only after calling
+ * latexila_post_processor_process_finish().
+ *
+ * Another solution would have been to pass the #LatexilaBuildView to the
+ * post-processor, so the filtered messages can directly be outputed to the
+ * build view as they come. But some post-processors don't know what to output
+ * directly. The latexmk post-processor can have a simplified output with only
+ * the LaTeX messages, in which case the detailed messages are also available,
+ * but this can be known only when all the stream has been processed.
+ *
+ * Obviously if the build view is passed to the post-processor, the latexmk
+ * post-processor can output its messages only at the end. But another reason to
+ * not pass the build view is for the unit tests. It is easier for the unit
+ * tests to check the returned #GNode than analyzing a #GtkTreeView. Of course
+ * it would be possible to keep also the messages in a #GNode and have this
+ * function only for the unit tests, but it takes more memory (unless a custom
+ * #GtkTreeModel is implemented), or another function is needed to configure
+ * whether a #GNode is kept in memory or not... It becomes a little too
+ * complicated, and doesn't really worth the effort as most users use latexmk.
+ *
+ * The current solution is "good enough".
+ *
+ * Returns: the tree of filtered messages.
+ */
+const GNode *
 latexila_post_processor_get_messages (LatexilaPostProcessor *pp)
 {
   g_return_val_if_fail (LATEXILA_IS_POST_PROCESSOR (pp), NULL);
diff --git a/src/liblatexila/latexila-post-processor.h b/src/liblatexila/latexila-post-processor.h
index 51e5bab..ee86baf 100644
--- a/src/liblatexila/latexila-post-processor.h
+++ b/src/liblatexila/latexila-post-processor.h
@@ -20,8 +20,7 @@
 #ifndef __LATEXILA_POST_PROCESSOR_H__
 #define __LATEXILA_POST_PROCESSOR_H__
 
-#include <glib.h>
-#include <glib-object.h>
+#include <gio/gio.h>
 #include "latexila-types.h"
 
 G_BEGIN_DECLS
@@ -66,10 +65,10 @@ struct _LatexilaPostProcessorClass
 {
   GObjectClass parent_class;
 
-  void (* process) (LatexilaPostProcessor *post_processor,
-                    const gchar           *output);
+  void (* process_lines) (LatexilaPostProcessor  *pp,
+                          gchar                 **lines);
 
-  GSList * (* get_messages) (LatexilaPostProcessor *post_processor);
+  const GNode * (* get_messages) (LatexilaPostProcessor *pp);
 };
 
 GType                   latexila_post_processor_get_type              (void) G_GNUC_CONST;
@@ -79,12 +78,16 @@ gboolean                latexila_post_processor_get_type_from_name    (const gch
 
 const gchar *           latexila_post_processor_get_name_from_type    (LatexilaPostProcessorType type);
 
-LatexilaPostProcessor * latexila_post_processor_new                   (void);
+void                    latexila_post_processor_process_async         (LatexilaPostProcessor *pp,
+                                                                       GInputStream          *stream,
+                                                                       GCancellable          *cancellable,
+                                                                       GAsyncReadyCallback    callback,
+                                                                       gpointer               user_data);
 
-void                    latexila_post_processor_process               (LatexilaPostProcessor *post_processor,
-                                                                       const gchar           *output);
+void                    latexila_post_processor_process_finish        (LatexilaPostProcessor *pp,
+                                                                       GAsyncResult          *result);
 
-GSList *                latexila_post_processor_get_messages          (LatexilaPostProcessor 
*post_processor);
+const GNode *           latexila_post_processor_get_messages          (LatexilaPostProcessor *pp);
 
 G_END_DECLS