[gegl] tests: add optional operations test - convert gegl-tester output to TAP format - add optional test

[Øyvind "pippin" Kolås <ok src gnome org>]

commit f1b2e3ff29d7eef0c92358ab24dd28a7f4394796
Author: John <jtm home gmail com>
Date:   Fri Feb 26 16:25:36 2021 +0000

    tests: add optional operations test
     - convert gegl-tester output to TAP format
     - add optional test using gegl-tester

 meson_options.txt            |  6 ++++
 tests/meson.build            |  9 ++++--
 tests/operations/meson.build | 72 ++++++++++++++++++++++++++++++++++++++++++++
 tools/gegl-tester.c          | 38 ++++++++++++++---------
 4 files changed, 107 insertions(+), 18 deletions(-)
---
diff --git a/meson_options.txt b/meson_options.txt
index 7e8c18e72..e8cb37d61 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -33,6 +33,12 @@ option('parallel-tests',
   description: 'run tests in parallel or sequentially'
 )
 
+option('operation-test',
+  type: 'boolean',
+  value: 'false',
+  description: 'include gegl-tester in test suite'
+)
+
 # optional dependencies
 option('gdk-pixbuf',    type: 'feature', value: 'auto')
 option('gexiv2',        type: 'feature', value: 'auto')
diff --git a/tests/meson.build b/tests/meson.build
index c9712c935..52c03490e 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -36,14 +36,17 @@ gegl_test_parallel = get_option('parallel-tests')
 
 # Tests
 subdir('simple')
+subdir('buffer')
+subdir('xml')
 subdir('mipmap')
 subdir('opencl')
+subdir('compositions')
 if pygobject3.found()
   subdir('python')
 endif
-subdir('xml')
+if get_option('operation-test')
+  subdir('operations')
+endif
 if avlibs_found and gexiv2.found()
   subdir('ff-load-save')
 endif
-subdir('compositions')
-subdir('buffer')
diff --git a/tests/operations/meson.build b/tests/operations/meson.build
new file mode 100644
index 000000000..397718233
--- /dev/null
+++ b/tests/operations/meson.build
@@ -0,0 +1,72 @@
+
+# Source operations
+exclude = [
+  'gegl:buffer-source',
+  'gegl:exr-load',
+  'gegl:ff-load',
+  'gegl:gegl-buffer-load',
+  'gegl:gif-load',
+  'gegl:icc-load',
+  'gegl:jp2-load',
+  'gegl:jpg-load',
+  'gegl:load',
+  'gegl:magick-load',
+  'gegl:open-buffer',
+  'gegl:pdf-load',
+  'gegl:pixbuf',
+  'gegl:png-load',
+  'gegl:ppm-load',
+  'gegl:rawbayer-load',
+  'gegl:raw-load',
+  'gegl:rgbe-load',
+  'gegl:svg-load',
+  'gegl:tiff-load',
+  'gegl:v4l',
+  'gegl:v4l2',
+  'gegl:webp-load',
+]
+# Programming operations
+exclude += [
+  'gegl:cache',
+  'gegl:cast-format',
+  'gegl:clone',
+  'gegl:convert-format',
+  'gegl:copy-buffer',
+  'gegl:image-compare',
+  'gegl:line-profile',
+  'gegl:nop',
+]
+# Other operations
+exclude += [
+  'gegl:exp-combine',
+  'gegl:gegl',
+  'gegl:introspect',
+  'gegl:lcms-from-profile',
+  'gegl:mblur',
+]
+
+# Crashing - workshop
+exclude += [
+  'gegl:alpha-inpaint',
+  'gegl:integral-image',
+]
+
+
+gegl_test_ops_env = gegl_test_env
+gegl_test_ops_env.set('BABL_TOLERANCE', '0.0')
+
+test(
+  'ops_stable',
+  gegl_tester,
+  args: [
+    '--data-directory', project_source_root / 'docs' / 'images',
+    '--output-directory', meson.current_build_dir(),
+    '--all',
+    '--exclusion-pattern', '|'.join(exclude),
+  ],
+  env: gegl_test_ops_env,
+  protocol: 'tap',
+  suite: 'operations',
+  is_parallel: gegl_test_parallel,
+  timeout: 300,
+)
diff --git a/tools/gegl-tester.c b/tools/gegl-tester.c
index a67339127..ad322f338 100644
--- a/tools/gegl-tester.c
+++ b/tools/gegl-tester.c
@@ -32,6 +32,7 @@ static gchar    *exclusion_pattern = "a^"; /* doesn't match anything by default
 static gboolean *output_all      = FALSE;
 static gint      failed          = 0;
 static GString  *failed_ops      = NULL;
+static gint      test_num        = 0;
 
 static const GOptionEntry options[] =
 {
@@ -208,8 +209,9 @@ process_operations (GType type)
         {
           GeglNode *composition;
 
-          g_printf ("%s: ", name); /* more information will follow
-                                        if we're testing */
+          test_num ++;
+          /* more information will follow if we're testing */
+          // g_printf ("# %s: %3i\n", name, test_num);
 
           if (xml)
             composition = gegl_node_new_from_xml (xml, data_dir);
@@ -218,7 +220,7 @@ process_operations (GType type)
 
           if (!composition)
             {
-              g_printf ("FAIL\n  Composition graph is flawed\n");
+              g_printf ("not ok %3i - Composition graph is flawed\n", test_num);
             }
           else if (output_all)
             {
@@ -242,7 +244,8 @@ process_operations (GType type)
                !(g_type_is_a (operations[i], GEGL_TYPE_OPERATION_SINK) ||
                  g_type_is_a (operations[i], GEGL_TYPE_OPERATION_TEMPORAL)))
         {
-          g_printf ("%s ", name);
+          test_num ++;
+          // g_printf ("# %s: %3i\n", name, test_num);
           standard_output (name);
         }
 
@@ -251,16 +254,18 @@ process_operations (GType type)
         gchar *output_path = operation_to_path (name, FALSE);
         gchar *gothash = compute_hash_for_path (output_path);
         if (g_str_equal (hash, gothash))
-          g_printf (" OK\n");
+          g_printf ("ok     %3i - %s\n", test_num, name);
         else if (hashB && g_str_equal (hashB, gothash))
-          g_printf (" OK (hash b)\n");
+          g_printf ("ok     %3i - %s (ref b)\n", test_num, name);
         else if (hashC && g_str_equal (hashC, gothash))
-          g_printf (" OK (hash c)\n");
+          g_printf ("ok     %3i - %s (ref c)\n", test_num, name);
+        else if (g_str_equal (hash, "unstable"))
+          g_printf ("not ok %3i - %s (unstable) # TODO reference is not reproducible\n", test_num, name);
         else
         {
-          g_printf (" FAIL %s != %s\n", gothash, hash);
+          g_printf ("not ok %3i - %s %s != %s\n", test_num, name, gothash, hash);
           failed ++;
-          g_string_append_printf (failed_ops, "%s %s != %s\n", name, gothash, hash);
+          g_string_append_printf (failed_ops, "#  %s %s != %s\n", name, gothash, hash);
         }
         g_free (gothash);
         g_free (output_path);
@@ -272,9 +277,9 @@ process_operations (GType type)
         gchar *gothash = compute_hash_for_path (output_path);
         if (g_str_equal (gothash, "9bbe341d798da4f7b181c903e6f442fd") ||
             g_str_equal (gothash, "ffb9e86edb25bc92e8d4e68f59bbb04b"))
-          g_printf (" reference is noop?\n");
+          g_printf ("not ok %3i - %s (noop) # TODO hash is noop\n", test_num, name);
         else
-          g_printf (" hash = %s\n", gothash);
+          g_printf ("not ok %3i - %s (no ref) # TODO hash = %s\n", test_num, name, gothash);
         g_free (gothash);
         g_free (output_path);
       }
@@ -312,12 +317,12 @@ main (gint    argc,
     }
   else if (output_all && !(data_dir && output_dir))
     {
-      g_printf ("Data and output directories must be specified\n");
+      g_printf ("Bail out! Data and output directories must be specified\n");
       return -1;
     }
   else if (!(output_all || (data_dir && output_dir)))
     {
-      g_printf ("Data and output directories must be specified\n");
+      g_printf ("Bail out! Data and output directories must be specified\n");
       return -1;
     }
   else
@@ -334,11 +339,14 @@ main (gint    argc,
 
   gegl_exit ();
 
-  g_printf ("\n\nwith%s opencl acceleration\n", gegl_cl_is_accelerated()?"":"out");
+    // TAP plan
+  g_printf("1..%i\n", test_num);
+
+  g_printf ("\n\n# with%s opencl acceleration\n", gegl_cl_is_accelerated()?"":"out");
   if (failed != 0)
   {
 
-    g_print ("Maybe see bug https://bugzilla.gnome.org/show_bug.cgi?id=780226\n%i operations producing 
unexpected hashes:\n%s\n", failed, failed_ops->str);
+    g_print ("# Maybe see bug https://bugzilla.gnome.org/show_bug.cgi?id=780226\n# %i operations producing 
unexpected hashes:\n%s\n", failed, failed_ops->str);
     // return -1;
   }
   g_string_free (failed_ops, TRUE);