[gtk/wip/smcv/reftest-tolerance: 4/4] reftests: Allow ignoring individual tests or accepting minor differences

[Simon McVittie <smcv src gnome org>]

commit 6b73dcfff6fdc07411a1d7a422f874bd1043a6cc
Author: Simon McVittie <smcv debian org>
Date:   Sat Feb 13 16:19:10 2021 +0000

    reftests: Allow ignoring individual tests or accepting minor differences
    
    Based on an earlier patch by Michael Biebl, as used in Debian's GTK 3
    packaging, with additional inspiration from librsvg's reftests.
    
    Signed-off-by: Simon McVittie <smcv debian org>

 testsuite/gsk/compare-render.c     | 29 ++++++++++++++++++++++++++++-
 testsuite/reftests/gtk-reftest.c   | 29 ++++++++++++++++++++++++++++-
 testsuite/reftests/image-compare.c |  2 +-
 3 files changed, 57 insertions(+), 3 deletions(-)
---
diff --git a/testsuite/gsk/compare-render.c b/testsuite/gsk/compare-render.c
index da6f9e2eb2..c9f35edf94 100644
--- a/testsuite/gsk/compare-render.c
+++ b/testsuite/gsk/compare-render.c
@@ -98,6 +98,12 @@ get_output_file (const char *file,
   return result;
 }
 
+static char *
+get_test_keyfile (const char *node_file)
+{
+  return file_replace_extension (node_file, ".node", ".keyfile");
+}
+
 static void
 save_image (cairo_surface_t *surface,
             const char      *test_name,
@@ -242,12 +248,33 @@ main (int argc, char **argv)
 
       if (diff_surface)
         {
+          char *keyfile_path = get_test_keyfile (node_file);
+          GKeyFile *keyfile = g_key_file_new ();
+          guint64 diff_allowed;
+          guint64 pixels_allowed;
+          gboolean may_fail;
+
+          /* deliberately ignoring error: treat as empty */
+          g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, NULL);
+          diff_allowed = g_key_file_get_uint64 (keyfile, "reftest", "max-diff-level", NULL);
+          pixels_allowed = g_key_file_get_uint64 (keyfile, "reftest", "max-diff-pixels", NULL);
+          may_fail = g_key_file_get_boolean (keyfile, "reftest", "may-fail", NULL);
+
           g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n",
                    pixels_changed, pixels, max_diff);
 
           save_image (diff_surface, node_file, ".diff.png");
           cairo_surface_destroy (diff_surface);
-          success = FALSE;
+
+          if (max_diff <= diff_allowed && pixels_changed <= pixels_allowed)
+            g_print ("that's close enough\n");
+          else if (may_fail)
+            g_print ("ignoring failure due to may-fail flag\n");
+          else
+            success = FALSE;
+
+          g_key_file_unref (keyfile);
+          g_free (keyfile_path);
         }
     }
 
diff --git a/testsuite/reftests/gtk-reftest.c b/testsuite/reftests/gtk-reftest.c
index 6ef17aa12a..526d834bab 100644
--- a/testsuite/reftests/gtk-reftest.c
+++ b/testsuite/reftests/gtk-reftest.c
@@ -290,6 +290,12 @@ save_image (cairo_surface_t *surface,
   g_free (filename);
 }
 
+static char *
+get_test_keyfile (const char *ui_file)
+{
+  return get_test_file (ui_file, ".keyfile", FALSE);
+}
+
 static void
 test_ui_file (GFile *file)
 {
@@ -326,10 +332,31 @@ test_ui_file (GFile *file)
 
   if (diff_image)
     {
+      char *keyfile_path = get_test_keyfile (ui_file);
+      GKeyFile *keyfile = g_key_file_new ();
+      guint64 diff_allowed;
+      guint64 pixels_allowed;
+      gboolean may_fail;
+
+      /* deliberately ignoring error: treat as empty */
+      g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, NULL);
+      diff_allowed = g_key_file_get_uint64 (keyfile, "reftest", "max-diff-level", NULL);
+      pixels_allowed = g_key_file_get_uint64 (keyfile, "reftest", "max-diff-pixels", NULL);
+      may_fail = g_key_file_get_boolean (keyfile, "reftest", "may-fail", NULL);
+
       g_test_message ("%u (out of %u) pixels differ from reference by up to %u levels",
                       pixels_changed, pixels, max_diff);
       save_image (diff_image, ui_file, ".diff.png");
-      g_test_fail ();
+
+      if (max_diff <= diff_allowed && pixels_changed <= pixels_allowed)
+        g_test_incomplete ("that's close enough");
+      else if (may_fail)
+        g_test_incomplete ("ignoring failure due to may-fail flag");
+      else
+        g_test_fail ();
+
+      g_key_file_unref (keyfile);
+      g_free (keyfile_path);
     }
 
   remove_extra_css (provider);
diff --git a/testsuite/reftests/image-compare.c b/testsuite/reftests/image-compare.c
index 07bb79e4e0..714f4fcadc 100644
--- a/testsuite/reftests/image-compare.c
+++ b/testsuite/reftests/image-compare.c
@@ -41,7 +41,7 @@ main (int argc, char **argv)
   image1 = cairo_image_surface_create_from_png (argv[1]);
   image2 = cairo_image_surface_create_from_png (argv[2]);
 
-  diff = reftest_compare_surfaces (image1, image2);
+  diff = reftest_compare_surfaces (image1, image2, NULL, NULL, NULL);
 
   if (opt_filename && diff)
     cairo_surface_write_to_png (diff, opt_filename);