[gtk/wip/smcv/reftest-tolerance: 3/4] reftest_compare_surfaces: Report how much the images differ

[Simon McVittie <smcv src gnome org>]

commit 724908065b17fd0ee894512feab04e085d989a7e
Author: Simon McVittie <smcv debian org>
Date:   Sat Feb 13 18:26:24 2021 +0000

    reftest_compare_surfaces: Report how much the images differ
    
    Some of the reftests don't produce *identical* results on all
    architectures, but do produce results that are visually
    indistinguishable. Report how many pixels differ and by how much, so we
    can get an idea of what's a rounding error and what's a serious problem.
    
    Signed-off-by: Simon McVittie <smcv debian org>

 testsuite/gsk/compare-render.c       | 10 +++++++++-
 testsuite/reftests/gtk-reftest.c     |  9 ++++++++-
 testsuite/reftests/reftest-compare.c | 28 +++++++++++++++++++++++++---
 testsuite/reftests/reftest-compare.h |  5 ++++-
 4 files changed, 46 insertions(+), 6 deletions(-)
---
diff --git a/testsuite/gsk/compare-render.c b/testsuite/gsk/compare-render.c
index 462953ef9f..da6f9e2eb2 100644
--- a/testsuite/gsk/compare-render.c
+++ b/testsuite/gsk/compare-render.c
@@ -232,11 +232,19 @@ main (int argc, char **argv)
     }
   else
     {
+      guint max_diff = 0;
+      guint pixels_changed = 0;
+      guint pixels = 0;
+
       /* Now compare the two */
-      diff_surface = reftest_compare_surfaces (rendered_surface, reference_surface);
+      diff_surface = reftest_compare_surfaces (rendered_surface, reference_surface,
+                                               &max_diff, &pixels_changed, &pixels);
 
       if (diff_surface)
         {
+          g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n",
+                   pixels_changed, pixels, max_diff);
+
           save_image (diff_surface, node_file, ".diff.png");
           cairo_surface_destroy (diff_surface);
           success = FALSE;
diff --git a/testsuite/reftests/gtk-reftest.c b/testsuite/reftests/gtk-reftest.c
index 7e2be7ee1d..6ef17aa12a 100644
--- a/testsuite/reftests/gtk-reftest.c
+++ b/testsuite/reftests/gtk-reftest.c
@@ -296,6 +296,9 @@ test_ui_file (GFile *file)
   char *ui_file, *reference_file;
   cairo_surface_t *ui_image, *reference_image, *diff_image;
   GtkStyleProvider *provider;
+  guint max_diff = 0;
+  guint pixels_changed = 0;
+  guint pixels = 0;
 
   ui_file = g_file_get_path (file);
 
@@ -315,12 +318,16 @@ test_ui_file (GFile *file)
     }
   g_free (reference_file);
 
-  diff_image = reftest_compare_surfaces (ui_image, reference_image);
+  diff_image = reftest_compare_surfaces (ui_image, reference_image,
+                                         &max_diff, &pixels_changed, &pixels);
 
   save_image (ui_image, ui_file, ".out.png");
   save_image (reference_image, ui_file, ".ref.png");
+
   if (diff_image)
     {
+      g_test_message ("%u (out of %u) pixels differ from reference by up to %u levels",
+                      pixels_changed, pixels, max_diff);
       save_image (diff_image, ui_file, ".diff.png");
       g_test_fail ();
     }
diff --git a/testsuite/reftests/reftest-compare.c b/testsuite/reftests/reftest-compare.c
index 34285ace72..274fd52f36 100644
--- a/testsuite/reftests/reftest-compare.c
+++ b/testsuite/reftests/reftest-compare.c
@@ -83,12 +83,16 @@ buffer_diff_core (const guchar *buf_a,
                  const guchar *buf_b,
                   int           stride_b,
                  int           width,
-                 int           height)
+                 int           height,
+                  guint        *max_diff_out,
+                  guint        *pixels_changed_out)
 {
   int x, y;
   guchar *buf_diff = NULL;
   int stride_diff = 0;
   cairo_surface_t *diff = NULL;
+  guint max_diff = 0;
+  guint pixels_changed = 0;
 
   for (y = 0; y < height; y++)
     {
@@ -124,6 +128,10 @@ buffer_diff_core (const guchar *buf_a,
               guint channel_diff;
 
               channel_diff = ABS (value_a - value_b);
+
+              if (channel_diff > max_diff)
+                max_diff = channel_diff;
+
               channel_diff *= 4;  /* emphasize */
               if (channel_diff)
                 channel_diff += 128; /* make sure it's visible */
@@ -132,6 +140,8 @@ buffer_diff_core (const guchar *buf_a,
               diff_pixel |= channel_diff << (channel * 8);
             }
 
+          pixels_changed++;
+
           if ((diff_pixel & 0x00ffffff) == 0)
             {
               /* alpha only difference, convert to luminance */
@@ -143,12 +153,21 @@ buffer_diff_core (const guchar *buf_a,
       }
   }
 
+  if (max_diff_out != NULL)
+    *max_diff_out = max_diff;
+
+  if (pixels_changed_out != NULL)
+    *pixels_changed_out = pixels_changed;
+
   return diff;
 }
 
 cairo_surface_t *
 reftest_compare_surfaces (cairo_surface_t *surface1,
-                          cairo_surface_t *surface2)
+                          cairo_surface_t *surface2,
+                          guint           *max_diff_out,
+                          guint           *pixels_changed_out,
+                          guint           *pixels_out)
 {
   int w1, h1, w2, h2, w, h;
   cairo_surface_t *coerced1, *coerced2, *diff;
@@ -164,11 +183,14 @@ reftest_compare_surfaces (cairo_surface_t *surface1,
                            cairo_image_surface_get_stride (coerced1),
                            cairo_image_surface_get_data (coerced2),
                            cairo_image_surface_get_stride (coerced2),
-                           w, h);
+                           w, h, max_diff_out, pixels_changed_out);
 
   cairo_surface_destroy (coerced1);
   cairo_surface_destroy (coerced2);
 
+  if (pixels_out != NULL)
+    *pixels_out = w * h;
+
   return diff;
 }
 
diff --git a/testsuite/reftests/reftest-compare.h b/testsuite/reftests/reftest-compare.h
index 551b1c5a92..c6e001c505 100644
--- a/testsuite/reftests/reftest-compare.h
+++ b/testsuite/reftests/reftest-compare.h
@@ -24,7 +24,10 @@ G_BEGIN_DECLS
 
 G_MODULE_EXPORT
 cairo_surface_t *       reftest_compare_surfaces        (cairo_surface_t        *surface1,
-                                                         cairo_surface_t        *surface2);
+                                                         cairo_surface_t        *surface2,
+                                                         guint                  *max_diff_out,
+                                                         guint                  *pixels_changed_out,
+                                                         guint                  *pixels_out);
 
 G_END_DECLS