[gegl] buffer: improve linear/cubic samplers box-filter criterion

[Ell <ell src gnome org>]

commit d970b1158e6529e0ba86aec65714119531ecc4c4
Author: Ell <ell_se yahoo com>
Date:   Thu Nov 15 16:25:41 2018 -0500

    buffer: improve linear/cubic samplers box-filter criterion
    
    When the linear and cubic samplers are given a scale matrix,
    perform box filtering whenever the norm of any of the transformed
    basis vectors is greater-than or equal-to 2, that is, roughly
    speaking, whenever the image is scaled down by a factor of 2 or
    more in *either* direction.  This replaces the old criterion,
    checking whether the absolute value of the scale matrix's
    determinent, corresponding to the sampled area, is greater-than or
    equal-to 4, that is, roughly speaking, whether the image is scaled
    down by a factor of 2 or more in *both* directions.
    
    This is especially important for GIMP's warp tool (which, since
    last commit, uses a scale matrix), since it very commonly produces
    uneven scaling.

 gegl/buffer/gegl-sampler.h | 204 +++++++++++++++++++++++----------------------
 1 file changed, 106 insertions(+), 98 deletions(-)
---
diff --git a/gegl/buffer/gegl-sampler.h b/gegl/buffer/gegl-sampler.h
index a3f11bbe2..54d2724fc 100644
--- a/gegl/buffer/gegl-sampler.h
+++ b/gegl/buffer/gegl-sampler.h
@@ -230,121 +230,129 @@ _gegl_sampler_box_get (GeglSampler*    restrict  self,
                        gint                      n_samples)
 {
   gint channels = self->interpolate_components;
-  if (scale && fabs (gegl_buffer_matrix2_determinant (scale)) >= 4.0)
+  if (scale)
     {
-      gfloat  result[channels];
-      gdouble uv_samples_inv;
+      const gdouble u_norm2 = scale->coeff[0][0] * scale->coeff[0][0] +
+                              scale->coeff[1][0] * scale->coeff[1][0];
+      const gdouble v_norm2 = scale->coeff[0][1] * scale->coeff[0][1] +
+                              scale->coeff[1][1] * scale->coeff[1][1];
 
-      for (gint c = 0; c < channels; c++)
-        result[c] = 0.0f;
-
-      if (! self->point_sampler)
+      if (u_norm2 >= 4.0 || v_norm2 >= 4.0)
         {
-          self->point_sampler = gegl_buffer_sampler_new (self->buffer,
-                                                         self->format,
-                                                         point_sampler_type);
-          self->point_sampler_get_fun =
-            gegl_sampler_get_fun (self->point_sampler);
-        }
+          gfloat  result[channels];
+          gdouble uv_samples_inv;
 
-      if (gegl_buffer_matrix2_is_scale (scale))
-        {
-          const gdouble u_norm         = fabs (scale->coeff[0][0]);
-          const gdouble v_norm         = fabs (scale->coeff[1][1]);
-          const gint    u_samples      = ceil (MIN (u_norm, n_samples));
-          const gint    v_samples      = ceil (MIN (v_norm, n_samples));
-          const gdouble u_samples_inv  = 1.0 / u_samples;
-          const gdouble v_samples_inv  = 1.0 / v_samples;
-          const gdouble u_dx           = scale->coeff[0][0] * u_samples_inv;
-          const gdouble v_dy           = scale->coeff[1][1] * v_samples_inv;
-          gdouble       x0             = absolute_x - (scale->coeff[0][0] - u_dx) /
-                                                      2.0;
-          gdouble       y0             = absolute_y - (scale->coeff[1][1] - v_dy) /
-                                                      2.0;
-          gint          u;
-          gint          v;
-
-          uv_samples_inv = u_samples_inv * v_samples_inv;
-
-          for (v = 0; v < v_samples; v++)
+          for (gint c = 0; c < channels; c++)
+            result[c] = 0.0f;
+
+          if (! self->point_sampler)
             {
-              gdouble x = x0;
-              gdouble y = y0;
+              self->point_sampler = gegl_buffer_sampler_new (self->buffer,
+                                                             self->format,
+                                                             point_sampler_type);
+              self->point_sampler_get_fun =
+                gegl_sampler_get_fun (self->point_sampler);
+            }
 
-              for (u = 0; u < u_samples; u++)
+          if (gegl_buffer_matrix2_is_scale (scale))
+            {
+              const gdouble u_norm         = fabs (scale->coeff[0][0]);
+              const gdouble v_norm         = fabs (scale->coeff[1][1]);
+              const gint    u_norm_i       = ceil (u_norm);
+              const gint    v_norm_i       = ceil (v_norm);
+              const gint    u_samples      = CLAMP (u_norm_i, 1, n_samples);
+              const gint    v_samples      = CLAMP (v_norm_i, 1, n_samples);
+              const gdouble u_samples_inv  = 1.0 / u_samples;
+              const gdouble v_samples_inv  = 1.0 / v_samples;
+              const gdouble u_dx           = scale->coeff[0][0] * u_samples_inv;
+              const gdouble v_dy           = scale->coeff[1][1] * v_samples_inv;
+              gdouble       x0             = absolute_x - (scale->coeff[0][0] - u_dx) /
+                                                          2.0;
+              gdouble       y0             = absolute_y - (scale->coeff[1][1] - v_dy) /
+                                                          2.0;
+              gint          u;
+              gint          v;
+
+              uv_samples_inv = u_samples_inv * v_samples_inv;
+
+              for (v = 0; v < v_samples; v++)
                 {
-                  int c;
-                  gfloat input[4];
-                  self->point_sampler_get_fun (self->point_sampler,
-                                               x, y, NULL, input, repeat_mode);
-                  for (c = 0; c < 4; c++)
-                    result[c] += input[c];
-
-                  x += u_dx;
+                  gdouble x = x0;
+                  gdouble y = y0;
+
+                  for (u = 0; u < u_samples; u++)
+                    {
+                      int c;
+                      gfloat input[4];
+                      self->point_sampler_get_fun (self->point_sampler,
+                                                   x, y, NULL, input, repeat_mode);
+                      for (c = 0; c < 4; c++)
+                        result[c] += input[c];
+
+                      x += u_dx;
+                    }
+
+                  y0 += v_dy;
                 }
-
-              y0 += v_dy;
             }
-        }
-      else
-        {
-          const gdouble u_norm         = sqrt (scale->coeff[0][0] * scale->coeff[0][0] +
-                                               scale->coeff[1][0] * scale->coeff[1][0]);
-          const gdouble v_norm         = sqrt (scale->coeff[0][1] * scale->coeff[0][1] +
-                                               scale->coeff[1][1] * scale->coeff[1][1]);
-          const gint    u_samples      = ceil (MIN (u_norm, n_samples));
-          const gint    v_samples      = ceil (MIN (v_norm, n_samples));
-          const gdouble u_samples_inv  = 1.0 / u_samples;
-          const gdouble v_samples_inv  = 1.0 / v_samples;
-          const gdouble u_dx           = scale->coeff[0][0] * u_samples_inv;
-          const gdouble u_dy           = scale->coeff[1][0] * u_samples_inv;
-          const gdouble v_dx           = scale->coeff[0][1] * v_samples_inv;
-          const gdouble v_dy           = scale->coeff[1][1] * v_samples_inv;
-          gdouble       x0             = absolute_x - (scale->coeff[0][0] - u_dx +
-                                                       scale->coeff[0][1] - v_dx) /
-                                                      2.0;
-          gdouble       y0             = absolute_y - (scale->coeff[1][0] - u_dy +
-                                                       scale->coeff[1][1] - v_dy) /
-                                                      2.0;
-          gint          u;
-          gint          v;
-
-          uv_samples_inv = u_samples_inv * v_samples_inv;
-
-          for (v = 0; v < v_samples; v++)
+          else
             {
-              gdouble x = x0;
-              gdouble y = y0;
-
-              for (u = 0; u < u_samples; u++)
+              const gdouble u_norm         = sqrt (u_norm2);
+              const gdouble v_norm         = sqrt (v_norm2);
+              const gint    u_norm_i       = ceil (u_norm);
+              const gint    v_norm_i       = ceil (v_norm);
+              const gint    u_samples      = CLAMP (u_norm_i, 1, n_samples);
+              const gint    v_samples      = CLAMP (v_norm_i, 1, n_samples);
+              const gdouble u_samples_inv  = 1.0 / u_samples;
+              const gdouble v_samples_inv  = 1.0 / v_samples;
+              const gdouble u_dx           = scale->coeff[0][0] * u_samples_inv;
+              const gdouble u_dy           = scale->coeff[1][0] * u_samples_inv;
+              const gdouble v_dx           = scale->coeff[0][1] * v_samples_inv;
+              const gdouble v_dy           = scale->coeff[1][1] * v_samples_inv;
+              gdouble       x0             = absolute_x - (scale->coeff[0][0] - u_dx +
+                                                           scale->coeff[0][1] - v_dx) /
+                                                          2.0;
+              gdouble       y0             = absolute_y - (scale->coeff[1][0] - u_dy +
+                                                           scale->coeff[1][1] - v_dy) /
+                                                          2.0;
+              gint          u;
+              gint          v;
+
+              uv_samples_inv = u_samples_inv * v_samples_inv;
+
+              for (v = 0; v < v_samples; v++)
                 {
-                  int c;
-                  gfloat input[channels];
-                  self->point_sampler_get_fun (self->point_sampler,
-                                               x, y, NULL, input, repeat_mode);
-                  for (c = 0; c < channels; c++)
-                    result[c] += input[c];
-
-                  x += u_dx;
-                  y += u_dy;
+                  gdouble x = x0;
+                  gdouble y = y0;
+
+                  for (u = 0; u < u_samples; u++)
+                    {
+                      int c;
+                      gfloat input[channels];
+                      self->point_sampler_get_fun (self->point_sampler,
+                                                   x, y, NULL, input, repeat_mode);
+                      for (c = 0; c < channels; c++)
+                        result[c] += input[c];
+
+                      x += u_dx;
+                      y += u_dy;
+                    }
+
+                  x0 += v_dx;
+                  y0 += v_dy;
                 }
-
-              x0 += v_dx;
-              y0 += v_dy;
             }
-        }
 
-      for (gint c = 0; c < channels; c++)
-        result[c] *= uv_samples_inv;
+          for (gint c = 0; c < channels; c++)
+            result[c] *= uv_samples_inv;
 
-      babl_process (self->fish, result, output, 1);
+          babl_process (self->fish, result, output, 1);
 
-      return TRUE;
-    }
-  else
-    {
-      return FALSE;
+          return TRUE;
+        }
     }
+
+  return FALSE;
 }
 
 G_END_DECLS