[gimp] Bug 316479 - The Perspective Tool creates an empy image...

[Michael Natterer <mitch src gnome org>]

commit 768d06614f203bf555bbda1f6186d4730ae2f8b5
Author: Alexis Wilhelm <alexiswilhelm%40gmail.com>
Date:   Tue Nov 8 17:15:16 2016 +0100

    Bug 316479 - The Perspective Tool creates an empy image...
    
    ...instead of transforming it
    
    Add gimp_matrix3_will_explode() which determines if a transform
    matrix will blow up something in a rectangle to infinity, and use
    the function so set both the GIMP and GEGL code paths to clip the
    transform to the input size.

 app/core/gimpdrawable-transform.c    |    4 +++
 app/gegl/gimp-gegl-apply-operation.c |    9 ++++++-
 app/gegl/gimp-gegl-apply-operation.h |    1 +
 libgimpmath/gimpmath.def             |    1 +
 libgimpmath/gimpmatrix.c             |   36 ++++++++++++++++++++++++++++++++++
 libgimpmath/gimpmatrix.h             |    6 +++++
 6 files changed, 55 insertions(+), 2 deletions(-)
---
diff --git a/app/core/gimpdrawable-transform.c b/app/core/gimpdrawable-transform.c
index a4baa00..034603a 100644
--- a/app/core/gimpdrawable-transform.c
+++ b/app/core/gimpdrawable-transform.c
@@ -116,6 +116,9 @@ gimp_drawable_transform_buffer_affine (GimpDrawable           *drawable,
       ! babl_format_has_alpha (gegl_buffer_get_format (orig_buffer)))
     clip_result = GIMP_TRANSFORM_RESIZE_CLIP;
 
+  if (gimp_matrix3_will_explode (&m, u1, v1, u2, v2))
+    clip_result = GIMP_TRANSFORM_RESIZE_CLIP;
+
   /*  Find the bounding coordinates of target */
   gimp_transform_resize_boundary (&m, clip_result,
                                   u1, v1, u2, v2,
@@ -133,6 +136,7 @@ gimp_drawable_transform_buffer_affine (GimpDrawable           *drawable,
   gimp_gegl_apply_transform (orig_buffer, progress, NULL,
                              new_buffer,
                              interpolation_type,
+                             clip_result,
                              &gegl_matrix);
 
   *new_offset_x = x1;
diff --git a/app/gegl/gimp-gegl-apply-operation.c b/app/gegl/gimp-gegl-apply-operation.c
index a86e2de..41440d7 100644
--- a/app/gegl/gimp-gegl-apply-operation.c
+++ b/app/gegl/gimp-gegl-apply-operation.c
@@ -685,17 +685,22 @@ gimp_gegl_apply_transform (GeglBuffer            *src_buffer,
                            const gchar           *undo_desc,
                            GeglBuffer            *dest_buffer,
                            GimpInterpolationType  interpolation_type,
+                           GimpTransformResize    clip_result,
                            GimpMatrix3           *transform)
 {
   GeglNode *node;
+  gboolean  clip_to_input;
 
   g_return_if_fail (GEGL_IS_BUFFER (src_buffer));
   g_return_if_fail (progress == NULL || GIMP_IS_PROGRESS (progress));
   g_return_if_fail (GEGL_IS_BUFFER (dest_buffer));
 
+  clip_to_input = (clip_result == GIMP_TRANSFORM_RESIZE_CLIP);
+
   node = gegl_node_new_child (NULL,
-                              "operation", "gegl:transform",
-                              "sampler",   interpolation_type,
+                              "operation",     "gegl:transform",
+                              "sampler",       interpolation_type,
+                              "clip-to-input", clip_to_input,
                               NULL);
 
   gimp_gegl_node_set_matrix (node, transform);
diff --git a/app/gegl/gimp-gegl-apply-operation.h b/app/gegl/gimp-gegl-apply-operation.h
index acca14b..9039b5b 100644
--- a/app/gegl/gimp-gegl-apply-operation.h
+++ b/app/gegl/gimp-gegl-apply-operation.h
@@ -153,6 +153,7 @@ void   gimp_gegl_apply_transform       (GeglBuffer             *src_buffer,
                                         const gchar            *undo_desc,
                                         GeglBuffer             *dest_buffer,
                                         GimpInterpolationType   interpolation_type,
+                                        GimpTransformResize     clip_result,
                                         GimpMatrix3            *transform);
 
 
diff --git a/libgimpmath/gimpmath.def b/libgimpmath/gimpmath.def
index 1a48940..039a03d 100644
--- a/libgimpmath/gimpmath.def
+++ b/libgimpmath/gimpmath.def
@@ -16,6 +16,7 @@ EXPORTS
        gimp_matrix3_scale
        gimp_matrix3_transform_point
        gimp_matrix3_translate
+       gimp_matrix3_will_explode
        gimp_matrix3_xshear
        gimp_matrix3_yshear
        gimp_matrix4_to_deg
diff --git a/libgimpmath/gimpmatrix.c b/libgimpmath/gimpmatrix.c
index 48c2210..457057d 100644
--- a/libgimpmath/gimpmatrix.c
+++ b/libgimpmath/gimpmatrix.c
@@ -866,6 +866,42 @@ gimp_matrix3_is_simple (const GimpMatrix3 *matrix)
 }
 
 /**
+ * gimp_matrix3_will_explode:
+ * @m:  The matrix that is to be tested.
+ * @u1: The rectangle's left coordinate.
+ * @v1: The rectangle's top coordinate.
+ * @u2: The rectangle's right coordinate.
+ * @v2: The rectangle's bottom coordinate.
+ *
+ * Checks if the given transformation maps a point of the rectangle to
+ * infinity, or something equally stupid.
+ *
+ * Returns: %TRUE if the transformation will fail, %FALSE otherwise
+ *
+ * Since: 2.10
+ */
+gboolean
+gimp_matrix3_will_explode (const GimpMatrix3 *m,
+                           gdouble            u1,
+                           gdouble            v1,
+                           gdouble            u2,
+                           gdouble            v2)
+{
+  const gdouble a  = m->coeff[2][0];
+  const gdouble b  = m->coeff[2][1];
+  const gdouble c  = m->coeff[2][2];
+  const gdouble d1 = a * u1 + b * v1 + c;
+  const gdouble d2 = a * u1 + b * v2 + c;
+  const gdouble d3 = a * u2 + b * v1 + c;
+  const gdouble d4 = a * u2 + b * v2 + c;
+
+  /* We are safe if all 4 corners of the region are on the same side
+   * of the a.u+b.v+c=0 line, ie. if d1..d4 have the same sign.
+   */
+  return ! (d1 * d2 > 0 && d1 * d3 > 0 && d1 * d4 > 0);
+}
+
+/**
  * gimp_matrix4_to_deg:
  * @matrix:
  * @a:
diff --git a/libgimpmath/gimpmatrix.h b/libgimpmath/gimpmatrix.h
index e435d88..23a7587 100644
--- a/libgimpmath/gimpmatrix.h
+++ b/libgimpmath/gimpmatrix.h
@@ -111,6 +111,12 @@ gboolean      gimp_matrix3_is_diagonal     (const GimpMatrix3 *matrix);
 gboolean      gimp_matrix3_is_affine       (const GimpMatrix3 *matrix);
 gboolean      gimp_matrix3_is_simple       (const GimpMatrix3 *matrix);
 
+gboolean      gimp_matrix3_will_explode    (const GimpMatrix3 *matrix,
+                                            gdouble            u1,
+                                            gdouble            v1,
+                                            gdouble            u2,
+                                            gdouble            v2);
+
 void          gimp_matrix3_transform_point (const GimpMatrix3 *matrix,
                                             gdouble            x,
                                             gdouble            y,