[gimp] Issue #2090 - Crash when using transform tools

[Ell <ell src gnome org>]

commit be7906c05cc0437428d157cdd2db2cf208670d7c
Author: Ell <ell_se yahoo com>
Date:   Sat Mar 30 11:11:10 2019 -0400

    Issue #2090 - Crash when using transform tools
    
    In GimpTransformGridTool, avoid producing non-finite coordinate
    and angle values.  In particular, this fixes a crash in
    gimp_transform_grid_tool_get_cursor() as a result of a NaN angle
    value being converted to a negative integer, hitting an assert.

 app/display/gimptooltransformgrid.c | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)
---
diff --git a/app/display/gimptooltransformgrid.c b/app/display/gimptooltransformgrid.c
index df162b4a29..eb649222c3 100644
--- a/app/display/gimptooltransformgrid.c
+++ b/app/display/gimptooltransformgrid.c
@@ -887,11 +887,11 @@ calcangle (GimpVector2 a,
 
   length = norm (a) * norm (b);
 
-  angle = acos (dotprod (a, b)/length);
+  angle = acos (SAFE_CLAMP (dotprod (a, b) / length, -1.0, +1.0));
   angle2 = b.y;
   b.y = -b.x;
   b.x = angle2;
-  angle2 = acos (dotprod (a, b)/length);
+  angle2 = acos (SAFE_CLAMP (dotprod (a, b) / length, -1.0, +1.0));
 
   return ((angle2 > G_PI / 2.0) ? angle : 2.0 * G_PI - angle);
 }
@@ -1750,12 +1750,6 @@ gimp_tool_transform_grid_motion (GimpToolWidget   *widget,
         }
     }
 
-  for (i = 0; i < 4; i++)
-    {
-      *x[i] = newpos[i].x;
-      *y[i] = newpos[i].y;
-    }
-
   /* this will have been set to TRUE if an operation used the pivot in
    * addition to being a user option
    */
@@ -1768,6 +1762,22 @@ gimp_tool_transform_grid_motion (GimpToolWidget   *widget,
       pivot = vectoradd (pivot, delta);
     }
 
+  /* make sure the new coordinates are valid */
+  for (i = 0; i < 4; i++)
+    {
+      if (! isfinite (newpos[i].x) || ! isfinite (newpos[i].y))
+        return;
+    }
+
+  if (! isfinite (pivot.x) || ! isfinite (pivot.y))
+    return;
+
+  for (i = 0; i < 4; i++)
+    {
+      *x[i] = newpos[i].x;
+      *y[i] = newpos[i].y;
+    }
+
   /* set unconditionally: if options get toggled during operation, we
    * have to move pivot back
    */