[gimp/gimp-2-10] app: in 3D Transform tool, fix rotation direction when rotating arouns local Z axis

[Ell <ell src gnome org>]

commit c4d0c0bd6d00fbfb128b8f837bb464433fb7c028
Author: Ell <ell_se yahoo com>
Date:   Fri Jan 10 22:15:56 2020 +0200

    app: in 3D Transform tool, fix rotation direction when rotating arouns local Z axis
    
    In GimpToolTransform3DGrid, fix the test for determining which side
    of the plane is facing the viewer, affecting the direction of
    rotation when rotating around the local Z axis.
    
    (cherry picked from commit bcaaa453c4303faf2a5c30440ee174212b210163)

 app/display/gimptooltransform3dgrid.c | 97 ++++++++++++++++++++++-------------
 1 file changed, 62 insertions(+), 35 deletions(-)
---
diff --git a/app/display/gimptooltransform3dgrid.c b/app/display/gimptooltransform3dgrid.c
index 41a10b9012..9e3dca7796 100644
--- a/app/display/gimptooltransform3dgrid.c
+++ b/app/display/gimptooltransform3dgrid.c
@@ -952,41 +952,68 @@ gimp_tool_transform_3d_grid_motion_rotate (GimpToolTransform3DGrid *grid,
     }
   else
     {
-      GimpVector2 o, u, v;
-
-      gimp_matrix4_identity (&matrix);
-
-      gimp_transform_3d_matrix4_rotate_euler (&matrix,
-                                              priv->rotation_order,
-                                              priv->angle_x,
-                                              priv->angle_y,
-                                              priv->angle_z,
-                                              0.0, 0.0, 0.0);
-
-      z_sign = matrix.coeff[2][2] >= 0.0 ? +1.0 : -1.0;
-
-      gimp_matrix3_transform_point (&priv->orig_transform,
-                                    priv->pivot_x, priv->pivot_y,
-                                    &o.x, &o.y);
-      gimp_matrix3_transform_point (&priv->orig_transform,
-                                    priv->pivot_x + 1.0, priv->pivot_y,
-                                    &u.x, &u.y);
-      gimp_matrix3_transform_point (&priv->orig_transform,
-                                    priv->pivot_x, priv->pivot_y + 1.0,
-                                    &v.x, &v.y);
-
-      gimp_vector2_sub (&u, &u, &o);
-      gimp_vector2_sub (&v, &v, &o);
-
-      gimp_vector2_normalize (&u);
-      gimp_vector2_normalize (&v);
-
-      basis_inv.coeff[0][0] = u.x;
-      basis_inv.coeff[1][0] = u.y;
-      basis_inv.coeff[0][1] = v.x;
-      basis_inv.coeff[1][1] = v.y;
-
-      gimp_matrix2_invert (&basis_inv);
+      {
+        GimpVector3 o, n, c;
+
+        gimp_matrix4_identity (&matrix);
+
+        gimp_transform_3d_matrix4_rotate_euler (&matrix,
+                                                priv->rotation_order,
+                                                priv->angle_x,
+                                                priv->angle_y,
+                                                priv->angle_z,
+                                                priv->pivot_x,
+                                                priv->pivot_y,
+                                                priv->pivot_z);
+
+        gimp_transform_3d_matrix4_translate (&matrix,
+                                             priv->offset_x,
+                                             priv->offset_y,
+                                             priv->offset_z);
+
+        gimp_matrix4_transform_point (&matrix,
+                                      0.0,  0.0,  0.0,
+                                      &o.x, &o.y, &o.z);
+        gimp_matrix4_transform_point (&matrix,
+                                      0.0,  0.0,  1.0,
+                                      &n.x, &n.y, &n.z);
+
+        c.x = priv->camera_x;
+        c.y = priv->camera_y;
+        c.z = priv->camera_z;
+
+        gimp_vector3_sub (&n, &n, &o);
+        gimp_vector3_sub (&c, &c, &o);
+
+        z_sign = gimp_vector3_inner_product (&c, &n) <= 0.0 ? +1.0 : -1.0;
+      }
+
+      {
+        GimpVector2 o, u, v;
+
+        gimp_matrix3_transform_point (&priv->orig_transform,
+                                      priv->pivot_x, priv->pivot_y,
+                                      &o.x, &o.y);
+        gimp_matrix3_transform_point (&priv->orig_transform,
+                                      priv->pivot_x + 1.0, priv->pivot_y,
+                                      &u.x, &u.y);
+        gimp_matrix3_transform_point (&priv->orig_transform,
+                                      priv->pivot_x, priv->pivot_y + 1.0,
+                                      &v.x, &v.y);
+
+        gimp_vector2_sub (&u, &u, &o);
+        gimp_vector2_sub (&v, &v, &o);
+
+        gimp_vector2_normalize (&u);
+        gimp_vector2_normalize (&v);
+
+        basis_inv.coeff[0][0] = u.x;
+        basis_inv.coeff[1][0] = u.y;
+        basis_inv.coeff[0][1] = v.x;
+        basis_inv.coeff[1][1] = v.y;
+
+        gimp_matrix2_invert (&basis_inv);
+      }
     }
 
   if (! priv->z_axis)