[mutter] monitor-transform: Fix transform() and drop relative_transform()

[Marge Bot <marge-bot src gnome org>]

commit 29cafe6f6c57f8aa6709deed03f1afc061cd82f8
Author: Robert Mader <robert mader posteo de>
Date:   Fri Jul 1 16:03:20 2022 +0200

    monitor-transform: Fix transform() and drop relative_transform()
    
    The following implicit definition for `transform()` did not
    correctly apply:
    ```
    a * b = c
    c * invert(b) = a
    ```
    
    Crucially the following did not apply for `FLIPPED-90`
    and `FLIPPED-270`:
    ```
    a * invert(a) = identity
    ```
    
    Fix this by applying the operations, first the flip, then the
    rotation, in this order and add tests to ensure correct results
    for the requirement above.
    
    Also drop `relative_transform()` as it only had a single user and
    can be replaced by `transform()`:
    ```
    invert(a) * b = c
    a * c = b
    ```
    
    As this is not very intuitive, ensure in tests as well.
    
    Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2459>

 src/backends/meta-monitor-transform.c             | 67 +++++++++++++----------
 src/backends/meta-monitor-transform.h             |  3 -
 src/backends/native/meta-cursor-renderer-native.c |  5 +-
 src/tests/monitor-transform-tests.c               | 29 ++++++++++
 4 files changed, 70 insertions(+), 34 deletions(-)
---
diff --git a/src/backends/meta-monitor-transform.c b/src/backends/meta-monitor-transform.c
index e4a010e502..5706ad5de7 100644
--- a/src/backends/meta-monitor-transform.c
+++ b/src/backends/meta-monitor-transform.c
@@ -58,45 +58,54 @@ meta_monitor_transform_invert (MetaMonitorTransform transform)
   return 0;
 }
 
+static MetaMonitorTransform
+meta_monitor_transform_flip (MetaMonitorTransform transform)
+{
+  switch (transform)
+    {
+    case META_MONITOR_TRANSFORM_NORMAL:
+      return META_MONITOR_TRANSFORM_FLIPPED;
+    case META_MONITOR_TRANSFORM_90:
+      return META_MONITOR_TRANSFORM_FLIPPED_270;
+    case META_MONITOR_TRANSFORM_180:
+      return META_MONITOR_TRANSFORM_FLIPPED_180;
+    case META_MONITOR_TRANSFORM_270:
+      return META_MONITOR_TRANSFORM_FLIPPED_90;
+    case META_MONITOR_TRANSFORM_FLIPPED:
+      return META_MONITOR_TRANSFORM_NORMAL;
+    case META_MONITOR_TRANSFORM_FLIPPED_90:
+      return META_MONITOR_TRANSFORM_270;
+    case META_MONITOR_TRANSFORM_FLIPPED_180:
+      return META_MONITOR_TRANSFORM_180;
+    case META_MONITOR_TRANSFORM_FLIPPED_270:
+      return META_MONITOR_TRANSFORM_90;
+    }
+  g_assert_not_reached ();
+  return 0;
+}
+
 MetaMonitorTransform
 meta_monitor_transform_transform (MetaMonitorTransform transform,
                                   MetaMonitorTransform other)
 {
   MetaMonitorTransform new_transform;
+  gboolean needs_flip = FALSE;
 
-  new_transform = (transform + other) % META_MONITOR_TRANSFORM_FLIPPED;
-  if (meta_monitor_transform_is_flipped (transform) !=
-      meta_monitor_transform_is_flipped (other))
-    new_transform += META_MONITOR_TRANSFORM_FLIPPED;
-
-  return new_transform;
-}
+  if (meta_monitor_transform_is_flipped (other))
+    new_transform = meta_monitor_transform_flip (transform);
+  else
+    new_transform = transform;
 
-/**
- * meta_monitor_transform_relative_transform:
- * @transform: The transform to start from
- * @other: The transform to go to
- *
- * Return value: a transform to get from @transform to @other
- */
-MetaMonitorTransform
-meta_monitor_transform_relative_transform (MetaMonitorTransform transform,
-                                           MetaMonitorTransform other)
-{
-  MetaMonitorTransform relative_transform;
+  if (meta_monitor_transform_is_flipped (new_transform))
+    needs_flip = TRUE;
 
-  relative_transform = ((other % META_MONITOR_TRANSFORM_FLIPPED -
-                         transform % META_MONITOR_TRANSFORM_FLIPPED) %
-                        META_MONITOR_TRANSFORM_FLIPPED);
+  new_transform += other;
+  new_transform %= META_MONITOR_TRANSFORM_FLIPPED;
 
-  if (meta_monitor_transform_is_flipped (transform) !=
-      meta_monitor_transform_is_flipped (other))
-    {
-      relative_transform = (meta_monitor_transform_invert (relative_transform) +
-                            META_MONITOR_TRANSFORM_FLIPPED);
-    }
+  if (needs_flip)
+    new_transform += META_MONITOR_TRANSFORM_FLIPPED;
 
-  return relative_transform;
+  return new_transform;
 }
 
 void
diff --git a/src/backends/meta-monitor-transform.h b/src/backends/meta-monitor-transform.h
index e027324f72..3449c14d7e 100644
--- a/src/backends/meta-monitor-transform.h
+++ b/src/backends/meta-monitor-transform.h
@@ -64,9 +64,6 @@ META_EXPORT_TEST
 MetaMonitorTransform meta_monitor_transform_transform (MetaMonitorTransform transform,
                                                        MetaMonitorTransform other);
 
-MetaMonitorTransform meta_monitor_transform_relative_transform (MetaMonitorTransform transform,
-                                                                MetaMonitorTransform other);
-
 void meta_monitor_transform_transform_point (MetaMonitorTransform  transform,
                                              int                   area_width,
                                              int                   area_height,
diff --git a/src/backends/native/meta-cursor-renderer-native.c 
b/src/backends/native/meta-cursor-renderer-native.c
index 7da9b2c5ab..9e852e82ae 100644
--- a/src/backends/native/meta-cursor-renderer-native.c
+++ b/src/backends/native/meta-cursor-renderer-native.c
@@ -816,8 +816,9 @@ get_common_crtc_sprite_transform_for_logical_monitors (MetaCursorRenderer   *ren
         {
           MetaMonitor *monitor = l_mon->data;
 
-          tmp_transform = meta_monitor_transform_relative_transform (
-            meta_cursor_sprite_get_texture_transform (cursor_sprite),
+          tmp_transform = meta_monitor_transform_transform (
+            meta_monitor_transform_invert (
+              meta_cursor_sprite_get_texture_transform (cursor_sprite)),
             meta_monitor_logical_to_crtc_transform (monitor, logical_transform));
 
           if (has_visible_crtc_sprite && transform != tmp_transform)
diff --git a/src/tests/monitor-transform-tests.c b/src/tests/monitor-transform-tests.c
index 871a06b101..3f1a2a757d 100644
--- a/src/tests/monitor-transform-tests.c
+++ b/src/tests/monitor-transform-tests.c
@@ -83,6 +83,7 @@ test_transform (void)
     },
   };
   int i;
+  MetaMonitorTransform transform;
 
   for (i = 0; i < G_N_ELEMENTS (tests); i++)
     {
@@ -92,6 +93,34 @@ test_transform (void)
                                                  tests[i].other);
       g_assert_cmpint (result, ==, tests[i].expect);
     }
+
+  for (transform = 0; transform <= META_MONITOR_TRANSFORM_FLIPPED_270; transform++)
+    {
+      MetaMonitorTransform other;
+      MetaMonitorTransform result1;
+
+      result1 =
+        meta_monitor_transform_transform (transform,
+                                          meta_monitor_transform_invert (transform));
+      g_assert_cmpint (result1, ==, META_MONITOR_TRANSFORM_NORMAL);
+
+      for (other = 0; other <= META_MONITOR_TRANSFORM_FLIPPED_270; other++)
+        {
+          MetaMonitorTransform result2;
+
+          result1 = meta_monitor_transform_transform (transform, other);
+          result2 =
+            meta_monitor_transform_transform (result1,
+                                              meta_monitor_transform_invert (other));
+          g_assert_cmpint (result2, ==, transform);
+
+          result1 =
+            meta_monitor_transform_transform (meta_monitor_transform_invert (transform),
+                                              other);
+          result2 = meta_monitor_transform_transform (transform, result1);
+          g_assert_cmpint (result2, ==, other);
+        }
+    }
 }
 
 void