[mutter/gnome-3-36] shaped-texture: Fix damage propagation for rotated transforms with viewport

[Marge Bot <marge-bot src gnome org>]

commit 1448836bb9e16c99a6d8ca3879c5a77991383fa5
Author: Robert Mader <robert mader posteo de>
Date:   Tue Apr 20 14:56:56 2021 +0200

    shaped-texture: Fix damage propagation for rotated transforms with viewport
    
    When a viewport source rect or destination size is set, `stex->dst_width`
    gives us the the cropped and/or scaled size. At this step, we need the
    uncropped/unscaled size however.
    
    Note: this is only ever relevant if buffer transform and viewport are
    used together - otherwise the values are the same.
    
    
    (cherry picked from commit 7d3e2c5f9c09fae50450df691eeba0f067de8cff)
    
    Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1838>

 src/compositor/meta-shaped-texture.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)
---
diff --git a/src/compositor/meta-shaped-texture.c b/src/compositor/meta-shaped-texture.c
index adf213f769..dca1c74766 100644
--- a/src/compositor/meta-shaped-texture.c
+++ b/src/compositor/meta-shaped-texture.c
@@ -897,6 +897,8 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
                                  cairo_rectangle_int_t *clip)
 {
   MetaMonitorTransform inverted_transform;
+  int scaled_and_transformed_width;
+  int scaled_and_transformed_height;
 
   if (stex->texture == NULL)
     return FALSE;
@@ -913,12 +915,21 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
                                META_ROUNDING_STRATEGY_SHRINK,
                                clip);
 
+  if (meta_monitor_transform_is_rotated (stex->transform))
+    {
+      scaled_and_transformed_width = stex->tex_height / stex->buffer_scale;
+      scaled_and_transformed_height = stex->tex_width / stex->buffer_scale;
+    }
+  else
+    {
+      scaled_and_transformed_width = stex->tex_width / stex->buffer_scale;
+      scaled_and_transformed_height = stex->tex_height / stex->buffer_scale;
+    }
   inverted_transform = meta_monitor_transform_invert (stex->transform);
-  meta_shaped_texture_ensure_size_valid (stex);
   meta_rectangle_transform (clip,
                             inverted_transform,
-                            stex->dst_width,
-                            stex->dst_height,
+                            scaled_and_transformed_width,
+                            scaled_and_transformed_height,
                             clip);
 
   if (stex->has_viewport_src_rect || stex->has_viewport_dst_size)
@@ -939,8 +950,8 @@ meta_shaped_texture_update_area (MetaShapedTexture     *stex,
           viewport = (graphene_rect_t) {
             .origin.x = 0,
             .origin.y = 0,
-            .size.width = stex->tex_width / stex->buffer_scale,
-            .size.height = stex->tex_height / stex->buffer_scale,
+            .size.width = scaled_and_transformed_width,
+            .size.height = scaled_and_transformed_height,
           };
         }