[mutter] clutter/offscreen-effect: Set the viewport correctly

[Marge Bot <marge-bot src gnome org>]

commit 41bf0181b953ceab732b359eae5fbc83e6aa52ff
Author: Daniel van Vugt <daniel van vugt canonical com>
Date:   Thu Feb 13 16:27:32 2020 +0800

    clutter/offscreen-effect: Set the viewport correctly
    
    Previously we were setting the FBO's viewport to be the same dimensions as
    the stage itself for compatibility. This works for most cases, but not if
    the actor is larger than the stage. In that case it could cause excessive
    clipping if the actor's transformed screen position was negative, which
    is seen in https://gitlab.gnome.org/GNOME/gnome-shell/issues/2087
    
    Also if a small actor paints to its negative dimensions (like a box-shadow)
    then we might be missing those pixels on the left or top, even though
    they're inside the paint volume.
    
    Now we set the viewport dimensions to match the area we're actually
    rendering so the FBO contents are never over or under clipped.
    
    Fixes: https://gitlab.gnome.org/GNOME/gnome-shell/-/issues/3068
    
    Although if you try using shadows larger than that (like in
    gnome-shell#1090) then you will also need gnome-shell!1417.
    
    Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1053>

 clutter/clutter/clutter-offscreen-effect.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
---
diff --git a/clutter/clutter/clutter-offscreen-effect.c b/clutter/clutter/clutter-offscreen-effect.c
index 6f599670cb..3e72d89fdc 100644
--- a/clutter/clutter/clutter-offscreen-effect.c
+++ b/clutter/clutter/clutter-offscreen-effect.c
@@ -300,7 +300,7 @@ clutter_offscreen_effect_pre_paint (ClutterEffect       *effect,
   CoglFramebuffer *offscreen;
   ClutterActorBox raw_box, box;
   ClutterActor *stage;
-  graphene_matrix_t projection, modelview;
+  graphene_matrix_t projection, modelview, transform;
   const ClutterPaintVolume *volume;
   gfloat stage_width, stage_height;
   gfloat target_width = -1, target_height = -1;
@@ -368,16 +368,25 @@ clutter_offscreen_effect_pre_paint (ClutterEffect       *effect,
    * contents on screen...
    */
   clutter_actor_get_transform (priv->stage, &modelview);
+  graphene_matrix_init_translate (&transform,
+                                  &GRAPHENE_POINT3D_INIT (-priv->fbo_offset_x,
+                                                          -priv->fbo_offset_y,
+                                                          0.0));
+  graphene_matrix_scale (&transform,
+                         stage_width / target_width,
+                         stage_height / target_height,
+                         1.0);
+  graphene_matrix_multiply (&transform, &modelview, &modelview);
   cogl_framebuffer_set_modelview_matrix (offscreen, &modelview);
 
-  /* Set up the viewport so that it has the same size as the stage (avoid
-   * distortion), but translated to account for the FBO offset...
+  /* Set up the viewport so that it has the minimal size required to render any
+   * pixel in the FBO without clipping.
    */
   cogl_framebuffer_set_viewport (offscreen,
-                                 -priv->fbo_offset_x,
-                                 -priv->fbo_offset_y,
-                                 stage_width,
-                                 stage_height);
+                                 0,
+                                 0,
+                                 target_width,
+                                 target_height);
 
   /* Copy the stage's projection matrix across to the offscreen */
   _clutter_stage_get_projection_matrix (CLUTTER_STAGE (priv->stage),