[gimp] app: fix layer group effective mode for different mask modes

[Ell <ell src gnome org>]

commit 8c81c6e1539d0729a218b2e729c295ee27e1c1a7
Author: Ell <ell_se yahoo com>
Date:   Fri Feb 21 22:26:47 2020 +0200

    app: fix layer group effective mode for different mask modes
    
    In gimp_group_layer_get_effective_mode(), don't take the mask into
    account if GimpLayer::apply-mask is FALSE.

 app/core/gimpgrouplayer.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)
---
diff --git a/app/core/gimpgrouplayer.c b/app/core/gimpgrouplayer.c
index 2d1a7c7aa8..88d5d05099 100644
--- a/app/core/gimpgrouplayer.c
+++ b/app/core/gimpgrouplayer.c
@@ -1182,8 +1182,6 @@ gimp_group_layer_mask_changed (GimpLayer *layer)
 
   g_warn_if_fail (GET_PRIVATE (layer)->suspend_mask == 0);
 
-  gimp_layer_update_effective_mode (layer);
-
   /* if we've already computed a bounding box, update it now, since the mask
    * limits the bounding box to the group's size.  if we haven't computed a
    * bounding box yet we can skip this, and, in fact, we have to, or else the
@@ -1226,7 +1224,8 @@ gimp_group_layer_get_effective_mode (GimpLayer              *layer,
        *
        *   - and,
        *
-       *     - the group's opacity is 100%, and it has no mask; or,
+       *     - the group's opacity is 100%, and it has no mask (or the mask
+       *       isn't applied); or,
        *
        *     - the group's composite space equals the active children's
        *       composite space.
@@ -1301,9 +1300,14 @@ gimp_group_layer_get_effective_mode (GimpLayer              *layer,
 
       if (reduce)
         {
+          gboolean has_mask;
+
+          has_mask = gimp_layer_get_mask (layer) &&
+                     gimp_layer_get_apply_mask (layer);
+
           if (first                                                  ||
               (gimp_layer_get_opacity (layer) == GIMP_OPACITY_OPAQUE &&
-               ! gimp_layer_get_mask (layer))                        ||
+               ! has_mask)                                           ||
               *composite_space == gimp_layer_get_real_composite_space (layer))
             {
               /* strength reduction succeeded! */