[gimp/gimp-2-10] app: fix layer effective mode for different mask modes

[Ell <ell src gnome org>]

commit 603d0a6274582f8211b4e723696d02eae73199b8
Author: Ell <ell_se yahoo com>
Date:   Fri Feb 21 22:05:44 2020 +0200

    app: fix layer effective mode for different mask modes
    
    In GimpLayer, update the effective mode, and excludes-backdrop,
    when any of the mask-related properties change, and, when show-mask
    is active, set the effective mode to NORMAL.
    
    (cherry picked from commit b2a1a7cb01ff9b5a1cc31470494797869d4c320b)

 app/core/gimplayer.c | 39 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 5 deletions(-)
---
diff --git a/app/core/gimplayer.c b/app/core/gimplayer.c
index 5d785d21b6..e0f1ca00e7 100644
--- a/app/core/gimplayer.c
+++ b/app/core/gimplayer.c
@@ -1903,6 +1903,9 @@ gimp_layer_add_mask (GimpLayer      *layer,
 
   gimp_drawable_update_bounding_box (GIMP_DRAWABLE (layer));
 
+  gimp_layer_update_effective_mode (layer);
+  gimp_layer_update_excludes_backdrop (layer);
+
   if (gimp_layer_get_apply_mask (layer) ||
       gimp_layer_get_show_mask (layer))
     {
@@ -2222,6 +2225,9 @@ gimp_layer_apply_mask (GimpLayer         *layer,
 
   gimp_drawable_update_bounding_box (GIMP_DRAWABLE (layer));
 
+  gimp_layer_update_effective_mode (layer);
+  gimp_layer_update_excludes_backdrop (layer);
+
   /*  If applying actually changed the view  */
   if (view_changed)
     {
@@ -2278,6 +2284,9 @@ gimp_layer_set_apply_mask (GimpLayer *layer,
 
       gimp_drawable_update_bounding_box (GIMP_DRAWABLE (layer));
 
+      gimp_layer_update_effective_mode (layer);
+      gimp_layer_update_excludes_backdrop (layer);
+
       gimp_drawable_update (GIMP_DRAWABLE (layer), 0, 0, -1, -1);
 
       g_signal_emit (layer, layer_signals[APPLY_MASK_CHANGED], 0);
@@ -2364,6 +2373,9 @@ gimp_layer_set_show_mask (GimpLayer *layer,
           gimp_layer_update_mode_node (layer);
         }
 
+      gimp_layer_update_effective_mode (layer);
+      gimp_layer_update_excludes_backdrop (layer);
+
       gimp_drawable_update (GIMP_DRAWABLE (layer), 0, 0, -1, -1);
 
       g_signal_emit (layer, layer_signals[SHOW_MASK_CHANGED], 0);
@@ -2851,11 +2863,28 @@ gimp_layer_update_effective_mode (GimpLayer *layer)
 
   g_return_if_fail (GIMP_IS_LAYER (layer));
 
-  GIMP_LAYER_GET_CLASS (layer)->get_effective_mode (layer,
-                                                    &mode,
-                                                    &blend_space,
-                                                    &composite_space,
-                                                    &composite_mode);
+  if (layer->mask && layer->show_mask)
+    {
+      mode            = GIMP_LAYER_MODE_NORMAL;
+      blend_space     = GIMP_LAYER_COLOR_SPACE_AUTO;
+      composite_space = GIMP_LAYER_COLOR_SPACE_AUTO;
+      composite_mode  = GIMP_LAYER_COMPOSITE_AUTO;
+
+      /* This makes sure that masks of LEGACY-mode layers are
+       * composited in PERCEPTUAL space, and non-LEGACY layers in
+       * LINEAR space, or whatever composite space was chosen in the
+       * layer attributes dialog
+       */
+      composite_space = gimp_layer_get_real_composite_space (layer);
+    }
+  else
+    {
+      GIMP_LAYER_GET_CLASS (layer)->get_effective_mode (layer,
+                                                        &mode,
+                                                        &blend_space,
+                                                        &composite_space,
+                                                        &composite_mode);
+    }
 
   if (mode            != layer->effective_mode            ||
       blend_space     != layer->effective_blend_space     ||