[gimp/gimp-2-10] app: disregard composite space in source-only trivial layer modes

[Ell <ell src gnome org>]

commit 7a8d0c4afb156baf720be7dc875af3661e8e087f
Author: Ell <ell_se yahoo com>
Date:   Fri May 24 02:39:15 2019 -0400

    app: disregard composite space in source-only trivial layer modes
    
    Extend last commit to also disregard the composite space when the
    layer mode is trivial and only the source region is included in
    compositing, since, in this case, the source color is unmodified.
    
    (cherry picked from commit c2021d3c5b72e53849f4b38c6613a15e7651152b)

 app/operations/layer-modes/gimp-layer-modes.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)
---
diff --git a/app/operations/layer-modes/gimp-layer-modes.c b/app/operations/layer-modes/gimp-layer-modes.c
index bec2a5db20..2064ae539c 100644
--- a/app/operations/layer-modes/gimp-layer-modes.c
+++ b/app/operations/layer-modes/gimp-layer-modes.c
@@ -1429,6 +1429,8 @@ gimp_layer_mode_get_format (GimpLayerMode           mode,
                             GimpLayerCompositeMode  composite_mode,
                             const Babl             *preferred_format)
 {
+  GimpLayerCompositeRegion composite_region;
+
   /* for now, all modes perform i/o in the composite space. */
   (void) mode;
   (void) blend_space;
@@ -1439,22 +1441,26 @@ gimp_layer_mode_get_format (GimpLayerMode           mode,
   if (composite_mode == GIMP_LAYER_COMPOSITE_AUTO)
     composite_mode = gimp_layer_mode_get_composite_mode (mode);
 
+  composite_region = gimp_layer_mode_get_included_region (mode, composite_mode);
+
   if (gimp_layer_mode_is_alpha_only (mode))
     {
-      switch (composite_mode)
+      if (composite_region != GIMP_LAYER_COMPOSITE_REGION_UNION)
         {
-        case GIMP_LAYER_COMPOSITE_AUTO:
-        case GIMP_LAYER_COMPOSITE_UNION:
-          break;
-
-        case GIMP_LAYER_COMPOSITE_CLIP_TO_BACKDROP:
-        case GIMP_LAYER_COMPOSITE_CLIP_TO_LAYER:
-        case GIMP_LAYER_COMPOSITE_INTERSECTION:
           /* alpha-only layer modes don't combine colors in non-union composite
            * modes, hence we can disregard the composite space.
            */
           composite_space = GIMP_LAYER_COLOR_SPACE_AUTO;
-          break;
+        }
+    }
+  else if (gimp_layer_mode_is_trivial (mode))
+    {
+      if (! (composite_region & GIMP_LAYER_COMPOSITE_REGION_DESTINATION))
+        {
+          /* trivial layer modes don't combine colors when only the source
+           * region is included, hence we can disregard the composite space.
+           */
+          composite_space = GIMP_LAYER_COLOR_SPACE_AUTO;
         }
     }