[gimp] app: disregard composite space in source-only trivial layer modes

[Ell <ell src gnome org>]

commit c2021d3c5b72e53849f4b38c6613a15e7651152b
Author: Ell <ell_se yahoo com>
Date:   Fri May 24 02:39:15 2019 -0400

    app: disregard composite space in source-only trivial layer modes
    
    Extend last commit to also disregard the composite space when the
    layer mode is trivial and only the source region is included in
    compositing, since, in this case, the source color is unmodified.

 app/operations/layer-modes/gimp-layer-modes.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)
---
diff --git a/app/operations/layer-modes/gimp-layer-modes.c b/app/operations/layer-modes/gimp-layer-modes.c
index b59b30506a..7778dbca2d 100644
--- a/app/operations/layer-modes/gimp-layer-modes.c
+++ b/app/operations/layer-modes/gimp-layer-modes.c
@@ -1429,6 +1429,8 @@ gimp_layer_mode_get_format (GimpLayerMode           mode,
                             GimpLayerCompositeMode  composite_mode,
                             const Babl             *preferred_format)
 {
+  GimpLayerCompositeRegion composite_region;
+
   /* for now, all modes perform i/o in the composite space. */
   (void) mode;
   (void) blend_space;
@@ -1439,22 +1441,26 @@ gimp_layer_mode_get_format (GimpLayerMode           mode,
   if (composite_mode == GIMP_LAYER_COMPOSITE_AUTO)
     composite_mode = gimp_layer_mode_get_composite_mode (mode);
 
+  composite_region = gimp_layer_mode_get_included_region (mode, composite_mode);
+
   if (gimp_layer_mode_is_alpha_only (mode))
     {
-      switch (composite_mode)
+      if (composite_region != GIMP_LAYER_COMPOSITE_REGION_UNION)
         {
-        case GIMP_LAYER_COMPOSITE_AUTO:
-        case GIMP_LAYER_COMPOSITE_UNION:
-          break;
-
-        case GIMP_LAYER_COMPOSITE_CLIP_TO_BACKDROP:
-        case GIMP_LAYER_COMPOSITE_CLIP_TO_LAYER:
-        case GIMP_LAYER_COMPOSITE_INTERSECTION:
           /* alpha-only layer modes don't combine colors in non-union composite
            * modes, hence we can disregard the composite space.
            */
           composite_space = GIMP_LAYER_COLOR_SPACE_AUTO;
-          break;
+        }
+    }
+  else if (gimp_layer_mode_is_trivial (mode))
+    {
+      if (! (composite_region & GIMP_LAYER_COMPOSITE_REGION_DESTINATION))
+        {
+          /* trivial layer modes don't combine colors when only the source
+           * region is included, hence we can disregard the composite space.
+           */
+          composite_space = GIMP_LAYER_COLOR_SPACE_AUTO;
         }
     }