[gimp/gimp-2-10] app: disregard composite space in non-union alpha-only layer modes

[Ell <ell src gnome org>]

commit 731368619203524666bc24768adb93d454706ab7
Author: Ell <ell_se yahoo com>
Date:   Fri May 24 01:33:53 2019 -0400

    app: disregard composite space in non-union alpha-only layer modes
    
    In gimp_layer_mode_get_format(), disregard the requested composite
    space when selecting the format, if the input layer mode is alpha-
    only, and the requested composite mode is not UNION, since, in this
    case, the layer mode doesn't combine the layer/backdrop colors, and
    rather only modifies the alpha of one of them.  This allows us to
    use the preferred format, avoiding gamma conversion.
    
    This particularly improves the performance of the Eraser tool in
    perceptual images.
    
    (cherry picked from commit a5962e4049461d2f7b29ab840e58ad4fab2b430c)

 app/core/gimpfilloptions.c                         |  2 ++
 app/gegl/gimp-gegl-nodes.c                         |  3 ++-
 app/operations/layer-modes/gimp-layer-modes.c      | 31 +++++++++++++++++++---
 app/operations/layer-modes/gimp-layer-modes.h      |  3 ++-
 .../layer-modes/gimpoperationlayermode.c           |  3 ++-
 app/paint/gimpbrushcore.c                          |  8 ++++--
 app/paint/gimpink.c                                |  8 ++++--
 app/paint/gimppaintcore-loops.cc                   |  3 ++-
 8 files changed, 49 insertions(+), 12 deletions(-)
---
diff --git a/app/core/gimpfilloptions.c b/app/core/gimpfilloptions.c
index fc87149e98..d10b6e4572 100644
--- a/app/core/gimpfilloptions.c
+++ b/app/core/gimpfilloptions.c
@@ -473,6 +473,8 @@ gimp_fill_options_get_format (GimpFillOptions *options,
   return gimp_layer_mode_get_format (gimp_context_get_paint_mode (context),
                                      GIMP_LAYER_COLOR_SPACE_AUTO,
                                      GIMP_LAYER_COLOR_SPACE_AUTO,
+                                     gimp_layer_mode_get_paint_composite_mode (
+                                       gimp_context_get_paint_mode (context)),
                                      gimp_drawable_get_format (drawable));
 }
 
diff --git a/app/gegl/gimp-gegl-nodes.c b/app/gegl/gimp-gegl-nodes.c
index 66fbc9f169..9470834679 100644
--- a/app/gegl/gimp-gegl-nodes.c
+++ b/app/gegl/gimp-gegl-nodes.c
@@ -57,8 +57,9 @@ gimp_gegl_create_flatten_node (const GimpRGB       *background,
                                "value",     c,
                                "format",    gimp_layer_mode_get_format (
                                               GIMP_LAYER_MODE_NORMAL,
-                                              composite_space,
                                               GIMP_LAYER_COLOR_SPACE_AUTO,
+                                              composite_space,
+                                              GIMP_LAYER_COMPOSITE_AUTO,
                                               NULL),
                                NULL);
   g_object_unref (c);
diff --git a/app/operations/layer-modes/gimp-layer-modes.c b/app/operations/layer-modes/gimp-layer-modes.c
index da2897c2c1..bec2a5db20 100644
--- a/app/operations/layer-modes/gimp-layer-modes.c
+++ b/app/operations/layer-modes/gimp-layer-modes.c
@@ -1423,10 +1423,11 @@ gimp_layer_mode_get_for_group (GimpLayerMode       old_mode,
 }
 
 const Babl *
-gimp_layer_mode_get_format (GimpLayerMode        mode,
-                            GimpLayerColorSpace  composite_space,
-                            GimpLayerColorSpace  blend_space,
-                            const Babl          *preferred_format)
+gimp_layer_mode_get_format (GimpLayerMode           mode,
+                            GimpLayerColorSpace     blend_space,
+                            GimpLayerColorSpace     composite_space,
+                            GimpLayerCompositeMode  composite_mode,
+                            const Babl             *preferred_format)
 {
   /* for now, all modes perform i/o in the composite space. */
   (void) mode;
@@ -1435,6 +1436,28 @@ gimp_layer_mode_get_format (GimpLayerMode        mode,
   if (composite_space == GIMP_LAYER_COLOR_SPACE_AUTO)
     composite_space = gimp_layer_mode_get_composite_space (mode);
 
+  if (composite_mode == GIMP_LAYER_COMPOSITE_AUTO)
+    composite_mode = gimp_layer_mode_get_composite_mode (mode);
+
+  if (gimp_layer_mode_is_alpha_only (mode))
+    {
+      switch (composite_mode)
+        {
+        case GIMP_LAYER_COMPOSITE_AUTO:
+        case GIMP_LAYER_COMPOSITE_UNION:
+          break;
+
+        case GIMP_LAYER_COMPOSITE_CLIP_TO_BACKDROP:
+        case GIMP_LAYER_COMPOSITE_CLIP_TO_LAYER:
+        case GIMP_LAYER_COMPOSITE_INTERSECTION:
+          /* alpha-only layer modes don't combine colors in non-union composite
+           * modes, hence we can disregard the composite space.
+           */
+          composite_space = GIMP_LAYER_COLOR_SPACE_AUTO;
+          break;
+        }
+    }
+
   switch (composite_space)
     {
     case GIMP_LAYER_COLOR_SPACE_AUTO:
diff --git a/app/operations/layer-modes/gimp-layer-modes.h b/app/operations/layer-modes/gimp-layer-modes.h
index fa340c1d8f..30c5e152a4 100644
--- a/app/operations/layer-modes/gimp-layer-modes.h
+++ b/app/operations/layer-modes/gimp-layer-modes.h
@@ -61,8 +61,9 @@ gboolean                   gimp_layer_mode_get_for_group              (GimpLayer
                                                                        GimpLayerMode          *new_mode);
 
 const Babl               * gimp_layer_mode_get_format                 (GimpLayerMode           mode,
-                                                                       GimpLayerColorSpace     
composite_space,
                                                                        GimpLayerColorSpace     blend_space,
+                                                                       GimpLayerColorSpace     
composite_space,
+                                                                       GimpLayerCompositeMode  
composite_mode,
                                                                        const Babl             
*preferred_format);
 
 GimpLayerCompositeRegion   gimp_layer_mode_get_included_region        (GimpLayerMode           mode,
diff --git a/app/operations/layer-modes/gimpoperationlayermode.c 
b/app/operations/layer-modes/gimpoperationlayermode.c
index b2e679b792..6fa277e4de 100644
--- a/app/operations/layer-modes/gimpoperationlayermode.c
+++ b/app/operations/layer-modes/gimpoperationlayermode.c
@@ -359,8 +359,9 @@ gimp_operation_layer_mode_prepare (GeglOperation *operation)
     }
 
   format = gimp_layer_mode_get_format (self->layer_mode,
-                                       self->composite_space,
                                        self->blend_space,
+                                       self->composite_space,
+                                       self->composite_mode,
                                        preferred_format);
 
   gegl_operation_set_format (operation, "input",  format);
diff --git a/app/paint/gimpbrushcore.c b/app/paint/gimpbrushcore.c
index 6f67da979b..d1fed9a5d5 100644
--- a/app/paint/gimpbrushcore.c
+++ b/app/paint/gimpbrushcore.c
@@ -816,12 +816,16 @@ gimp_brush_core_get_paint_buffer (GimpPaintCore    *paint_core,
   /*  configure the canvas buffer  */
   if ((x2 - x1) && (y2 - y1))
     {
-      GimpTempBuf *temp_buf;
-      const Babl  *format;
+      GimpTempBuf            *temp_buf;
+      const Babl             *format;
+      GimpLayerCompositeMode  composite_mode;
+
+      composite_mode = gimp_layer_mode_get_paint_composite_mode (paint_mode);
 
       format = gimp_layer_mode_get_format (paint_mode,
                                            GIMP_LAYER_COLOR_SPACE_AUTO,
                                            GIMP_LAYER_COLOR_SPACE_AUTO,
+                                           composite_mode,
                                            gimp_drawable_get_format (drawable));
 
       if (paint_core->paint_buffer                                       &&
diff --git a/app/paint/gimpink.c b/app/paint/gimpink.c
index 452389d7dd..9573e83f47 100644
--- a/app/paint/gimpink.c
+++ b/app/paint/gimpink.c
@@ -258,12 +258,16 @@ gimp_ink_get_paint_buffer (GimpPaintCore    *paint_core,
   /*  configure the canvas buffer  */
   if ((x2 - x1) && (y2 - y1))
     {
-      GimpTempBuf *temp_buf;
-      const Babl  *format;
+      GimpTempBuf            *temp_buf;
+      const Babl             *format;
+      GimpLayerCompositeMode  composite_mode;
+
+      composite_mode = gimp_layer_mode_get_paint_composite_mode (paint_mode);
 
       format = gimp_layer_mode_get_format (paint_mode,
                                            GIMP_LAYER_COLOR_SPACE_AUTO,
                                            GIMP_LAYER_COLOR_SPACE_AUTO,
+                                           composite_mode,
                                            gimp_drawable_get_format (drawable));
 
       temp_buf = gimp_temp_buf_new ((x2 - x1), (y2 - y1),
diff --git a/app/paint/gimppaintcore-loops.cc b/app/paint/gimppaintcore-loops.cc
index 3c06ec1d25..2180c547be 100644
--- a/app/paint/gimppaintcore-loops.cc
+++ b/app/paint/gimppaintcore-loops.cc
@@ -1886,8 +1886,9 @@ struct DoLayerBlend : Base
     layer_mode.real_composite_mode = layer_mode.composite_mode;
 
     iterator_format = gimp_layer_mode_get_format (params->paint_mode,
-                                                  layer_mode.composite_space,
                                                   layer_mode.blend_space,
+                                                  layer_mode.composite_space,
+                                                  layer_mode.composite_mode,
                                                   gimp_temp_buf_get_format (params->paint_buf));
 
     g_return_if_fail (gimp_temp_buf_get_format (params->paint_buf) == iterator_format);