[gimp] Bug 791519 - Unexpected selection from channel

[Michael Natterer <mitch src gnome org>]

commit bb4ac7c8293ad994f82b257b32fc27c3eafd2b26
Author: Michael Natterer <mitch gimp org>
Date:   Sun Apr 8 19:16:58 2018 +0200

    Bug 791519 - Unexpected selection from channel
    
    Storing selections and layer masks as linear grayscale, but channels
    as whatever-the-layers-are caused severe problems in images with
    gamma-corrected layers: when combining channels with the selection,
    they would go thorugh a gamma conversion before being combined, giving
    unexpected results.
    
    This commit changes all channels to always be linear, except in 8-bit
    images, where they continue to be "Y' u8", for compatibility with old
    XCF files, and because linear 8-bit can't really be used in
    compositing (channels can be visible too).
    
    To fix channel -> selection combinations also for these images, add a
    small hack to gimp_gegl_mask_combine_buffer() which makes sure the
    to-be-combined channel's pixels are always read as-is, without any
    gamma conversion. After changing channels to linear, this makes no
    difference except in the 8-bit images where we need this hack.

 app/core/gimpimage.c              |   13 ++++++++++---
 app/gegl/gimp-gegl-mask-combine.c |   19 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 4 deletions(-)
---
diff --git a/app/core/gimpimage.c b/app/core/gimpimage.c
index 2ba4b3d..7176369 100644
--- a/app/core/gimpimage.c
+++ b/app/core/gimpimage.c
@@ -1836,11 +1836,18 @@ gimp_image_get_layer_format (GimpImage *image,
 const Babl *
 gimp_image_get_channel_format (GimpImage *image)
 {
+  GimpPrecision precision;
+
   g_return_val_if_fail (GIMP_IS_IMAGE (image), NULL);
 
-  return gimp_image_get_format (image, GIMP_GRAY,
-                                gimp_image_get_precision (image),
-                                FALSE);
+  precision = gimp_image_get_precision (image);
+
+  if (precision == GIMP_PRECISION_U8_GAMMA)
+    return gimp_image_get_format (image, GIMP_GRAY,
+                                  gimp_image_get_precision (image),
+                                  FALSE);
+
+  return gimp_babl_mask_format (precision);
 }
 
 const Babl *
diff --git a/app/gegl/gimp-gegl-mask-combine.c b/app/gegl/gimp-gegl-mask-combine.c
index ec8f179..f826208 100644
--- a/app/gegl/gimp-gegl-mask-combine.c
+++ b/app/gegl/gimp-gegl-mask-combine.c
@@ -27,6 +27,7 @@
 
 #include "gimp-gegl-types.h"
 
+#include "gimp-babl.h"
 #include "gimp-gegl-mask-combine.h"
 
 
@@ -376,6 +377,7 @@ gimp_gegl_mask_combine_buffer (GeglBuffer     *mask,
 {
   GeglBufferIterator *iter;
   GeglRectangle       rect;
+  const Babl         *add_on_format;
   gint                x, y, w, h;
 
   g_return_val_if_fail (GEGL_IS_BUFFER (mask), FALSE);
@@ -402,8 +404,23 @@ gimp_gegl_mask_combine_buffer (GeglBuffer     *mask,
   rect.x -= off_x;
   rect.y -= off_y;
 
+  /*  This is a hack: all selections/layer masks/channels are always
+   *  linear except for channels in 8-bit images. We don't want these
+   *  "Y' u8" to be converted to "Y float" because that would cause a
+   *  gamma canversion and give unexpected results for
+   *  "add/subtract/etc channel from selection". Instead, use all
+   *  channel values "as-is", which makes no differce except in the
+   *  8-bit case where we need it.
+   *
+   *  See https://bugzilla.gnome.org/show_bug.cgi?id=791519
+   */
+  if (gimp_babl_format_get_linear (gegl_buffer_get_format (add_on)))
+    add_on_format = babl_format ("Y float");
+  else
+    add_on_format = babl_format ("Y' float");
+
   gegl_buffer_iterator_add (iter, add_on, &rect, 0,
-                            babl_format ("Y float"),
+                            add_on_format,
                             GEGL_ACCESS_READ, GEGL_ABYSS_NONE);
 
   switch (op)