[gimp/gimp-2-10] app: out-of-gamut for grayscale images too in GimpFgBgEditor.

[Jehan <jehanp src gnome org>]

commit b409a781cdd6626bc93279c0da6dd2de6edf1cbf
Author: Jehan <jehan girinstud io>
Date:   Wed Feb 13 15:48:27 2019 +0100

    app: out-of-gamut for grayscale images too in GimpFgBgEditor.
    
    I realized that the same issue as for indexed images could also apply to
    grayscale. If your fg/bg colors are not gray, it should not be expected
    for them to be paintable. So let's give the out-of-gamut hint.
    
    (cherry picked from commit 9090de96f2ec1aa48e3f2fb4750b0e4d4227f4e2)

 app/widgets/gimpfgbgeditor.c | 58 +++++++++++++++++++++++++++++---------------
 1 file changed, 39 insertions(+), 19 deletions(-)
---
diff --git a/app/widgets/gimpfgbgeditor.c b/app/widgets/gimpfgbgeditor.c
index f0f34aa96c..38331845b0 100644
--- a/app/widgets/gimpfgbgeditor.c
+++ b/app/widgets/gimpfgbgeditor.c
@@ -44,6 +44,7 @@
 #include "gimpfgbgeditor.h"
 #include "gimpwidgets-utils.h"
 
+#define CHANNEL_EPSILON 1e-3
 
 enum
 {
@@ -258,18 +259,19 @@ static gboolean
 gimp_fg_bg_editor_expose (GtkWidget      *widget,
                           GdkEventExpose *eevent)
 {
-  GimpFgBgEditor *editor = GIMP_FG_BG_EDITOR (widget);
-  GtkStyle       *style  = gtk_widget_get_style (widget);
-  GdkWindow      *window = gtk_widget_get_window (widget);
-  cairo_t        *cr;
-  GimpPalette    *colormap_palette = NULL;
-  GtkAllocation   allocation;
-  gint            width, height;
-  gint            default_w, default_h;
-  gint            swap_w, swap_h;
-  gint            rect_w, rect_h;
-  GimpRGB         color;
-  GimpRGB         transformed_color;
+  GimpFgBgEditor    *editor = GIMP_FG_BG_EDITOR (widget);
+  GtkStyle          *style  = gtk_widget_get_style (widget);
+  GdkWindow         *window = gtk_widget_get_window (widget);
+  cairo_t           *cr;
+  GimpPalette       *colormap_palette = NULL;
+  GtkAllocation      allocation;
+  gint               width, height;
+  gint               default_w, default_h;
+  gint               swap_w, swap_h;
+  gint               rect_w, rect_h;
+  GimpRGB            color;
+  GimpRGB            transformed_color;
+  GimpImageBaseType  base_type = GIMP_RGB;
 
   if (! gtk_widget_is_drawable (widget))
     return FALSE;
@@ -337,9 +339,13 @@ gimp_fg_bg_editor_expose (GtkWidget      *widget,
   if (! editor->transform)
     gimp_fg_bg_editor_create_transform (editor);
 
-  if (gimp_context_get_image (editor->context) &&
-      gimp_image_get_base_type (gimp_context_get_image (editor->context)) == GIMP_INDEXED)
-    colormap_palette = gimp_image_get_colormap_palette (editor->active_image);
+  if (editor->active_image)
+    {
+      base_type = gimp_image_get_base_type (editor->active_image);
+
+      if (base_type == GIMP_INDEXED)
+        colormap_palette = gimp_image_get_colormap_palette (editor->active_image);
+    }
 
   /*  draw the background area  */
 
@@ -368,10 +374,17 @@ gimp_fg_bg_editor_expose (GtkWidget      *widget,
 
 
       if (editor->color_config &&
+          /* Common out-of-gamut case */
           ((color.r < 0.0 || color.r > 1.0 ||
             color.g < 0.0 || color.g > 1.0 ||
             color.b < 0.0 || color.b > 1.0) ||
-           (colormap_palette && ! gimp_palette_find_entry (colormap_palette, &color, NULL))))
+           /* Indexed images */
+           (colormap_palette && ! gimp_palette_find_entry (colormap_palette, &color, NULL)) ||
+           /* Grayscale images */
+           (base_type == GIMP_GRAY &&
+            (ABS (color.r - color.g) > CHANNEL_EPSILON ||
+             ABS (color.r - color.b) > CHANNEL_EPSILON ||
+             ABS (color.g - color.b) > CHANNEL_EPSILON))))
         {
           gint side = MIN (rect_w, rect_h) * 2 / 3;
 
@@ -419,10 +432,17 @@ gimp_fg_bg_editor_expose (GtkWidget      *widget,
       cairo_fill (cr);
 
       if (editor->color_config &&
+          /* Common out-of-gamut case */
           ((color.r < 0.0 || color.r > 1.0 ||
-           color.g < 0.0 || color.g > 1.0 ||
-           color.b < 0.0 || color.b > 1.0) ||
-           (colormap_palette && ! gimp_palette_find_entry (colormap_palette, &color, NULL))))
+            color.g < 0.0 || color.g > 1.0 ||
+            color.b < 0.0 || color.b > 1.0) ||
+           /* Indexed images */
+           (colormap_palette && ! gimp_palette_find_entry (colormap_palette, &color, NULL)) ||
+           /* Grayscale images */
+           (base_type == GIMP_GRAY &&
+            (ABS (color.r - color.g) > CHANNEL_EPSILON ||
+             ABS (color.r - color.b) > CHANNEL_EPSILON ||
+             ABS (color.g - color.b) > CHANNEL_EPSILON))))
         {
           gint side = MIN (rect_w, rect_h) * 2 / 3;