[gimp/gimp-2-10] app: fix incorrect drawable format after applying a filter that resizes the drawable

[Ell <ell src gnome org>]

commit b0a6cd64125f3826797e8c6342a896a5856ac044
Author: Ell <ell_se yahoo com>
Date:   Fri Feb 21 11:54:18 2020 +0200

    app: fix incorrect drawable format after applying a filter that resizes the drawable
    
    In gimp_drawable_merge_filter(), fix the format of newly allocated
    buffers when applying a filter that changes the size of the
    drawable, and the requested format is either unspecified, or
    matches the existing drawable format.  The new buffer would
    previously be created with a "RGBA float" format in this case,
    regardless of the image precision, resulting in a potential
    mismatch between the drawable format and the image precision.
    
    This cna lead to inconsistent buffer data when saved as XCF, which
    is later rejected on load.  See issue #4643.
    
    (cherry picked from commit ae6f6ba42ad7561f993d101467e4b3d2365664f8)

 app/core/gimpdrawable-filters.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
---
diff --git a/app/core/gimpdrawable-filters.c b/app/core/gimpdrawable-filters.c
index 5e1dc8bb51..762a870e2c 100644
--- a/app/core/gimpdrawable-filters.c
+++ b/app/core/gimpdrawable-filters.c
@@ -138,8 +138,8 @@ gimp_drawable_merge_filter (GimpDrawable *drawable,
   applicator  = gimp_filter_get_applicator (filter);
   dest_buffer = gimp_drawable_get_buffer (drawable);
 
-  if (format == gimp_drawable_get_format (drawable))
-    format = NULL;
+  if (! format)
+    format = gimp_drawable_get_format (drawable);
 
   rect = gegl_node_get_bounding_box (gimp_filter_get_node (filter));
 
@@ -158,7 +158,7 @@ gimp_drawable_merge_filter (GimpDrawable *drawable,
           return TRUE;
         }
 
-      if (format)
+      if (format != gimp_drawable_get_format (drawable))
         {
           buffer = gegl_buffer_new (gegl_buffer_get_extent (dest_buffer),
                                     format);
@@ -201,7 +201,7 @@ gimp_drawable_merge_filter (GimpDrawable *drawable,
       applicator_output_format = gimp_applicator_get_output_format (applicator);
 
       gimp_applicator_set_cache (applicator, FALSE);
-      if (applicator_output_format == gimp_drawable_get_format (drawable))
+      if (applicator_output_format == format)
         gimp_applicator_set_output_format (applicator, NULL);
     }