[gimp/gimp-2-10] plug-ins: fix #7429 Error loading PSD file: Unsupported or invalid layer mask height

[Jacob Boerema <jboerema src gnome org>]

commit 7ebff6ed2871d8516a401e1bbec72639e4d010b1
Author: Jacob Boerema <jgboerema gmail com>
Date:   Tue Oct 26 12:28:03 2021 -0400

    plug-ins: fix #7429 Error loading PSD file: Unsupported or invalid layer mask height
    
    Our psd plug-in checks for valid layer mask sizes. Apparently a psd
    layer mask where the rendered flag is set can have invalid dimensions
    0, 0, 0, -1.
    We will skip the sanity checks for rendered layers and at the same time
    move the debug statements up to have relevant debug info available even
    if our sanity checks stop further processing.
    
    (cherry picked from commit b27bf5caa982597deb4cc3cd74d7674116950034)

 plug-ins/file-psd/psd-load.c | 60 +++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 28 deletions(-)
---
diff --git a/plug-ins/file-psd/psd-load.c b/plug-ins/file-psd/psd-load.c
index ac5c6cc205..b212450413 100644
--- a/plug-ins/file-psd/psd-load.c
+++ b/plug-ins/file-psd/psd-load.c
@@ -811,34 +811,6 @@ read_layer_info (PSDimage  *img_a,
                     }
                 }
 
-              /* sanity checks */
-              if (lyr_a[lidx]->layer_mask.bottom < lyr_a[lidx]->layer_mask.top ||
-                  lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top > GIMP_MAX_IMAGE_SIZE)
-                {
-                  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
-                               _("Unsupported or invalid layer mask height: %d"),
-                               lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top);
-                  return NULL;
-                }
-              if (lyr_a[lidx]->layer_mask.right < lyr_a[lidx]->layer_mask.left ||
-                  lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left > GIMP_MAX_IMAGE_SIZE)
-                {
-                  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
-                               _("Unsupported or invalid layer mask width: %d"),
-                               lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left);
-                  return NULL;
-                }
-
-              if ((lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left) >
-                  G_MAXINT32 / MAX (lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top, 1))
-                {
-                  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
-                               _("Unsupported or invalid layer mask size: %dx%d"),
-                               lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left,
-                               lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top);
-                  return NULL;
-                }
-
               IFDBG(2) g_debug ("Layer mask coords %d %d %d %d",
                                 lyr_a[lidx]->layer_mask.left,
                                 lyr_a[lidx]->layer_mask.top,
@@ -853,6 +825,38 @@ read_layer_info (PSDimage  *img_a,
                                 lyr_a[lidx]->layer_mask.flags,
                                 lyr_a[lidx]->layer_mask.extra_flags,
                                 lyr_a[lidx]->layer_mask.mask_params);
+
+              /* Rendered masks can have invalid dimensions: 0, 0, 0, -1 */
+              if (! lyr_a[lidx]->layer_mask.mask_flags.rendered)
+                {
+                  /* sanity checks */
+                  if (lyr_a[lidx]->layer_mask.bottom < lyr_a[lidx]->layer_mask.top ||
+                      lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top > GIMP_MAX_IMAGE_SIZE)
+                    {
+                      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                                   _("Unsupported or invalid layer mask height: %d"),
+                                   lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top);
+                      return NULL;
+                    }
+                  if (lyr_a[lidx]->layer_mask.right < lyr_a[lidx]->layer_mask.left ||
+                      lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left > GIMP_MAX_IMAGE_SIZE)
+                    {
+                      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                                   _("Unsupported or invalid layer mask width: %d"),
+                                   lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left);
+                      return NULL;
+                    }
+
+                  if ((lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left) >
+                      G_MAXINT32 / MAX (lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top, 1))
+                    {
+                      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                                   _("Unsupported or invalid layer mask size: %dx%d"),
+                                   lyr_a[lidx]->layer_mask.right - lyr_a[lidx]->layer_mask.left,
+                                   lyr_a[lidx]->layer_mask.bottom - lyr_a[lidx]->layer_mask.top);
+                      return NULL;
+                    }
+                }
             }
 
           /* Layer blending ranges */           /* FIXME  */