[gimp/gimp-2-10] plug-ins: fix #8411 crash when attempting to open PSD

[Jacob Boerema <jboerema src gnome org>]

commit 8352d80de6c3587a40ef5e4cfc195a2afa53039c
Author: Jacob Boerema <jgboerema gmail com>
Date:   Wed Aug 3 17:32:34 2022 -0400

    plug-ins: fix #8411 crash when attempting to open PSD
    
    Apparently some layers in PSD files can have extra channels which we
    did not account for when computing destination offsets, causing crashes.
    
    So let's make sure we don't include the "extra" channels when computing
    the offsets by introducing base_channels. We also move the alpha channel
    in the spot of the first extra channel (even though it seems, based on
    the one example we have, that the extra channel might be the same as
    the alpha channel).
    
    (cherry picked from commit 5fbd06629de33191ba58450942aee4385829ef89)
    
    # Conflicts:
    #       plug-ins/file-psd/psd-load.c

 plug-ins/file-psd/psd-load.c | 48 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 40 insertions(+), 8 deletions(-)
---
diff --git a/plug-ins/file-psd/psd-load.c b/plug-ins/file-psd/psd-load.c
index 8d53d384c2..d5a0b2ad44 100644
--- a/plug-ins/file-psd/psd-load.c
+++ b/plug-ins/file-psd/psd-load.c
@@ -1268,7 +1268,7 @@ add_layers (gint32     image_id,
   gint32                parent_group_id = -1;
   guint16               alpha_chn;
   guint16               user_mask_chn;
-  guint16               layer_channels;
+  guint16               layer_channels, base_channels;
   guint16               channel_idx[MAX_CHANNELS];
   guint16              *rle_pack_len;
   guint16               bps;
@@ -1498,6 +1498,13 @@ add_layers (gint32     image_id,
           else
             parent_group_id = -1; /* root */
 
+          if (img_a->color_mode == PSD_CMYK)
+            base_channels = 4;
+          else if (img_a->color_mode == PSD_RGB || img_a->color_mode == PSD_LAB)
+            base_channels = 3;
+          else
+            base_channels = 1;
+
           IFDBG(3) g_debug ("Re-hash channel indices");
           for (cidx = 0; cidx < lyr_a[lidx]->num_channels; ++cidx)
             {
@@ -1513,15 +1520,40 @@ add_layers (gint32     image_id,
                 }
               else if (lyr_chn[cidx]->data)
                 {
-                  channel_idx[layer_channels] = cidx;   /* Assumes in sane order */
-                  layer_channels++;                     /* RGB, Lab, CMYK etc.   */
+                  if (layer_channels < base_channels)
+                    {
+                      channel_idx[layer_channels] = cidx;   /* Assumes in sane order */
+                      layer_channels++;                     /* RGB, Lab, CMYK etc.   */
+                    }
+                  else
+                    {
+                      /* channel_idx[base_channels] is reserved for alpha channel,
+                       * but this layer apparently has extra channels.
+                       * From the one example I have (see #8411) it looks like
+                       * that channel is the same as the alpha channel. */
+                      IFDBG(2) g_debug ("This layer has an extra channel (id: %d)", lyr_chn[cidx]->id);
+                      channel_idx[layer_channels+1] = cidx;   /* Assumes in sane order */
+                      layer_channels += 2;                    /* RGB, Lab, CMYK etc.   */
+                    }
+                }
+              else
+                {
+                  IFDBG(4) g_debug ("Channel %d (id: %d) has no data", cidx, lyr_chn[cidx]->id);
                 }
             }
 
           if (alpha)
             {
-              channel_idx[layer_channels] = alpha_chn;
-              layer_channels++;
+              if (layer_channels <= base_channels)
+                {
+                  channel_idx[layer_channels] = alpha_chn;
+                  layer_channels++;
+                }
+              else
+                {
+                  channel_idx[base_channels] = alpha_chn;
+                }
+              base_channels++;
             }
 
           /* Create the layer */
@@ -1687,15 +1719,15 @@ add_layers (gint32     image_id,
                           const GeglRectangle *roi      = &iter->items[0].roi;
                           guint8              *dst0     = iter->items[0].data;
                           gint                 src_step = bps;
-                          gint                 dst_step = bps * layer_channels;
+                          gint                 dst_step = bps * base_channels;
 
                           if (img_a->color_mode == PSD_CMYK)
                             {
-                              dst0 = gegl_scratch_alloc (layer_channels *
+                              dst0 = gegl_scratch_alloc (base_channels *
                                                          iter->length);
                             }
 
-                          for (cidx = 0; cidx < layer_channels; ++cidx)
+                          for (cidx = 0; cidx < base_channels; ++cidx)
                             {
                               gint b;