[gimp/gimp-2-6] Bug 687392 - Memory corruption vulnerability when reading XWD files

[Nils Philippsen <nphilipp src gnome org>]

commit e525cf82fb25873df4e1ca5bbc0eb2340df09095
Author: Michael Natterer <mitch gimp org>
Date:   Thu Nov 8 00:16:31 2012 +0100

    Bug 687392 - Memory corruption vulnerability when reading XWD files
    
    Applied and enhanced patch from andres which makes file-xwd detect
    this kind of file corruption and abort loading with an error message.
    (cherry picked from commit 0b35f6a082a0b3c372c568ea6bde39a4796acde2)

 plug-ins/common/file-xwd.c |   78 +++++++++++++++++++++++++++++--------------
 1 files changed, 52 insertions(+), 26 deletions(-)
---
diff --git a/plug-ins/common/file-xwd.c b/plug-ins/common/file-xwd.c
index 0eb82af..e4866a6 100644
--- a/plug-ins/common/file-xwd.c
+++ b/plug-ins/common/file-xwd.c
@@ -186,11 +186,13 @@ static gint32 load_xwd_f2_d16_b16 (const gchar *,
 static gint32 load_xwd_f2_d24_b32 (const gchar *,
                                    FILE *,
                                    L_XWDFILEHEADER *,
-                                   L_XWDCOLOR *);
+                                   L_XWDCOLOR *,
+                                   GError **);
 static gint32 load_xwd_f1_d24_b1  (const gchar *,
                                    FILE *,
                                    L_XWDFILEHEADER *,
-                                   L_XWDCOLOR *);
+                                   L_XWDCOLOR *,
+                                   GError **);
 
 static L_CARD32 read_card32  (FILE *,
                               gint *);
@@ -536,7 +538,8 @@ load_image (const gchar  *filename,
     case 1:    /* Single plane pixmap */
       if ((depth <= 24) && (bpp == 1))
         {
-          image_ID = load_xwd_f1_d24_b1 (filename, ifp, &xwdhdr, xwdcolmap);
+          image_ID = load_xwd_f1_d24_b1 (filename, ifp, &xwdhdr, xwdcolmap,
+                                         error);
         }
       break;
 
@@ -555,7 +558,8 @@ load_image (const gchar  *filename,
         }
       else if ((depth <= 24) && ((bpp == 24) || (bpp == 32)))
         {
-          image_ID = load_xwd_f2_d24_b32 (filename, ifp, &xwdhdr, xwdcolmap);
+          image_ID = load_xwd_f2_d24_b32 (filename, ifp, &xwdhdr, xwdcolmap,
+                                          error);
         }
       break;
     }
@@ -565,7 +569,7 @@ load_image (const gchar  *filename,
   if (xwdcolmap)
     g_free (xwdcolmap);
 
-  if (image_ID == -1)
+  if (image_ID == -1 && ! (error && *error))
     g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
                  _("XWD-file %s has format %d, depth %d and bits per pixel %d. "
                    "Currently this is not supported."),
@@ -1615,10 +1619,11 @@ load_xwd_f2_d16_b16 (const gchar     *filename,
 /* Load XWD with pixmap_format 2, pixmap_depth up to 24, bits_per_pixel 24/32 */
 
 static gint32
-load_xwd_f2_d24_b32 (const gchar     *filename,
-                     FILE            *ifp,
-                     L_XWDFILEHEADER *xwdhdr,
-                     L_XWDCOLOR      *xwdcolmap)
+load_xwd_f2_d24_b32 (const gchar      *filename,
+                     FILE             *ifp,
+                     L_XWDFILEHEADER  *xwdhdr,
+                     L_XWDCOLOR       *xwdcolmap,
+                     GError          **error)
 {
   register guchar *dest, lsbyte_first;
   gint             width, height, linepad, i, j, c0, c1, c2, c3;
@@ -1643,12 +1648,6 @@ load_xwd_f2_d24_b32 (const gchar     *filename,
   width  = xwdhdr->l_pixmap_width;
   height = xwdhdr->l_pixmap_height;
 
-  image_ID = create_new_image (filename, width, height, GIMP_RGB,
-                               &layer_ID, &drawable, &pixel_rgn);
-
-  tile_height = gimp_tile_height ();
-  data = g_malloc (tile_height * width * 3);
-
   redmask   = xwdhdr->l_red_mask;
   greenmask = xwdhdr->l_green_mask;
   bluemask  = xwdhdr->l_blue_mask;
@@ -1676,6 +1675,22 @@ load_xwd_f2_d24_b32 (const gchar     *filename,
   maxblue = 0; while (bluemask >> (blueshift + maxblue)) maxblue++;
   maxblue = (1 << maxblue) - 1;
 
+  if (maxred   > sizeof (redmap)   ||
+      maxgreen > sizeof (greenmap) ||
+      maxblue  > sizeof (bluemap))
+    {
+      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                   _("XWD-file %s is corrupt."),
+                   gimp_filename_to_utf8 (filename));
+      return -1;
+    }
+
+  image_ID = create_new_image (filename, width, height, GIMP_RGB,
+                               &layer_ID, &drawable, &pixel_rgn);
+
+  tile_height = gimp_tile_height ();
+  data = g_malloc (tile_height * width * 3);
+
   /* Set map-arrays for red, green, blue */
   for (red = 0; red <= maxred; red++)
     redmap[red] = (red * 255) / maxred;
@@ -1815,10 +1830,11 @@ load_xwd_f2_d24_b32 (const gchar     *filename,
 /* Load XWD with pixmap_format 1, pixmap_depth up to 24, bits_per_pixel 1 */
 
 static gint32
-load_xwd_f1_d24_b1 (const gchar     *filename,
-                    FILE            *ifp,
-                    L_XWDFILEHEADER *xwdhdr,
-                    L_XWDCOLOR      *xwdcolmap)
+load_xwd_f1_d24_b1 (const gchar      *filename,
+                    FILE             *ifp,
+                    L_XWDFILEHEADER  *xwdhdr,
+                    L_XWDCOLOR       *xwdcolmap,
+                    GError          **error)
 {
   register guchar *dest, outmask, inmask, do_reverse;
   gint             width, height, i, j, plane, fromright;
@@ -1853,13 +1869,6 @@ load_xwd_f1_d24_b1 (const gchar     *filename,
   indexed         = (xwdhdr->l_pixmap_depth <= 8);
   bytes_per_pixel = (indexed ? 1 : 3);
 
-  image_ID = create_new_image (filename, width, height,
-                               indexed ? GIMP_INDEXED : GIMP_RGB,
-                               &layer_ID, &drawable, &pixel_rgn);
-
-  tile_height = gimp_tile_height ();
-  data = g_malloc (tile_height * width * bytes_per_pixel);
-
   for (j = 0; j < 256; j++)   /* Create an array for reversing bits */
     {
       inmask = 0;
@@ -1903,6 +1912,16 @@ load_xwd_f1_d24_b1 (const gchar     *filename,
       maxblue = 0; while (bluemask >> (blueshift + maxblue)) maxblue++;
       maxblue = (1 << maxblue) - 1;
 
+      if (maxred   > sizeof (redmap)   ||
+          maxgreen > sizeof (greenmap) ||
+          maxblue  > sizeof (bluemap))
+        {
+          g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                       _("XWD-file %s is corrupt."),
+                       gimp_filename_to_utf8 (filename));
+          return -1;
+        }
+
       /* Set map-arrays for red, green, blue */
       for (red = 0; red <= maxred; red++)
         redmap[red] = (red * 255) / maxred;
@@ -1912,6 +1931,13 @@ load_xwd_f1_d24_b1 (const gchar     *filename,
         bluemap[blue] = (blue * 255) / maxblue;
     }
 
+  image_ID = create_new_image (filename, width, height,
+                               indexed ? GIMP_INDEXED : GIMP_RGB,
+                               &layer_ID, &drawable, &pixel_rgn);
+
+  tile_height = gimp_tile_height ();
+  data = g_malloc (tile_height * width * bytes_per_pixel);
+
   ncols = xwdhdr->l_colormap_entries;
   if (xwdhdr->l_ncolors < ncols)
     ncols = xwdhdr->l_ncolors;