[gegl] ppm-load: CVE-2012-4433: don't overflow memory allocation

[Nils Philippsen <nphilipp src gnome org>]

commit 1e92e5235ded0415d555aa86066b8e4041ee5a53
Author: Nils Philippsen <nils redhat com>
Date:   Tue Oct 16 16:58:27 2012 +0200

    ppm-load: CVE-2012-4433: don't overflow memory allocation
    
    Carefully selected width/height values could cause the size of a later
    allocation to overflow, resulting in a buffer much too small to store
    the data which would then written beyond its end.

 operations/external/ppm-load.c |   29 +++++++++++++++++++++++++----
 1 files changed, 25 insertions(+), 4 deletions(-)
---
diff --git a/operations/external/ppm-load.c b/operations/external/ppm-load.c
index efe6d56..3d6bce7 100644
--- a/operations/external/ppm-load.c
+++ b/operations/external/ppm-load.c
@@ -84,7 +84,6 @@ ppm_load_read_header(FILE       *fp,
     /* Get Width and Height */
     img->width  = strtol (header,&ptr,0);
     img->height = atoi (ptr);
-    img->numsamples = img->width * img->height * CHANNEL_COUNT;
 
     fgets (header,MAX_CHARS_IN_ROW,fp);
     maxval = strtol (header,&ptr,0);
@@ -109,6 +108,16 @@ ppm_load_read_header(FILE       *fp,
       g_warning ("%s: Programmer stupidity error", G_STRLOC);
     }
 
+    /* Later on, img->numsamples is multiplied with img->bpc to allocate
+     * memory. Ensure it doesn't overflow. */
+    if (!img->width || !img->height ||
+        G_MAXSIZE / img->width / img->height / CHANNEL_COUNT < img->bpc)
+      {
+        g_warning ("Illegal width/height: %ld/%ld", img->width, img->height);
+        return FALSE;
+      }
+    img->numsamples = img->width * img->height * CHANNEL_COUNT;
+
     return TRUE;
 }
 
@@ -229,12 +238,24 @@ process (GeglOperation       *operation,
   if (!ppm_load_read_header (fp, &img))
     goto out;
 
-  rect.height = img.height;
-  rect.width = img.width;
-
   /* Allocating Array Size */
+
+  /* Should use g_try_malloc(), but this causes crashes elsewhere because the
+   * error signalled by returning FALSE isn't properly acted upon. Therefore
+   * g_malloc() is used here which aborts if the requested memory size can't be
+   * allocated causing a controlled crash. */
   img.data = (guchar*) g_malloc (img.numsamples * img.bpc);
 
+  /* No-op without g_try_malloc(), see above. */
+  if (! img.data)
+    {
+      g_warning ("Couldn't allocate %" G_GSIZE_FORMAT " bytes, giving up.", ((gsize)img.numsamples * img.bpc));
+      goto out;
+    }
+
+  rect.height = img.height;
+  rect.width = img.width;
+
   switch (img.bpc)
     {
     case 1: