[gimp] Bug 789613 - core: Fix integer overflow on 64 bit systems

[Michael Natterer <mitch src gnome org>]

commit f98d1b3a76e7031410d7b26bb74da23a1bb20a6c
Author: Tobias Stoeckmann <tobias stoeckmann org>
Date:   Tue Oct 31 12:02:26 2017 +0100

    Bug 789613 - core: Fix integer overflow on 64 bit systems
    
    The C language only promotes data values up to (un)signed int,
    which is 32 bit, if no larger data type is used within the
    calculation. Having a multiplication of two gint variables,
    even if the expected target variable is of type gsize (64 bit),
    leads to a possible integer overflow.
    
    This bug can be triggered in gimp_temp_buf_new, which is used
    to allocate memory for given supplied dimensions and bytes per
    pixel. If triggered, less memory than needed is allocated and
    therefore allows out of boundary accesses, either resulting in
    possible code execution or information leakage.
    
    While at it, make sure that the supplied format can actually be
    resolved to a bytes per pixel value. If not, return NULL.
    
    Signed-off-by: Tobias Stoeckmann <tobias stoeckmann org>

 app/core/gimptempbuf.c |   13 +++++++++----
 1 files changed, 9 insertions(+), 4 deletions(-)
---
diff --git a/app/core/gimptempbuf.c b/app/core/gimptempbuf.c
index f03e735..6ea2450 100644
--- a/app/core/gimptempbuf.c
+++ b/app/core/gimptempbuf.c
@@ -48,18 +48,22 @@ gimp_temp_buf_new (gint        width,
                    const Babl *format)
 {
   GimpTempBuf *temp;
+  gint         bpp;
 
-  g_return_val_if_fail (width > 0 && height > 0, NULL);
   g_return_val_if_fail (format != NULL, NULL);
 
+  bpp = babl_format_get_bytes_per_pixel (format);
+
+  g_return_val_if_fail (width > 0 && height > 0 && bpp > 0, NULL);
+  g_return_val_if_fail (G_MAXSIZE / width / height / bpp > 0, NULL);
+
   temp = g_slice_new (GimpTempBuf);
 
   temp->ref_count = 1;
   temp->width     = width;
   temp->height    = height;
   temp->format    = format;
-  temp->data      = gegl_malloc (width * height *
-                                 babl_format_get_bytes_per_pixel (format));
+  temp->data      = gegl_malloc ((gsize) width * height * bpp);
 
   return temp;
 }
@@ -246,7 +250,8 @@ gimp_temp_buf_get_data (const GimpTempBuf *buf)
 gsize
 gimp_temp_buf_get_data_size (const GimpTempBuf *buf)
 {
-  return babl_format_get_bytes_per_pixel (buf->format) * buf->width * buf->height;
+  return (gsize) babl_format_get_bytes_per_pixel (buf->format) *
+                 buf->width * buf->height;
 }
 
 guchar *