[gdk-pixbuf] pixops: Be more careful about integer overflow

[Matthias Clasen <matthiasc src gnome org>]

commit ffec86ed5010c5a2be14f47b33bcf4ed3169a199
Author: Matthias Clasen <mclasen redhat com>
Date:   Mon Jul 13 00:33:40 2015 -0400

    pixops: Be more careful about integer overflow
    
    Our loader code is supposed to handle out-of-memory and overflow
    situations gracefully, reporting errors instead of aborting. But
    if you load an image at a specific size, we also execute our
    scaling code, which was not careful enough about overflow in some
    places.
    
    This commit makes the scaling code silently return if it fails to
    allocate filter tables. This is the best we can do, since
    gdk_pixbuf_scale() is not taking a GError.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=752297

 gdk-pixbuf/pixops/pixops.c |   22 +++++++++++++++++-----
 1 files changed, 17 insertions(+), 5 deletions(-)
---
diff --git a/gdk-pixbuf/pixops/pixops.c b/gdk-pixbuf/pixops/pixops.c
index 29a1c14..ce51745 100644
--- a/gdk-pixbuf/pixops/pixops.c
+++ b/gdk-pixbuf/pixops/pixops.c
@@ -1272,7 +1272,16 @@ make_filter_table (PixopsFilter *filter)
   int i_offset, j_offset;
   int n_x = filter->x.n;
   int n_y = filter->y.n;
-  int *weights = g_new (int, SUBSAMPLE * SUBSAMPLE * n_x * n_y);
+  gsize n_weights;
+  int *weights;
+
+  n_weights = SUBSAMPLE * SUBSAMPLE * n_x * n_y;
+  if (n_weights / (SUBSAMPLE * SUBSAMPLE * n_x) != n_y)
+    return NULL; /* overflow, bail */
+
+  weights = g_try_new (int, n_weights);
+  if (!weights)
+    return NULL; /* overflow, bail */
 
   for (i_offset=0; i_offset < SUBSAMPLE; i_offset++)
     for (j_offset=0; j_offset < SUBSAMPLE; j_offset++)
@@ -1347,8 +1356,11 @@ pixops_process (guchar         *dest_buf,
   if (x_step == 0 || y_step == 0)
     return; /* overflow, bail out */
 
-  line_bufs = g_new (guchar *, filter->y.n);
   filter_weights = make_filter_table (filter);
+  if (!filter_weights)
+    return; /* overflow, bail out */
+
+  line_bufs = g_new (guchar *, filter->y.n);
 
   check_shift = check_size ? get_check_shift (check_size) : 0;
 
@@ -1468,7 +1480,7 @@ tile_make_weights (PixopsFilterDimension *dim,
                   double                 scale)
 {
   int n = ceil (1 / scale + 1);
-  double *pixel_weights = g_new (double, SUBSAMPLE * n);
+  double *pixel_weights = g_malloc_n (sizeof (double) * SUBSAMPLE, n);
   int offset;
   int i;
 
@@ -1526,7 +1538,7 @@ bilinear_magnify_make_weights (PixopsFilterDimension *dim,
     }
 
   dim->n = n;
-  dim->weights = g_new (double, SUBSAMPLE * n);
+  dim->weights = g_malloc_n (sizeof (double) * SUBSAMPLE, n);
 
   pixel_weights = dim->weights;
 
@@ -1617,7 +1629,7 @@ bilinear_box_make_weights (PixopsFilterDimension *dim,
                           double                 scale)
 {
   int n = ceil (1/scale + 3.0);
-  double *pixel_weights = g_new (double, SUBSAMPLE * n);
+  double *pixel_weights = g_malloc_n (sizeof (double) * SUBSAMPLE, n);
   double w;
   int offset, i;