[gdk-pixbuf] pixops: Fail make_weights functions on OOM

[Benjamin Otte <otte src gnome org>]

commit 19f9685dbff7d1f929c61cf99188df917a18811d
Author: Benjamin Otte <otte redhat com>
Date:   Sat Sep 19 21:24:34 2015 +0200

    pixops: Fail make_weights functions on OOM
    
    The weights could grow very large under certain circumstances, in
    particular in security-relevant conditions, including the testsuite.
    By allowing the weight allocation to fail, this can be worked around.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=754387

 gdk-pixbuf/pixops/pixops.c |   75 ++++++++++++++++++++++++++++++++------------
 1 files changed, 55 insertions(+), 20 deletions(-)
---
diff --git a/gdk-pixbuf/pixops/pixops.c b/gdk-pixbuf/pixops/pixops.c
index e41b286..4cdb5df 100644
--- a/gdk-pixbuf/pixops/pixops.c
+++ b/gdk-pixbuf/pixops/pixops.c
@@ -1478,15 +1478,19 @@ pixops_process (guchar         *dest_buf,
 /* Compute weights for reconstruction by replication followed by
  * sampling with a box filter
  */
-static void
+static gboolean
 tile_make_weights (PixopsFilterDimension *dim,
                   double                 scale)
 {
   int n = ceil (1 / scale + 1);
-  double *pixel_weights = g_malloc_n (sizeof (double) * SUBSAMPLE, n);
+  double *pixel_weights;
   int offset;
   int i;
 
+  pixel_weights = g_try_malloc_n (sizeof (double) * SUBSAMPLE, n);
+  if (pixel_weights == NULL)
+    return FALSE;
+
   dim->n = n;
   dim->offset = 0;
   dim->weights = pixel_weights;
@@ -1514,13 +1518,15 @@ tile_make_weights (PixopsFilterDimension *dim,
             }
        }
     }
+
+  return TRUE;
 }
 
 /* Compute weights for a filter that, for minification
  * is the same as 'tiles', and for magnification, is bilinear
  * reconstruction followed by a sampling with a delta function.
  */
-static void
+static gboolean
 bilinear_magnify_make_weights (PixopsFilterDimension *dim,
                               double                 scale)
 {
@@ -1541,7 +1547,9 @@ bilinear_magnify_make_weights (PixopsFilterDimension *dim,
     }
 
   dim->n = n;
-  dim->weights = g_malloc_n (sizeof (double) * SUBSAMPLE, n);
+  dim->weights = g_try_malloc_n (sizeof (double) * SUBSAMPLE, n);
+  if (dim->weights == NULL)
+    return FALSE;
 
   pixel_weights = dim->weights;
 
@@ -1581,6 +1589,8 @@ bilinear_magnify_make_weights (PixopsFilterDimension *dim,
             }
         }
     }
+
+  return TRUE;
 }
 
 /* Computes the integral from b0 to b1 of
@@ -1627,15 +1637,19 @@ linear_box_half (double b0, double b1)
 /* Compute weights for reconstructing with bilinear
  * interpolation, then sampling with a box filter
  */
-static void
+static gboolean
 bilinear_box_make_weights (PixopsFilterDimension *dim,
                           double                 scale)
 {
   int n = ceil (1/scale + 3.0);
-  double *pixel_weights = g_malloc_n (sizeof (double) * SUBSAMPLE, n);
+  double *pixel_weights;
   double w;
   int offset, i;
 
+  pixel_weights = g_malloc_n (sizeof (double) * SUBSAMPLE, n);
+  if (pixel_weights == NULL)
+    return FALSE;
+
   dim->offset = -1.0;
   dim->n = n;
   dim->weights = pixel_weights;
@@ -1653,9 +1667,11 @@ bilinear_box_make_weights (PixopsFilterDimension *dim,
           *(pixel_weights++) = w * scale;
         }
     }
+
+  return TRUE;
 }
 
-static void
+static gboolean
 make_weights (PixopsFilter     *filter,
              PixopsInterpType  interp_type,          
              double            scale_x,
@@ -1664,23 +1680,39 @@ make_weights (PixopsFilter     *filter,
   switch (interp_type)
     {
     case PIXOPS_INTERP_NEAREST:
+    default:
       g_assert_not_reached ();
-      break;
+      return FALSE;
 
     case PIXOPS_INTERP_TILES:
-      tile_make_weights (&filter->x, scale_x);
-      tile_make_weights (&filter->y, scale_y);
-      break;
+      if (!tile_make_weights (&filter->x, scale_x))
+        return FALSE;
+      if (!tile_make_weights (&filter->y, scale_y))
+        {
+          g_free (filter->x.weights);
+          return FALSE;
+        }
+      return TRUE;
       
     case PIXOPS_INTERP_BILINEAR:
-      bilinear_magnify_make_weights (&filter->x, scale_x);
-      bilinear_magnify_make_weights (&filter->y, scale_y);
-      break;
+      if (!bilinear_magnify_make_weights (&filter->x, scale_x))
+        return FALSE;
+      if (!bilinear_magnify_make_weights (&filter->y, scale_y))
+        {
+          g_free (filter->x.weights);
+          return FALSE;
+        }
+      return TRUE;
       
     case PIXOPS_INTERP_HYPER:
-      bilinear_box_make_weights (&filter->x, scale_x);
-      bilinear_box_make_weights (&filter->y, scale_y);
-      break;
+      if (!bilinear_box_make_weights (&filter->x, scale_x))
+        return FALSE;
+      if (!bilinear_box_make_weights (&filter->y, scale_y))
+        {
+          g_free (filter->x.weights);
+          return FALSE;
+        }
+      return TRUE;
     }
 }
 
@@ -1735,7 +1767,8 @@ _pixops_composite_color_real (guchar          *dest_buf,
     }
   
   filter.overall_alpha = overall_alpha / 255.;
-  make_weights (&filter, interp_type, scale_x, scale_y);
+  if (!make_weights (&filter, interp_type, scale_x, scale_y))
+    return;
 
 #ifdef USE_MMX
   if (filter.x.n == 2 && filter.y.n == 2 &&
@@ -1890,7 +1923,8 @@ _pixops_composite_real (guchar          *dest_buf,
     }
   
   filter.overall_alpha = overall_alpha / 255.;
-  make_weights (&filter, interp_type, scale_x, scale_y);
+  if (!make_weights (&filter, interp_type, scale_x, scale_y))
+    return;
 
   if (filter.x.n == 2 && filter.y.n == 2 && dest_channels == 4 &&
       src_channels == 4 && src_has_alpha && !dest_has_alpha)
@@ -2297,7 +2331,8 @@ _pixops_scale_real (guchar        *dest_buf,
     }
   
   filter.overall_alpha = 1.0;
-  make_weights (&filter, interp_type, scale_x, scale_y);
+  if (!make_weights (&filter, interp_type, scale_x, scale_y))
+    return;
 
   if (filter.x.n == 2 && filter.y.n == 2 && dest_channels == 3 && src_channels == 3)
     {