[gegl] buffer: fix 32bit integer boxfilter overflow

[Øyvind "pippin" Kolås <ok src gnome org>]

commit 687508cc37902287e0b66d291320a65d47315ed7
Author: Øyvind Kolås <pippin gimp org>
Date:   Fri Oct 12 15:44:37 2018 +0200

    buffer: fix 32bit integer boxfilter overflow
    
    In the existing implementation we overflow due to the imprecision introduced by
    using single precision floating point in the template. We introduce a temp type
    that can be used for computations, as well as do an upper bounds check when
    doing the rounding/converting of the final result. This fixes overflows but not
    the imprecision; that only applies for 32bit integer precision and in code
    paths primarily used for display rather than processing.
    
    Fixing issue #103

 gegl/buffer/gegl-algorithms-boxfilter.inc | 70 ++++++++++++++++---------------
 gegl/buffer/gegl-algorithms.c             | 19 ++++++++-
 2 files changed, 54 insertions(+), 35 deletions(-)
---
diff --git a/gegl/buffer/gegl-algorithms-boxfilter.inc b/gegl/buffer/gegl-algorithms-boxfilter.inc
index 5a5704f43..778bee844 100644
--- a/gegl/buffer/gegl-algorithms-boxfilter.inc
+++ b/gegl/buffer/gegl-algorithms-boxfilter.inc
@@ -8,6 +8,7 @@ BOXFILTER_FUNCNAME (guchar                    *dest_buf,
                     const gint                 bpp,
                     const gint                 d_rowstride)
 {
+#define TT BOXFILTER_TEMP_TYPE
   const BOXFILTER_TYPE *src[9];
   gint  components = bpp / sizeof(BOXFILTER_TYPE);
 
@@ -36,7 +37,7 @@ BOXFILTER_FUNCNAME (guchar                    *dest_buf,
       gfloat top_weight, middle_weight, bottom_weight;
       const gfloat sy = (dst_rect->y + y + .5) / scale - src_rect->y;
       const gint     ii = int_floorf (sy);
-      BOXFILTER_TYPE             *dst = (BOXFILTER_TYPE*)(dest_buf + y * d_rowstride);
+      BOXFILTER_TYPE *dst = (BOXFILTER_TYPE*)(dest_buf + y * d_rowstride);
       const guchar  *src_base = source_buf + ii * s_rowstride;
 
       top_weight    = .5 - scale * (sy - ii);
@@ -82,21 +83,21 @@ BOXFILTER_FUNCNAME (guchar                    *dest_buf,
               const gfloat b = bottom_weight;
 
               dst[0] = BOXFILTER_ROUND(
-                (src[0][0] * t + src[3][0] * m + src[6][0] * b) * l +
-                (src[1][0] * t + src[4][0] * m + src[7][0] * b) * c +
-                (src[2][0] * t + src[5][0] * m + src[8][0] * b) * r);
+                ((TT)src[0][0] * t + (TT)src[3][0] * m + (TT)src[6][0] * b) * l +
+                ((TT)src[1][0] * t + (TT)src[4][0] * m + (TT)src[7][0] * b) * c +
+                ((TT)src[2][0] * t + (TT)src[5][0] * m + (TT)src[8][0] * b) * r);
               dst[1] = BOXFILTER_ROUND(
-                (src[0][1] * t + src[3][1] * m + src[6][1] * b) * l +
-                (src[1][1] * t + src[4][1] * m + src[7][1] * b) * c +
-                (src[2][1] * t + src[5][1] * m + src[8][1] * b) * r);
+                ((TT)src[0][1] * t + (TT)src[3][1] * m + (TT)src[6][1] * b) * l +
+                ((TT)src[1][1] * t + (TT)src[4][1] * m + (TT)src[7][1] * b) * c +
+                ((TT)src[2][1] * t + (TT)src[5][1] * m + (TT)src[8][1] * b) * r);
               dst[2] = BOXFILTER_ROUND(
-                (src[0][2] * t + src[3][2] * m + src[6][2] * b) * l +
-                (src[1][2] * t + src[4][2] * m + src[7][2] * b) * c +
-                (src[2][2] * t + src[5][2] * m + src[8][2] * b) * r);
+                ((TT)src[0][2] * t + (TT)src[3][2] * m + (TT)src[6][2] * b) * l +
+                ((TT)src[1][2] * t + (TT)src[4][2] * m + (TT)src[7][2] * b) * c +
+                ((TT)src[2][2] * t + (TT)src[5][2] * m + (TT)src[8][2] * b) * r);
               dst[3] = BOXFILTER_ROUND(
-                (src[0][3] * t + src[3][3] * m + src[6][3] * b) * l +
-                (src[1][3] * t + src[4][3] * m + src[7][3] * b) * c +
-                (src[2][3] * t + src[5][3] * m + src[8][3] * b) * r);
+                ((TT)src[0][3] * t + (TT)src[3][3] * m + (TT)src[6][3] * b) * l +
+                ((TT)src[1][3] * t + (TT)src[4][3] * m + (TT)src[7][3] * b) * c +
+                ((TT)src[2][3] * t + (TT)src[5][3] * m + (TT)src[8][3] * b) * r);
               }
             dst += 4;
             }
@@ -123,17 +124,17 @@ BOXFILTER_FUNCNAME (guchar                    *dest_buf,
               const gfloat b = bottom_weight;
 
               dst[0] = BOXFILTER_ROUND(
-                (src[0][0] * t + src[3][0] * m + src[6][0] * b) * l +
-                (src[1][0] * t + src[4][0] * m + src[7][0] * b) * c +
-                (src[2][0] * t + src[5][0] * m + src[8][0] * b) * r);
+                ((TT)src[0][0] * t + (TT)src[3][0] * m + (TT)src[6][0] * b) * l +
+                ((TT)src[1][0] * t + (TT)src[4][0] * m + (TT)src[7][0] * b) * c +
+                ((TT)src[2][0] * t + (TT)src[5][0] * m + (TT)src[8][0] * b) * r);
               dst[1] = BOXFILTER_ROUND(
-                (src[0][1] * t + src[3][1] * m + src[6][1] * b) * l +
-                (src[1][1] * t + src[4][1] * m + src[7][1] * b) * c +
-                (src[2][1] * t + src[5][1] * m + src[8][1] * b) * r);
+                ((TT)src[0][1] * t + (TT)src[3][1] * m + (TT)src[6][1] * b) * l +
+                ((TT)src[1][1] * t + (TT)src[4][1] * m + (TT)src[7][1] * b) * c +
+                ((TT)src[2][1] * t + (TT)src[5][1] * m + (TT)src[8][1] * b) * r);
               dst[2] = BOXFILTER_ROUND(
-                (src[0][2] * t + src[3][2] * m + src[6][2] * b) * l +
-                (src[1][2] * t + src[4][2] * m + src[7][2] * b) * c +
-                (src[2][2] * t + src[5][2] * m + src[8][2] * b) * r);
+                ((TT)src[0][2] * t + (TT)src[3][2] * m + (TT)src[6][2] * b) * l +
+                ((TT)src[1][2] * t + (TT)src[4][2] * m + (TT)src[7][2] * b) * c +
+                ((TT)src[2][2] * t + (TT)src[5][2] * m + (TT)src[8][2] * b) * r);
             }
             dst += 3;
             }
@@ -160,13 +161,13 @@ BOXFILTER_FUNCNAME (guchar                    *dest_buf,
               const gfloat b = bottom_weight;
 
               dst[0] = BOXFILTER_ROUND(
-                (src[0][0] * t + src[3][0] * m + src[6][0] * b) * l +
-                (src[1][0] * t + src[4][0] * m + src[7][0] * b) * c +
-                (src[2][0] * t + src[5][0] * m + src[8][0] * b) * r);
+                ((TT)src[0][0] * t + (TT)src[3][0] * m + (TT)src[6][0] * b) * l +
+                ((TT)src[1][0] * t + (TT)src[4][0] * m + (TT)src[7][0] * b) * c +
+                ((TT)src[2][0] * t + (TT)src[5][0] * m + (TT)src[8][0] * b) * r);
               dst[1] = BOXFILTER_ROUND(
-                (src[0][1] * t + src[3][1] * m + src[6][1] * b) * l +
-                (src[1][1] * t + src[4][1] * m + src[7][1] * b) * c +
-                (src[2][1] * t + src[5][1] * m + src[8][1] * b) * r);
+                ((TT)src[0][1] * t + (TT)src[3][1] * m + (TT)src[6][1] * b) * l +
+                ((TT)src[1][1] * t + (TT)src[4][1] * m + (TT)src[7][1] * b) * c +
+                ((TT)src[2][1] * t + (TT)src[5][1] * m + (TT)src[8][1] * b) * r);
             }
             dst += 2;
             }
@@ -193,9 +194,9 @@ BOXFILTER_FUNCNAME (guchar                    *dest_buf,
               const gfloat b = bottom_weight;
 
               dst[0] = BOXFILTER_ROUND(
-                (src[0][0] * t + src[3][0] * m + src[6][0] * b) * l +
-                (src[1][0] * t + src[4][0] * m + src[7][0] * b) * c +
-                (src[2][0] * t + src[5][0] * m + src[8][0] * b) * r);
+                ((TT)src[0][0] * t + (TT)src[3][0] * m + (TT)src[6][0] * b) * l +
+                ((TT)src[1][0] * t + (TT)src[4][0] * m + (TT)src[7][0] * b) * c +
+                ((TT)src[2][0] * t + (TT)src[5][0] * m + (TT)src[8][0] * b) * r);
             }
             dst += 1;
             }
@@ -224,9 +225,9 @@ BOXFILTER_FUNCNAME (guchar                    *dest_buf,
               for (gint i = 0; i < components; ++i)
                 {
                   dst[i] = BOXFILTER_ROUND(
-                    (src[0][i] * t + src[3][i] * m + src[6][i] * b) * l +
-                    (src[1][i] * t + src[4][i] * m + src[7][i] * b) * c +
-                    (src[2][i] * t + src[5][i] * m + src[8][i] * b) * r);
+                    ((TT)src[0][i] * t + (TT)src[3][i] * m + (TT)src[6][i] * b) * l +
+                    ((TT)src[1][i] * t + (TT)src[4][i] * m + (TT)src[7][i] * b) * c +
+                    ((TT)src[2][i] * t + (TT)src[5][i] * m + (TT)src[8][i] * b) * r);
                 }
               }
             dst += components;
@@ -234,4 +235,5 @@ BOXFILTER_FUNCNAME (guchar                    *dest_buf,
         break;
       }
     }
+#undef TT
 }
diff --git a/gegl/buffer/gegl-algorithms.c b/gegl/buffer/gegl-algorithms.c
index ad76681de..dcb8db7a5 100644
--- a/gegl/buffer/gegl-algorithms.c
+++ b/gegl/buffer/gegl-algorithms.c
@@ -1289,41 +1289,58 @@ gegl_resample_nearest (guchar              *dst,
 
 #define BOXFILTER_FUNCNAME   gegl_resample_boxfilter_double
 #define BOXFILTER_TYPE       gdouble
+#define BOXFILTER_TEMP_TYPE  gdouble
 #define BOXFILTER_ROUND(val) (val)
 #include "gegl-algorithms-boxfilter.inc"
 #undef BOXFILTER_FUNCNAME
 #undef BOXFILTER_TYPE
+#undef BOXFILTER_TEMP_TYPE
 #undef BOXFILTER_ROUND
 
 #define BOXFILTER_FUNCNAME   gegl_resample_boxfilter_float
 #define BOXFILTER_TYPE       gfloat
+#define BOXFILTER_TEMP_TYPE  gfloat
 #define BOXFILTER_ROUND(val) (val)
 #include "gegl-algorithms-boxfilter.inc"
 #undef BOXFILTER_FUNCNAME
 #undef BOXFILTER_TYPE
+#undef BOXFILTER_TEMP_TYPE
 #undef BOXFILTER_ROUND
 
 #define BOXFILTER_FUNCNAME   gegl_resample_boxfilter_u8
 #define BOXFILTER_TYPE       guchar
+#define BOXFILTER_TEMP_TYPE  guchar
 #define BOXFILTER_ROUND(val) ((int)((val)+0.5))
 #include "gegl-algorithms-boxfilter.inc"
 #undef BOXFILTER_FUNCNAME
 #undef BOXFILTER_TYPE
+#undef BOXFILTER_TEMP_TYPE
 #undef BOXFILTER_ROUND
 
 #define BOXFILTER_FUNCNAME   gegl_resample_boxfilter_u16
 #define BOXFILTER_TYPE       guint16
+#define BOXFILTER_TEMP_TYPE  guint16
 #define BOXFILTER_ROUND(val) ((int)((val)+0.5))
 #include "gegl-algorithms-boxfilter.inc"
 #undef BOXFILTER_FUNCNAME
 #undef BOXFILTER_TYPE
+#undef BOXFILTER_TEMP_TYPE
 #undef BOXFILTER_ROUND
 
+static inline guint32 _gegl_trunc_u32(guint64 value)
+{
+  if ((guint64) value > ((1l<<32)-1))
+   return (1l<<32)-1;
+  return value;
+}
+
 #define BOXFILTER_FUNCNAME   gegl_resample_boxfilter_u32
 #define BOXFILTER_TYPE       guint32
-#define BOXFILTER_ROUND(val) ((int)((val)+0.5))
+#define BOXFILTER_TEMP_TYPE  guint64
+#define BOXFILTER_ROUND(val) _gegl_trunc_u32((val)+0.5)
 #include "gegl-algorithms-boxfilter.inc"
 #undef BOXFILTER_FUNCNAME
+#undef BOXFILTER_TEMP_TYPE
 #undef BOXFILTER_TYPE
 #undef BOXFILTER_ROUND