[pango] [HB] Fix possible int overflows during sanitize

[Behdad Esfahbod <behdad src gnome org>]

commit 10bc378f81fb2fd21fd7ef5f8d6242ad18b88132
Author: Behdad Esfahbod <behdad behdad org>
Date:   Fri Aug 14 17:31:16 2009 -0400

    [HB] Fix possible int overflows during sanitize

 pango/opentype/TODO                         |    1 -
 pango/opentype/hb-open-type-private.hh      |   22 ++++++++++++++++++++++
 pango/opentype/hb-ot-layout-gpos-private.hh |   15 ++++++++-------
 3 files changed, 30 insertions(+), 8 deletions(-)
---
diff --git a/pango/opentype/TODO b/pango/opentype/TODO
index 54560aa..d2b067d 100644
--- a/pango/opentype/TODO
+++ b/pango/opentype/TODO
@@ -3,4 +3,3 @@
 - cmap14 support in get_glyph callback
 - size_t?
 - Figure out compiler selection (add test for link to libstdc++)
-- Audit sanitize for int overflows
diff --git a/pango/opentype/hb-open-type-private.hh b/pango/opentype/hb-open-type-private.hh
index ae8dbb6..b5118fc 100644
--- a/pango/opentype/hb-open-type-private.hh
+++ b/pango/opentype/hb-open-type-private.hh
@@ -196,6 +196,26 @@ _hb_sanitize_check (SANITIZE_ARG_DEF,
 }
 
 static HB_GNUC_UNUSED inline bool
+_hb_sanitize_array (SANITIZE_ARG_DEF,
+		    const char *base,
+		    unsigned int record_size,
+		    unsigned int len)
+{
+  bool overflows = len >= ((unsigned int) -1) / record_size;
+
+#if HB_DEBUG
+  if (sanitize_depth < HB_DEBUG) \
+    fprintf (stderr, "SANITIZE(%p) %-*d-> array [%p..%p] (%d*%d=%ld bytes) in [%p..%p] -> %s\n", \
+	     base,
+	     sanitize_depth, sanitize_depth,
+	     base, base + (record_size * len), record_size, len, (unsigned long) record_size * len,
+	     context->start, context->end,
+	     !overflows ? "does not overflow" : "OVERFLOWS FAIL");
+#endif
+  return HB_LIKELY (!overflows) && _hb_sanitize_check (SANITIZE_ARG, base, record_size * len);
+}
+
+static HB_GNUC_UNUSED inline bool
 _hb_sanitize_edit (SANITIZE_ARG_DEF,
 		   const char *base HB_GNUC_UNUSED,
 		   unsigned int len HB_GNUC_UNUSED)
@@ -232,6 +252,8 @@ _hb_sanitize_edit (SANITIZE_ARG_DEF,
 /* TODO Optimize this if L is fixed (gcc magic) */
 #define SANITIZE_MEM(B,L) HB_LIKELY (_hb_sanitize_check (SANITIZE_ARG, CONST_CHARP(B), (L)))
 
+#define SANITIZE_ARRAY(A,S,L) HB_LIKELY (_hb_sanitize_array (SANITIZE_ARG, CONST_CHARP(A), S, L))
+
 #define NEUTER(Var, Val) \
 	(SANITIZE_OBJ (Var) && \
 	 _hb_sanitize_edit (SANITIZE_ARG, CONST_CHARP(&(Var)), sizeof (Var)) && \
diff --git a/pango/opentype/hb-ot-layout-gpos-private.hh b/pango/opentype/hb-ot-layout-gpos-private.hh
index 6b20716..f45c852 100644
--- a/pango/opentype/hb-ot-layout-gpos-private.hh
+++ b/pango/opentype/hb-ot-layout-gpos-private.hh
@@ -561,11 +561,12 @@ struct PairPosFormat2
 
   inline bool sanitize (SANITIZE_ARG_DEF) {
     SANITIZE_DEBUG ();
-    return SANITIZE_SELF () && SANITIZE_THIS (coverage) &&
-	   SANITIZE_THIS2 (classDef1, classDef2) &&
-	   SANITIZE_MEM (values,
-			 (valueFormat1.get_size () + valueFormat2.get_size ()) *
-			 class1Count * class2Count); /* XXX overflow (in other places too) */
+    if (!(SANITIZE_SELF () && SANITIZE_THIS (coverage) &&
+	  SANITIZE_THIS2 (classDef1, classDef2))) return false;
+
+    unsigned int record_size =valueFormat1.get_size () + valueFormat2.get_size ();
+    unsigned int len = class1Count * class2Count;
+    return SANITIZE_ARRAY (values, record_size, len);
   }
 
   private:
@@ -883,7 +884,7 @@ struct BaseArray
     SANITIZE_DEBUG ();
     if (!SANITIZE_SELF ()) return false;
     unsigned int count = cols * len;
-    if (!SANITIZE_MEM (matrix, sizeof (matrix[0]) * count)) return false;
+    if (!SANITIZE_ARRAY (matrix, sizeof (matrix[0]), count)) return false;
     for (unsigned int i = 0; i < count; i++)
       if (!SANITIZE_THIS (matrix[i])) return false;
     return true;
@@ -1175,7 +1176,7 @@ struct Mark2Array
     SANITIZE_DEBUG ();
     if (!SANITIZE_SELF ()) return false;
     unsigned int count = cols * len;
-    if (!SANITIZE_MEM (matrix, sizeof (matrix[0]) * count)) return false;
+    if (!SANITIZE_ARRAY (matrix, sizeof (matrix[0]), count)) return false;
     for (unsigned int i = 0; i < count; i++)
       if (!SANITIZE_THIS (matrix[i])) return false;
     return true;