[glib: 1/2] gvariant-parser: Fix error handling when type coalescing fails

[Philip Withnall <pwithnall src gnome org>]

commit c927c59a8426be62f9e9b19e40c862f88b4c17be
Author: Philip Withnall <withnall endlessm com>
Date:   Tue Feb 5 15:50:15 2019 +0000

    gvariant-parser: Fix error handling when type coalescing fails
    
    When parsing GVariant text format strings, we do a limited form of type
    inference. The algorithm for type inference for nested array child types
    is not complete, however (and making it complete, at least with a naive
    implementation, would make it O(N^2), which is not worth it) and so some
    text format arrays were triggering an assertion failure in the error
    handling code.
    
    Fix that by making the error handling code a little more relaxed, in the
    knowledge that our type inference algorithm is not complete. See the
    comment added to the code.
    
    This includes a test case, provided by oss-fuzz.
    
    oss-fuzz#11578
    
    Signed-off-by: Philip Withnall <withnall endlessm com>

 glib/gvariant-parser.c | 28 ++++++++++++++++++++++++++--
 glib/tests/gvariant.c  |  1 +
 2 files changed, 27 insertions(+), 2 deletions(-)
---
diff --git a/glib/gvariant-parser.c b/glib/gvariant-parser.c
index 2b02ec90f..3b4956a9c 100644
--- a/glib/gvariant-parser.c
+++ b/glib/gvariant-parser.c
@@ -671,6 +671,20 @@ ast_array_get_pattern (AST    **array,
   gchar *pattern;
   gint i;
 
+  /* Find the pattern which applies to all children in the array, by l-folding a
+   * coalesce operation. This will not always work: for example, the GVariant:
+   *    [[0], [], [nothing]]
+   * has patterns:
+   *    MaMN, Ma*, Mam*
+   * which pairwise coalesce as:
+   *    MaMN + Ma* = MaN
+   *    MaN + Mam* = (doesn’t coalesce)
+   *
+   * However, the pattern MamN coalesces with all three child patterns. Finding
+   * this pattern would require trying all O(n_items^2) pairs, though, which is
+   * expensive. Just let it fail, and require the user to provide type
+   * annotations.
+   */
   pattern = ast_get_pattern (array[0], error);
 
   if (pattern == NULL)
@@ -705,8 +719,18 @@ ast_array_get_pattern (AST    **array,
               gchar *tmp2;
               gchar *m;
 
-              /* if 'j' reaches 'i' then we failed to find the pair */
-              g_assert (j < i);
+              /* if 'j' reaches 'i' then we failed to find the pair, which can
+               * happen due to only trying pairwise coalesces in order rather
+               * than between all pairs (see above). so just report an error
+               * for i. */
+              if (j >= i)
+                {
+                  ast_set_error (array[i], error, NULL,
+                                 G_VARIANT_PARSE_ERROR_NO_COMMON_TYPE,
+                                 "unable to find a common type");
+                  g_free (tmp);
+                  return NULL;
+                }
 
               tmp2 = ast_get_pattern (array[j], NULL);
               g_assert (tmp2 != NULL);
diff --git a/glib/tests/gvariant.c b/glib/tests/gvariant.c
index 33caaf04a..80bc7caf4 100644
--- a/glib/tests/gvariant.c
+++ b/glib/tests/gvariant.c
@@ -3943,6 +3943,7 @@ test_parse_failures (void)
     "[4, 5, '']",               "1-2,7-9:",        "common type",
     "[[4], [], ['']]",          "1-4,10-14:",      "common type",
     "[[], [4], ['']]",          "5-8,10-14:",      "common type",
+    "[[0], [], [nothing]]",     "10-19:",          "common type",
     "just",                     "4:",              "expected value",
     "nothing",                  "0-7:",            "unable to infer",
     "just [4, '']",             "6-7,9-11:",       "common type",