[pygobject] [gi] Support nested objects and empty sequences in GLib.Variant building

[Martin Pitt <martinpitt src gnome org>]

commit 6d8ff4d5bdda5480089543869535cc3ee83da2f5
Author: Martin Pitt <martin pitt ubuntu com>
Date:   Wed Jan 19 11:41:11 2011 +0100

    [gi] Support nested objects and empty sequences in GLib.Variant building
    
    The GVariant constructor (in the overrides) previously did not support empty
    arrays/dictionaries or nested structures. Rewrite the VariantCreator class to
    be fully recursive and determine the element types of arrays/dictionaries.
    
    This now also allows you to use actual tuples as input values for GVariant
    tuple types. Taking values from the flat argument list is still supported for
    convenience, though.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=639939

 gi/overrides/GLib.py    |  229 +++++++++++++++++++++++++----------------------
 tests/test_overrides.py |  159 +++++++++++++++++++++++++++++++--
 2 files changed, 273 insertions(+), 115 deletions(-)
---
diff --git a/gi/overrides/GLib.py b/gi/overrides/GLib.py
index 100574e..fce03d9 100644
--- a/gi/overrides/GLib.py
+++ b/gi/overrides/GLib.py
@@ -44,108 +44,126 @@ class _VariantCreator(object):
         'v': GLib.Variant.new_variant,
     }
 
-    def __init__(self, format_string, args):
-        self._format_string = format_string
-        self._args = args
-
-    def create(self):
-        if self._format_string_is_leaf():
-            return self._new_variant_leaf()
-
-        format_char = self._pop_format_char()
-        arg = self._pop_arg()
-
-        if format_char == 'm':
-            raise NotImplementedError()
-        else:
-            builder = GLib.VariantBuilder()
-            if format_char == '(':
-                builder.init(variant_type_from_string('r'))
-            elif format_char == '{':
-                builder.init(variant_type_from_string('{?*}'))
-            else:
-                raise NotImplementedError()
-            format_char = self._pop_format_char()
-            while format_char not in [')', '}']:
-                builder.add_value(Variant(format_char, arg))
-                format_char = self._pop_format_char()
-                if self._args:
-                    arg = self._pop_arg()
-            return builder.end()
-
-    def _format_string_is_leaf(self):
-        format_char = self._format_string[0]
-        return not format_char in ['m', '(', '{']
-
-    def _format_string_is_nnp(self):
-        format_char = self._format_string[0]
-        return format_char in ['a', 's', 'o', 'g', '^', '@', '*', '?', 'r',
-                               'v', '&']
-
-    def _new_variant_leaf(self):
-        if self._format_string_is_nnp():
-            return self._new_variant_nnp()
-
-        format_char = self._pop_format_char()
-        arg = self._pop_arg()
-
-        return _VariantCreator._LEAF_CONSTRUCTORS[format_char](arg)
-
-    def _new_variant_nnp(self):
-        format_char = self._pop_format_char()
-        arg = self._pop_arg()
-
-        if format_char == '&':
-            format_char = self._pop_format_char()
-
-        if format_char == 'a':
-            builder = GLib.VariantBuilder()
-            builder.init(variant_type_from_string('a*'))
-
-            element_format_string = self._pop_leaf_format_string()
-
-            if isinstance(arg, dict):
-                for element in arg.items():
-                    value = Variant(element_format_string, *element)
-                    builder.add_value(value)
+    def _create(self, format, args):
+        '''Create a GVariant object from given format and argument list.
+
+        This method recursively calls itself for complex structures (arrays,
+        dictionaries, boxed).
+
+        Return a tuple (variant, rest_format, rest_args) with the generated
+        GVariant, the remainder of the format string, and the remainder of the
+        arguments.
+
+        If args is None, then this won't actually consume any arguments, and
+        just parse the format string and generate empty GVariant structures.
+        This is required for creating empty dictionaries or arrays.
+        '''
+        # leaves (simple types)
+        constructor = self._LEAF_CONSTRUCTORS.get(format[0])
+        if constructor:
+            if args is not None:
+                if not args:
+                    raise TypeError('not enough arguments for GVariant format string')
+                v = constructor(args[0])
+                return (constructor(args[0]), format[1:], args[1:])
             else:
-                for element in arg:
-                    value = Variant(element_format_string, element)
-                    builder.add_value(value)
-            return builder.end()
-        elif format_char == '^':
-            raise NotImplementedError()
-        elif format_char == '@':
-            raise NotImplementedError()
-        elif format_char == '*':
-            raise NotImplementedError()
-        elif format_char == 'r':
-            raise NotImplementedError()
-        elif format_char == '?':
-            raise NotImplementedError()
+                return (None, format[1:], None)
+
+        if format[0] == '(':
+            return self._create_tuple(format, args)
+
+        if format.startswith('a{'):
+            return self._create_dict(format, args)
+
+        if format[0] == 'a':
+            return self._create_array(format, args)
+
+        raise NotImplementedError('cannot handle GVariant type ' + format)
+
+    def _create_tuple(self, format, args):
+        '''Handle the case where the outermost type of format is a tuple.
+
+        Tuples might come in as actual tuples in args, or as a flat list, so
+        this needs to handle both cases.
+        '''
+        format = format[1:] # eat the '('
+        builder = GLib.VariantBuilder()
+        builder.init(variant_type_from_string('r'))
+        if args is not None and args and type(args[0]) == type(()):
+            # tuple in args
+            for i in xrange(len(args[0])):
+                (v, format, _) = self._create(format, args[0][i:])
+                builder.add_value(v)
+            args = args[1:]
         else:
-            return _VariantCreator._LEAF_CONSTRUCTORS[format_char](arg)
-
-    def _pop_format_char(self):
-        format_char = self._format_string[0]
-        self._format_string = self._format_string[1:]
-        return format_char
-
-    def _pop_leaf_format_string(self):
-        # FIXME: This will break when the leaf is inside a tuple or dict entry
-        format_string = self._format_string
-        self._format_string = ''
-        return format_string
-
-    def _pop_arg(self):
-        arg = self._args[0]
-        self._args = self._args[1:]
-        return arg
+            # flat list
+            while format[0] != ')':
+                (v, format, args) = self._create(format, args)
+                builder.add_value(v)
+        return (builder.end(), format[1:], args)
+
+    def _create_dict(self, format, args):
+        '''Handle the case where the outermost type of format is a dict.'''
+
+        builder = GLib.VariantBuilder()
+        if args is None or not args[0]:
+            # empty value: we need to call _create() to parse the subtype,
+            # and specify the element type precisely
+            rest_format = self._create(format[2:], None)[1]
+            rest_format = self._create(rest_format, None)[1]
+            if not rest_format.startswith('}'):
+                raise ValueError('dictionary type string not closed with }')
+            rest_format = rest_format[1:] # eat the }
+            element_type = format[:len(format) - len(rest_format)]
+            builder.init(variant_type_from_string(element_type))
+        else:
+            builder.init(variant_type_from_string('a{?*}'))
+            for k, v in args[0].iteritems():
+                (key_v, rest_format, _) = self._create(format[2:], [k])
+                (val_v, rest_format, _) = self._create(rest_format, [v])
+
+                if not rest_format.startswith('}'):
+                    raise ValueError('dictionary type string not closed with }')
+                rest_format = rest_format[1:] # eat the }
+
+                entry = GLib.VariantBuilder()
+                entry.init(variant_type_from_string('{?*}'))
+                entry.add_value(key_v)
+                entry.add_value(val_v)
+                builder.add_value(entry.end())
+
+        if args is not None:
+            args = args[1:]
+        return (builder.end(), rest_format, args)
+
+    def _create_array(self, format, args):
+        '''Handle the case where the outermost type of format is an array.'''
+
+        builder = GLib.VariantBuilder()
+        if args is None or not args[0]:
+            # empty value: we need to call _create() to parse the subtype,
+            # and specify the element type precisely
+            rest_format = self._create(format[1:], None)[1]
+            element_type = format[:len(format) - len(rest_format)]
+            builder.init(variant_type_from_string(element_type))
+        else:
+            builder.init(variant_type_from_string('a*'))
+            for i in xrange(len(args[0])):
+                (v, rest_format, _) = self._create(format[1:], args[0][i:])
+                builder.add_value(v)
+        if args is not None:
+            args = args[1:]
+        return (builder.end(), rest_format, args)
 
 class Variant(GLib.Variant):
     def __new__(cls, format_string, *args):
-        creator = _VariantCreator(format_string, args)
-        return creator.create()
+        creator = _VariantCreator()
+        (v, rest_format, restargs) = creator._create(format_string, list(args))
+        if rest_format:
+            raise TypeError('invalid remaining format string: "%s"' % rest_format)
+        if restargs:
+            raise TypeError('too many arguments for format string: "%s"' % str(restargs))
+        return v
 
     def __repr__(self):
         return '<GLib.Variant(%s)>' % getattr(self, 'print')(True)
@@ -193,7 +211,7 @@ class Variant(GLib.Variant):
             return [self.get_child_value(i).unpack() 
                     for i in xrange(self.n_children())]
 
-        raise NotImplementedError, 'unsupported GVariant type ' + self.get_type_string()
+        raise NotImplementedError('unsupported GVariant type ' + self.get_type_string())
 
     #
     # Pythonic iterators
@@ -203,7 +221,7 @@ class Variant(GLib.Variant):
             return len(self.get_string())
         if self.get_type_string().startswith('a') or self.get_type_string().startswith('('):
             return self.n_children()
-        raise TypeError, 'GVariant type %s does not have a length' % self.get_type_string()
+        raise TypeError('GVariant type %s does not have a length' % self.get_type_string())
 
     def __getitem__(self, key):
         # dict
@@ -211,7 +229,7 @@ class Variant(GLib.Variant):
             try:
                 val = self.lookup_value(key, variant_type_from_string('*'))
                 if val is None:
-                    raise KeyError, key
+                    raise KeyError(key)
                 return val.unpack()
             except TypeError:
                 # lookup_value() only works for string keys, which is certainly
@@ -221,25 +239,22 @@ class Variant(GLib.Variant):
                     v = self.get_child_value(i)
                     if v.get_child_value(0).unpack() == key:
                         return v.get_child_value(1).unpack()
-                raise KeyError, key
+                raise KeyError(key)
 
         # array/tuple
         if self.get_type_string().startswith('a') or self.get_type_string().startswith('('):
-            try:
-                key = int(key)
-            except ValueError, e:
-                raise TypeError, str(e)
+            key = int(key)
             if key < 0:
                 key = self.n_children() + key
             if key < 0 or key >= self.n_children():
-                raise IndexError, 'list index out of range'
+                raise IndexError('list index out of range')
             return self.get_child_value(key).unpack()
 
         # string
         if self.get_type_string() in ['s', 'o', 'g']:
             return self.get_string().__getitem__(key)
 
-        raise TypeError, 'GVariant type %s is not a container' % self.get_type_string()
+        raise TypeError('GVariant type %s is not a container' % self.get_type_string())
 
     def keys(self):
         if not self.get_type_string().startswith('a{'):
diff --git a/tests/test_overrides.py b/tests/test_overrides.py
index 8517055..c14dca6 100644
--- a/tests/test_overrides.py
+++ b/tests/test_overrides.py
@@ -18,34 +18,177 @@ import gi.types
 class TestGLib(unittest.TestCase):
 
     def test_gvariant_create(self):
+        # simple values
+
         variant = GLib.Variant('i', 42)
         self.assertTrue(isinstance(variant, GLib.Variant))
         self.assertEquals(variant.get_int32(), 42)
 
+        variant = GLib.Variant('s', '')
+        self.assertTrue(isinstance(variant, GLib.Variant))
+        self.assertEquals(variant.get_string(), '')
+
+        variant = GLib.Variant('s', 'hello')
+        self.assertTrue(isinstance(variant, GLib.Variant))
+        self.assertEquals(variant.get_string(), 'hello')
+
+        # tuples
+
+        variant = GLib.Variant('()')
+        self.assertEqual(variant.get_type_string(), '()')
+        self.assertEquals(variant.n_children(), 0)
+
+        # canonical arguments
+        variant = GLib.Variant('(ss)', ('mec', 'mac'))
+        self.assertEqual(variant.get_type_string(), '(ss)')
+        self.assertTrue(isinstance(variant, GLib.Variant))
+        self.assertTrue(isinstance(variant.get_child_value(0), GLib.Variant))
+        self.assertTrue(isinstance(variant.get_child_value(1), GLib.Variant))
+        self.assertEquals(variant.get_child_value(0).get_string(), 'mec')
+        self.assertEquals(variant.get_child_value(1).get_string(), 'mac')
+
+        # flat arguments
         variant = GLib.Variant('(ss)', 'mec', 'mac')
+        self.assertEqual(variant.get_type_string(), '(ss)')
         self.assertTrue(isinstance(variant, GLib.Variant))
         self.assertTrue(isinstance(variant.get_child_value(0), GLib.Variant))
         self.assertTrue(isinstance(variant.get_child_value(1), GLib.Variant))
         self.assertEquals(variant.get_child_value(0).get_string(), 'mec')
         self.assertEquals(variant.get_child_value(1).get_string(), 'mac')
 
-        variant = GLib.Variant('a{si}', {'key1': 1, 'key2': 2})
+        # nested tuples (canonical)
+        variant = GLib.Variant('((si)(ub))', (('hello', -1), (42, True)))
+        self.assertEqual(variant.get_type_string(), '((si)(ub))')
+        self.assertEqual(variant.unpack(), (('hello', -1), (42L, True)))
+
+        # nested tuples (flat)
+        variant = GLib.Variant('((si)(ub))', 'hello', -1, 42, True)
+        self.assertEqual(variant.get_type_string(), '((si)(ub))')
+        self.assertEqual(variant.unpack(), (('hello', -1), (42L, True)))
+
+        # dictionaries
+
+        variant = GLib.Variant('a{si}', {})
+        self.assertTrue(isinstance(variant, GLib.Variant))
+        self.assertEqual(variant.get_type_string(), 'a{si}')
+        self.assertEquals(variant.n_children(), 0)
+
+        variant = GLib.Variant('a{si}', {'': 1, 'key1': 2, 'key2': 3})
+        self.assertEqual(variant.get_type_string(), 'a{si}')
         self.assertTrue(isinstance(variant, GLib.Variant))
         self.assertTrue(isinstance(variant.get_child_value(0), GLib.Variant))
         self.assertTrue(isinstance(variant.get_child_value(1), GLib.Variant))
-        # Looks like order is not preserved
-        self.assertEquals(variant.get_child_value(1).get_child_value(0).get_string(), 'key1')
-        self.assertEquals(variant.get_child_value(1).get_child_value(1).get_int32(), 1)
-        self.assertEquals(variant.get_child_value(0).get_child_value(0).get_string(), 'key2')
-        self.assertEquals(variant.get_child_value(0).get_child_value(1).get_int32(), 2)
+        self.assertTrue(isinstance(variant.get_child_value(2), GLib.Variant))
+        self.assertEqual(variant.unpack(), {'': 1, 'key1': 2, 'key2': 3})
+
+        # nested dictionaries
+        variant = GLib.Variant('a{sa{si}}', {})
+        self.assertTrue(isinstance(variant, GLib.Variant))
+        self.assertEqual(variant.get_type_string(), 'a{sa{si}}')
+        self.assertEquals(variant.n_children(), 0)
+
+        d = {'':     {'': 1, 'keyn1': 2},
+             'key1': {'key11': 11, 'key12': 12}}
+        variant = GLib.Variant('a{sa{si}}', d)
+        self.assertEqual(variant.get_type_string(), 'a{sa{si}}')
+        self.assertTrue(isinstance(variant, GLib.Variant))
+        self.assertEqual(variant.unpack(), d)
+
+        # arrays
+
+        variant = GLib.Variant('ai', [])
+        self.assertEqual(variant.get_type_string(), 'ai')
+        self.assertEquals(variant.n_children(), 0)
 
         variant = GLib.Variant('ai', [1, 2])
+        self.assertEqual(variant.get_type_string(), 'ai')
         self.assertTrue(isinstance(variant, GLib.Variant))
         self.assertTrue(isinstance(variant.get_child_value(0), GLib.Variant))
         self.assertTrue(isinstance(variant.get_child_value(1), GLib.Variant))
         self.assertEquals(variant.get_child_value(0).get_int32(), 1)
         self.assertEquals(variant.get_child_value(1).get_int32(), 2)
 
+        variant = GLib.Variant('as', [])
+        self.assertEqual(variant.get_type_string(), 'as')
+        self.assertEquals(variant.n_children(), 0)
+
+        variant = GLib.Variant('as', [''])
+        self.assertEqual(variant.get_type_string(), 'as')
+        self.assertTrue(isinstance(variant, GLib.Variant))
+        self.assertTrue(isinstance(variant.get_child_value(0), GLib.Variant))
+        self.assertEquals(variant.get_child_value(0).get_string(), '')
+
+        variant = GLib.Variant('as', ['hello', 'world'])
+        self.assertEqual(variant.get_type_string(), 'as')
+        self.assertTrue(isinstance(variant, GLib.Variant))
+        self.assertTrue(isinstance(variant.get_child_value(0), GLib.Variant))
+        self.assertTrue(isinstance(variant.get_child_value(1), GLib.Variant))
+        self.assertEquals(variant.get_child_value(0).get_string(), 'hello')
+        self.assertEquals(variant.get_child_value(1).get_string(), 'world')
+
+        # nested arrays
+        variant = GLib.Variant('aai', [])
+        self.assertEqual(variant.get_type_string(), 'aai')
+        self.assertEquals(variant.n_children(), 0)
+
+        variant = GLib.Variant('aai', [[]])
+        self.assertEqual(variant.get_type_string(), 'aai')
+        self.assertEquals(variant.n_children(), 1)
+        self.assertEquals(variant.get_child_value(0).n_children(), 0)
+
+        variant = GLib.Variant('aai', [[1, 2], [3, 4, 5]])
+        self.assertEqual(variant.get_type_string(), 'aai')
+        self.assertEquals(variant.unpack(), [[1, 2], [3, 4, 5]])
+
+        #
+        # complex types
+        #
+
+        variant = GLib.Variant('(as)', [])
+        self.assertEqual(variant.get_type_string(), '(as)')
+        self.assertEquals(variant.n_children(), 1)
+        self.assertEquals(variant.get_child_value(0).n_children(), 0)
+
+        variant = GLib.Variant('(as)', [''])
+        self.assertEqual(variant.get_type_string(), '(as)')
+        self.assertEquals(variant.n_children(), 1)
+        self.assertEquals(variant.get_child_value(0).n_children(), 1)
+        self.assertEquals(variant.get_child_value(0).get_child_value(0).get_string(), '')
+
+        variant = GLib.Variant('(as)', ['hello'])
+        self.assertEqual(variant.get_type_string(), '(as)')
+        self.assertEquals(variant.n_children(), 1)
+        self.assertEquals(variant.get_child_value(0).n_children(), 1)
+        self.assertEquals(variant.get_child_value(0).get_child_value(0).get_string(), 'hello')
+
+        obj = {'a1': (1, True), 'a2': (2, False)}
+        variant = GLib.Variant('a{s(ib)}', obj)
+        self.assertEqual(variant.get_type_string(), 'a{s(ib)}')
+        self.assertEqual(variant.unpack(), obj)
+
+        obj = (1, {'a': {'a1': True, 'a2': False},
+                   'b': {'b1': False},
+                   'c': {}
+                  },
+               'foo')
+        variant = GLib.Variant('(ia{sa{sb}}s)', obj)
+        self.assertEqual(variant.get_type_string(), '(ia{sa{sb}}s)')
+        self.assertEqual(variant.unpack(), obj)
+
+    def test_gvariant_create_errors(self):
+        # excess arguments
+        self.assertRaises(TypeError, GLib.Variant, 'i', 42, 3)
+
+        # not enough arguments
+        self.assertRaises(TypeError, GLib.Variant, '(ii)', 42)
+
+        # data type mismatch
+        self.assertRaises(TypeError, GLib.Variant, 'i', 'hello')
+        self.assertRaises(TypeError, GLib.Variant, 's', 42)
+
+        # unimplemented data type
+        self.assertRaises(NotImplementedError, GLib.Variant, 'Q', 1)
+
     def test_gvariant_unpack(self):
         # simple values
         res = GLib.Variant.new_int32(-42).unpack()
@@ -92,7 +235,7 @@ class TestGLib(unittest.TestCase):
         self.assertEqual(v[-2], -1)
         self.assertRaises(IndexError, v.__getitem__, 2)
         self.assertRaises(IndexError, v.__getitem__, -3)
-        self.assertRaises(TypeError, v.__getitem__, 'a')
+        self.assertRaises(ValueError, v.__getitem__, 'a')
 
         # array iteration
         self.assertEqual([x for x in v], [-1, 3])
@@ -108,7 +251,7 @@ class TestGLib(unittest.TestCase):
         self.assertEqual(v[-2], -1)
         self.assertRaises(IndexError, v.__getitem__, 2)
         self.assertRaises(IndexError, v.__getitem__, -3)
-        self.assertRaises(TypeError, v.__getitem__, 'a')
+        self.assertRaises(ValueError, v.__getitem__, 'a')
 
         # tuple iteration
         self.assertEqual([x for x in v], [-1, 'hello'])