[gobject-introspection/wip/transformer] New file finaltransformer, does most of 2nd/final pass

[Colin Walters <walters src gnome org>]

commit a86c5f04b78f7ed18e085ea4b8b46c6c8fd87539
Author: Colin Walters <walters verbum org>
Date:   Fri Jul 23 05:29:29 2010 -0400

    New file finaltransformer, does most of 2nd/final pass
    
    glibtransformer is now just basically the GType XML dump processing
    and a little bit of early analysis.

 giscanner/Makefile.am         |    1 +
 giscanner/annotationparser.py |    4 +-
 giscanner/finaltransformer.py |  440 +++++++++++++++++++++++++++++++++++++++++
 giscanner/glibtransformer.py  |  436 ----------------------------------------
 giscanner/scannermain.py      |   11 +-
 5 files changed, 449 insertions(+), 443 deletions(-)
---
diff --git a/giscanner/Makefile.am b/giscanner/Makefile.am
index 642bcbc..6a83826 100644
--- a/giscanner/Makefile.am
+++ b/giscanner/Makefile.am
@@ -38,6 +38,7 @@ pkgpyexec_PYTHON = 		\
 	cachestore.py		\
 	config.py		\
 	dumper.py		\
+	finaltransformer.py	\
 	girparser.py		\
 	girwriter.py		\
 	glibast.py		\
diff --git a/giscanner/annotationparser.py b/giscanner/annotationparser.py
index ab19574..f3bb6f2 100644
--- a/giscanner/annotationparser.py
+++ b/giscanner/annotationparser.py
@@ -154,10 +154,10 @@ class AnnotationParser(object):
     OPTION_RE = re.compile(r'\([A-Za-z]+[^(]*\)')
     RETURNS_RE = re.compile(r'^return(s?)( value)?:', re.IGNORECASE)
 
-    def __init__(self, namespace, source_scanner, transformer):
+    def __init__(self, source_scanner, transformer):
         self._blocks = {}
-        self._namespace = namespace
         self._transformer = transformer
+        self._namespace = transformer.namespace
         for comment in source_scanner.get_comments():
             self._parse_comment(comment)
 
diff --git a/giscanner/finaltransformer.py b/giscanner/finaltransformer.py
new file mode 100644
index 0000000..da15c89
--- /dev/null
+++ b/giscanner/finaltransformer.py
@@ -0,0 +1,440 @@
+# -*- Mode: Python -*-
+# Copyright (C) 2010 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+
+import os
+import sys
+import re
+import tempfile
+import shutil
+import subprocess
+
+from .ast import (Alias, Bitfield, Callable, Callback, Class, Constant, Enum,
+                  Function, Interface, Member, Namespace, Node, Parameter,
+                  Property, Record, Return, Type, TypeContainer, Union,
+                  Class, Field, VFunction, default_array_types,
+                  TYPE_ANY, TYPE_GTYPE, TYPE_UINT8, PARAM_TRANSFER_FULL, Array, List,
+                  Map, Varargs, type_names)
+from .glibast import (GLibBoxed, GLibEnum, GLibEnumMember, GLibFlags,
+                      GLibInterface, GLibObject, GLibSignal, GLibBoxedStruct,
+                      GLibBoxedUnion, GLibBoxedOther, GLibRecord)
+from .utils import to_underscores, to_underscores_noprefix
+
+class FinalTransformer(object):
+
+    def __init__(self, transformer):
+        self._transformer = transformer
+        self._namespace = transformer.namespace
+        self._uscore_type_names = {}
+
+    # Public API
+
+    def transform(self):
+        # We have a rough tree which should have all
+        # of the types we know about.  Let's attempt closure; walk
+        # over all of the Type() types and see if they match up
+        # with something.
+        self._namespace.walk(self._pass_type_resolution)
+
+        # Generate a reverse mapping "bar_baz" -> BarBaz
+        for node in self._namespace.itervalues():
+            uscored = to_underscores_noprefix(node.name).lower()
+            if isinstance(node, (Class, Interface, Record, Union, GLibBoxedOther)):
+                self._uscore_type_names[uscored] = node
+
+        for node in list(self._namespace.itervalues()):
+            if isinstance(node, Function):
+                # Discover which toplevel functions are actually methods
+                self._pair_function(node)
+            if isinstance(node, (Class, Interface)):
+                self._pair_class_virtuals(node)
+
+        self._namespace.walk(self._pass3)
+
+        # TODO - merge into pass3
+        for node in self._namespace.itervalues():
+            self._resolve_quarks()
+
+        self._namespace.walk(self._analyze_node)
+        self._namespace.walk(self._introspectable_pass2)
+
+    # Private
+
+    def _get_no_uscore_prefixed_name(self, type_name):
+        # Besides the straight underscore conversion, we also try
+        # removing the underscores from the namespace as a possible C
+        # mapping; e.g. it's webkit_web_view, not web_kit_web_view
+        suffix = self._transformer.remove_prefix(type_name)
+        prefix = type_name[:-len(suffix)]
+        return (prefix + '_' + to_underscores(suffix)).lower()
+
+    def _pass_type_resolution(self, node, chain):
+        if isinstance(node, Alias):
+            self._transformer.resolve_type(node.target)
+        if isinstance(node, Callable):
+            for parameter in node.parameters:
+                self._transformer.resolve_type(parameter.type)
+            self._transformer.resolve_type(node.retval.type)
+        if isinstance(node, Constant):
+            self._transformer.resolve_type(node.value_type)
+        if isinstance(node, (Class, Interface, Record, Union)):
+            for field in node.fields:
+                if field.anonymous_node:
+                    field.anonymous_node.walk(self._pass_type_resolution, chain)
+                else:
+                    self._transformer.resolve_type(field.type)
+        if isinstance(node, (Class, Interface)):
+            resolved_parent = None
+            for parent in node.parent_chain:
+                try:
+                    self._transformer.resolve_type(parent)
+                except ValueError, e:
+                    continue
+                node.parent = parent
+                break
+            for prop in node.properties:
+                self._transformer.resolve_type(prop.type)
+            for sig in node.signals:
+                for param in sig.parameters:
+                    self._transformer.resolve_type(param.type)
+        if isinstance(node, Class):
+            for iface in node.interfaces:
+                self._transformer.resolve_type(iface)
+        if isinstance(node, Interface):
+            for iface in node.prerequisites:
+                self._transformer.resolve_type(iface)
+        return True
+
+    def _resolve_quarks(self):
+        # self._uscore_type_names is an authoritative mapping of types
+        # to underscored versions, since it is based on get_type() methods;
+        # but only covers enums that are registered as GObject enums.
+        # Create a fallback mapping based on all known enums in this module.
+        uscore_enums = {}
+        for enum in self._namespace.itervalues():
+            if not isinstance(enum, Enum):
+                continue
+            type_name = enum.symbol
+            uscored = to_underscores(type_name).lower()
+
+            uscore_enums[uscored] = enum
+
+            no_uscore_prefixed = self._get_no_uscore_prefixed_name(type_name)
+            if no_uscore_prefixed not in uscore_enums:
+                uscore_enums[no_uscore_prefixed] = enum
+
+        for node in self._namespace.itervalues():
+            if not isinstance(node, Function):
+                continue
+            if node.retval.type.target_giname != 'GLib.Quark':
+                continue
+            short = node.symbol[:-len('_quark')]
+            if short == "g_io_error":
+                # Special case; GIOError was already taken forcing GIOErrorEnum
+                enum = self._names.type_names["GIOErrorEnum"][1]
+            else:
+                enum = self._uscore_type_names.get(short)
+                if enum is None:
+                    enum = uscore_enums.get(short)
+            if enum is not None:
+                enum.error_quark = node.symbol
+            else:
+                self._transformer.log_node_warning(node,
+"""Couldn't find corresponding enumeration""")
+
+    def _split_uscored_by_type(self, uscored):
+        """'uscored' should be an un-prefixed uscore string.  This
+function searches through the namespace for the longest type which
+prefixes uscored, and returns (type, suffix).  Example, assuming
+namespace Gtk, type is TextBuffer:
+
+_split_uscored_by_type(text_buffer_try_new) -> (Class(TextBuffer), 'try_new')"""
+        node = None
+        count = 0
+        prev_split_count = -1
+        while True:
+            components = uscored.rsplit('_', count)
+            if len(components) == prev_split_count:
+                return None
+            prev_split_count = len(components)
+            type_string = components[0]
+            node = self._uscore_type_names.get(type_string)
+            if node:
+                return (node, '_'.join(components[1:]))
+            count += 1
+
+    def _pair_function(self, func):
+        """Check to see whether a toplevel function should be a
+method or constructor of some type."""
+        if func.symbol.endswith('_get_type') or func.symbol.startswith('_'):
+            return
+        (ns, subsymbol) = self._transformer.split_csymbol(func.symbol)
+        assert ns == self._namespace
+        if self._pair_constructor(func, subsymbol):
+            return
+        elif self._pair_method(func, subsymbol):
+            return
+        elif self._pair_static_method(func, subsymbol):
+            return
+
+    def _uscored_identifier_for_type(self, typeval):
+        """Given a Type(target_giname='Foo.BarBaz'), return 'bar_baz'."""
+        name = typeval.get_giname()
+        return to_underscores_noprefix(name).lower()
+
+    def _pair_method(self, func, subsymbol):
+        if not func.parameters:
+            return False
+        first = func.parameters[0]
+        target = self._transformer.lookup_typenode(first.type)
+        if not isinstance(target, (Class, Interface, Record, Union, GLibBoxedOther)):
+            return False
+        uscored = self._uscored_identifier_for_type(first.type)
+        if not subsymbol.startswith(uscored):
+            return False
+        del func.parameters[0]
+        subsym_idx = func.symbol.find(subsymbol)
+        self._namespace.float(func)
+        func.name = func.symbol[(subsym_idx + len(uscored) + 1):]
+        target.methods.append(func)
+        func.is_method = True
+        return True
+
+    def _pair_static_method(self, func, subsymbol):
+        split = self._split_uscored_by_type(subsymbol)
+        if split is None:
+            return False
+        (node, funcname) = split
+        if not isinstance(node, (Class, Interface, Record, Union, GLibBoxedOther)):
+            return False
+        self._namespace.float(func)
+        func.name = funcname
+        node.static_methods.append(func)
+
+    def _pair_constructor(self, func, subsymbol):
+        if not (func.symbol.find('_new_') >= 0 or func.symbol.endswith('_new')):
+            return False
+        target = self._transformer.lookup_typenode(func.retval.type)
+        if not isinstance(target, (Class, Record, Union, GLibBoxedOther)):
+            return False
+        new_idx = func.symbol.rfind('_new')
+        assert (new_idx >= 0)
+        prefix = func.symbol[:new_idx]
+        split = self._split_uscored_by_type(subsymbol)
+        if split is None:
+            # TODO - need a e.g. (method) annotation
+            self._transformer.log_node_warning(func,
+                "Can't find matching type for constructor; symbol=%r" % (func.symbol, ))
+            return False
+        (origin_node, funcname) = split
+        if isinstance(target, Class):
+            parent = origin_node
+            while parent:
+                if parent == target:
+                    break
+                parent = self._transformer.lookup_typenode(parent.parent)
+                if parent is None:
+                    self._transformer.log_node_warning(func,
+                                                       "Return value is not superclass for constructor; symbol=%r constructed=%r return=%r" % (func.symbol, str(origin_node.create_type()), str(func.retval.type)))
+                    return False
+        else:
+            if origin_node != target:
+                self._transformer.log_node_warning(func,
+                                                   "Constructor return type mismatch symbol=%r constructed=%r return=%r" % (func.symbol, str(origin_node.create_type()), str(func.retval.type)))
+                return False
+        self._namespace.float(func)
+        func.name = funcname
+        target.constructors.append(func)
+        return True
+
+    def _handle_closure(self, param, closure_idx, closure_param):
+        if (closure_param.type is TYPE_ANY and
+            closure_param.name.endswith('data')):
+            param.closure_name = closure_param.argname
+            param.closure_index = closure_idx
+            return True
+        return False
+
+    def _handle_destroy(self, param, destroy_idx, destroy_param):
+        if destroy_param.type.target_giname == 'GLib.DestroyNotify':
+            param.destroy_name = destroy_param.argname
+            param.destroy_index = destroy_idx
+            return True
+        return False
+
+    def _pair_class_virtuals(self, node):
+        """Look for virtual methods from the class structure."""
+        if not node.glib_type_struct:
+            self._transformer.log_node_warning(node,
+                "Failed to find class structure for %r" % (node.name, ))
+            return
+
+        node_type = node.create_type()
+        class_struct = self._transformer.lookup_typenode(node.glib_type_struct)
+        
+        # Object class fields are assumed to be read-only
+        # (see also _introspect_object and transformer.py)
+        for field in class_struct.fields:
+            if isinstance(field, Field):
+                field.writable = False
+
+        # Loop through fields to determine which are virtual
+        # functions and which are signal slots by
+        # assuming everything that doesn't share a name
+        # with a known signal is a virtual slot.
+        for field in class_struct.fields:
+            if not isinstance(field.anonymous_node, Callback):
+                continue
+            callback = field.anonymous_node
+            # Check the first parameter is the object
+            if len(callback.parameters) == 0:
+                continue
+            firstparam_type = callback.parameters[0].type
+            if firstparam_type != node_type:
+                continue
+            # Also double check we don't have a signal with this
+            # name.
+            matched_signal = False
+            for signal in node.signals:
+                if signal.name.replace('-', '_') == callback.name:
+                    matched_signal = True
+                    break
+            if matched_signal:
+                continue
+            vfunc = VFunction.from_callback(callback)
+            vfunc.inherit_file_positions(callback)
+            node.virtual_methods.append(vfunc)
+
+        # Take the set of virtual methods we found, and try
+        # to pair up with any matching methods using the
+        # name+signature.
+        for vfunc in node.virtual_methods:
+            for method in node.methods:
+                if method.name != vfunc.name:
+                    continue
+                if method.retval.type != vfunc.retval.type:
+                    continue
+                if len(method.parameters) != len(vfunc.parameters):
+                    continue
+                for i in xrange(len(method.parameters)):
+                    m_type = method.parameters[i].type
+                    v_type = vfunc.parameters[i].type
+                    if m_type != v_type:
+                        continue
+                vfunc.invoker = method.name
+
+    def _pass3(self, node, chain):
+        """Pass 3 is after we've loaded GType data and performed type
+        closure."""
+        if isinstance(node, Callable):
+            self._pass3_callable_callbacks(node)
+            self._pass3_callable_throws(node)
+        return True
+
+    def _pass3_callable_callbacks(self, node):
+        """Check to see if we have anything that looks like a
+        callback+user_data+GDestroyNotify set."""
+        params = node.parameters
+        for i, param in enumerate(params):
+            node = self._transformer.lookup_typenode(param.type)
+            if not isinstance(node, Callback):
+                continue
+
+            # set a default scope
+            if param.scope is None:
+                param.scope = 'call'
+
+            # j is the index where we look for closure/destroy to
+            # group with the callback param
+            j = i + 1
+            if j == len(params):
+                continue # no more args -> nothing to group
+            # look at the param directly following for either a
+            # closure or a destroy; only one of these will fire
+            had_closure = self._handle_closure(param, j, params[j])
+            had_destroy = self._handle_destroy(param, j, params[j])
+            j += 1
+            # are we out of params, or did we find neither?
+            if j == len(params) or (not had_closure and not had_destroy):
+                continue
+            # we found either a closure or a destroy; check the
+            # parameter following for the other
+            if not had_closure:
+                self._handle_closure(param, j, params[j])
+            if not had_destroy:
+                self._handle_destroy(param, j, params[j])
+
+    def _pass3_callable_throws(self, node):
+        """Check to see if we have anything that looks like a
+        callback+user_data+GDestroyNotify set."""
+        if not node.parameters:
+            return
+        last_param = node.parameters[-1]
+        # Checking type.name=='GLib.Error' generates false positives
+        # on methods that take a 'GError *'
+        if last_param.type.ctype == 'GError**':
+            node.parameters.pop()
+            node.throws = True
+
+    # Validation
+
+    def _interface_vfunc_check(self, node, stack):
+        if isinstance(node, GLibInterface):
+            for vfunc in node.virtual_methods:
+                if not vfunc.invoker:
+                    self._transformer.log_node_warning(vfunc,
+"""Virtual function %r has no known invoker""" % (vfunc.name, ),
+                    context=node)
+
+    def _introspectable_analysis(self, node, stack):
+        if isinstance(node, TypeContainer):
+            parent = stack[-1]
+            if isinstance(node.type, Varargs):
+                parent.introspectable = False
+            elif not isinstance(node.type, List) and \
+                 (node.type.target_giname == 'GLib.List'):
+                if isinstance(node, Parameter):
+                    self._transformer.log_node_warning(parent,
+"""Missing (element-type) annotation on argument %r""" % (node.name, ),
+                                                    context=parent)
+                else:
+                    self._transformer.log_node_warning(parent,
+"""Missing (element-type) annotation on return value""", context=parent)
+                parent.introspectable = False
+
+    def _analyze_node(self, obj, stack):
+        if isinstance(obj, Node) and obj.skip:
+            return False
+        # Combine one-pass checks here
+        self._interface_vfunc_check(obj, stack)
+        # Our first pass for scriptability
+        self._introspectable_analysis(obj, stack)
+        return True
+
+    def _introspectable_pass2(self, obj, stack):
+        if isinstance(obj, Node) and obj.skip:
+            return False
+        # In the second introspectable pass, we propagate introspectablity;
+        # for example, a varargs callback as an argument to a function
+        # makes the whole function unintrospectable
+        if isinstance(obj, TypeContainer):
+            parent = stack[-1]
+            target = self._transformer.lookup_typenode(obj.type)
+            if target and not target.introspectable:
+                parent.introspectable = False
+        return True
diff --git a/giscanner/glibtransformer.py b/giscanner/glibtransformer.py
index b75d546..42fc6ef 100644
--- a/giscanner/glibtransformer.py
+++ b/giscanner/glibtransformer.py
@@ -77,7 +77,6 @@ class GLibTransformer(object):
         self._uscore_type_names = {}
         self._binary = None
         self._get_type_functions = []
-        self._error_quark_functions = []
         self._gtype_data = {}
         self._boxed_types = {}
         self._private_internal_types = {}
@@ -132,133 +131,8 @@ class GLibTransformer(object):
             elif isinstance(node, Record):
                 self._pair_class_record(node)
 
-        # We have a rough tree which should have all
-        # of the types we know about.  Let's attempt closure; walk
-        # over all of the Type() types and see if they match up
-        # with something.
-        self._namespace.walk(self._pass_type_resolution)
-
-        # Generate a reverse mapping "bar_baz" -> BarBaz
-        for node in self._namespace.itervalues():
-            uscored = to_underscores_noprefix(node.name).lower()
-            if isinstance(node, (Class, Interface, Record, Union, GLibBoxedOther)):
-                self._uscore_type_names[uscored] = node
-
-        for node in list(self._namespace.itervalues()):
-            if isinstance(node, Function):
-                # Discover which toplevel functions are actually methods
-                self._pair_function(node)
-            if isinstance(node, (Class, Interface)):
-                self._pair_class_virtuals(node)
-
-        self._namespace.walk(self._pass3)
-
-        # TODO - merge into pass3
-        for node in self._namespace.itervalues():
-            self._resolve_quarks()
-
-        return self._namespace
-
-    # Private
-
-    def _get_no_uscore_prefixed_name(self, type_name):
-        # Besides the straight underscore conversion, we also try
-        # removing the underscores from the namespace as a possible C
-        # mapping; e.g. it's webkit_web_view, not web_kit_web_view
-        suffix = self._transformer.remove_prefix(type_name)
-        prefix = type_name[:-len(suffix)]
-        return (prefix + '_' + to_underscores(suffix)).lower()
-
-    def _pass_type_resolution(self, node, chain):
-        if isinstance(node, Alias):
-            self._transformer.resolve_type(node.target)
-        if isinstance(node, Callable):
-            for parameter in node.parameters:
-                self._transformer.resolve_type(parameter.type)
-            self._transformer.resolve_type(node.retval.type)
-        if isinstance(node, Constant):
-            self._transformer.resolve_type(node.value_type)
-        if isinstance(node, (Class, Interface, Record, Union)):
-            for field in node.fields:
-                if field.anonymous_node:
-                    field.anonymous_node.walk(self._pass_type_resolution, chain)
-                else:
-                    self._transformer.resolve_type(field.type)
-        if isinstance(node, (Class, Interface)):
-            resolved_parent = None
-            for parent in node.parent_chain:
-                try:
-                    self._transformer.resolve_type(parent)
-                except ValueError, e:
-                    continue
-                node.parent = parent
-                break
-            for prop in node.properties:
-                self._transformer.resolve_type(prop.type)
-            for sig in node.signals:
-                for param in sig.parameters:
-                    self._transformer.resolve_type(param.type)
-        if isinstance(node, Class):
-            for iface in node.interfaces:
-                self._transformer.resolve_type(iface)
-        if isinstance(node, Interface):
-            for iface in node.prerequisites:
-                self._transformer.resolve_type(iface)
-        return True
-
-    def _resolve_quarks(self):
-        # self._uscore_type_names is an authoritative mapping of types
-        # to underscored versions, since it is based on get_type() methods;
-        # but only covers enums that are registered as GObject enums.
-        # Create a fallback mapping based on all known enums in this module.
-        uscore_enums = {}
-        for enum in self._namespace.itervalues():
-            if not isinstance(enum, Enum):
-                continue
-            type_name = enum.symbol
-            uscored = to_underscores(type_name).lower()
-
-            uscore_enums[uscored] = enum
-
-            no_uscore_prefixed = self._get_no_uscore_prefixed_name(type_name)
-            if no_uscore_prefixed not in uscore_enums:
-                uscore_enums[no_uscore_prefixed] = enum
-
-        for node in self._error_quark_functions:
-            short = node.symbol[:-len('_quark')]
-            if short == "g_io_error":
-                # Special case; GIOError was already taken forcing GIOErrorEnum
-                enum = self._names.type_names["GIOErrorEnum"][1]
-            else:
-                enum = self._uscore_type_names.get(short)
-                if enum is None:
-                    enum = uscore_enums.get(short)
-            if enum is not None:
-                enum.error_quark = node.symbol
-            else:
-                self._transformer.log_node_warning(node,
-"""Couldn't find corresponding enumeration""")
-
     # Helper functions
 
-    def _resolve_gtypename(self, gtype_name):
-        try:
-            return self._transformer.gtypename_to_giname(gtype_name,
-                                                         self._names)
-        except KeyError, e:
-            return Unresolved(gtype_name)
-
-    def _resolve_gtypename_chain(self, gtype_names):
-        """Like _resolve_gtypename, but grab the first one that resolves.
-        If none of them do, return an Unresolved for the first."""
-        for gtype_name in gtype_names:
-            try:
-                return self._transformer.gtypename_to_giname(gtype_name,
-                                                             self._names)
-            except KeyError, e:
-                continue
-        return Unresolved(gtype_names[0])
-
     def _execute_binary_get_tree(self):
         """Load the library (or executable), returning an XML
 blob containing data gleaned from GObject's primitive introspection."""
@@ -320,8 +194,6 @@ blob containing data gleaned from GObject's primitive introspection."""
             return
         elif symbol.endswith('_get_type'):
             self._initparse_get_type_function(func)
-        elif symbol.endswith('_error_quark'):
-            self._initparse_error_quark_function(func)
 
     def _initparse_get_type_function(self, func):
         if self._namespace.name == 'GLib':
@@ -343,128 +215,6 @@ blob containing data gleaned from GObject's primitive introspection."""
         self._get_type_functions.append(func.symbol)
         return True
 
-    def _initparse_error_quark_function(self, func):
-        if func.parameters:
-            return False
-        if func.retval.type.target_giname != 'GLib.Quark':
-            return False
-
-        self._error_quark_functions.append(func)
-        return True
-
-    def _name_is_internal_gtype(self, giname):
-        try:
-            node = self._namespace.get(giname)
-            return isinstance(node, (GLibObject, GLibInterface,
-                                     GLibBoxed, GLibEnum, GLibFlags))
-        except KeyError, e:
-            return False
-
-    def _split_uscored_by_type(self, uscored):
-        """'uscored' should be an un-prefixed uscore string.  This
-function searches through the namespace for the longest type which
-prefixes uscored, and returns (type, suffix).  Example, assuming
-namespace Gtk, type is TextBuffer:
-
-_split_uscored_by_type(text_buffer_try_new) -> (Class(TextBuffer), 'try_new')"""
-        node = None
-        count = 0
-        prev_split_count = -1
-        while True:
-            components = uscored.rsplit('_', count)
-            if len(components) == prev_split_count:
-                return None
-            prev_split_count = len(components)
-            type_string = components[0]
-            node = self._uscore_type_names.get(type_string)
-            if node:
-                return (node, '_'.join(components[1:]))
-            count += 1
-
-    def _pair_function(self, func):
-        """Check to see whether a toplevel function should be a
-method or constructor of some type."""
-        if func.symbol.endswith('_get_type') or func.symbol.startswith('_'):
-            return
-        (ns, subsymbol) = self._transformer.split_csymbol(func.symbol)
-        assert ns == self._namespace
-        if self._pair_constructor(func, subsymbol):
-            return
-        elif self._pair_method(func, subsymbol):
-            return
-        elif self._pair_static_method(func, subsymbol):
-            return
-
-    def _uscored_identifier_for_type(self, typeval):
-        """Given a Type(target_giname='Foo.BarBaz'), return 'bar_baz'."""
-        name = typeval.get_giname()
-        return to_underscores_noprefix(name).lower()
-
-    def _pair_method(self, func, subsymbol):
-        if not func.parameters:
-            return False
-        first = func.parameters[0]
-        target = self._transformer.lookup_typenode(first.type)
-        if not isinstance(target, (Class, Interface, Record, Union, GLibBoxedOther)):
-            return False
-        uscored = self._uscored_identifier_for_type(first.type)
-        if not subsymbol.startswith(uscored):
-            return False
-        del func.parameters[0]
-        subsym_idx = func.symbol.find(subsymbol)
-        self._namespace.float(func)
-        func.name = func.symbol[(subsym_idx + len(uscored) + 1):]
-        target.methods.append(func)
-        func.is_method = True
-        return True
-
-    def _pair_static_method(self, func, subsymbol):
-        split = self._split_uscored_by_type(subsymbol)
-        if split is None:
-            return False
-        (node, funcname) = split
-        if not isinstance(node, (Class, Interface, Record, Union, GLibBoxedOther)):
-            return False
-        self._namespace.float(func)
-        func.name = funcname
-        node.static_methods.append(func)
-
-    def _pair_constructor(self, func, subsymbol):
-        if not (func.symbol.find('_new_') >= 0 or func.symbol.endswith('_new')):
-            return False
-        target = self._transformer.lookup_typenode(func.retval.type)
-        if not isinstance(target, (Class, Record, Union, GLibBoxedOther)):
-            return False
-        new_idx = func.symbol.rfind('_new')
-        assert (new_idx >= 0)
-        prefix = func.symbol[:new_idx]
-        split = self._split_uscored_by_type(subsymbol)
-        if split is None:
-            # TODO - need a e.g. (method) annotation
-            self._transformer.log_node_warning(func,
-                "Can't find matching type for constructor; symbol=%r" % (func.symbol, ))
-            return False
-        (origin_node, funcname) = split
-        if isinstance(target, Class):
-            parent = origin_node
-            while parent:
-                if parent == target:
-                    break
-                parent = self._transformer.lookup_typenode(parent.parent)
-                if parent is None:
-                    self._transformer.log_node_warning(func,
-                                                       "Return value is not superclass for constructor; symbol=%r constructed=%r return=%r" % (func.symbol, str(origin_node.create_type()), str(func.retval.type)))
-                    return False
-        else:
-            if origin_node != target:
-                self._transformer.log_node_warning(func,
-                                                   "Constructor return type mismatch symbol=%r constructed=%r return=%r" % (func.symbol, str(origin_node.create_type()), str(func.retval.type)))
-                return False
-        self._namespace.float(func)
-        func.name = funcname
-        target.constructors.append(func)
-        return True
-
     def _initparse_gobject_record(self, record):
         # Special handling for when we're parsing GObject
         internal_names = ("Object", 'InitiallyUnowned')
@@ -656,24 +406,6 @@ method or constructor of some type."""
             return False
         self._namespace.append(boxed_item, replace=True)
 
-    # Node walking
-
-
-    def _handle_closure(self, param, closure_idx, closure_param):
-        if (closure_param.type is TYPE_ANY and
-            closure_param.name.endswith('data')):
-            param.closure_name = closure_param.argname
-            param.closure_index = closure_idx
-            return True
-        return False
-
-    def _handle_destroy(self, param, destroy_idx, destroy_param):
-        if destroy_param.type.target_giname == 'GLib.DestroyNotify':
-            param.destroy_name = destroy_param.argname
-            param.destroy_index = destroy_idx
-            return True
-        return False
-
     def _strip_class_suffix(self, name):
         if (name.endswith('Class') or
             name.endswith('Iface')):
@@ -700,171 +432,3 @@ method or constructor of some type."""
         pair_class.glib_type_struct = gclass_struct.create_type()
         pair_class.inherit_file_positions(class_struct)
         gclass_struct.is_gtype_struct_for = pair_class.create_type()
-
-    def _pair_class_virtuals(self, node):
-        """Look for virtual methods from the class structure."""
-        if not node.glib_type_struct:
-            self._transformer.log_node_warning(node,
-                "Failed to find class structure for %r" % (node.name, ))
-            return
-
-        node_type = node.create_type()
-        class_struct = self._transformer.lookup_typenode(node.glib_type_struct)
-        
-        # Object class fields are assumed to be read-only
-        # (see also _introspect_object and transformer.py)
-        for field in class_struct.fields:
-            if isinstance(field, Field):
-                field.writable = False
-
-        # Loop through fields to determine which are virtual
-        # functions and which are signal slots by
-        # assuming everything that doesn't share a name
-        # with a known signal is a virtual slot.
-        for field in class_struct.fields:
-            if not isinstance(field.anonymous_node, Callback):
-                continue
-            callback = field.anonymous_node
-            # Check the first parameter is the object
-            if len(callback.parameters) == 0:
-                continue
-            firstparam_type = callback.parameters[0].type
-            if firstparam_type != node_type:
-                continue
-            # Also double check we don't have a signal with this
-            # name.
-            matched_signal = False
-            for signal in node.signals:
-                if signal.name.replace('-', '_') == callback.name:
-                    matched_signal = True
-                    break
-            if matched_signal:
-                continue
-            vfunc = VFunction.from_callback(callback)
-            vfunc.inherit_file_positions(callback)
-            node.virtual_methods.append(vfunc)
-
-        # Take the set of virtual methods we found, and try
-        # to pair up with any matching methods using the
-        # name+signature.
-        for vfunc in node.virtual_methods:
-            for method in node.methods:
-                if method.name != vfunc.name:
-                    continue
-                if method.retval.type != vfunc.retval.type:
-                    continue
-                if len(method.parameters) != len(vfunc.parameters):
-                    continue
-                for i in xrange(len(method.parameters)):
-                    m_type = method.parameters[i].type
-                    v_type = vfunc.parameters[i].type
-                    if m_type != v_type:
-                        continue
-                vfunc.invoker = method.name
-
-    def _pass3(self, node, chain):
-        """Pass 3 is after we've loaded GType data and performed type
-        closure."""
-        if isinstance(node, Callable):
-            self._pass3_callable_callbacks(node)
-            self._pass3_callable_throws(node)
-        return True
-
-    def _pass3_callable_callbacks(self, node):
-        """Check to see if we have anything that looks like a
-        callback+user_data+GDestroyNotify set."""
-        params = node.parameters
-        for i, param in enumerate(params):
-            node = self._transformer.lookup_typenode(param.type)
-            if not isinstance(node, Callback):
-                continue
-
-            # set a default scope
-            if param.scope is None:
-                param.scope = 'call'
-
-            # j is the index where we look for closure/destroy to
-            # group with the callback param
-            j = i + 1
-            if j == len(params):
-                continue # no more args -> nothing to group
-            # look at the param directly following for either a
-            # closure or a destroy; only one of these will fire
-            had_closure = self._handle_closure(param, j, params[j])
-            had_destroy = self._handle_destroy(param, j, params[j])
-            j += 1
-            # are we out of params, or did we find neither?
-            if j == len(params) or (not had_closure and not had_destroy):
-                continue
-            # we found either a closure or a destroy; check the
-            # parameter following for the other
-            if not had_closure:
-                self._handle_closure(param, j, params[j])
-            if not had_destroy:
-                self._handle_destroy(param, j, params[j])
-
-    def _pass3_callable_throws(self, node):
-        """Check to see if we have anything that looks like a
-        callback+user_data+GDestroyNotify set."""
-        if not node.parameters:
-            return
-        last_param = node.parameters[-1]
-        # Checking type.name=='GLib.Error' generates false positives
-        # on methods that take a 'GError *'
-        if last_param.type.ctype == 'GError**':
-            node.parameters.pop()
-            node.throws = True
-
-    # Validation
-
-    def _interface_vfunc_check(self, node, stack):
-        if isinstance(node, GLibInterface):
-            for vfunc in node.virtual_methods:
-                if not vfunc.invoker:
-                    self._transformer.log_node_warning(vfunc,
-"""Virtual function %r has no known invoker""" % (vfunc.name, ),
-                    context=node)
-
-    def _introspectable_analysis(self, node, stack):
-        if isinstance(node, TypeContainer):
-            parent = stack[-1]
-            if isinstance(node.type, Varargs):
-                parent.introspectable = False
-            elif not isinstance(node.type, List) and \
-                 (node.type.target_giname == 'GLib.List'):
-                if isinstance(node, Parameter):
-                    self._transformer.log_node_warning(parent,
-"""Missing (element-type) annotation on argument %r""" % (node.name, ),
-                                                    context=parent)
-                else:
-                    self._transformer.log_node_warning(parent,
-"""Missing (element-type) annotation on return value""", context=parent)
-                parent.introspectable = False
-
-    def _analyze_node(self, obj, stack):
-        if isinstance(obj, Node) and obj.skip:
-            return False
-        # Combine one-pass checks here
-        self._interface_vfunc_check(obj, stack)
-        # Our first pass for scriptability
-        self._introspectable_analysis(obj, stack)
-        return True
-
-    def _introspectable_pass2(self, obj, stack):
-        if isinstance(obj, Node) and obj.skip:
-            return False
-        # In the second introspectable pass, we propagate introspectablity;
-        # for example, a varargs callback as an argument to a function
-        # makes the whole function unintrospectable
-        if isinstance(obj, TypeContainer):
-            parent = stack[-1]
-            target = self._transformer.lookup_typenode(obj.type)
-            if target and not target.introspectable:
-                parent.introspectable = False
-        return True
-
-    # This function is called at the very end, before we hand back the
-    # completed namespace to the writer.  Add static analysis checks here.
-    def final_analyze(self):
-        self._namespace.walk(self._analyze_node)
-        self._namespace.walk(self._introspectable_pass2)
diff --git a/giscanner/scannermain.py b/giscanner/scannermain.py
index 2afbff6..1df48e1 100644
--- a/giscanner/scannermain.py
+++ b/giscanner/scannermain.py
@@ -35,6 +35,7 @@ from giscanner.minixpath import myxpath, xpath_assert
 from giscanner.sourcescanner import SourceScanner
 from giscanner.shlibs import resolve_shlibs
 from giscanner.transformer import Transformer
+from giscanner.finaltransformer import FinalTransformer
 from giscanner.girparser import GIRParser
 from giscanner.girwriter import GIRWriter
 from giscanner.utils import files_are_identical
@@ -288,16 +289,16 @@ def scanner_main(args):
     shlibs = resolve_shlibs(options, binary, libraries)
 
     glibtransformer.set_introspection_binary(binary)
+    glibtransformer.parse()
 
-    namespace = glibtransformer.parse()
-
-    ap = AnnotationParser(namespace, ss, transformer)
+    ap = AnnotationParser(ss, transformer)
     try:
         ap.parse()
     except InvalidAnnotationError, e:
         raise SystemExit("ERROR in annotation: %s" % (str(e), ))
 
-    glibtransformer.final_analyze()
+    final = FinalTransformer(transformer)
+    final.transform()
 
     if options.warn_fatal and transformer.did_warn():
         return 1
@@ -307,7 +308,7 @@ def scanner_main(args):
         exported_packages = options.packages_export
     else:
         exported_packages = options.packages
-    writer = Writer(namespace, shlibs, transformer.get_includes(),
+    writer = Writer(transformer.namespace, shlibs, transformer.get_includes(),
                     exported_packages, options.c_includes)
     data = writer.get_xml()
     if options.output: