gobject-introspection r842 - in trunk: . giscanner tools

[johan svn gnome org]

Author: johan
Date: Thu Oct 30 17:12:51 2008
New Revision: 842
URL: http://svn.gnome.org/viewvc/gobject-introspection?rev=842&view=rev

Log:
2008-10-30  Johan Dahlin  <jdahlin async com br>

    * giscanner/girparser.py:
    Remove arguments from the constructor, move them to
    separate accessors. Add a new parse_tree method
    which takes an element tree instance.
    * tools/g-ir-scanner:
    Update callsite for this

    * giscanner/Makefile.am:
    * giscanner/cachestore.py:
    * giscanner/transformer.py:
    Cache the include parsing. Saves ~25% time when
    creating vte (which includes everything up to gtk+).



Added:
   trunk/giscanner/cachestore.py   (contents, props changed)
Modified:
   trunk/ChangeLog
   trunk/giscanner/Makefile.am
   trunk/giscanner/girparser.py
   trunk/giscanner/transformer.py
   trunk/tools/g-ir-scanner

Modified: trunk/giscanner/Makefile.am
==============================================================================
--- trunk/giscanner/Makefile.am	(original)
+++ trunk/giscanner/Makefile.am	Thu Oct 30 17:12:51 2008
@@ -36,6 +36,7 @@
 pkgpyexec_PYTHON = 		\
 	__init__.py		\
 	ast.py			\
+	cachestore.py		\
 	cgobject.py		\
 	config.py		\
 	girparser.py		\

Added: trunk/giscanner/cachestore.py
==============================================================================
--- (empty file)
+++ trunk/giscanner/cachestore.py	Thu Oct 30 17:12:51 2008
@@ -0,0 +1,87 @@
+# -*- Mode: Python -*-
+# GObject-Introspection - a framework for introspecting GObject libraries
+# Copyright (C) 2008  Johan Dahlin
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+
+import cPickle
+import hashlib
+import os
+import errno
+
+
+def _get_cachedir():
+    cachedir = os.path.join(os.environ['HOME'], '.cache')
+    if not os.path.exists(cachedir):
+        os.mkdir(cachedir, 0755)
+
+    scannerdir = os.path.join(cachedir, 'g-ir-scanner')
+    if not os.path.exists(scannerdir):
+        os.mkdir(scannerdir, 0755)
+    # If it exists and is a file, don't cache at all
+    elif not os.path.isdir(scannerdir):
+        return None
+    return scannerdir
+
+
+class CacheStore(object):
+
+    def __init__(self):
+        try:
+            self._directory = _get_cachedir()
+        except OSError, e:
+            if e.errno != errno.EPERM:
+                raise
+            self._directory = None
+
+    def _get_filename(self, filename):
+        # If we couldn't create the directory we're probably
+        # on a read only home directory where we just disable
+        # the cache all together.
+        if self._directory is None:
+            return
+        hexdigest = hashlib.sha1(filename).hexdigest()
+        return os.path.join(self._directory, hexdigest)
+
+    def _cache_is_valid(self, store_filename, filename):
+        return (os.stat(store_filename).st_mtime >=
+                os.stat(filename).st_mtime)
+
+    def store(self, filename, data):
+        store_filename = self._get_filename(filename)
+        if store_filename is None:
+            return
+        if (os.path.exists(store_filename) and
+            self._cache_is_valid(store_filename, filename)):
+            return None
+        fd = open(store_filename, 'w')
+        cPickle.dump(data, fd)
+
+    def load(self, filename):
+        store_filename = self._get_filename(filename)
+        if store_filename is None:
+            return
+        try:
+            fd = open(store_filename)
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                return None
+            raise
+        if not self._cache_is_valid(store_filename, filename):
+            return None
+        data = cPickle.load(fd)
+        return data

Modified: trunk/giscanner/girparser.py
==============================================================================
--- trunk/giscanner/girparser.py	(original)
+++ trunk/giscanner/girparser.py	Thu Oct 30 17:12:51 2008
@@ -46,26 +46,23 @@
 
 class GIRParser(object):
 
-    def __init__(self, filename,
-                 initial_parse=True,
-                 include_parsing=False):
+    def __init__(self):
+        self._include_parsing = False
+        self._shared_libraries = []
         self._includes = set()
         self._namespace = None
-        self._shared_libraries = []
-        self._include_parsing = include_parsing
-        self._tree = parse(filename)
-
-        if initial_parse:
-            self.parse()
 
     # Public API
 
-    def parse(self):
-        self._includes.clear()
-        del self._namespace
-        del self._shared_libraries[:]
+    def parse(self, filename):
+        tree = parse(filename)
+        self.parse_tree(tree)
 
-        self._parse_api(self._tree.getroot())
+    def parse_tree(self, tree):
+        self._includes.clear()
+        self._namespace = None
+        self._shared_libraries = []
+        self._parse_api(tree.getroot())
 
     def get_namespace(self):
         return self._namespace
@@ -77,7 +74,10 @@
         return self._includes
 
     def get_doc(self):
-        return self._tree
+        return parse(self._filename)
+
+    def set_include_parsing(self, include_parsing):
+        self._include_parsing = include_parsing
 
     # Private
 
@@ -159,8 +159,8 @@
             obj.fields.append(self._parse_function_common(callback, Callback))
         for field in node.findall(_corens('field')):
             obj.fields.append(self._parse_field(field))
-        for property in node.findall(_corens('property')):
-            obj.properties.append(self._parse_property(property))
+        for prop in node.findall(_corens('property')):
+            obj.properties.append(self._parse_property(prop))
         for signal in node.findall(_glibns('signal')):
             obj.signals.append(self._parse_function_common(signal, Function))
 
@@ -237,7 +237,7 @@
                                     node.attrib.get(_cns('type')))
         else:
             union = Union(node.attrib['name'],
-                           node.attrib.get(_cns('type')))
+                          node.attrib.get(_cns('type')))
         self._add_node(union)
 
         if self._include_parsing:

Modified: trunk/giscanner/transformer.py
==============================================================================
--- trunk/giscanner/transformer.py	(original)
+++ trunk/giscanner/transformer.py	Thu Oct 30 17:12:51 2008
@@ -65,7 +65,9 @@
 
 class Transformer(object):
 
-    def __init__(self, generator, namespace_name, namespace_version):
+    def __init__(self, cachestore, generator,
+                 namespace_name, namespace_version):
+        self._cachestore = cachestore
         self.generator = generator
         self._namespace = Namespace(namespace_name, namespace_version)
         self._names = Names()
@@ -123,11 +125,19 @@
                              % (girname, searchdirs))
 
     def _parse_include(self, filename):
-        parser = GIRParser(filename, include_parsing=True)
+        parser = self._cachestore.load(filename)
+        if parser is None:
+            parser = GIRParser()
+            parser.set_include_parsing(True)
+            parser.parse(filename)
+            self._cachestore.store(filename, parser)
+
         for include in parser.get_includes():
             self.register_include(include)
-        nsname = parser.get_namespace().name
-        for node in parser.get_namespace().nodes:
+
+        namespace = parser.get_namespace()
+        nsname = namespace.name
+        for node in namespace.nodes:
             if isinstance(node, Alias):
                 self._names.aliases[node.name] = (nsname, node)
             elif isinstance(node, (GLibBoxed, Interface, Class)):

Modified: trunk/tools/g-ir-scanner
==============================================================================
--- trunk/tools/g-ir-scanner	(original)
+++ trunk/tools/g-ir-scanner	Thu Oct 30 17:12:51 2008
@@ -36,6 +36,7 @@
 sys.path.insert(0, path)
 
 from giscanner.ast import Include
+from giscanner.cachestore import CacheStore
 from giscanner.glibtransformer import GLibTransformer
 from giscanner.minixpath import myxpath, xpath_assert
 from giscanner.sourcescanner import SourceScanner
@@ -115,15 +116,19 @@
     from giscanner.girparser import GIRParser
     from giscanner.girwriter import GIRWriter
     from giscanner.girparser import C_NS
+    from xml.etree.cElementTree import parse
+
     c_ns_key = '{%s}' % (C_NS, )
 
-    parser = GIRParser(path, initial_parse=False)
-    doc = parser.get_doc()
-    for node in doc.getiterator():
+    tree = parse(path)
+    root = tree.getroot()
+    for node in root.getiterator():
         for attrib in list(node.attrib):
             if attrib.startswith(c_ns_key):
                 del node.attrib[attrib]
-    parser.parse()
+    parser = GIRParser()
+    parser.parse_tree(tree)
+
     writer = GIRWriter(parser.get_namespace(),
                        parser.get_shared_libraries(),
                        parser.get_includes())
@@ -135,8 +140,8 @@
     from giscanner.girwriter import GIRWriter
     from xml.etree.cElementTree import parse
 
-    parser = GIRParser(path, initial_parse=False)
-    root = parser.get_doc().getroot()
+    tree = parse(path)
+    root = tree.getroot()
     injectDoc = parse(open(additions))
     for node in injectDoc.getroot():
         injectPath = node.attrib['path']
@@ -145,7 +150,9 @@
             raise ValueError("Couldn't find path %r" % (injectPath, ))
         for child in node:
             target.append(child)
-    parser.parse()
+
+    parser = GIRParser()
+    parser.parse_tree(tree)
     writer = GIRWriter(parser.get_namespace(),
                        parser.get_shared_libraries(),
                        parser.get_includes())
@@ -248,6 +255,7 @@
                 _error('%s: no such a file or directory' % (arg, ))
             filenames.append(arg)
 
+    cachestore = CacheStore()
     # Run the preprocessor, tokenize and construct simple
     # objects representing the raw C symbols
     ss = SourceScanner()
@@ -258,7 +266,7 @@
     ss.parse_macros(filenames)
 
     # Transform the C symbols into AST nodes
-    transformer = Transformer(ss,
+    transformer = Transformer(cachestore, ss,
                               options.namespace_name,
                               options.namespace_version)
     if options.strip_prefix: