[library-web] moved some generic (non library.gnome.org specific) stuff to app.py



commit 098715479e7c3f92071fcf1ceda23899eba62f9c
Author: Frédéric Péters <fpeters 0d be>
Date:   Thu Aug 6 22:03:31 2009 +0200

    moved some generic (non library.gnome.org specific) stuff to app.py

 src/app.py           |  208 ++++++++++++++++++++++++++++++++++++++++++++++++++
 src/document.py      |    4 +-
 src/lgo.py           |  168 +---------------------------------------
 src/modtypes/base.py |    2 +-
 src/overlay.py       |    2 +-
 src/utils.py         |   22 -----
 6 files changed, 217 insertions(+), 189 deletions(-)
---
diff --git a/src/app.py b/src/app.py
new file mode 100644
index 0000000..1d65557
--- /dev/null
+++ b/src/app.py
@@ -0,0 +1,208 @@
+# libgo - script to build library.gnome.org
+# Copyright (C) 2007-2009  Frederic Peters
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301  USA
+
+import __builtin__
+import glob
+import logging
+import os
+from optparse import OptionParser
+import stat
+import urllib2
+
+from config import Config
+import utils
+
+class App:
+    indexes_xsl_file = os.path.join(data_dir, 'xslt', 'indexes.xsl')
+    javascript_dir = os.path.join(data_dir, 'js')
+    skin_dir = os.path.join(data_dir, 'skin')
+
+    rebuild_all = False
+    debug = False
+
+    def __init__(self):
+        __builtin__.__dict__['app'] = self
+        self.documents = []
+
+        parser = OptionParser()
+        parser.add_option('-c', '--config', dest = 'config')
+        parser.add_option('-v', '--verbose',
+            action = 'count', dest = 'verbose', default = 0,
+            help = 'verbosity level (more -v for more verbose)')
+        parser.add_option('--rebuild', dest = 'rebuild_module',
+            help = 'rebuild documentation from FILENAME', metavar = 'FILENAME')
+        parser.add_option('--rebuild-all',
+            action = 'store_true', dest = 'rebuild_all',
+            help = 'rebuild all documents (even those that were already built)')
+        parser.add_option('--rebuild-language', dest = 'rebuild_language',
+            help = 'rebuild all documents in LANGUAGE', metavar = 'LANGUAGE')
+        parser.add_option('--skip-extra-tarballs',
+            action = 'store_false', dest = 'skip_extra_tarballs',
+            help = "don't look for documentation extra tarballs")
+        self.options, args = parser.parse_args()
+
+        logging.basicConfig(level = 10 + logging.CRITICAL - self.options.verbose*10,
+            formatter = utils.LogFormatter())
+        logging.getLogger().handlers[0].setFormatter(utils.LogFormatter())
+
+        self.debug = (self.options.verbose >= 5)
+        self.rebuild_all = self.options.rebuild_all
+        self.rebuild_language = self.options.rebuild_language
+
+        if self.options.config:
+            self.config = Config(filename = self.options.config)
+        else:
+            self.config = Config()
+
+        self.check_sanity()
+
+    def check_sanity(self):
+        for filename in [os.path.join(data_dir, 'overlay.xml'),
+                    os.path.join(data_dir, 'catalog.xml')]:
+            if not os.path.exists(filename):
+                print >> sys.stderr, '%s is missing, you should run make' % filename
+                sys.exit(1)
+
+        if not self.config.output_dir.endswith(os.path.sep):
+            logging.warning('output dir should end with slash')
+            self.config.output_dir += os.path.sep
+
+    def copy_static_files(self):
+        if not os.path.exists(os.path.join(self.config.output_dir, 'js')):
+            os.makedirs(os.path.join(self.config.output_dir, 'js'))
+        if not os.path.exists(os.path.join(self.config.output_dir, 'skin')):
+            os.makedirs(os.path.join(self.config.output_dir, 'skin'))
+        if not os.path.exists(os.path.join(self.config.output_dir, 'skin/icons')):
+            os.makedirs(os.path.join(self.config.output_dir, 'skin/icons'))
+
+        for src in glob.glob('%s/*.js' % self.javascript_dir):
+            dst = os.path.join(self.config.output_dir, 'js', os.path.basename(src))
+            if not os.path.exists(dst) or \
+                    os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
+                open(dst, 'w').write(open(src, 'r').read())
+
+        for src in glob.glob('%s/*.css' % self.skin_dir) + \
+                        glob.glob('%s/*.png' % self.skin_dir) + \
+                        glob.glob('%s/*.gif' % self.skin_dir):
+            dst = os.path.join(self.config.output_dir, 'skin', os.path.basename(src))
+            if not os.path.exists(dst) or \
+                    os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
+                open(dst, 'w').write(open(src, 'r').read())
+
+        for src in glob.glob('%s/icons/*.png' % self.skin_dir):
+            dst = os.path.join(self.config.output_dir, 'skin/icons', os.path.basename(src))
+            if not os.path.exists(dst) or \
+                    os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
+                open(dst, 'w').write(open(src, 'r').read())
+
+    def download(self, url):
+        parsed_url = urllib2.urlparse.urlparse(url)
+        if parsed_url[0] == 'file':
+            return parsed_url[2]
+        filename = '/'.join(parsed_url[1:3])
+        cache_filename = os.path.join(self.config.download_cache_dir, filename)
+        cache_dir = os.path.split(cache_filename)[0]
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+        if not os.path.exists(cache_filename):
+            logging.info('downloading %s' % url)
+            try:
+                s = urllib2.urlopen(url).read()
+            except urllib2.HTTPError, e:
+                logging.warning('error %s downloading %s' % (e.code, url))
+                return None
+            except urllib2.URLError, e:
+                logging.warning('error (URLError) downloading %s' % url)
+                return None
+            open(cache_filename, 'w').write(s)
+        return cache_filename
+
+    def apply_overlay(self):
+        logging.info('Applying overlay')
+        for doc in self.documents:
+            self.overlay.apply(doc)
+        self.documents.extend(self.overlay.get_new_docs())
+
+    def generate_symbols_files(self):
+        if not (self.config.symbols_dbm_filepath or self.config.symbols_sqlite_filepath):
+            return
+        logging.info('getting all symbols')
+
+        if self.rebuild_all:
+            if self.config.symbols_dbm_filepath and os.path.exists(
+                    self.config.symbols_dbm_filepath):
+                os.unlink(self.config.symbols_dbm_filepath)
+
+        if self.config.symbols_sqlite_filepath and os.path.exists(
+                self.config.symbols_sqlite_filepath):
+            os.unlink(self.config.symbols_sqlite_filepath)
+
+        if self.config.symbols_dbm_filepath:
+            cmd = [self.config.httxt2dbm_path, '-i', '-', '-o', self.config.symbols_dbm_filepath]
+            logging.debug('executing %s' % ' '.join(cmd))
+            try:
+                httxt2dbm = subprocess.Popen(cmd, stdin = subprocess.PIPE).stdin
+            except OSError:
+                logging.error('failed to generate dbm symbols file (OSError)')
+                return
+        else:
+            httxt2dbm = None
+
+        if sqlite and self.config.symbols_sqlite_filepath:
+            sqlcon = sqlite.connect(self.config.symbols_sqlite_filepath, isolation_level=None)
+            sqlcon.execute('create table symbols(symbol, path)')
+            sqlcur = sqlcon.cursor()
+        else:
+            sqlcon = None
+            sqlcur = None
+
+        def symbols_iterator():
+            for doc in self.documents:
+                if doc.category != 'api':
+                    continue
+                if not doc.module or not doc.path:
+                    continue
+
+                web_dir = os.path.join(app.config.output_dir, doc.path[1:])
+
+                devhelp_path = os.path.join(web_dir, '%s.devhelp2' % doc.module)
+                if os.path.exists(devhelp_path):
+                    tree = ET.parse(devhelp_path)
+                    for keyword in tree.findall('//{http://www.devhelp.net/book}keyword'):
+                        key = keyword.attrib.get('name').replace('()', '').strip()
+                        if not key or ' ' in key:
+                            # ignore keys with spaces in their name
+                            continue
+                        value = os.path.join(doc.path, keyword.attrib.get('link'))
+                        if httxt2dbm:
+                            print >> httxt2dbm, key, value
+                        yield (key, value)
+            return
+
+        # if active, the dbm symbol file will be generated while iterating for
+        # SQLite.
+        if sqlcur:
+            sqlcur.executemany('insert into symbols values (?, ?)', symbols_iterator())
+            sqlcur.execute('create index symbols_idx on symbols(symbol)')
+        else:
+            for x in symbols_iterator():
+                pass
+
+        if httxt2dbm:
+            httxt2dbm.close()
+
diff --git a/src/document.py b/src/document.py
index 1ee0b82..9402d5e 100644
--- a/src/document.py
+++ b/src/document.py
@@ -31,7 +31,7 @@ try:
 except ImportError:
     html5lib = None
 
-from utils import version_cmp, download
+from utils import version_cmp
 
 
 def assert_elementtree_node(node):
@@ -246,5 +246,5 @@ class RemoteDocument(Document):
     def download(self, href):
         # TODO: add some support (think <local update="daily"/>) so the file
         # can be "watched" for changes
-        return download(href)
+        return app.download(href)
 
diff --git a/src/lgo.py b/src/lgo.py
index 62c4804..23eff85 100755
--- a/src/lgo.py
+++ b/src/lgo.py
@@ -30,8 +30,6 @@ try:
 except ImportError:
     import xml.etree.ElementTree as ET
 import tarfile
-import glob
-import tempfile
 import stat
 import subprocess
 import dbm
@@ -47,10 +45,9 @@ except ImportError:
 data_dir = os.path.join(os.path.dirname(__file__), '../data')
 __builtin__.__dict__['data_dir'] = data_dir
 
-from config import Config
 import errors
 import utils
-from utils import version_cmp, is_version_number, download
+from utils import version_cmp, is_version_number
 
 from document import Document
 from overlay import Overlay
@@ -59,8 +56,8 @@ from modtypes.gnomedocbook import GnomeDocbookModule
 from modtypes.gtkdoc import GtkDocModule
 from modtypes.htmlfiles import HtmlFilesModule
 
+from app import App
 
-app = None
 
 # timeout for downloads, so it doesn't hang on connecting to sourceforge
 socket.setdefaulttimeout(10)
@@ -131,51 +128,9 @@ class FtpDotGnomeDotOrg:
             return []
 
 
-class Lgo:
+class Lgo(App):
     '''Main Application Class'''
 
-    indexes_xsl_file = os.path.join(data_dir, 'xslt', 'indexes.xsl')
-    javascript_dir = os.path.join(data_dir, 'js')
-    skin_dir = os.path.join(data_dir, 'skin')
-
-    rebuild_all = False
-    debug = False
-
-    def __init__(self):
-        self.documents = []
-
-        parser = OptionParser()
-        parser.add_option('-c', '--config', dest = 'config')
-        parser.add_option('-v', '--verbose',
-            action = 'count', dest = 'verbose', default = 0,
-            help = 'verbosity level (more -v for more verbose)')
-        parser.add_option('--rebuild', dest = 'rebuild_module',
-            help = 'rebuild documentation from FILENAME', metavar = 'FILENAME')
-        parser.add_option('--rebuild-all',
-            action = 'store_true', dest = 'rebuild_all',
-            help = 'rebuild all documents (even those that were already built)')
-        parser.add_option('--rebuild-language', dest = 'rebuild_language',
-            help = 'rebuild all documents in LANGUAGE', metavar = 'LANGUAGE')
-        parser.add_option('--skip-extra-tarballs',
-            action = 'store_false', dest = 'skip_extra_tarballs',
-            help = "don't look for documentation extra tarballs")
-        self.options, args = parser.parse_args()
-
-        logging.basicConfig(level = 10 + logging.CRITICAL - self.options.verbose*10,
-            formatter = utils.LogFormatter())
-        logging.getLogger().handlers[0].setFormatter(utils.LogFormatter())
-
-        self.debug = (self.options.verbose >= 5)
-        self.rebuild_all = self.options.rebuild_all
-        self.rebuild_language = self.options.rebuild_language
-
-        if self.options.config:
-            self.config = Config(filename = self.options.config)
-        else:
-            self.config = Config()
-
-        self.check_sanity()
-    
     def run(self):
         self.ftp_gnome_org = FtpDotGnomeDotOrg(self.config)
         self.overlay = Overlay(os.path.join(data_dir, 'overlay.xml'))
@@ -198,17 +153,6 @@ class Lgo:
         self.generate_symbols_files()
         self.generate_static_pages()
 
-    def check_sanity(self):
-        for filename in [os.path.join(data_dir, 'overlay.xml'),
-                    os.path.join(data_dir, 'catalog.xml')]:
-            if not os.path.exists(filename):
-                print >> sys.stderr, '%s is missing, you should run make' % filename
-                sys.exit(1)
-
-        if not self.config.output_dir.endswith(os.path.sep):
-            logging.warning('output dir should end with slash')
-            self.config.output_dir += os.path.sep
-
     def get_yelp_categories(self):
         logging.info('Getting categories from Yelp')
 
@@ -243,38 +187,13 @@ class Lgo:
                 self.toc_mapping[subject.attrib['category']] = sub_id
 
     def copy_static_files(self):
-        if not os.path.exists(os.path.join(self.config.output_dir, 'js')):
-            os.makedirs(os.path.join(self.config.output_dir, 'js'))
-        if not os.path.exists(os.path.join(self.config.output_dir, 'skin')):
-            os.makedirs(os.path.join(self.config.output_dir, 'skin'))
-        if not os.path.exists(os.path.join(self.config.output_dir, 'skin/icons')):
-            os.makedirs(os.path.join(self.config.output_dir, 'skin/icons'))
+        App.copy_static_files(self)
 
         src = os.path.join(data_dir, 'gnome-library-search.xml')
         dst = os.path.join(self.config.output_dir, 'gnome-library-search.xml')
         if not os.path.exists(dst) or \
                 os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
             open(dst, 'w').write(open(src, 'r').read())
-        
-        for src in glob.glob('%s/*.js' % self.javascript_dir):
-            dst = os.path.join(self.config.output_dir, 'js', os.path.basename(src))
-            if not os.path.exists(dst) or \
-                    os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
-                open(dst, 'w').write(open(src, 'r').read())
-
-        for src in glob.glob('%s/*.css' % self.skin_dir) + \
-                        glob.glob('%s/*.png' % self.skin_dir) + \
-                        glob.glob('%s/*.gif' % self.skin_dir):
-            dst = os.path.join(self.config.output_dir, 'skin', os.path.basename(src))
-            if not os.path.exists(dst) or \
-                    os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
-                open(dst, 'w').write(open(src, 'r').read())
-
-        for src in glob.glob('%s/icons/*.png' % self.skin_dir):
-            dst = os.path.join(self.config.output_dir, 'skin/icons', os.path.basename(src))
-            if not os.path.exists(dst) or \
-                    os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
-                open(dst, 'w').write(open(src, 'r').read())
 
     def process_releases(self):
         '''Download GNOME releases'''
@@ -383,7 +302,7 @@ class Lgo:
                 logging.error('error downloading %s' % url)
                 return
         else:
-            filename = download(url)
+            filename = App.download(self, url)
 
         return filename
 
@@ -562,14 +481,6 @@ class Lgo:
             for doc_module in self.extract_modules(filename, nightly = True):
                 doc_module.process()
 
-
-    def apply_overlay(self):
-        logging.info('Applying overlay')
-        for doc in self.documents:
-            self.overlay.apply(doc)
-
-        self.documents.extend(self.overlay.get_new_docs())
-
     def generate_indexes(self):
         logging.info('generating indexes')
         indexes = ET.Element('indexes')
@@ -744,78 +655,9 @@ class Lgo:
             if rc != 0:
                 logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
 
-    def generate_symbols_files(self):
-        if not (self.config.symbols_dbm_filepath or self.config.symbols_sqlite_filepath):
-            return
-        logging.info('getting all symbols')
-
-        if self.rebuild_all:
-            if self.config.symbols_dbm_filepath and os.path.exists(
-                    self.config.symbols_dbm_filepath):
-                os.unlink(self.config.symbols_dbm_filepath)
-
-        if self.config.symbols_sqlite_filepath and os.path.exists(
-                self.config.symbols_sqlite_filepath):
-            os.unlink(self.config.symbols_sqlite_filepath)
-
-        if self.config.symbols_dbm_filepath:
-            cmd = [self.config.httxt2dbm_path, '-i', '-', '-o', self.config.symbols_dbm_filepath]
-            logging.debug('executing %s' % ' '.join(cmd))
-            try:
-                httxt2dbm = subprocess.Popen(cmd, stdin = subprocess.PIPE).stdin
-            except OSError:
-                logging.error('failed to generate dbm symbols file (OSError)')
-                return
-        else:
-            httxt2dbm = None
-
-        if sqlite and self.config.symbols_sqlite_filepath:
-            sqlcon = sqlite.connect(self.config.symbols_sqlite_filepath, isolation_level=None)
-            sqlcon.execute('create table symbols(symbol, path)')
-            sqlcur = sqlcon.cursor()
-        else:
-            sqlcon = None
-            sqlcur = None
-
-        def symbols_iterator():
-            for doc in self.documents:
-                if doc.category != 'api':
-                    continue
-                if not doc.module or not doc.path:
-                    continue
-            
-                web_dir = os.path.join(app.config.output_dir, doc.path[1:])
-
-                devhelp_path = os.path.join(web_dir, '%s.devhelp2' % doc.module)
-                if os.path.exists(devhelp_path):
-                    tree = ET.parse(devhelp_path)
-                    for keyword in tree.findall('//{http://www.devhelp.net/book}keyword'):
-                        key = keyword.attrib.get('name').replace('()', '').strip()
-                        if not key or ' ' in key:
-                            # ignore keys with spaces in their name
-                            continue
-                        value = os.path.join(doc.path, keyword.attrib.get('link'))
-                        if httxt2dbm:
-                            print >> httxt2dbm, key, value
-                        yield (key, value)
-            return
-
-        # if active, the dbm symbol file will be generated while iterating for
-        # SQLite.
-        if sqlcur:
-            sqlcur.executemany('insert into symbols values (?, ?)', symbols_iterator())
-            sqlcur.execute('create index symbols_idx on symbols(symbol)')
-        else:
-            for x in symbols_iterator():
-                pass
-
-        if httxt2dbm:
-            httxt2dbm.close()
-
 
 if __name__ == '__main__':
     app = Lgo()
     app.Document = Document
-    __builtin__.__dict__['app'] = app
     app.run()
 
diff --git a/src/modtypes/base.py b/src/modtypes/base.py
index d6ee4c5..c43f29b 100644
--- a/src/modtypes/base.py
+++ b/src/modtypes/base.py
@@ -22,7 +22,7 @@ import re
 import stat
 import tarfile
 
-from utils import version_cmp, is_version_number, download
+from utils import version_cmp, is_version_number
 
 licence_modules = ['fdl', 'gpl', 'lgpl']
 
diff --git a/src/overlay.py b/src/overlay.py
index 29ef9c7..b3ad6c5 100644
--- a/src/overlay.py
+++ b/src/overlay.py
@@ -22,7 +22,7 @@ except ImportError:
     import xml.etree.ElementTree as ET
 
 from document import RemoteDocument
-from utils import version_cmp, is_version_number, download
+from utils import version_cmp, is_version_number
 
 
 class SubIndex:
diff --git a/src/utils.py b/src/utils.py
index 5f8db39..dfbde93 100644
--- a/src/utils.py
+++ b/src/utils.py
@@ -79,28 +79,6 @@ def version_cmp(x, y):
 def is_version_number(v):
     return re.match('\d+\.\d+', v) is not None
 
-def download(href):
-    parsed_url = urllib2.urlparse.urlparse(href)
-    if parsed_url[0] == 'file':
-        return parsed_url[2]
-    filename = '/'.join(parsed_url[1:3])
-    cache_filename = os.path.join(app.config.download_cache_dir, filename)
-    cache_dir = os.path.split(cache_filename)[0]
-    if not os.path.exists(cache_dir):
-        os.makedirs(cache_dir)
-    if not os.path.exists(cache_filename):
-        logging.info('downloading %s' % href)
-        try:
-            s = urllib2.urlopen(href).read()
-        except urllib2.HTTPError, e:
-            logging.warning('error %s downloading %s' % (e.code, href))
-            return None
-        except urllib2.URLError, e:
-            logging.warning('error (URLError) downloading %s' % href)
-            return None
-        open(cache_filename, 'w').write(s)
-    return cache_filename
-
 class LogFormatter(logging.Formatter):
     '''Class used for formatting log messages'''
 



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]