[library-web/libmaemo] Adding central but missing file
- From: Frederic Peters <fpeters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [library-web/libmaemo] Adding central but missing file
- Date: Tue, 8 Jun 2010 11:34:46 +0000 (UTC)
commit af78ff62f28df6bcd00d44c4958d7ce376fe4f86
Author: Frédéric Péters <fpeters 0d be>
Date: Tue Jun 8 13:33:58 2010 +0200
Adding central but missing file
src/libmaemo.py | 481 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 481 insertions(+), 0 deletions(-)
---
diff --git a/src/libmaemo.py b/src/libmaemo.py
new file mode 100755
index 0000000..fc1e870
--- /dev/null
+++ b/src/libmaemo.py
@@ -0,0 +1,481 @@
+#!/usr/bin/env python
+#
+# libmaemo - script to build library.maemo.org
+# Copyright (C) 2007-2009 Frederic Peters
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 USA
+
+import gzip
+import os
+import sys
+import re
+import urllib2
+from cStringIO import StringIO
+from optparse import OptionParser
+import logging
+try:
+ import elementtree.ElementTree as ET
+except ImportError:
+ import xml.etree.ElementTree as ET
+import tarfile
+import stat
+import subprocess
+import dbm
+import rfc822
+import shutil
+import socket
+import __builtin__
+
+import apt_pkg
+apt_pkg.init()
+
+
+data_dir = os.path.join(os.path.dirname(__file__), '../data')
+__builtin__.__dict__['data_dir'] = data_dir
+
+import errors
+import utils
+from utils import version_cmp, is_version_number
+
+from document import Document
+from overlay import Overlay
+
+from modtypes.gnomedocbook import GnomeDocbookModule
+from modtypes.gtkdoc import GtkDocModule
+from modtypes.htmlfiles import HtmlFilesModule
+from modtypes.mallard import MallardModule
+
+HtmlFilesModule.html2html_xsl_file = os.path.join(data_dir, 'maemo', 'html2html.xsl')
+HtmlFilesModule.related_xsl_files = [
+ HtmlFilesModule.html2html_xsl_file, os.path.join(data_dir, 'maemo', 'heading.xsl')]
+
+
+class DoxygenHtmlFilesModule(HtmlFilesModule):
+ transform_mode = 'doxygen'
+ extra_xslt_attribs = {'extracss': 'doxygen.css'}
+ HtmlFilesModule.related_xsl_files = [
+ HtmlFilesModule.html2html_xsl_file,
+ os.path.join(data_dir, 'maemo', 'heading.xsl')]
+
+class GtkDocHtmlFilesModule(HtmlFilesModule):
+ transform_mode = 'gtk-doc'
+ extra_xslt_attribs = {'extracss': 'gtkdoc.css'}
+ html2html_xsl_file = os.path.join(data_dir, 'maemo', 'html2html-gtkdoc.xsl')
+ HtmlFilesModule.related_xsl_files = [
+ html2html_xsl_file,
+ HtmlFilesModule.html2html_xsl_file,
+ os.path.join(data_dir, 'maemo', 'heading.xsl')]
+
+
+from app import App
+
+# timeout for downloads, so it doesn't hang on connecting to sourceforge
+socket.setdefaulttimeout(10)
+
+class LibMaemo(App):
+ '''Main Application Class'''
+
+ indexes_xsl_file = os.path.join(data_dir, 'maemo', 'indexes.xsl')
+ javascript_dir = os.path.join(data_dir, 'js')
+ skin_dir = os.path.join(data_dir, 'maemo', 'skin')
+
+ releases = [
+ {'codename': 'fremantle',
+ 'repositories': [
+ ('repository.maemo.org', 'fremantle/sdk', 'free'),
+ ('repository.maemo.org/extras-devel', 'fremantle', 'free')
+ ]
+ }
+ ]
+
+ def run(self):
+ self.overlay = Overlay(os.path.join(data_dir, 'overlay.xml'))
+
+ self.config.create_tarballs = False # XXX: support for tarballs
+
+ if self.options.rebuild_module:
+ self.rebuild_all = True
+ for doc_module in self.extract_modules(self.options.rebuild_module):
+ doc_module.process()
+ sys.exit(0)
+
+ self.copy_static_files()
+ self.process_releases()
+ #self.apply_overlay()
+ self.generate_indexes()
+
+ def process_releases(self):
+ '''Download Maemo SDK releases'''
+
+ for release in self.releases:
+ self.process_release(release)
+ self.generate_symbols_files(release)
+
+ def process_release(self, release):
+ codename = release.get('codename')
+ gtkdoc_modules = []
+ doxygen_modules = []
+
+ # all_packages is a dict with key: package name, value: entry in Source
+ # file; it is used to contain only a single version (the latest one) of
+ # the packages.
+ all_packages = {}
+
+ for repository in release.get('repositories'):
+ sources_url = 'http://%s/dists/%s/%s/source/Sources' % repository
+ sources = self.download(sources_url)
+ fp = file(sources)
+
+ while True:
+ m = rfc822.Message(fp)
+ if not m:
+ break
+ package_name = m.getheader('Package')
+ if package_name in all_packages:
+ if apt_pkg.VersionCompare(
+ all_packages[package_name].getheader('Version'),
+ m.getheader('Version')) < 0:
+ all_packages[package_name] = m
+ else:
+ all_packages[package_name] = m
+
+ for package_name, m in all_packages.items():
+ build_depends = m.getheader('Build-Depends', '')
+ build_depends_indep = m.getheader('Build-Depends-Indep', '')
+ binaries = m.getheader('Binary')
+ if 'gtk-doc-tools' in build_depends + build_depends_indep:
+ gtkdoc_modules.extend([x.strip() for x in binaries.split(',')])
+ elif 'doxygen' in build_depends + build_depends_indep:
+ doxygen_modules.extend([x.strip() for x in binaries.split(',')])
+
+ doc_modules = []
+ release['doc_modules'] = doc_modules
+
+ package_extract_dir = os.path.join(self.config.private_dir, 'packages')
+ fallback_used = []
+ for repository in release.get('repositories'):
+ packages_url = 'http://%s/dists/%s/%s/binary-i386/Packages' % repository
+ packages = self.download(packages_url)
+ fp = file(packages)
+ while True:
+ m = rfc822.Message(fp)
+ if not m:
+ break
+ package_name = m.getheader('Package')
+ if not package_name in gtkdoc_modules + doxygen_modules:
+ continue
+ package_version = m.getheader('Version')
+
+ source_package_name = m.getheader('Source') or m.getheader('Package')
+ source_package_name = source_package_name.split(' ')[0]
+ if package_version != all_packages[source_package_name].getheader('Version'):
+ continue
+
+ if ':' in package_version:
+ # chop epoch
+ package_version = package_version[package_version.index(':')+1:]
+ package_url = 'http://%s/%s' % (repository[0], m.getheader('Filename'))
+ package_filename = self.download(package_url)
+ mtime = os.stat(package_filename)[stat.ST_MTIME]
+ dist_dir = os.path.join(package_extract_dir, package_name + '-' + package_version)
+ if not os.path.exists(dist_dir):
+ logging.debug('extracting %s %s' % (package_name, package_version))
+ os.makedirs(dist_dir)
+ os.system('dpkg -x %s %s' % (package_filename, dist_dir))
+
+ for basename, dirnames, filenames in os.walk(dist_dir):
+ if 'index.html' in filenames and len(
+ [x for x in filenames if x.endswith('.html')]) > 1:
+ index_html = file(os.path.join(basename, 'index.html')).read()
+ forced_index_filename = None
+ if '<frameset' in index_html and os.path.exists(
+ os.path.join(basename, 'main.html')):
+ # detect doxygen frame mode
+ index_html = file(os.path.join(basename, 'main.html')).read()
+ forced_index_filename = 'main.html'
+ if '<!-- Generated by Doxygen' in index_html:
+ doc = DoxygenHtmlFilesModule()
+ doc.index_filename = forced_index_filename
+ elif '<meta name="generator" content="GTK-Doc' in index_html:
+ doc = GtkDocHtmlFilesModule()
+
+ doc.languages = ['C']
+ doc.packagename = package_name
+ doc.modulename = os.path.basename(basename)
+ if doc.modulename in ('html', 'api', 'doxygen'):
+ # if the documentation isn't set in a directory
+ # with a distinctive name, use the source package
+ # name as module name
+ doc.modulename = m.getheader('Source') or m.getheader('Package')
+ if doc.modulename in fallback_used:
+ logging.error(
+ 'Source with two documents to rename: %r' % doc.modulename)
+ fallback_used.append(doc.modulename)
+ doc.dirname = basename
+ doc.mtime_tarball = mtime
+ doc.version = package_version
+ doc.one_dot_version = re.match(r'\d+\.\d+', doc.version).group()
+ doc.channel = codename
+ doc_modules.append(doc)
+ doc.modtype = None
+
+ del all_packages
+
+ for doc_module in doc_modules:
+ self.set_module_web_path(doc_module)
+ logging.info('processing %s (from %s)' % (doc_module.modulename, doc_module.packagename))
+ doc_module.process()
+
+
+ def generate_indexes(self):
+ logging.info('generating indexes')
+ indexes = ET.Element('indexes')
+
+ # get all possible languages
+ languages = {}
+ for doc in app.documents:
+ for lang in doc.languages:
+ if lang == 'C':
+ continue # ignore
+ if self.config.languages and not lang in self.config.languages:
+ continue
+ languages[lang] = True
+
+ for lang in languages.keys():
+ home = ET.SubElement(indexes, 'home')
+ home.set('lang', lang)
+
+ for release in self.releases:
+ codename = release.get('codename')
+ docs = [x for x in app.documents if x.path.startswith('/' + codename)]
+
+ if not docs:
+ continue
+
+ for lang in languages.keys():
+ logging.debug('generating index for lang %s' % lang)
+
+ sections = {}
+ for x in docs:
+ sections['Other'] = True
+ x.toc_id = 'Other'
+ x.subsection = 'Other'
+ sections = sections.keys()
+ sections.sort()
+
+ docs.sort(lambda x,y: cmp(x.title.get(lang), y.title.get(lang)))
+
+ subindexes = self.overlay.get_subindexes(release)
+ if not subindexes:
+ index = ET.SubElement(indexes, 'index')
+ index.set('lang', lang)
+ index.set('channel', codename)
+ for section in sections:
+ section_docs = [x for x in docs if x.toc_id == section]
+ if not section_docs:
+ continue
+ self.create_section(index, section, section_docs, lang)
+ else:
+ remaining_sections = sections[:]
+ subindex_index = ET.SubElement(indexes, 'index')
+ subindex_index.set('lang', lang)
+ subindex_index.set('channel', codename)
+
+ for subindex in subindexes:
+ local_sections = [x for x in sections if x in subindex.sections]
+ if not local_sections:
+ continue
+
+ index = subindex.create_element(subindex_index,
+ release, lang)
+
+ for section in local_sections:
+ remaining_sections.remove(section)
+ section_docs = [x for x in docs if x.toc_id == section]
+ if not section_docs:
+ continue
+ self.create_section(index, section, section_docs, lang)
+ if remaining_sections:
+ logging.warn('%s channel is missing some sections: %s' % (
+ codename, ', '.join(remaining_sections)))
+
+ idx_dirname = os.path.join(self.config.private_dir, 'indexes')
+ if not os.path.exists(idx_dirname):
+ os.makedirs(idx_dirname)
+
+ tree = ET.ElementTree(indexes)
+ tree.write(os.path.join(idx_dirname, 'maemo-indexes.xml'))
+
+ self.generate_html_indexes()
+
+ def create_section(self, index, section, section_docs, lang):
+ section_node = ET.SubElement(index, 'section')
+ section_node.set('toc_id', section)
+ section_node.set('weight', str(
+ self.overlay.get_section_weight(section)))
+
+ subsections = {}
+ for x in section_docs:
+ subsections[x.subsection] = True
+ subsections = subsections.keys()
+
+ for subsection in subsections:
+ subsection_docs = [x for x in section_docs if x.subsection == subsection]
+
+ if subsection is None:
+ parent_elem = section_node
+ else:
+ parent_elem = ET.SubElement(section_node, 'section')
+ parent_elem.set('title', subsection)
+ parent_elem.set('weight', str(
+ self.overlay.get_section_weight(subsection)))
+
+ for doc in subsection_docs:
+ logging.debug('generating index for module %s' % doc.module)
+ if lang in doc.languages:
+ # document is available in the requested
+ # language, perfect.
+ doc_lang = lang
+ elif lang[:2] in doc.languages:
+ # mapping to "general" language, for example
+ # from en_GB to en, from fr_BE to fr...
+ doc_lang = lang[:2]
+ elif [x for x in doc.languages if x[:2] == lang]:
+ # mapping to "country" language, for
+ # example from pt to pt_BR
+ doc_lang = [x for x in doc.languages if x[:2] == lang][0]
+ else:
+ # fallback to English
+ doc_lang = 'en'
+
+ doc.create_element(parent_elem, doc_lang,
+ original_language = lang)
+
+ def generate_html_indexes(self):
+ idx_filename = os.path.join(self.config.private_dir, 'indexes', 'maemo-indexes.xml')
+
+ cmd = ['xsltproc', '--output', self.config.output_dir,
+ '--nonet', '--xinclude',
+ self.indexes_xsl_file, idx_filename]
+
+ if self.debug:
+ cmd.insert(-2, '--param')
+ cmd.insert(-2, 'libgo.debug')
+ cmd.insert(-2, 'true()')
+ if self.config.symbols_dbm_filepath:
+ cmd.insert(-2, '--param')
+ cmd.insert(-2, 'libgo.dbm_support')
+ cmd.insert(-2, 'true()')
+
+ logging.debug('executing %s' % ' '.join(cmd))
+ rc = subprocess.call(cmd)
+ if rc != 0:
+ logging.warn('%s failed with error %d' % (' '.join(cmd), rc))
+
+ def get_module_web_output_dir(self, module, versioned=False):
+ # XXX: don't hardcode dist
+ return os.path.join(self.config.output_dir, 'fremantle', module.modulename)
+
+ def set_module_web_path(self, module):
+ # XXX: don't hardcode dist
+ module.path = '/' + os.path.join('fremantle', module.modulename)
+
+ def generate_symbols_files(self, release):
+ if not (self.config.symbols_dbm_filepath or self.config.symbols_sqlite_filepath):
+ return
+ codename = release.get('codename')
+ try:
+ symbols_dbm_filepath_skel = self.config.symbols_dbm_filepath
+ symbols_sqlite_filepath_skel = self.config.symbols_sqlite_filepath
+ if symbols_dbm_filepath_skel:
+ self.config.symbols_dbm_filepath = symbols_dbm_filepath_skel.replace(
+ '$codename', codename)
+ if symbols_sqlite_filepath_skel:
+ self.config.symbols_sqlite_filepath = symbols_sqlite_filepath_skel.replace(
+ '$codename', codename)
+ App.generate_symbols_files(self)
+ finally:
+ self.config.symbols_dbm_filepath = symbols_dbm_filepath_skel
+ self.config.symbols_sqlite_filepath = symbols_sqlite_filepath_skel
+
+ def symbols_iterator(self, httxt2dbm=None):
+ regex_doxygen_symbols1 = re.compile(
+ r'<a.{0,200}?class="el".{0,200}?href="(.*?)".{0,200}?>([A-Za-z].*?)<')
+ regex_doxygen_symbols2 = re.compile(
+ r'<a.{0,200}?href="(.*?)".{0,200}?class="el".{0,200}?>([A-Za-z].*?)<')
+ for doc in self.documents:
+ web_dir = os.path.join(app.config.output_dir, doc.path[1:])
+
+ devhelp_path = os.path.join(web_dir, '%s.devhelp2' % doc.module)
+ if os.path.exists(devhelp_path) or os.path.exists(devhelp_path + '.gz'):
+ logging.debug('getting symbols out of module with devhelp file: %s' % doc.module)
+ if os.path.exists(devhelp_path):
+ tree = ET.parse(devhelp_path)
+ else:
+ tree = ET.parse(gzip.GzipFile(devhelp_path + '.gz'))
+ for keyword in tree.findall('//{http://www.devhelp.net/book}keyword'):
+ key = keyword.attrib.get('name').replace('()', '').strip()
+ if not key or ' ' in key:
+ # ignore keys with spaces in their name
+ continue
+ value = os.path.join(doc.path, keyword.attrib.get('link'))
+ if httxt2dbm:
+ print >> httxt2dbm, key, value
+ yield (key, value)
+ elif os.path.exists(os.path.join(web_dir, 'doxygen.png')):
+ logging.debug('getting symbols out of doxygen module: %s' % doc.module)
+ done = {}
+ for filename in os.listdir(web_dir):
+ if not filename.endswith('.html'):
+ continue
+ if filename.endswith('-source.html'):
+ continue
+ s = file(os.path.join(web_dir, filename)).read()
+ print os.path.join(web_dir, filename)
+ for value, key in regex_doxygen_symbols1.findall(s) + \
+ regex_doxygen_symbols2.findall(s):
+ if value[0] == '#':
+ continue
+ if value.endswith('-source.html') or '-source.html#' in value:
+ continue
+ if key.lower() in ('null', 'true', 'false', 'bool'):
+ # ignore common macros
+ continue
+ if key.endswith('.c') or key.endswith('.h'):
+ # ignore pointer to source files
+ continue
+ if ' ' in key or '/' in key:
+ # ignore keys with spaces or slashes
+ continue
+ if key.endswith('()'):
+ # remove parenthesis
+ key = key[:-2]
+ if key in done:
+ continue
+ done[key] = True
+ value = os.path.join(doc.path, value)
+ if httxt2dbm:
+ print >> httxt2dbm, key, value
+ key = unicode(key, 'ascii', 'ignore').encode('ascii')
+ yield (key, value)
+ return
+
+
+if __name__ == '__main__':
+ app = LibMaemo()
+ app.Document = Document
+ app.run()
+
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
