[library-web/new-website-style] Possibility to create local pages out of wiki pages

[Frederic Peters <fpeters src gnome org>]

commit 7f034b7182a6665a49fd3535246a411132280303
Author: Frédéric Péters <fpeters 0d be>
Date:   Tue Mar 29 20:46:16 2011 +0530

    Possibility to create local pages out of wiki pages

 data/overlay.xml.in |    8 ++++++++
 src/document.py     |    2 ++
 src/lgo.py          |    4 ++--
 src/overlay.py      |   42 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 2 deletions(-)
---
diff --git a/data/overlay.xml.in b/data/overlay.xml.in
index 0f3b522..86a9b14 100644
--- a/data/overlay.xml.in
+++ b/data/overlay.xml.in
@@ -1154,6 +1154,14 @@
       <href>http://live.gnome.org/Git</href>
     </document>
 
+    <extrawikidocs>
+      <!-- create a serie of documents out of -->
+      <channel>devel</channel>
+      <category>devel-guides</category>
+      <href>http://live.gnome.org/DevGnomeOrg</href>
+      <attributes h2hmode="content-id" hidehref="true"/>
+    </extrawikidocs>
+
   </documents>
 
   <quirks doc_module="evolution" channel="users">
diff --git a/src/document.py b/src/document.py
index af05717..3447383 100644
--- a/src/document.py
+++ b/src/document.py
@@ -251,6 +251,8 @@ class RemoteDocument(Document):
     def download(self, href):
         # TODO: add some support (think <local update="daily"/>) so the file
         # can be "watched" for changes
+        if self.overlay.find('local').attrib.get('nocache'):
+            return app.download(href, use_cache=False)
         return app.download(href)
 
     def create_element(self, parent, language, original_language = None):
diff --git a/src/lgo.py b/src/lgo.py
index c419dd4..e8c920e 100755
--- a/src/lgo.py
+++ b/src/lgo.py
@@ -247,7 +247,7 @@ class Lgo(App):
                     moduleset.seek(0)
                     self.process_latest_moduleset(moduleset)
 
-    def download(self, url):
+    def download(self, url, use_cache=True):
         if url.startswith('gnome://'):
             # special schema for modules on ftp.gnome.org; URL scheme is
             # gnome://<modulename>?min=<minimum version>
@@ -318,7 +318,7 @@ class Lgo(App):
                 logging.error('error downloading %s' % url)
                 return
         else:
-            filename = App.download(self, url)
+            filename = App.download(self, url, use_cache=use_cache)
 
         return filename
 
diff --git a/src/overlay.py b/src/overlay.py
index f26a4f4..e1d9de0 100644
--- a/src/overlay.py
+++ b/src/overlay.py
@@ -21,6 +21,10 @@ try:
 except ImportError:
     import xml.etree.ElementTree as ET
 
+import html5lib
+import logging
+import urlparse
+
 from document import RemoteDocument
 from utils import version_cmp, is_version_number
 
@@ -76,6 +80,8 @@ class Overlay:
         self.quirks = {}
         self.extra_devel_releases = {}
 
+        self.create_extra_wiki_docs()
+
         for doc in self.tree.findall('/documents/document'):
             if 'doc_module' in doc.attrib:
                 # modifying an existing document
@@ -109,6 +115,42 @@ class Overlay:
         for quirks in self.tree.findall('/quirks'):
             self.quirks[(quirks.attrib['doc_module'], quirks.attrib['channel'])] = quirks
 
+    def create_extra_wiki_docs(self):
+        documents_node = self.tree.find('documents')
+        for extra in self.tree.findall('/documents/extrawikidocs'):
+            href = extra.find('href').text
+            content = app.download(href + '?action=print', use_cache=False)
+
+            # parse the wiki page and get all links in content
+            parser = html5lib.HTMLParser()
+            doc = parser.parse(file(content))
+            del doc.childNodes[:-1]
+            html = ET.fromstring(doc.toxml())
+            links = ET.ElementTree(html).findall('//a')
+
+            channel = extra.find('channel').text
+            category = extra.find('category').text
+            attributes = extra.find('attributes')
+
+            for link in links:
+                if link.attrib.get('title'):
+                    # heuristic to eliminate generated links
+                    continue
+                doc_href = urlparse.urljoin(href, link.attrib.get('href'))
+                logging.info('adding extra document from wiki: %s' % doc_href)
+                title = link.text
+                doc_node = ET.SubElement(documents_node, 'document')
+                doc_node.attrib['doc_module'] = doc_href.split('/')[-1]
+                doc_node.attrib['channel'] = channel
+                doc_node.attrib['category'] = category
+                ET.SubElement(doc_node, 'new')
+                ET.SubElement(doc_node, 'title').text = title
+                ET.SubElement(doc_node, 'href').text = doc_href + '?action=print'
+                local_node = ET.SubElement(doc_node, 'local')
+                for key, value in attributes.attrib.items():
+                    local_node.attrib[key] = value
+                local_node.attrib['nocache'] = 'true'
+
     def apply(self, document):
         if (document.channel, document.toc_id) in self.toc_mapping:
             document.subsection = self.toc_mapping[(document.channel, document.toc_id)]