[library-web: 1/3] Add support for gi-docgen modules

[Matthias Clasen <matthiasc src gnome org>]

commit 0f3b909305842986559b927d6163a2224e8695d9
Author: Emmanuele Bassi <ebassi gnome org>
Date:   Thu Mar 18 17:40:59 2021 +0000

    Add support for gi-docgen modules
    
    This is a tiny bandaid over the mess that is library-web.
    
    We look for a toml file (or a toml file template) in the tarball; if we
    find it, we check that it's a gi-docgen project configuration by
    looking for the `[library]` section. The basename of the project's
    configuration file is assumed to be the name of the documentation
    module, which will then be used to match the various metadata expressed
    in the overlay file.
    
    Once we find a gi-docgen project configuration file, we look for HTML
    files in its proximity—either under a directory with the same name as
    the `namespace` key, or a directory with the same basename as the toml
    file; this matches the behaviours of GTK and Pango.
    
    Once we find an `index.html`, we start copying everything as is to the
    target directory.

 src/lgo.py               | 16 ++++++++-
 src/modtypes/base.py     |  5 ++-
 src/modtypes/gidocgen.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 106 insertions(+), 2 deletions(-)
---
diff --git a/src/lgo.py b/src/lgo.py
old mode 100755
new mode 100644
index ac412510..9b3a3f9c
--- a/src/lgo.py
+++ b/src/lgo.py
@@ -54,6 +54,7 @@ from modtypes.gnomedocbook import GnomeDocbookModule
 from modtypes.gtkdoc import GtkDocModule
 from modtypes.htmlfiles import HtmlFilesModule
 from modtypes.mallard import MallardModule
+from modtypes.gidocgen import GIDocGenModule
 
 from app import App
 
@@ -706,6 +707,7 @@ class Lgo(App):
         found_meson = False
         found_cmake = False
         found_autotools = False
+        found_gidocgen = False
         
         try:
             for tarinfo in tar:
@@ -767,6 +769,7 @@ class Lgo(App):
                                 cmakelists=cmakelists, nightly=nightly)
                     else:
                         continue
+
                     found_cmake = True
 
                 elif os.path.split(tarinfo.name)[-1] == 'meson.build' and not found_autotools:
@@ -781,6 +784,18 @@ class Lgo(App):
 
                     found_meson = True
 
+                elif os.path.splitext(os.path.split(tarinfo.name)[-1])[1] in ('.toml', '.toml.in'):
+                    fd = tar.extractfile(tarinfo)
+                    toml = fd.read()
+                    if re.findall(r"\s*\[library\]\s*", toml, re.DOTALL):
+                        logging.debug('found usage of gi-docgen in %s' % tarinfo.name)
+                        doc = GIDocGenModule.create_from_tar(tar, tarinfo,
+                                toml=toml, nightly=nightly)
+                    else:
+                        continue
+
+                    found_gidocgen = True
+
                 else:
                     for more_doc in more_tarball_docs[:]:
                         if not tarinfo.isdir():
@@ -1085,4 +1100,3 @@ if __name__ == '__main__':
     app = Lgo()
     app.Document = Document
     app.run()
-
diff --git a/src/modtypes/base.py b/src/modtypes/base.py
index ac6eefa8..b72548f9 100644
--- a/src/modtypes/base.py
+++ b/src/modtypes/base.py
@@ -45,7 +45,7 @@ class DocModule(object):
     mallard_merge = None
 
     def create_from_tar(cls, tar, tarinfo, makefile_am=None, nightly=False,
-            meson_build=None, cmakelists=None):
+            meson_build=None, cmakelists=None, toml=None):
         self = cls()
         if tarinfo.isdir():
             self.dirname = tarinfo.name
@@ -82,6 +82,9 @@ class DocModule(object):
             match = re.findall(r"gnome.gtkdoc\s*\(\s*'([\w-]+)'", meson_build, re.DOTALL)
             assert match
             self.modulename = match[0]
+        elif toml:
+            self.toml = toml
+            self.modulename = os.path.splitext(os.path.basename(tarinfo.name))[0]
         else:
             self.makefile_am = makefile_am
             pass # raise Exception('need makefile.am or meson.build')
diff --git a/src/modtypes/gidocgen.py b/src/modtypes/gidocgen.py
new file mode 100644
index 00000000..0d930f58
--- /dev/null
+++ b/src/modtypes/gidocgen.py
@@ -0,0 +1,87 @@
+# libgo - script to build library.gnome.org
+# Copyright 2021  GNOME Foundation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301  USA
+
+import logging
+import os
+import re
+import shutil
+import stat
+import subprocess
+import tempfile
+
+from base import DocModule
+
+class GIDocGenModule(DocModule):
+    '''Class for documentation generated by gi-docgen'''
+
+    def __str__(self):
+        return 'GIDocGen module at %s' % self.dirname
+
+    def process(self):
+        doc_module = self.modulename
+        ext_dirname = os.path.join(app.config.private_dir, 'extracts')
+
+        web_output_dir = app.get_module_web_output_dir(self)
+        if not os.path.exists(web_output_dir):
+            os.makedirs(web_output_dir)
+
+        namespace = re.findall(r'\s*namespace\s*=\s*"([\w\-]+)"', self.toml)[0].split()
+        if not namespace:
+            namespace = doc_module
+
+        htmldir = os.path.join(ext_dirname, self.dirname, namespace)
+
+        # simply copy HTML files shipped in tarball
+        logging.debug('copying files shipped with tarball')
+        for filename in os.listdir(htmldir):
+            src = os.path.join(htmldir, filename)
+            dst = os.path.join(web_output_dir, filename)
+            ext = os.path.splitext(filename)
+            if ext not in ['.html', '.css', '.js', '.devhelp2']:
+                continue
+            if not os.path.exists(dst) or os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
+                open(dst, 'w').write(open(src, 'r').read())
+
+        # copy non-HTML files
+        for filename in os.listdir(htmldir):
+            src = os.path.join(htmldir, filename)
+            dst = os.path.join(web_output_dir, filename)
+            ext = os.path.splitext(filename)
+            if ext in ['.html', '.css', '.js', '.devhelp2']:
+                continue
+            if os.path.isdir(src):
+                if os.path.exists(dst):
+                    shutil.rmtree(dst)
+                shutil.copytree(src, dst)
+            else:
+                if not os.path.exists(dst) or os.stat(src)[stat.ST_MTIME] > os.stat(dst)[stat.ST_MTIME]:
+                    open(dst, 'w').write(open(src, 'r').read())
+
+        doc = self.get_libgo_document(['en'])
+        if not doc:
+            return
+
+        if not os.path.exists(os.path.join(web_output_dir, 'index.html')):
+            return
+
+        title = re.findall('<title>(.*)</title>',
+                file(os.path.join(web_output_dir, 'index.html')).read())
+        if title:
+            doc.title = {'en': title[0]}
+
+        self.install_version_symlinks(doc)