[library-web] add experimental support for meson and documentation tarballs

[Frederic Peters <fpeters src gnome org>]

commit 94d3623053822444fd3c43c80521e6b85cf78b54
Author: Frédéric Péters <fpeters 0d be>
Date:   Fri Jan 12 20:13:34 2018 +0100

    add experimental support for meson and documentation tarballs
    
    <doc-tarball-location>https://download.gnome.org/whatever/gtk-docs-VERSION.tar.xz</doc-tarball-location>

 src/lgo.py             |   16 ++++++-
 src/modtypes/base.py   |   11 ++++-
 src/modtypes/gtkdoc.py |  110 ++++++++++++++++++++++++++++++++----------------
 src/overlay.py         |    9 ++++
 4 files changed, 106 insertions(+), 40 deletions(-)
---
diff --git a/src/lgo.py b/src/lgo.py
index f6c1a9c..b2f11a6 100755
--- a/src/lgo.py
+++ b/src/lgo.py
@@ -745,9 +745,14 @@ class Lgo(App):
                     logging.debug('skipping because it has $( in its modulename')
                     continue
 
-                if not doc.modulename or doc.modulename in self.config.blacklist:
-                    logging.debug('skipping because in blacklist')
-                    continue
+            elif os.path.split(tarinfo.name)[-1] == 'meson.build':
+                fd = tar.extractfile(tarinfo)
+                meson_build = fd.read()
+                if 'gnome.gtkdoc' in meson_build:
+                    logging.debug('found usage of gtk-doc in %s' % tarinfo.name)
+                    doc = GtkDocModule.create_from_tar(tar, tarinfo,
+                            meson_build=meson_build, nightly=nightly)
+
             else:
                 for more_doc in more_tarball_docs[:]:
                     if not tarinfo.isdir():
@@ -760,6 +765,11 @@ class Lgo(App):
                         more_tarball_docs.remove(more_doc)
                         continue
 
+            if doc and (not doc.modulename or doc.modulename in self.config.blacklist):
+                logging.debug('skipping because in blacklist')
+                continue
+
+
             if doc:
                 doc.filename = filename
                 doc.mtime_tarball = mtime
diff --git a/src/modtypes/base.py b/src/modtypes/base.py
index b98f466..5bc99d9 100644
--- a/src/modtypes/base.py
+++ b/src/modtypes/base.py
@@ -28,6 +28,7 @@ from utils import version_cmp, is_version_number
 class DocModule(object):
     '''Base class for documentation shipped in tarballs'''
     makefile_am = None
+    meson_build = None
 
     filename = None
     dirname = None
@@ -42,7 +43,8 @@ class DocModule(object):
     git_version = False
     mallard_merge = None
 
-    def create_from_tar(cls, tar, tarinfo, makefile_am, nightly = False):
+    def create_from_tar(cls, tar, tarinfo, makefile_am=None, nightly=False,
+            meson_build=None):
         self = cls()
         if tarinfo.isdir():
             self.dirname = tarinfo.name
@@ -69,6 +71,13 @@ class DocModule(object):
             # remove the version part, so libsoup-2.4 is handled just like
             # another version of libsoup
             self.modulename = re.sub('-\d+\.\d+$', '', self.modulename)
+        elif meson_build:
+            self.meson_build = meson_build
+            match = re.findall(r"gnome.gtkdoc\s*\(\s*'(\w+)'", meson_build, re.DOTALL)
+            assert match
+            self.modulename = match[0]
+        else:
+            raise Exception('need makefile.am or meson.build')
         self.version = os.path.splitext(tar.name)[0].split('-')[-1]
         if self.version.endswith('.tar'):
             self.version = self.version[:-4]
diff --git a/src/modtypes/gtkdoc.py b/src/modtypes/gtkdoc.py
index 7fbb538..35f65dc 100644
--- a/src/modtypes/gtkdoc.py
+++ b/src/modtypes/gtkdoc.py
@@ -16,6 +16,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 # 02110-1301  USA
 
+import collections
 import glob
 import logging
 import os
@@ -51,15 +52,18 @@ class GtkDocModule(DocModule):
         doc_module = self.modulename
         ext_dirname = os.path.join(app.config.private_dir, 'extracts')
 
-        main_sgml_file = re.findall(r'DOC_MAIN_SGML_FILE\s?=\s?(.*)',
-                self.makefile_am)[0].strip()
-        main_sgml_file = main_sgml_file.replace('$(DOC_MODULE)', doc_module)
+        if self.meson_build:
+            main_sgml_file = None  # will force external HTML
+        else:
+            main_sgml_file = re.findall(r'DOC_MAIN_SGML_FILE\s?=\s?(.*)',
+                    self.makefile_am)[0].strip()
+            main_sgml_file = main_sgml_file.replace('$(DOC_MODULE)', doc_module)
 
-        try:
-            html_images = re.findall('HTML_IMAGES\s+=\s+(.*)', self.makefile_am)[0].split()
-        except IndexError:
-            html_images = []
-        html_images = [x.replace('$(srcdir)/', '') for x in html_images]
+            try:
+                html_images = re.findall('HTML_IMAGES\s+=\s+(.*)', self.makefile_am)[0].split()
+            except IndexError:
+                html_images = []
+            html_images = [x.replace('$(srcdir)/', '') for x in html_images]
 
         web_output_dir = app.get_module_web_output_dir(self)
         if not os.path.exists(web_output_dir):
@@ -76,31 +80,36 @@ class GtkDocModule(DocModule):
             logging.debug('using already generated doc')
         else:
             logging.info('generating doc in %s' % web_output_dir[len(app.config.output_dir):])
-            cmd = ['xsltproc', '--output', web_output_dir + '/',
-                    '--nonet', '--xinclude',
-                    '--stringparam', 'libgo.lang', 'en',
-                    '--stringparam', 'gtkdoc.bookname', doc_module,
-                    '--stringparam', 'gtkdoc.version', '"(~lgo)"',
-                    '--stringparam', 'libgo.channel', self.channel,
-                    self.gtkdoc_xsl_file,
-                    os.path.join(ext_dirname, self.dirname, main_sgml_file)]
-
-            if app.config.symbols_dbm_filepath:
-                cmd.insert(-2, '--param')
-                cmd.insert(-2, 'libgo.dbm_support')
-                cmd.insert(-2, 'true()')
-
-            logging.debug('executing %s' % ' '.join(cmd))
-            xsltproc = subprocess.Popen(cmd, stdin = subprocess.PIPE, stderr = subprocess.PIPE)
-            stdout, stderr = xsltproc.communicate()
-            if re.findall('XInclude error : could not load.*and no fallback was found', stderr):
-                logging.warn('XInclude error, creating fake xsltproc return code')
-                xsltproc.returncode = 6
-
-            quirks = app.overlay.get_quirks(self)
-            if 'force-html-transform' in quirks:
-                logging.info('module marked as using html files, creating fake xsltproc return code')
-                xsltproc.returncode = 6
+            if main_sgml_file:
+                cmd = ['xsltproc', '--output', web_output_dir + '/',
+                        '--nonet', '--xinclude',
+                        '--stringparam', 'libgo.lang', 'en',
+                        '--stringparam', 'gtkdoc.bookname', doc_module,
+                        '--stringparam', 'gtkdoc.version', '"(~lgo)"',
+                        '--stringparam', 'libgo.channel', self.channel,
+                        self.gtkdoc_xsl_file,
+                        os.path.join(ext_dirname, self.dirname, main_sgml_file)]
+
+                if app.config.symbols_dbm_filepath:
+                    cmd.insert(-2, '--param')
+                    cmd.insert(-2, 'libgo.dbm_support')
+                    cmd.insert(-2, 'true()')
+
+                logging.debug('executing %s' % ' '.join(cmd))
+                xsltproc = subprocess.Popen(cmd, stdin = subprocess.PIPE, stderr = subprocess.PIPE)
+                stdout, stderr = xsltproc.communicate()
+                if re.findall('XInclude error : could not load.*and no fallback was found', stderr):
+                    logging.warn('XInclude error, creating fake xsltproc return code')
+                    xsltproc.returncode = 6
+
+                if 'force-html-transform' in quirks:
+                    logging.info('module marked as using html files, creating fake xsltproc return code')
+                    xsltproc.returncode = 6
+
+            else:
+                # simulate xsltproc failure
+                xsltproc = collections.namedtuple('ReturnCode', ('returncode',))(returncode=6)
+                cmd = ['fake (meson)']
 
             if xsltproc.returncode != 0:
                 logging.warn('%s failed with error %d' % (' '.join(cmd), xsltproc.returncode))
@@ -112,9 +121,38 @@ class GtkDocModule(DocModule):
                     htmlfiles_dir = os.path.join(ext_dirname, self.dirname, 'html')
                     if not os.path.exists(htmlfiles_dir):
                         # tarball shipped without an HTML version of the
-                        # documentation, can't do anything here :/
-                        logging.error('skipped %s as it is missing an html/ dir' % doc_module)
-                        return
+                        # documentation, maybe the overlay file specified
+                        # an external location?
+                        tarball_location = app.overlay.get_doc_tarball_location(self.modulename)
+                        if tarball_location:
+                            tarball_location = tarball_location.replace('VERSION', self.version)
+                            htmlfiles_dir = os.path.join(ext_dirname, 'doc-tarballs', self.dirname, 'html')
+                            if not os.path.exists(htmlfiles_dir):
+                                logging.debug('extracting %s' % tarball_location)
+                                os.makedirs(htmlfiles_dir)
+                                doc_tarball = app.download(tarball_location)
+                                tar2 = tarfile.open(doc_tarball, 'r')
+                                ext2_dirname = os.path.join(ext_dirname, 'doc-tarballs', 
self.dirname.split('/')[0])
+                                for tar2info in tar2.getmembers():
+                                    dest = os.path.join(ext2_dirname, tar2info.name)
+                                    if tar2info.isdir() and not os.path.exists(dest):
+                                        os.makedirs(dest)
+                                    elif tar2info.isreg():
+                                        if not os.path.exists(os.path.dirname(dest)):
+                                            os.makedirs(os.path.dirname(dest))
+                                        open(dest, 'w').write(tar2.extractfile(tar2info).read())
+                                tar2.close()
+
+                            html_images = []
+                            for basename, dirnames, filenames in os.walk(htmlfiles_dir):
+                                for filename in filenames:
+                                    if os.path.splitext(filename)[-1] in ('.png', '.gif', '.jpg', '.jpeg', 
'.svg'):
+                                        html_images.append(os.path.join(basename[len(htmlfiles_dir):], 
filename))
+
+                        else:
+                            #can't do anything here :/
+                            logging.error('skipped %s as it is missing an html/ dir' % doc_module)
+                            return
                     try:
                         import html5lib
                     except ImportError:
diff --git a/src/overlay.py b/src/overlay.py
index 9e3b609..fa47f08 100644
--- a/src/overlay.py
+++ b/src/overlay.py
@@ -203,6 +203,15 @@ class Overlay:
                 return doc.attrib.get('channel')
         return current_channel
 
+    def get_doc_tarball_location(self, module):
+        for doc in self.tree.findall('/documents/document'):
+            if doc.attrib.get('doc_module') != module:
+                continue
+            if doc.find('doc-tarball-location') is not None:
+                return doc.find('doc-tarball-location').text
+        return None
+
+
     def get_new_docs(self):
         l = []
         for overlay in self.new_docs: