[tracker-miners/sam/website] website: Include reference documentation

[Sam Thursfield <sthursfield src gnome org>]

commit 8fb78f78e7709059dfff26fe4a536a649450cba6
Author: Sam Thursfield <sam afuera me uk>
Date:   Sat Mar 28 16:43:22 2020 +0100

    website: Include reference documentation
    
    The latest API reference documentation is now built and included in
    the Tracker website, available under the /docs/api-preview/ prefix.
    
    This allows us to browse and reference the documentation during
    Tracker 3.0 development, as well as providing a sort of work around
    for https://gitlab.gnome.org/GNOME/tracker/-/issues/100.
    
    A warning is added to each documentation page advising that it is
    a preview built from Git.

 .gitlab-ci.yml        |  8 ++++-
 docs/website/build.py | 81 ++++++++++++++++++++++++++++++++++++++++++++-------
 docs/website/docs.md  | 19 ++++++++++--
 3 files changed, 94 insertions(+), 14 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cde36bb43..46d41c5d2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -89,7 +89,13 @@ pages:
   before_script:
     - pip3 install mkdocs mkdocs-cinder
   script:
-    - ./docs/website/build.py
+    # Build tracker and tracker-miners to generate the API documentation.
+    - su tracker -c '.gitlab-ci/checkout-tracker.sh'
+    - su tracker -c 'mkdir build; cd build; meson .. --prefix=/tmp/tracker -Dtracker_core=subproject; ninja 
install'
+    # Generate the website using mkdocs.
+    - |
+      tracker_commit=$(git -C ./subprojects/tracker rev-parse HEAD)
+      ./docs/website/build.py --api-docs=/tmp/tracker/share/gtk-doc/html --tracker-commit=${tracker_commit}
   artifacts:
     paths:
       - public
diff --git a/docs/website/build.py b/docs/website/build.py
index ffd708fd2..49c086529 100755
--- a/docs/website/build.py
+++ b/docs/website/build.py
@@ -19,35 +19,96 @@
 # Boston, MA  02110-1301, USA.
 
 
+import argparse
 import logging
 import pathlib
+import shutil
 import subprocess
 import sys
+import tempfile
+
+log = logging.getLogger('build.py')
 
-referencedocs_path = pathlib.Path('/opt/tracker3/share/gtk-doc/html/')
 output_path = pathlib.Path('public')
 mkdocs_root = pathlib.Path(__file__).parent.parent
 
 
+def argument_parser():
+    parser = argparse.ArgumentParser(
+        description="Tracker website build script")
+    parser.add_argument('--debug', dest='debug', action='store_true',
+                        help="Enable detailed logging to stderr")
+    parser.add_argument('--api-docs', required=True, metavar='PATH',
+                        help="Path that contains API documentation. Usually "
+                             "$prefix/share/gtk-doc/html/")
+    parser.add_argument('--tracker-commit', required=True, metavar='COMMIT',
+                        help="Commit ID of tracker.git repo used to build")
+    return parser
+
+
+def apidocs_header(tracker_commit):
+    return f"""<!-- Inserted by {__file__} -->
+    <div class="warning">
+        <p>This is a documentation preview for the next version of Tracker,
+           generated from <a 
href="https://gitlab.gnome.org/GNOME/tracker/commit/{tracker_commit}/";>tracker.git commit 
{tracker_commit[:7]}</a>.</p>
+        <p>See the <a href="../..">Tracker website</a> for more documentation.</p>
+    </div>"""
+
+
+def add_apidocs_header(text, filename):
+    """Add a header to the documentation preview files."""
+
+    # We insert the header before any of these
+    markers = [
+        '<div class="book">',
+        '<div class="chapter">',
+        '<div class="index">',
+        '<div class="glossary">',
+        '<div class="part">',
+        '<div class="refentry">',
+        '<div class="section">',
+    ]
+
+    with open(filename, encoding='utf8') as f_in:
+        with tempfile.NamedTemporaryFile(delete=False, mode='w', encoding='utf8') as f_out:
+            for line in f_in:
+                for marker in markers:
+                    if line.find(marker) != -1:
+                        f_out.write(text)
+                f_out.write(line)
+    shutil.move(f_out.name, filename)
+
+
 def main():
-    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+    args = argument_parser().parse_args()
+
+    if args.debug:
+        logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+    else:
+        logging.basicConfig(stream=sys.stderr, level=logging.INFO)
 
     if output_path.exists():
         raise RuntimeError(f"Output path '{output_path}' already exists.")
 
-    if not referencedocs_path.exists():
-        raise RuntimeError(f"Did not find directory '{referencedocs_path}'")
-
+    log.info("Building website")
     mkdocs_config = mkdocs_root.joinpath('mkdocs.yml')
     subprocess.run(['mkdocs', 'build', '--config-file', mkdocs_config,
                     '--site-dir', output_path.absolute()])
 
-    #echo "Copying reference documentation from $referencedocsdir directory."
-    #mkdir -p public/docs/api-preview/
-    #cp -a $referencedocsdir/libtracker-sparql/ ./public/docs/api-preview
-    #cp -a $referencedocsdir/ontology/ ./public/docs/api-preview
+    apidocs_src = pathlib.Path(args.api_docs)
+
+    log.info("Copying API reference documentation from %s", apidocs_src)
+    apidocs_dest = output_path.joinpath('docs/api-preview')
+    apidocs_dest.mkdir(parents=True)
+    for name in ['libtracker-sparql', 'ontology']:
+        shutil.copytree(apidocs_src.joinpath(name), apidocs_dest.joinpath(name))
+
+    log.info("Adding preview header to API reference documentation")
+    text = apidocs_header(args.tracker_commit)
+    for filename in apidocs_dest.rglob('*.html'):
+        add_apidocs_header(text, filename)
 
-    logging.info("Documentation available in public/ directory.")
+    log.info("Documentation available in public/ directory.")
 
 
 try:
diff --git a/docs/website/docs.md b/docs/website/docs.md
index a156e9291..beed96a56 100644
--- a/docs/website/docs.md
+++ b/docs/website/docs.md
@@ -14,7 +14,7 @@ The `tracker` commandline tool provides direct access to Tracker. Use the
 ## Developer Documentation
     
 Application and platform developers using Tracker will interact with Tracker
-using one or more of the shared libraries at provides:
+using one or more of the shared libraries it provides:
 
  * [libtracker-sparql](https://developer.gnome.org/libtracker-sparql/stable/) is
    used to read and write data from the Tracker store using SPARQL.
@@ -24,9 +24,22 @@ using one or more of the shared libraries at provides:
    be used if you want to implement a new data provider while reusing existing
    Tracker code.
 
-Developers are expected to read and write data from the Tracker store using the
-SPARQL.
+WARNING: The documention linked above is for out of date verions of Tracker 2.x.
+This is due to an [infrastructure
+issue](https://gitlab.gnome.org/Infrastructure/library-web/issues/50). See also
+[this bug](https://gitlab.gnome.org/GNOME/tracker/-/issues/100).
+
 The following documentation may be useful:
 
  * [Tracker ontology documentation](https://developer.gnome.org/ontology/stable/).
  * [Tracker documentation on wiki.gnome.org](https://wiki.gnome.org/Projects/Tracker).
+
+## Preview Documentation
+
+We provide an online version of the documentation for the latest in-development version
+of Tracker. You can browse it here:
+
+  * [libtracker-sparql](./api-preview/libtracker-sparql)
+  * [ontology](./api-preview/ontology)
+
+Be aware that some libraries from Tracker 2.0 will not be available for Tracker 3.0.