[tracker/sam/website-manpages] website: Include manpages

From: Sam Thursfield <sthursfield src gnome org>
To: commits-list gnome org
Cc:
Subject: [tracker/sam/website-manpages] website: Include manpages
Date: Sun, 12 Apr 2020 14:46:13 +0000 (UTC)
commit b49090f4b7e55437f0b0ccbb85db64e228f03957
Author: Sam Thursfield <sam afuera me uk>
Date:   Sun Apr 5 13:19:50 2020 +0200

    website: Include manpages
    
    The man pages are converted to XHTML using asciidoc and docbook,
    then postprocessed in Python using BeautifulSoup to include them
    in the mkdocs website output.

 .gitlab-ci.yml                              |  8 +--
 docs/manpages/meson.build                   |  2 +-
 docs/mkdocs.yml                             |  8 ++-
 docs/website/build.py                       | 99 +++++++++++++++++++++++++++--
 docs/website/docs/commandline.md.in         | 11 ++++
 docs/website/{docs.md => docs/developer.md} | 22 ++-----
 docs/website/docs/user.md                   | 10 +++
 docs/website/index.md                       |  4 +-
 8 files changed, 135 insertions(+), 29 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aeb209be6..8c62a82ad 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -85,8 +85,8 @@ pages:
   image: registry.gitlab.gnome.org/gnome/tracker-oci-images/amd64/fedora:latest
   dependencies: []
   before_script:
-    - dnf install -y python3-pip
-    - pip3 install mkdocs mkdocs-cinder
+    - dnf install -y python3-pip xmlto
+    - pip3 install beautifulsoup4 mkdocs mkdocs-cinder
   script:
     # Build tracker and install.
     - su tracker -c 'mkdir build; cd build; meson .. --prefix=/tmp/tracker; ninja install'
@@ -97,10 +97,10 @@ pages:
     - |
       tracker_commit=$CI_COMMIT_SHA1
       tracker_miners_commit=$(git -C ./extra/tracker-miners rev-parse HEAD)
-      ./docs/website/build.py --api-docs=/tmp/tracker/share/gtk-doc/html --tracker-commit=${tracker_commit}
+      ./docs/website/build.py --api-docs=/tmp/tracker/share/gtk-doc/html --tracker-commit=${tracker_commit} 
--man-pages ./docs/manpages/*.txt ./extra/tracker-miners/docs/manpages/*.txt
   artifacts:
     paths:
       - public
   only:
     - master
-    - sam/website
+    - /^sam\/website.*$/
diff --git a/docs/manpages/meson.build b/docs/manpages/meson.build
index 7f07099bc..413d1e591 100644
--- a/docs/manpages/meson.build
+++ b/docs/manpages/meson.build
@@ -27,8 +27,8 @@ foreach m : manpages
   custom_target(manpage,
     command: [xsltproc,
               '--stringparam', 'man.authors.section.enabled', '0',
-              '--output', '@OUTPUT@',
               '/etc/asciidoc/docbook-xsl/manpage.xsl', '@INPUT@'],
+    capture: true,
     input: xml,
     output: manpage,
     install: true,
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 6757bb3bc..b2838e14c 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -7,4 +7,10 @@ nav:
   - Overview: 'overview.md'
   - FAQ: 'faq.md'
   - Community: 'community.md'
-  - Documentation: 'docs.md'
+  - Documentation:
+      User documentation: 'docs/user.md'
+      Developer documentation: 'docs/developer.md'
+      Commandline documentation: 'docs/commandline.md'
+
+markdown_extensions:
+  - admonition
diff --git a/docs/website/build.py b/docs/website/build.py
index d8bf4e3e8..4d70b87e7 100755
--- a/docs/website/build.py
+++ b/docs/website/build.py
@@ -19,7 +19,10 @@
 # Boston, MA  02110-1301, USA.
 
 
+import bs4
+
 import argparse
+import contextlib
 import logging
 import pathlib
 import shutil
@@ -30,7 +33,13 @@ import tempfile
 log = logging.getLogger('build.py')
 
 output_path = pathlib.Path('public')
-mkdocs_root = pathlib.Path(__file__).parent.parent
+website_root = pathlib.Path(__file__).parent
+docs_root = website_root.parent
+source_root = docs_root.parent
+
+asciidoc = shutil.which('asciidoc')
+mkdocs = shutil.which('mkdocs')
+xmlto = shutil.which('xmlto')
 
 
 def argument_parser():
@@ -43,6 +52,8 @@ def argument_parser():
                              "$prefix/share/gtk-doc/html/")
     parser.add_argument('--tracker-commit', required=True, metavar='COMMIT',
                         help="Commit ID of tracker.git repo used to build")
+    parser.add_argument('--man-pages', nargs='+', required=True,
+                        help="List of Asciidoc manual page source")
     return parser
 
 
@@ -79,6 +90,54 @@ def add_apidocs_header(text, filename):
     shutil.move(f_out.name, filename)
 
 
+class Manpages():
+    def run(self, command):
+        command = [str(c) for c in command]
+        log.debug("Running: %s", ' '.join(command))
+        subprocess.run(command, check=True)
+
+    def generate_manpage_xml(self, in_path, out_path):
+        """Generate a docbook XML file for an Asciidoc manpage source file."""
+        self.run([asciidoc, '--backend', 'docbook', '--doctype', 'manpage',
+                 '--out-file', out_path, in_path])
+
+    def generate_manpage_html(self, in_path, out_path):
+        """Generate a HTML page from a docbook XML file"""
+        self.run([xmlto, 'xhtml-nochunks', '-o', out_path, in_path])
+
+    def generate_toplevel_markdown(self, in_path, out_path, html_files):
+        """Generate the master commandline.md page."""
+
+        includes = []
+        for path in sorted(html_files):
+            parser = bs4.BeautifulSoup(path.read_text(), 'html.parser')
+
+            title = parser.title.text
+            if title and title.startswith('tracker-'):
+                title = 'tracker ' + title[8:]
+
+            body = parser.body.contents[0]
+
+            includes.append("## %s\n\n%s\n---\n" % (title, body))
+
+        text = in_path.read_text()
+        text = text.format(
+            includes='\n'.join(includes)
+        )
+        out_path.write_text(text)
+
+
+@contextlib.contextmanager
+def tmpdir():
+    path = pathlib.Path(tempfile.mkdtemp())
+    log.debug("Created temporary directory %s", path)
+    try:
+        yield path
+    finally:
+        log.debug("Removed temporary directory %s", path)
+        shutil.rmtree(path)
+
+
 def main():
     args = argument_parser().parse_args()
 
@@ -90,10 +149,40 @@ def main():
     if output_path.exists():
         raise RuntimeError(f"Output path '{output_path}' already exists.")
 
+    if asciidoc is None:
+        raise RuntimeError("The 'asciidoc' tool is required.")
+    if xmlto is None:
+        raise RuntimeError("The 'xmlto' tool is required.")
+
+    log.info("Generating online man pages")
+    with tmpdir() as workdir:
+        manpages = Manpages()
+
+        htmldir = website_root.joinpath('docs')
+        htmlfiles = []
+
+        for man_page in args.man_pages:
+            asciidocpath = pathlib.Path(man_page)
+            xmlpath = workdir.joinpath(asciidocpath.stem + '.xml')
+
+            manpages.generate_manpage_xml(asciidocpath, xmlpath)
+            manpages.generate_manpage_html(xmlpath, htmldir)
+
+            htmlpath = htmldir.joinpath(xmlpath.with_suffix('.html').name)
+            htmlfiles.append(htmlpath)
+
+        template_in = website_root.joinpath('docs/commandline.md.in')
+        template_out = website_root.joinpath('docs/commandline.md')
+
+        manpages.generate_toplevel_markdown(template_in, template_out, htmlfiles)
+
+    #shutil.copy('/usr/share/asciidoc/stylesheets/docbook-xsl.css', manpage_output_path)
+
     log.info("Building website")
-    mkdocs_config = mkdocs_root.joinpath('mkdocs.yml')
-    subprocess.run(['mkdocs', 'build', '--config-file', mkdocs_config,
-                    '--site-dir', output_path.absolute()])
+    mkdocs_config = docs_root.joinpath('mkdocs.yml')
+    subprocess.run([mkdocs, 'build', '--config-file', mkdocs_config,
+                    '--site-dir', output_path.absolute()],
+                   check=True)
 
     apidocs_src = pathlib.Path(args.api_docs)
 
@@ -113,6 +202,6 @@ def main():
 
 try:
     main()
-except RuntimeError as e:
+except (RuntimeError, subprocess.CalledProcessError) as e:
     sys.stderr.write(f"ERROR: {e}\n")
     sys.exit(1)
diff --git a/docs/website/docs/commandline.md.in b/docs/website/docs/commandline.md.in
new file mode 100644
index 000000000..becc0ab7a
--- /dev/null
+++ b/docs/website/docs/commandline.md.in
@@ -0,0 +1,11 @@
+# Commandline reference
+
+You can control tracker using the `tracker` commandline tool. The various
+commands are documented below.
+
+This documentation is also available on your computer using the `man` command.
+
+!!! note
+    This documentation is for the latest in-development version of Tracker.
+
+{includes}
diff --git a/docs/website/docs.md b/docs/website/docs/developer.md
similarity index 71%
rename from docs/website/docs.md
rename to docs/website/docs/developer.md
index beed96a56..11951ccf4 100644
--- a/docs/website/docs.md
+++ b/docs/website/docs/developer.md
@@ -1,17 +1,4 @@
-# Documentation
-
-## User Documentation
-
-Tracker is a system component and most users will not need to interact with
-it directly.
-
-GNOME has documentation on how to
-[search for files in the file manager](https://developer.gnome.org/libtracker-sparql/stable/).
-
-The `tracker` commandline tool provides direct access to Tracker. Use the
-`--help` option for documentation of this tool.
-
-## Developer Documentation
+# Developer Documentation
     
 Application and platform developers using Tracker will interact with Tracker
 using one or more of the shared libraries it provides:
@@ -34,12 +21,15 @@ The following documentation may be useful:
  * [Tracker ontology documentation](https://developer.gnome.org/ontology/stable/).
  * [Tracker documentation on wiki.gnome.org](https://wiki.gnome.org/Projects/Tracker).
 
+For working on Tracker itself, read the [HACKING.md
+file](https://gitlab.gnome.org/GNOME/tracker/-/blob/master/HACKING.md).
+
 ## Preview Documentation
 
 We provide an online version of the documentation for the latest in-development version
 of Tracker. You can browse it here:
 
-  * [libtracker-sparql](./api-preview/libtracker-sparql)
-  * [ontology](./api-preview/ontology)
+  * [libtracker-sparql](../api-preview/libtracker-sparql/)
+  * [ontology](../api-preview/ontology/)
 
 Be aware that some libraries from Tracker 2.0 will not be available for Tracker 3.0.
diff --git a/docs/website/docs/user.md b/docs/website/docs/user.md
new file mode 100644
index 000000000..b75d78ce1
--- /dev/null
+++ b/docs/website/docs/user.md
@@ -0,0 +1,10 @@
+# User Documentation
+
+Tracker is a system component and most users will not need to interact with
+it directly.
+
+GNOME has documentation on how to
+[search for files in the file 
manager](https://help.gnome.org/users/gnome-help/unstable/files-search.html.en).
+
+The `tracker` commandline tool provides direct access to Tracker, and you
+can [read the documentation online](docs/commandline/).
diff --git a/docs/website/index.md b/docs/website/index.md
index 92becdd03..7e0f335d9 100644
--- a/docs/website/index.md
+++ b/docs/website/index.md
@@ -7,8 +7,8 @@ Are you having problems with Tracker in GNOME? Look at the [Frequently Asked
 Questions](faq).
 
 Are you interested in using or developing Tracker and want to find out more
-about it? Read the [overview](overview) first and then take a look at the
-[documentation](documentation).
+about it? Start with the [overview](overview), then see our documentation
+for [users](docs/user) and [developers](docs/developer).
 
 Do you want to contribute to Tracker? Join the [community](community)
 and look for some bugs marked 'Help wanted'
[Date Prev][Date Next] [Thread Prev][Thread Next] [Thread Index] [Date Index] [Author Index]