[gnome-builder] reStructuredText preview: add sphinx support

[Sébastien Lafargue <slafargue src gnome org>]

commit fb8c48baadd0a415ff9f0407e6beee18044b7121
Author: Sebastien Lafargue <slafargue gnome org>
Date:   Sun Mar 26 16:20:04 2017 +0200

    reStructuredText preview: add sphinx support
    
    When asking a preview from .rst file, we walk back in the file tree
    to search for a conf.py sphinx config file.
    (the limits are ten level up or the project tree basedir)
    
    the results of sphinx-build commands are keeped around
    for each basedir in the tmp folder in gnome-builder-sphinx-build-XXXXXX
    like folders (so we can update it after each key strokes or
    triggering another preview with the same basedir)

 .../html-preview/html_preview_plugin/__init__.py   |  359 +++++++++++++++++---
 1 files changed, 315 insertions(+), 44 deletions(-)
---
diff --git a/plugins/html-preview/html_preview_plugin/__init__.py 
b/plugins/html-preview/html_preview_plugin/__init__.py
index d41402c..5b51a4f 100644
--- a/plugins/html-preview/html_preview_plugin/__init__.py
+++ b/plugins/html-preview/html_preview_plugin/__init__.py
@@ -19,9 +19,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import builtins
 import gi
-import os
+import io
 import locale
+import os
+import shutil
+import sys
+import subprocess
+import threading
 
 gi.require_version('Gtk', '3.0')
 gi.require_version('Ide', '1.0')
@@ -41,15 +47,72 @@ except:
     pass
 
 can_preview_rst = True
+can_preview_sphinx = True
+old_open = None
 
 try:
     from docutils.core import publish_string
 except ImportError:
     can_preview_rst = False
 
+try:
+    import sphinx
+except ImportError:
+    can_preview_sphinx = False
+
+sphinx_states = {}
+sphinx_override = {}
+
+
+def add_override_file(path, content):
+    if path in sphinx_override:
+        return False
+    else:
+        sphinx_override[path] = content.encode('utf-8')
+        return True
+
+
+def remove_override_file(path):
+    try:
+        del sphinx_override[path]
+    except KeyError:
+        return False
+
+    return True
+
+
+def new_open(*args, **kwargs):
+    path = args[0]
+    if path in sphinx_override:
+        return io.BytesIO(sphinx_override[path])
+
+    return old_open(*args, **kwargs)
+
+old_open = builtins.open
+builtins.open = new_open
+
 _ = Ide.gettext
 
 
+def is_sphinx_installed():
+    with open(os.devnull, 'w') as devnull:
+        try:
+            if subprocess.call(['sphinx-build', '--version'],
+                               stdout=devnull, stderr=devnull) == 0:
+                return True
+        except FileNotFoundError:
+            pass
+
+        return False
+
+
+class SphinxState():
+    def __init__(self, builddir):
+        self.builddir = builddir
+        self.is_running = False
+        self.need_build = False
+
+
 class HtmlPreviewData(GObject.Object, Ide.ApplicationAddin):
     MARKDOWN_CSS = None
     MARKED_JS = None
@@ -60,6 +123,13 @@ class HtmlPreviewData(GObject.Object, Ide.ApplicationAddin):
         HtmlPreviewData.MARKED_JS = self.get_data('marked.js')
         HtmlPreviewData.MARKDOWN_VIEW_JS = self.get_data('markdown-view.js')
 
+    def do_unload(self, app):
+        for state in sphinx_states.items():
+            # Be extra sure that we are in the tmp dir
+            tmpdir = GLib.get_tmp_dir()
+            if state.builddir.startswith(tmpdir):
+                shutil.rmtree(state.builddir)
+
     def get_data(self, name):
         engine = Peas.Engine.get_default()
         info = engine.get_plugin_info('html_preview_plugin')
@@ -68,48 +138,168 @@ class HtmlPreviewData(GObject.Object, Ide.ApplicationAddin):
         return open(path, 'r').read()
 
 
-class HtmlPreviewAddin(GObject.Object, Ide.EditorViewAddin):
-    def do_load(self, editor):
-        self.workbench = editor.get_ancestor(Ide.Workbench)
+class HtmlWorkbenchAddin(GObject.Object, Ide.WorkbenchAddin):
+    def do_load(self, workbench):
+        self.workbench = workbench
 
-        self.action = Gio.SimpleAction(name='preview-as-html', enabled=True)
-        self.action.connect('activate', lambda *_: self.preview_activated(editor))
-
-        actions = editor.get_action_group('view')
-        actions.add_action(self.action)
+        group = Gio.SimpleActionGroup()
 
         self.install_action = Gio.SimpleAction(name='install-docutils', enabled=True)
-        self.install_action.connect('activate', lambda *_: self.install_docutils(editor))
+        self.install_action.connect('activate', lambda *_: self.install_docutils())
+        group.insert(self.install_action)
 
-        group = Gio.SimpleActionGroup()
+        self.install_action = Gio.SimpleAction(name='install-sphinx', enabled=True)
+        self.install_action.connect('activate', lambda *_: self.install_sphinx())
         group.insert(self.install_action)
 
         self.workbench.insert_action_group('html-preview', group)
 
-    def do_unload(self, editor):
-        actions = editor.get_action_group('view')
+    def do_unload(self, workbench):
+        self.workbench = None
+
+    def install_docutils(self):
+        transfer = Ide.PkconTransfer(packages=['python3-docutils'])
+        context = self.workbench.get_context()
+        manager = context.get_transfer_manager()
+
+        manager.execute_async(transfer, None, self.docutils_installed, None)
+
+    def install_sphinx(self):
+        transfer = Ide.PkconTransfer(packages=['python3-sphinx'])
+        context = self.workbench.get_context()
+        manager = context.get_transfer_manager()
+
+        manager.execute_async(transfer, None, self.sphinx_installed, None)
+
+    def docutils_installed(self, object, result, data):
+        global can_preview_rst
+        global publish_string
+
+        try:
+            from docutils.core import publish_string
+        except ImportError:
+            return
+
+        can_preview_rst = True
+        self.workbench.pop_message('org.gnome.builder.docutils.install')
+
+    def sphinx_installed(self, object, result, data):
+        global can_preview_sphinx
+        global sphinx
+
+        try:
+            import sphinx
+        except ImportError:
+            return
+
+        can_preview_sphinx = True
+        self.workbench.pop_message('org.gnome.builder.sphinx.install')
+
+
+class HtmlPreviewAddin(GObject.Object, Ide.EditorViewAddin):
+    def do_load(self, view):
+        self.workbench = view.get_ancestor(Ide.Workbench)
+        self.view = view
+        self.can_preview = False
+        self.sphinx_basedir = None
+        self.sphinx_builddir = None
+
+        self.action = Gio.SimpleAction(name='preview-as-html', enabled=True)
+        self.action.connect('activate', lambda *_: self.preview_activated(view))
+
+        actions = view.get_action_group('view')
+        actions.add_action(self.action)
+
+        document = view.get_document()
+        language = document.get_language()
+        language_id = language.get_id() if language else None
+
+        self.do_language_changed(language_id)
+
+    def do_unload(self, view):
+        actions = view.get_action_group('view')
         actions.remove_action('preview-as-html')
 
     def do_language_changed(self, language_id):
         enabled = (language_id in ('html', 'markdown', 'rst'))
         self.action.set_enabled(enabled)
+        self.lang_id = language_id
+        self.can_preview = enabled
+
+        if self.lang_id == 'rst':
+            if not self.sphinx_basedir:
+                document = self.view.get_document()
+                path = document.get_file().get_file().get_path()
+                self.sphinx_basedir = self.search_sphinx_base_dir(path)
+
+            if self.sphinx_basedir:
+                self.sphinx_builddir = self.setup_sphinx_states(self.sphinx_basedir)
 
-    def preview_activated(self, editor):
+        if not enabled:
+            self.sphinx_basedir = None
+            self.sphinx_builddir = None
+
+    def setup_sphinx_states(self, basedir):
+        global sphinx_states
+
+        if basedir in sphinx_states:
+            state = sphinx_states[basedir]
+            sphinx_builddir = state.builddir
+        else:
+            sphinx_builddir = GLib.Dir.make_tmp('gnome-builder-sphinx-build-XXXXXX')
+            state = SphinxState(sphinx_builddir)
+            sphinx_states[basedir] = state
+
+        return sphinx_builddir
+
+    def preview_activated(self, view):
         global can_preview_rst
 
-        document = editor.get_document()
-        language = document.get_language()
+        if self.lang_id == 'rst':
+            if self.sphinx_basedir:
+                if not can_preview_sphinx:
+                    self.show_missing_sphinx_message(view)
+                    return
+            elif not can_preview_rst:
+                self.show_missing_docutils_message(view)
+                return
+
+        document = view.get_document()
+        web_view = HtmlPreviewView(document,
+                                   self.sphinx_basedir,
+                                   self.sphinx_builddir,
+                                   visible=True)
+
+        stack = view.get_ancestor(Ide.LayoutStack)
+        stack.add(web_view)
+
+    def search_sphinx_base_dir(self, path):
+        context = self.workbench.get_context()
+        vcs = context.get_vcs()
+        working_dir = vcs.get_working_directory().get_path()
 
-        if language and language.get_id() == 'rst' and not can_preview_rst:
-            self.show_missing_message(editor)
-            return
+        try:
+            if os.path.commonpath([working_dir, path]) != working_dir:
+                working_dir = '/'
+        except:
+            working_dir = '/'
 
-        view = HtmlPreviewView(document, visible=True)
+        folder = os.path.dirname(path)
+        level = 10
 
-        stack = editor.get_ancestor(Ide.LayoutStack)
-        stack.add(view)
+        while level > 0:
+            files = os.scandir(folder)
+            for file in files:
+                if file.name == 'conf.py':
+                    return folder
 
-    def show_missing_message(self, editor):
+            if folder == working_dir:
+                return None
+
+            level -= 1
+            folder = os.path.dirname(folder)
+
+    def show_missing_docutils_message(self, view):
         message = Ide.WorkbenchMessage(
             id='org.gnome.builder.docutils.install',
             title=_('Your computer is missing python3-docutils'),
@@ -119,32 +309,28 @@ class HtmlPreviewAddin(GObject.Object, Ide.EditorViewAddin):
         message.add_action(_('Install'), 'html-preview.install-docutils')
         self.workbench.push_message(message)
 
-    def install_docutils(self, editor):
-        transfer = Ide.PkconTransfer(packages=['python3-docutils'])
-        context = self.workbench.get_context()
-        manager = context.get_transfer_manager()
-
-        manager.execute_async(transfer, None, self.docutils_installed, None)
-
-    def docutils_installed(self, object, result, data):
-        global can_preview_rst
-        global publish_string
-
-        try:
-            from docutils.core import publish_string
-        except ImportError:
-            return
+    def show_missing_sphinx_message(self, view):
+        message = Ide.WorkbenchMessage(
+            id='org.gnome.builder.sphinx.install',
+            title=_('Your computer is missing python3-sphinx'),
+            show_close_button=True,
+            visible=True)
 
-        can_preview_rst = True
-        self.workbench.pop_message('org.gnome.builder.docutils.install')
+        message.add_action(_('Install'), 'html-preview.install-sphinx')
+        self.workbench.push_message(message)
 
 
 class HtmlPreviewView(Ide.LayoutView):
     markdown = False
     rst = False
 
-    def __init__(self, document, *args, **kwargs):
+    def __init__(self, document, sphinx_basedir, sphinx_builddir, *args, **kwargs):
+        global old_open
+
         super().__init__(*args, **kwargs)
+
+        self.sphinx_basedir = sphinx_basedir
+        self.sphinx_builddir = sphinx_builddir
         self.document = document
 
         self.webview = WebKit2.WebView(visible=True, expand=True)
@@ -199,9 +385,69 @@ class HtmlPreviewView(Ide.LayoutView):
                               source_path=path,
                               destination_path=path)
 
+    def get_sphinx_rst_async(self, text, path, basedir, builddir, cancellable, callback):
+        task = Gio.Task.new(self, cancellable, callback)
+        threading.Thread(target=self.get_sphinx_rst_worker,
+                         args=[task, text, path, basedir, builddir],
+                         name='sphinx-rst-thread').start()
+
+    def purge_cache(self, basedir, builddir, document):
+        path = document.get_file().get_file().get_path()
+        rel_path = os.path.relpath(path, start=basedir)
+        rel_path_doctree = os.path.splitext(rel_path)[0] + '.doctree'
+        doctree_path = os.path.join(builddir, '.doctrees', rel_path_doctree)
+
+        tmpdir = GLib.get_tmp_dir()
+        if doctree_path.startswith(tmpdir):
+            try:
+                os.remove(doctree_path)
+            except:
+                pass
+
+    def get_sphinx_rst_worker(self, task, text, path, basedir, builddir):
+        add_override_file(path, text)
+
+        rel_path = os.path.relpath(path, start=basedir)
+        command = ['sphinx-build', '-Q', '-b', 'html', basedir, builddir, path]
+
+        rel_path_html = os.path.splitext(rel_path)[0] + '.html'
+        builddir_path = os.path.join(builddir, rel_path_html)
+
+        result = not sphinx.build_main(command)
+        remove_override_file(path)
+
+        if not result:
+            task.builddir_path = None
+            task.return_error(GLib.Error('\'sphinx-build\' command error for {}'.format(path)))
+            return
+
+        task.builddir_path = builddir_path
+        task.return_boolean(True)
+
+    def get_sphinx_rst_finish(self, result):
+        succes = result.propagate_boolean()
+        builddir_path = result.builddir_path
+
+        return builddir_path
+
+    def get_sphinx_state(self, basedir):
+        global sphinx_states
+
+        try:
+            state = sphinx_states[basedir]
+        except KeyError:
+            return None
+
+        return state
+
     def reload(self):
-        file = self.document.get_file().get_file()
-        base_uri = file.get_uri()
+        state = self.get_sphinx_state(self.sphinx_basedir)
+        if state and state.is_running:
+            state.need_build = True
+            return
+
+        gfile = self.document.get_file().get_file()
+        base_uri = gfile.get_uri()
 
         begin, end = self.document.get_bounds()
         text = self.document.get_text(begin, end, True)
@@ -209,9 +455,34 @@ class HtmlPreviewView(Ide.LayoutView):
         if self.markdown:
             text = self.get_markdown(text)
         elif self.rst:
-            text = self.get_rst(text, file.get_path()).decode("utf-8")
+            if self.sphinx_basedir:
+                self.purge_cache(self.sphinx_basedir, self.sphinx_builddir, self.document)
+                state.is_running = True
+
+                self.get_sphinx_rst_async(text,
+                                          gfile.get_path(),
+                                          self.sphinx_basedir,
+                                          self.sphinx_builddir,
+                                          None,
+                                          self.get_sphinx_rst_cb)
+
+                return
+            else:
+                text = self.get_rst(text, gfile.get_path()).decode("utf-8")
 
         self.webview.load_html(text, base_uri)
 
+    def get_sphinx_rst_cb(self, obj, result):
+        builddir_path = self.get_sphinx_rst_finish(result)
+        if builddir_path:
+            uri = 'file:///' + builddir_path
+            self.webview.load_uri(uri)
+
+        state = self.get_sphinx_state(self.sphinx_basedir)
+        state.is_running = False
+        if state.need_build:
+            state.need_build = False
+            self.reload()
+
     def on_changed(self, document):
         self.reload()