[devdocsgjs/main: 797/1867] salt_stack: finish scraper and filters

[Andy Holmes <andyholmes src gnome org>]

commit f3d5a6eae324c6a97e8d5e849db1d98fb0f6f69c
Author: Jasper van Merle <jaspervmerle gmail com>
Date:   Tue Aug 13 16:32:40 2019 +0200

    salt_stack: finish scraper and filters

 .../javascripts/templates/pages/about_tmpl.coffee  |  6 ++++
 lib/docs/filters/salt_stack/clean_html.rb          | 11 ++++++++
 lib/docs/filters/salt_stack/entries.rb             | 13 ++++++++-
 lib/docs/scrapers/salt_stack.rb                    | 32 +++++++++++++++-------
 4 files changed, 51 insertions(+), 11 deletions(-)
---
diff --git a/assets/javascripts/templates/pages/about_tmpl.coffee 
b/assets/javascripts/templates/pages/about_tmpl.coffee
index 5e9dc6cc..dea5c4da 100644
--- a/assets/javascripts/templates/pages/about_tmpl.coffee
+++ b/assets/javascripts/templates/pages/about_tmpl.coffee
@@ -638,6 +638,12 @@ credits = [
     'MIT',
     'https://raw.githubusercontent.com/rust-lang/rust/master/LICENSE-MIT'
   ], [
+    'Salt Stack',
+    '2019 SaltStack',
+    'Apache',
+    'https://raw.githubusercontent.com/saltstack/salt/develop/LICENSE'
+  ],
+  [
     'Sass',
     '2006-2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein',
     'MIT',
diff --git a/lib/docs/filters/salt_stack/clean_html.rb b/lib/docs/filters/salt_stack/clean_html.rb
index 0f084519..8c5cb6ca 100644
--- a/lib/docs/filters/salt_stack/clean_html.rb
+++ b/lib/docs/filters/salt_stack/clean_html.rb
@@ -4,6 +4,17 @@ module Docs
       def call
         css('.headerlink').remove
 
+        css('div[class^="highlight-"]').each do |node|
+          node.name = 'pre'
+          node['data-language'] = node['class'].scan(/highlight-([a-z]+)/i)[0][0]
+          node.content = node.content.strip
+        end
+
+        css('.function > dt').each do |node|
+          node.name = 'h3'
+          node.content = node.content
+        end
+
         doc
       end
     end
diff --git a/lib/docs/filters/salt_stack/entries.rb b/lib/docs/filters/salt_stack/entries.rb
index 23b8f46f..d346fa29 100644
--- a/lib/docs/filters/salt_stack/entries.rb
+++ b/lib/docs/filters/salt_stack/entries.rb
@@ -16,12 +16,23 @@ module Docs
       end
 
       def get_type
-        slug.split('/', 2).first
+        slug.split('/', 3)[1]
       end
 
       def include_default_entry?
         slug.split('/').last.start_with? 'salt'
       end
+
+      def additional_entries
+        entries = []
+
+        css('.function > h3').each do |node|
+          name = node.content.remove('salt.').split('(')[0] + '()'
+          entries << [name, node['id']]
+        end
+
+        entries
+      end
     end
   end
 end
diff --git a/lib/docs/scrapers/salt_stack.rb b/lib/docs/scrapers/salt_stack.rb
index ae4e47f3..390dd7bb 100644
--- a/lib/docs/scrapers/salt_stack.rb
+++ b/lib/docs/scrapers/salt_stack.rb
@@ -1,8 +1,19 @@
 module Docs
-  class SaltStack < UrlScraper
-    self.type = 'salt_stack'
-    self.release = '2018.3.2'
-    self.base_url = 'https://docs.saltstack.com/en/latest/ref/'
+  # The official documentation website is heavily rate-limited
+  #
+  # The documentation can be generated like this (replace 2019.2 with the correct tag):
+  # $ git clone https://github.com/saltstack/salt.git --branch 2019.2 --depth 1
+  # $ cd salt/doc
+  # $ pip install sphinx
+  # $ make html
+  #
+  # The generated html can be found in salt/doc/_build/html
+  class SaltStack < FileScraper
+    self.type = 'simple'
+    self.slug = 'salt_stack'
+    self.release = '2019.2.0'
+    self.base_url = 'https://docs.saltstack.com/en/latest/'
+    self.root_path = 'ref/index.html'
     self.links = {
       home: 'https://www.saltstack.com/',
       code: 'https://github.com/saltstack/salt'
@@ -10,15 +21,16 @@ module Docs
 
     html_filters.push 'salt_stack/clean_html', 'salt_stack/entries'
 
-    options[:only_patterns] = [
-      %r{^[^/]+/all/}
-    ]
-
-    options[:container] = '.body-content'
+    options[:only_patterns] = [/all\//]
+    options[:container] = '.body-content > .section'
 
     options[:attribution] = <<-HTML
-      &copy; 2018 SaltStack.<br>
+      &copy; 2019 SaltStack.<br>
       Licensed under the Apache License, Version 2.0.
     HTML
+
+    def get_latest_version(opts)
+      get_latest_github_release('saltstack', 'salt', opts)
+    end
   end
 end