salt_stack: finish scraper and filters

Author: jmerle

assets/javascripts/templates/pages/about_tmpl.coffee

@@ -638,6 +638,12 @@ credits = [
     'MIT',
     'https://raw.githubusercontent.com/rust-lang/rust/master/LICENSE-MIT'
   ], [
+    'Salt Stack',
+    '2019 SaltStack',
+    'Apache',
+    'https://raw.githubusercontent.com/saltstack/salt/develop/LICENSE'
+  ],
+  [
     'Sass',
     '2006-2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein',
     'MIT',

lib/docs/filters/salt_stack/clean_html.rb

@@ -4,6 +4,17 @@ class CleanHtmlFilter < Filter
       def call
         css('.headerlink').remove
 
+        css('div[class^="highlight-"]').each do |node|
+          node.name = 'pre'
+          node['data-language'] = node['class'].scan(/highlight-([a-z]+)/i)[0][0]
+          node.content = node.content.strip
+        end
+
+        css('.function > dt').each do |node|
+          node.name = 'h3'
+          node.content = node.content
+        end
+
         doc
       end
     end

lib/docs/filters/salt_stack/entries.rb

@@ -16,12 +16,23 @@ def get_name
       end
 
       def get_type
-        slug.split('/', 2).first
+        slug.split('/', 3)[1]
       end
 
       def include_default_entry?
         slug.split('/').last.start_with? 'salt'
       end
+
+      def additional_entries
+        entries = []
+
+        css('.function > h3').each do |node|
+          name = node.content.remove('salt.').split('(')[0] + '()'
+          entries << [name, node['id']]
+        end
+
+        entries
+      end
     end
   end
 end

lib/docs/scrapers/salt_stack.rb

@@ -1,24 +1,36 @@
 module Docs
-  class SaltStack < UrlScraper
-    self.type = 'salt_stack'
-    self.release = '2018.3.2'
-    self.base_url = 'https://docs.saltstack.com/en/latest/ref/'
+  # The official documentation website is heavily rate-limited
+  #
+  # The documentation can be generated like this (replace 2019.2 with the correct tag):
+  # $ git clone https://github.com/saltstack/salt.git --branch 2019.2 --depth 1
+  # $ cd salt/doc
+  # $ pip install sphinx
+  # $ make html
+  #
+  # The generated html can be found in salt/doc/_build/html
+  class SaltStack < FileScraper
+    self.type = 'simple'
+    self.slug = 'salt_stack'
+    self.release = '2019.2.0'
+    self.base_url = 'https://docs.saltstack.com/en/latest/'
+    self.root_path = 'ref/index.html'
     self.links = {
       home: 'https://www.saltstack.com/',
       code: 'https://github.com/saltstack/salt'
     }
 
     html_filters.push 'salt_stack/clean_html', 'salt_stack/entries'
 
-    options[:only_patterns] = [
-      %r{^[^/]+/all/}
-    ]
-
-    options[:container] = '.body-content'
+    options[:only_patterns] = [/all\//]
+    options[:container] = '.body-content > .section'
 
     options[:attribution] = <<-HTML
-      &copy; 2018 SaltStack.<br>
+      &copy; 2019 SaltStack.<br>
       Licensed under the Apache License, Version 2.0.
     HTML
+
+    def get_latest_version(opts)
+      get_latest_github_release('saltstack', 'salt', opts)
+    end
   end
 end