[devdocsgjs/main: 588/1867] Implement get_latest_version for the remaining scrapers

From: Andy Holmes <andyholmes src gnome org>
To: commits-list gnome org
Cc:
Subject: [devdocsgjs/main: 588/1867] Implement get_latest_version for the remaining scrapers
Date: Fri, 19 Nov 2021 23:47:18 +0000 (UTC)

commit 6ee1693134bbef44912a3a53cf250c6f82e98fbc
Author: Jasper van Merle <jaspervmerle gmail com>
Date:   Sat Mar 9 20:22:48 2019 +0100

    Implement get_latest_version for the remaining scrapers

 lib/docs/scrapers/nginx.rb            |  7 +++++++
 lib/docs/scrapers/nginx_lua_module.rb | 10 ++++++++++
 lib/docs/scrapers/nim.rb              |  6 ++++++
 lib/docs/scrapers/node.rb             |  6 ++++++
 lib/docs/scrapers/nokogiri2.rb        |  6 ++++++
 lib/docs/scrapers/npm.rb              |  6 ++++++
 lib/docs/scrapers/numpy.rb            |  6 ++++++
 lib/docs/scrapers/openjdk.rb          | 21 +++++++++++++++++++++
 lib/docs/scrapers/opentsdb.rb         |  6 ++++++
 lib/docs/scrapers/padrino.rb          |  6 ++++++
 lib/docs/scrapers/pandas.rb           |  7 +++++++
 lib/docs/scrapers/perl.rb             |  6 ++++++
 lib/docs/scrapers/phalcon.rb          |  6 ++++++
 lib/docs/scrapers/phaser.rb           |  6 ++++++
 lib/docs/scrapers/phoenix.rb          |  6 ++++++
 lib/docs/scrapers/php.rb              |  7 +++++++
 lib/docs/scrapers/phpunit.rb          |  7 +++++++
 lib/docs/scrapers/postgresql.rb       |  7 +++++++
 lib/docs/scrapers/pug.rb              |  4 ++++
 lib/docs/scrapers/puppeteer.rb        |  6 ++++++
 lib/docs/scrapers/pygame.rb           |  6 ++++++
 lib/docs/scrapers/python.rb           |  6 ++++++
 lib/docs/scrapers/q.rb                |  4 ++++
 lib/docs/scrapers/qt.rb               |  7 +++++++
 lib/docs/scrapers/ramda.rb            |  6 ++++++
 lib/docs/scrapers/react.rb            |  6 ++++++
 lib/docs/scrapers/react_native.rb     |  6 ++++++
 lib/docs/scrapers/redis.rb            |  7 +++++++
 lib/docs/scrapers/redux.rb            |  4 ++++
 lib/docs/scrapers/relay.rb            |  6 ++++++
 lib/docs/scrapers/requirejs.rb        |  4 ++++
 lib/docs/scrapers/rethinkdb.rb        |  6 ++++++
 lib/docs/scrapers/rust.rb             |  7 +++++++
 lib/docs/scrapers/sass.rb             |  6 ++++++
 lib/docs/scrapers/scikit_image.rb     |  6 ++++++
 lib/docs/scrapers/scikit_learn.rb     |  6 ++++++
 lib/docs/scrapers/sinon.rb            |  6 ++++++
 lib/docs/scrapers/socketio.rb         |  4 ++++
 lib/docs/scrapers/sqlite.rb           |  6 ++++++
 lib/docs/scrapers/statsmodels.rb      |  5 +++++
 lib/docs/scrapers/symfony.rb          |  6 ++++++
 lib/docs/scrapers/tcl_tk.rb           |  7 +++++++
 lib/docs/scrapers/tensorflow.rb       |  6 ++++++
 lib/docs/scrapers/terraform.rb        |  6 ++++++
 lib/docs/scrapers/twig.rb             |  6 ++++++
 lib/docs/scrapers/typescript.rb       |  6 ++++++
 lib/docs/scrapers/underscore.rb       |  6 ++++++
 lib/docs/scrapers/vagrant.rb          |  6 ++++++
 lib/docs/scrapers/vue.rb              |  6 ++++++
 lib/docs/scrapers/vulkan.rb           |  6 ++++++
 lib/docs/scrapers/webpack.rb          |  4 ++++
 lib/docs/scrapers/yarn.rb             |  6 ++++++
 lib/docs/scrapers/yii.rb              |  6 ++++++
 lib/tasks/updates.thor                |  4 ++--
 54 files changed, 335 insertions(+), 2 deletions(-)
---
diff --git a/lib/docs/scrapers/nginx.rb b/lib/docs/scrapers/nginx.rb
index 5a3dbf1e..07a4565f 100644
--- a/lib/docs/scrapers/nginx.rb
+++ b/lib/docs/scrapers/nginx.rb
@@ -25,5 +25,12 @@ module Docs
       &copy; 2011-2018 Nginx, Inc.<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://nginx.org/en/download.html', options) do |doc|
+        table = doc.at_css('#content > table').inner_html
+        block.call table.scan(/nginx-([0-9.]+)</)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/nginx_lua_module.rb b/lib/docs/scrapers/nginx_lua_module.rb
index f4943f62..954b09a8 100644
--- a/lib/docs/scrapers/nginx_lua_module.rb
+++ b/lib/docs/scrapers/nginx_lua_module.rb
@@ -4,6 +4,9 @@ module Docs
     self.slug = 'nginx_lua_module'
     self.release = '0.10.13'
     self.base_url = "https://github.com/openresty/lua-nginx-module/tree/v#{self.release}/";
+    self.links = {
+      code: 'https://github.com/openresty/lua-nginx-module'
+    }
 
     html_filters.push 'nginx_lua_module/clean_html', 'nginx_lua_module/entries', 'title'
 
@@ -15,5 +18,12 @@ module Docs
       &copy; 2009&ndash;2018 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_tags('openresty', 'lua-nginx-module', options) do |tags|
+        tag = tags.find { |tag| !tag['name'].include?('rc') }
+        block.call tag['name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/nim.rb b/lib/docs/scrapers/nim.rb
index cce16f1c..28d83049 100644
--- a/lib/docs/scrapers/nim.rb
+++ b/lib/docs/scrapers/nim.rb
@@ -17,5 +17,11 @@ module Docs
       &copy; 2006&ndash;2018 Andreas Rumpf<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://nim-lang.org/docs/overview.html', options) do |doc|
+        block.call doc.at_css('.container > .docinfo > tbody > tr:last-child > td').content.strip
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/node.rb b/lib/docs/scrapers/node.rb
index 66c6a456..36c14e9e 100644
--- a/lib/docs/scrapers/node.rb
+++ b/lib/docs/scrapers/node.rb
@@ -46,5 +46,11 @@ module Docs
       self.release = '4.9.1'
       self.base_url = 'https://nodejs.org/dist/latest-v4.x/docs/api/'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://nodejs.org/en/', options) do |doc|
+        block.call doc.at_css('#home-intro > .home-downloadblock:last-of-type > a')['data-version'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/nokogiri2.rb b/lib/docs/scrapers/nokogiri2.rb
index 084ad2fb..aca5d10c 100644
--- a/lib/docs/scrapers/nokogiri2.rb
+++ b/lib/docs/scrapers/nokogiri2.rb
@@ -19,5 +19,11 @@ module Docs
       Patrick Mahoney, Yoko Harada, Akinori Musha, John Shahid, Lars Kanis<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('sparklemotion', 'nokogiri', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/npm.rb b/lib/docs/scrapers/npm.rb
index 2c854e6e..7ef727c0 100644
--- a/lib/docs/scrapers/npm.rb
+++ b/lib/docs/scrapers/npm.rb
@@ -29,5 +29,11 @@ module Docs
       Licensed under the npm License.<br>
       npm is a trademark of npm, Inc.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('npm', 'cli', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb
index 1327fc02..1fcc05d6 100644
--- a/lib/docs/scrapers/numpy.rb
+++ b/lib/docs/scrapers/numpy.rb
@@ -49,5 +49,11 @@ module Docs
       self.release = '1.10.4'
       self.base_url = "https://docs.scipy.org/doc/numpy-#{self.release}/reference/";
     end
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('numpy', 'numpy', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/openjdk.rb b/lib/docs/scrapers/openjdk.rb
index 944ac416..e833c7c4 100644
--- a/lib/docs/scrapers/openjdk.rb
+++ b/lib/docs/scrapers/openjdk.rb
@@ -86,5 +86,26 @@ module Docs
     def read_file(path)
       File.read(path).force_encoding('iso-8859-1').encode('utf-8') rescue nil
     end
+
+    def get_latest_version(options, &block)
+      latest_version = 8
+      current_attempt = latest_version
+      attempts = 0
+
+      while attempts < 3
+        current_attempt += 1
+
+        fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc";, options) do |doc|
+          if doc.at_css('.perror').nil?
+            latest_version = current_attempt
+            attempts = 0
+          else
+            attempts += 1
+          end
+        end
+      end
+
+      block.call latest_version.to_s
+    end
   end
 end
diff --git a/lib/docs/scrapers/opentsdb.rb b/lib/docs/scrapers/opentsdb.rb
index 0372e331..54a77017 100644
--- a/lib/docs/scrapers/opentsdb.rb
+++ b/lib/docs/scrapers/opentsdb.rb
@@ -18,5 +18,11 @@ module Docs
       &copy; 2010&ndash;2016 The OpenTSDB Authors<br>
       Licensed under the GNU LGPLv2.1+ and GPLv3+ licenses.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('OpenTSDB', 'opentsdb', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/padrino.rb b/lib/docs/scrapers/padrino.rb
index 218f1731..78588ef4 100644
--- a/lib/docs/scrapers/padrino.rb
+++ b/lib/docs/scrapers/padrino.rb
@@ -23,5 +23,11 @@ module Docs
     stub 'index2' do
       request_one(url_for('index')).body
     end
+
+    def get_latest_version(options, &block)
+      get_github_tags('padrino', 'padrino-framework', options) do |tags|
+        block.call tags[0]['name']
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/pandas.rb b/lib/docs/scrapers/pandas.rb
index 1355a012..f50c4427 100644
--- a/lib/docs/scrapers/pandas.rb
+++ b/lib/docs/scrapers/pandas.rb
@@ -49,5 +49,12 @@ module Docs
       self.release = '0.18.1'
       self.base_url = "http://pandas.pydata.org/pandas-docs/version/#{self.release}/";
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', options) do |doc|
+        label = doc.at_css('.body > .section > p').content
+        block.call label.scan(/Version: ([0-9.]+)/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/perl.rb b/lib/docs/scrapers/perl.rb
index 142ceaa5..136a6d75 100644
--- a/lib/docs/scrapers/perl.rb
+++ b/lib/docs/scrapers/perl.rb
@@ -43,5 +43,11 @@ module Docs
       self.release = '5.20.2'
       self.base_url = "https://perldoc.perl.org/#{self.release}/";
     end
+
+    def get_latest_version(options, &block)
+      fetch('https://perldoc.perl.org/static/perlversion.js', options) do |body|
+        block.call body.scan(/>Perl ([0-9.]+)/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/phalcon.rb b/lib/docs/scrapers/phalcon.rb
index c2cb9242..82a9cf85 100644
--- a/lib/docs/scrapers/phalcon.rb
+++ b/lib/docs/scrapers/phalcon.rb
@@ -29,5 +29,11 @@ module Docs
       self.release = '2.0.13'
       self.base_url = 'https://docs.phalconphp.com/en/2.0.0/'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://docs.phalconphp.com/', options) do |doc|
+        block.call doc.at_css('.custom-select__version').content.strip.sub(/Version /, '')
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/phaser.rb b/lib/docs/scrapers/phaser.rb
index 5ae371c1..ef265e2e 100644
--- a/lib/docs/scrapers/phaser.rb
+++ b/lib/docs/scrapers/phaser.rb
@@ -25,5 +25,11 @@ module Docs
       &copy; 2016 Richard Davey, Photon Storm Ltd.<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('photonstorm', 'phaser', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/phoenix.rb b/lib/docs/scrapers/phoenix.rb
index 26b0144f..08948b3c 100644
--- a/lib/docs/scrapers/phoenix.rb
+++ b/lib/docs/scrapers/phoenix.rb
@@ -46,5 +46,11 @@ module Docs
         HTML
       end
     }
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', options) do |doc|
+        block.call doc.at_css('.sidebar-projectVersion').content.strip[1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/php.rb b/lib/docs/scrapers/php.rb
index d4a66b5b..b6aac039 100644
--- a/lib/docs/scrapers/php.rb
+++ b/lib/docs/scrapers/php.rb
@@ -66,5 +66,12 @@ module Docs
       &copy; 1997&ndash;2018 The PHP Documentation Group<br>
       Licensed under the Creative Commons Attribution License v3.0 or later.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', options) do |doc|
+        label = doc.at_css('tbody.gen-changelog > tr > td').content
+        block.call label.split(',').last.strip
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/phpunit.rb b/lib/docs/scrapers/phpunit.rb
index 665ca4c7..c5e8c396 100644
--- a/lib/docs/scrapers/phpunit.rb
+++ b/lib/docs/scrapers/phpunit.rb
@@ -37,5 +37,12 @@ module Docs
       self.release = '4.8'
       self.base_url = "https://phpunit.de/manual/#{release}/en/";
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://phpunit.readthedocs.io/', options) do |doc|
+        label = doc.at_css('.rst-current-version').content.strip
+        block.call label.scan(/v: ([0-9.]+)/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/postgresql.rb b/lib/docs/scrapers/postgresql.rb
index 109685ba..4946066b 100644
--- a/lib/docs/scrapers/postgresql.rb
+++ b/lib/docs/scrapers/postgresql.rb
@@ -80,5 +80,12 @@ module Docs
 
       html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://www.postgresql.org/docs/current/index.html', options) do |doc|
+        label = doc.at_css('#pgContentWrap h1.title').content
+        block.call label.scan(/([0-9.]+)/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/pug.rb b/lib/docs/scrapers/pug.rb
index 76949529..9084feb1 100644
--- a/lib/docs/scrapers/pug.rb
+++ b/lib/docs/scrapers/pug.rb
@@ -18,6 +18,10 @@ module Docs
       Licensed under the MIT license.
     HTML
 
+    def get_latest_version(options, &block)
+      get_npm_version('pug', options, &block)
+    end
+
     private
 
     def parse(response) # Hook here because Nokogori removes whitespace from textareas
diff --git a/lib/docs/scrapers/puppeteer.rb b/lib/docs/scrapers/puppeteer.rb
index 2e26c180..a1506c7a 100644
--- a/lib/docs/scrapers/puppeteer.rb
+++ b/lib/docs/scrapers/puppeteer.rb
@@ -14,5 +14,11 @@ module Docs
       &copy; 2017 Google Inc<br>
       Licensed under the Apache License 2.0.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_file_contents('GoogleChrome', 'puppeteer', 'README.md', options) do |contents|
+        block.call contents.scan(/\/v([0-9.]+)\/docs\/api\.md/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/pygame.rb b/lib/docs/scrapers/pygame.rb
index 892619e4..538c0722 100644
--- a/lib/docs/scrapers/pygame.rb
+++ b/lib/docs/scrapers/pygame.rb
@@ -17,5 +17,11 @@ module Docs
       &copy; Pygame Developpers.<br>
       Licensed under the GNU LGPL License version 2.1.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('pygame', 'pygame', options) do |release|
+        block.call release['tag_name']
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb
index aa4336d9..16ecc366 100644
--- a/lib/docs/scrapers/python.rb
+++ b/lib/docs/scrapers/python.rb
@@ -50,5 +50,11 @@ module Docs
 
       html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://docs.python.org/', options) do |doc|
+        block.call doc.at_css('.version_switcher_placeholder').content
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/q.rb b/lib/docs/scrapers/q.rb
index bb66c156..0a160488 100644
--- a/lib/docs/scrapers/q.rb
+++ b/lib/docs/scrapers/q.rb
@@ -19,5 +19,9 @@ module Docs
       &copy; 2009&ndash;2017 Kristopher Michael Kowal<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_npm_version('q', options, &block)
+    end
   end
 end
diff --git a/lib/docs/scrapers/qt.rb b/lib/docs/scrapers/qt.rb
index 412eca6d..41c835e9 100644
--- a/lib/docs/scrapers/qt.rb
+++ b/lib/docs/scrapers/qt.rb
@@ -117,5 +117,12 @@ module Docs
       self.release = '5.6'
       self.base_url = 'https://doc.qt.io/qt-5.6/'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://doc.qt.io/qt-5/index.html', options) do |doc|
+        label = doc.at_css('.mainContent h1.title').content
+        block.call label.sub(/Qt /, '')
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/ramda.rb b/lib/docs/scrapers/ramda.rb
index 09ff46c8..d3751e38 100644
--- a/lib/docs/scrapers/ramda.rb
+++ b/lib/docs/scrapers/ramda.rb
@@ -15,6 +15,12 @@ module Docs
       &copy; 2013&ndash;2016 Scott Sauyet and Michael Hurley<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://ramdajs.com/docs/', options) do |doc|
+        block.call doc.at_css('.navbar-brand > .version').content[1..-1]
+      end
+    end
   end
 end
 
diff --git a/lib/docs/scrapers/react.rb b/lib/docs/scrapers/react.rb
index 704f3189..54038244 100644
--- a/lib/docs/scrapers/react.rb
+++ b/lib/docs/scrapers/react.rb
@@ -30,5 +30,11 @@ module Docs
       &copy; 2013&ndash;present Facebook Inc.<br>
       Licensed under the Creative Commons Attribution 4.0 International Public License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://reactjs.org/docs/getting-started.html', options) do |doc|
+        block.call doc.at_css('a[href="/versions"]').content.strip[1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/react_native.rb b/lib/docs/scrapers/react_native.rb
index d1f94001..ac5bd250 100644
--- a/lib/docs/scrapers/react_native.rb
+++ b/lib/docs/scrapers/react_native.rb
@@ -30,5 +30,11 @@ module Docs
       &copy; 2015&ndash;2018 Facebook Inc.<br>
       Licensed under the Creative Commons Attribution 4.0 International Public License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', options) do |doc|
+        block.call doc.at_css('header > a > h3').content
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/redis.rb b/lib/docs/scrapers/redis.rb
index 201dbb47..89a67598 100644
--- a/lib/docs/scrapers/redis.rb
+++ b/lib/docs/scrapers/redis.rb
@@ -19,5 +19,12 @@ module Docs
       &copy; 2009&ndash;2018 Salvatore Sanfilippo<br>
       Licensed under the Creative Commons Attribution-ShareAlike License 4.0.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', options) do |body|
+        body = body.lines[1..-1].join
+        block.call body.scan(/Redis ([0-9.]+)/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/redux.rb b/lib/docs/scrapers/redux.rb
index cd7369ba..09738bb3 100644
--- a/lib/docs/scrapers/redux.rb
+++ b/lib/docs/scrapers/redux.rb
@@ -20,5 +20,9 @@ module Docs
     stub '' do
       request_one('http://redux.js.org/index.html').body
     end
+
+    def get_latest_version(options, &block)
+      get_npm_version('redux', options, &block)
+    end
   end
 end
diff --git a/lib/docs/scrapers/relay.rb b/lib/docs/scrapers/relay.rb
index a020e9d7..90423ac6 100644
--- a/lib/docs/scrapers/relay.rb
+++ b/lib/docs/scrapers/relay.rb
@@ -18,5 +18,11 @@ module Docs
       &copy; 2013&ndash;present Facebook Inc.<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('http://facebook.github.io/relay/en/', options) do |doc|
+        block.call doc.at_css('header > a > h3').content[1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/requirejs.rb b/lib/docs/scrapers/requirejs.rb
index 200f1f2d..cf417a3e 100644
--- a/lib/docs/scrapers/requirejs.rb
+++ b/lib/docs/scrapers/requirejs.rb
@@ -30,5 +30,9 @@ module Docs
       &copy; jQuery Foundation and other contributors<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_npm_version('requirejs', options, &block)
+    end
   end
 end
diff --git a/lib/docs/scrapers/rethinkdb.rb b/lib/docs/scrapers/rethinkdb.rb
index 023bbe77..27d913e0 100644
--- a/lib/docs/scrapers/rethinkdb.rb
+++ b/lib/docs/scrapers/rethinkdb.rb
@@ -58,6 +58,12 @@ module Docs
       CODE
     end
 
+    def get_latest_version(options, &block)
+      get_latest_github_release('rethinkdb', 'rethinkdb', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
+
     private
 
     def process_response?(response)
diff --git a/lib/docs/scrapers/rust.rb b/lib/docs/scrapers/rust.rb
index 1677e138..7c8d1519 100644
--- a/lib/docs/scrapers/rust.rb
+++ b/lib/docs/scrapers/rust.rb
@@ -39,6 +39,13 @@ module Docs
       Licensed under the Apache License, Version 2.0 or the MIT license, at your option.
     HTML
 
+    def get_latest_version(options, &block)
+      fetch_doc('https://www.rust-lang.org/', options) do |doc|
+        label = doc.at_css('.button-download + p > a').content
+        block.call label.sub(/Version /, '')
+      end
+    end
+
     private
 
     REDIRECT_RGX = /http-equiv="refresh"/i
diff --git a/lib/docs/scrapers/sass.rb b/lib/docs/scrapers/sass.rb
index 244a88e7..f493289b 100644
--- a/lib/docs/scrapers/sass.rb
+++ b/lib/docs/scrapers/sass.rb
@@ -23,5 +23,11 @@ module Docs
       &copy; 2006&ndash;2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_file_contents('sass', 'sass', 'VERSION', options) do |contents|
+        block.call contents.strip
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/scikit_image.rb b/lib/docs/scrapers/scikit_image.rb
index 2a60aaec..3c454bcb 100644
--- a/lib/docs/scrapers/scikit_image.rb
+++ b/lib/docs/scrapers/scikit_image.rb
@@ -20,5 +20,11 @@ module Docs
       &copy; 2011 the scikit-image team<br>
       Licensed under the BSD 3-clause License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_tags('scikit-image', 'scikit-image', options) do |tags|
+        block.call tags[0]['name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/scikit_learn.rb b/lib/docs/scrapers/scikit_learn.rb
index fe6a8665..0d7fc90d 100644
--- a/lib/docs/scrapers/scikit_learn.rb
+++ b/lib/docs/scrapers/scikit_learn.rb
@@ -25,5 +25,11 @@ module Docs
       Licensed under the 3-clause BSD License.
     HTML
 
+    def get_latest_version(options, &block)
+      fetch_doc('https://scikit-learn.org/stable/documentation.html', options) do |doc|
+        label = doc.at_css('.body h1').content
+        block.call label.scan(/([0-9.]+)/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/sinon.rb b/lib/docs/scrapers/sinon.rb
index 9e81cba9..a1bd06cc 100644
--- a/lib/docs/scrapers/sinon.rb
+++ b/lib/docs/scrapers/sinon.rb
@@ -52,5 +52,11 @@ module Docs
       self.release = '1.17.7'
       self.base_url = "https://sinonjs.org/releases/v#{release}/";
     end
+
+    def get_latest_version(options, &block)
+      fetch('https://sinonjs.org/', options) do |body|
+        block.call body.scan(/\/releases\/v([0-9.]+)/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/socketio.rb b/lib/docs/scrapers/socketio.rb
index 427098c4..8518ff53 100644
--- a/lib/docs/scrapers/socketio.rb
+++ b/lib/docs/scrapers/socketio.rb
@@ -20,5 +20,9 @@ module Docs
       &copy; 2014&ndash;2015 Automattic<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_npm_version('socket.io', options, &block)
+    end
   end
 end
diff --git a/lib/docs/scrapers/sqlite.rb b/lib/docs/scrapers/sqlite.rb
index 790acf83..27720111 100644
--- a/lib/docs/scrapers/sqlite.rb
+++ b/lib/docs/scrapers/sqlite.rb
@@ -41,6 +41,12 @@ module Docs
 
     options[:attribution] = 'SQLite is in the Public Domain.'
 
+    def get_latest_version(options, &block)
+      fetch_doc('https://sqlite.org/chronology.html', options) do |doc|
+        block.call doc.at_css('#chrontab > tbody > tr > td:last-child > a').content
+      end
+    end
+
     private
 
     def parse(response)
diff --git a/lib/docs/scrapers/statsmodels.rb b/lib/docs/scrapers/statsmodels.rb
index 255bde6c..0c9bfd06 100644
--- a/lib/docs/scrapers/statsmodels.rb
+++ b/lib/docs/scrapers/statsmodels.rb
@@ -21,5 +21,10 @@ module Docs
       Licensed under the 3-clause BSD License.
     HTML
 
+    def get_latest_version(options, &block)
+      fetch_doc('http://www.statsmodels.org/stable/', options) do |doc|
+        block.call doc.at_css('.sphinxsidebarwrapper h3 + p > b').content
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/symfony.rb b/lib/docs/scrapers/symfony.rb
index f5e4285a..acd105da 100644
--- a/lib/docs/scrapers/symfony.rb
+++ b/lib/docs/scrapers/symfony.rb
@@ -70,5 +70,11 @@ module Docs
       self.release = '2.7.35'
       self.base_url = "https://api.symfony.com/#{version}/";
     end
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('symfony', 'symfony', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/tcl_tk.rb b/lib/docs/scrapers/tcl_tk.rb
index 1e4b2567..3dcc5765 100644
--- a/lib/docs/scrapers/tcl_tk.rb
+++ b/lib/docs/scrapers/tcl_tk.rb
@@ -25,5 +25,12 @@ module Docs
     options[:attribution] = <<-HTML
       Licensed under <a href="http://tcl.tk/software/tcltk/license.html";>Tcl/Tk terms</a>
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://www.tcl.tk/man/tcl/contents.htm', options) do |doc|
+        label = doc.at_css('h2').content
+        block.call label.scan(/Tk([0-9.]+)/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/tensorflow.rb b/lib/docs/scrapers/tensorflow.rb
index 0296299f..dfd89875 100644
--- a/lib/docs/scrapers/tensorflow.rb
+++ b/lib/docs/scrapers/tensorflow.rb
@@ -56,6 +56,12 @@ module Docs
         /\Aextend/]
     end
 
+    def get_latest_version(options, &block)
+      get_latest_github_release('tensorflow', 'tensorflow', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
+
     private
 
     def parse(response)
diff --git a/lib/docs/scrapers/terraform.rb b/lib/docs/scrapers/terraform.rb
index ab12775d..73d47e86 100644
--- a/lib/docs/scrapers/terraform.rb
+++ b/lib/docs/scrapers/terraform.rb
@@ -18,5 +18,11 @@ module Docs
       &copy; 2018 HashiCorp</br>
       Licensed under the MPL 2.0 License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', options) do |contents|
+        block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/twig.rb b/lib/docs/scrapers/twig.rb
index 4781dcb2..58c7e2ef 100755
--- a/lib/docs/scrapers/twig.rb
+++ b/lib/docs/scrapers/twig.rb
@@ -28,5 +28,11 @@ module Docs
       self.release = '1.34.3'
       self.base_url = 'https://twig.symfony.com/doc/1.x/'
     end
+
+    def get_latest_version(options, &block)
+      get_github_tags('twigphp', 'Twig', options) do |tags|
+        block.call tags[0]['name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/typescript.rb b/lib/docs/scrapers/typescript.rb
index 1117314a..9e62d6a9 100644
--- a/lib/docs/scrapers/typescript.rb
+++ b/lib/docs/scrapers/typescript.rb
@@ -24,6 +24,12 @@ module Docs
       &copy; Microsoft and other contributors<br>
       Licensed under the Apache License, Version 2.0.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('Microsoft', 'TypeScript', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
 
diff --git a/lib/docs/scrapers/underscore.rb b/lib/docs/scrapers/underscore.rb
index 004a09e9..127dd5d3 100644
--- a/lib/docs/scrapers/underscore.rb
+++ b/lib/docs/scrapers/underscore.rb
@@ -20,5 +20,11 @@ module Docs
       &copy; 2009&ndash;2018 Jeremy Ashkenas, DocumentCloud and Investigative Reporters &amp; Editors<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://underscorejs.org/', options) do |doc|
+        block.call doc.at_css('.version').content[1...-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/vagrant.rb b/lib/docs/scrapers/vagrant.rb
index 955746aa..0e89dad1 100644
--- a/lib/docs/scrapers/vagrant.rb
+++ b/lib/docs/scrapers/vagrant.rb
@@ -18,5 +18,11 @@ module Docs
       &copy; 2010&ndash;2018 Mitchell Hashimoto<br>
       Licensed under the MPL 2.0 License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', options) do |contents|
+        block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/vue.rb b/lib/docs/scrapers/vue.rb
index f92991f2..cbb6b15e 100644
--- a/lib/docs/scrapers/vue.rb
+++ b/lib/docs/scrapers/vue.rb
@@ -32,5 +32,11 @@ module Docs
       self.root_path = '/guide/index.html'
       self.initial_paths = %w(/api/index.html)
     end
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('vuejs', 'vue', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/vulkan.rb b/lib/docs/scrapers/vulkan.rb
index e1fa4b4b..111e405f 100644
--- a/lib/docs/scrapers/vulkan.rb
+++ b/lib/docs/scrapers/vulkan.rb
@@ -20,5 +20,11 @@ module Docs
       Licensed under the Creative Commons Attribution 4.0 International License.<br>
       Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_tags('KhronosGroup', 'Vulkan-Docs', options) do |tags|
+        block.call tags[0]['name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/webpack.rb b/lib/docs/scrapers/webpack.rb
index 255c1bb4..9d7303ed 100644
--- a/lib/docs/scrapers/webpack.rb
+++ b/lib/docs/scrapers/webpack.rb
@@ -68,5 +68,9 @@ module Docs
         Licensed under the MIT License.
       HTML
     end
+
+    def get_latest_version(options, &block)
+      get_npm_version('webpack', options, &block)
+    end
   end
 end
diff --git a/lib/docs/scrapers/yarn.rb b/lib/docs/scrapers/yarn.rb
index 825a749f..26f7c0cc 100644
--- a/lib/docs/scrapers/yarn.rb
+++ b/lib/docs/scrapers/yarn.rb
@@ -20,5 +20,11 @@ module Docs
       &copy; 2016&ndash;present Yarn Contributors<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('yarnpkg', 'yarn', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
diff --git a/lib/docs/scrapers/yii.rb b/lib/docs/scrapers/yii.rb
index d99d3ac7..0d087abb 100755
--- a/lib/docs/scrapers/yii.rb
+++ b/lib/docs/scrapers/yii.rb
@@ -34,5 +34,11 @@ module Docs
 
       options[:container] = '.grid_9'
     end
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('yiisoft', 'yii2', options) do |release|
+        block.call release['tag_name']
+      end
+    end
   end
 end
diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor
index f370544c..6eff052e 100644
--- a/lib/tasks/updates.thor
+++ b/lib/tasks/updates.thor
@@ -58,7 +58,7 @@ class UpdatesCLI < Thor
 
   private
 
-  def check_doc(doc, options)
+  def check_doc(doc, opts)
     # Newer scraper versions always come before older scraper versions
     # Therefore, the first item's release value is the latest scraper version
     #
@@ -71,7 +71,7 @@ class UpdatesCLI < Thor
 
     logger.debug("Checking #{doc.name}")
 
-    instance.get_latest_version(options) do |latest_version|
+    instance.get_latest_version(opts) do |latest_version|
       return {
         name: doc.name,
         scraper_version: scraper_version,
[Date Prev][Date Next] [Thread Prev][Thread Next] [Thread Index] [Date Index] [Author Index]