[devdocsgjs/main: 584/1867] Finish get_latest_version for 81 scrapers and add uploading functionality
- From: Andy Holmes <andyholmes src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [devdocsgjs/main: 584/1867] Finish get_latest_version for 81 scrapers and add uploading functionality
- Date: Fri, 19 Nov 2021 23:47:17 +0000 (UTC)
commit 3dc17a9b29a33bb77f8b22ae5b9e8cf8ea9fd8f9
Author: Jasper van Merle <jaspervmerle gmail com>
Date: Sat Mar 9 02:36:05 2019 +0100
Finish get_latest_version for 81 scrapers and add uploading functionality
Gemfile | 1 +
Gemfile.lock | 3 +
lib/docs/core/scraper.rb | 56 ++++--
lib/docs/scrapers/angular.rb | 4 +-
lib/docs/scrapers/angularjs.rb | 4 +-
lib/docs/scrapers/ansible.rb | 4 +-
lib/docs/scrapers/apache.rb | 4 +-
lib/docs/scrapers/apache_pig.rb | 4 +-
lib/docs/scrapers/async.rb | 4 +-
lib/docs/scrapers/babel.rb | 4 +-
lib/docs/scrapers/backbone.rb | 4 +-
lib/docs/scrapers/bash.rb | 4 +-
lib/docs/scrapers/bluebird.rb | 4 +-
lib/docs/scrapers/bootstrap.rb | 6 +
lib/docs/scrapers/bottle.rb | 7 +
lib/docs/scrapers/bower.rb | 4 +
lib/docs/scrapers/cakephp.rb | 6 +
lib/docs/scrapers/chai.rb | 4 +
lib/docs/scrapers/chef.rb | 7 +
lib/docs/scrapers/clojure.rb | 6 +
lib/docs/scrapers/cmake.rb | 7 +
lib/docs/scrapers/codeception.rb | 6 +
lib/docs/scrapers/codeceptjs.rb | 4 +
lib/docs/scrapers/codeigniter.rb | 7 +
lib/docs/scrapers/coffeescript.rb | 4 +
lib/docs/scrapers/cordova.rb | 9 +
lib/docs/scrapers/crystal.rb | 6 +
lib/docs/scrapers/d.rb | 6 +
lib/docs/scrapers/d3.rb | 4 +
lib/docs/scrapers/dart.rb | 7 +
lib/docs/scrapers/django.rb | 6 +
lib/docs/scrapers/docker.rb | 7 +
lib/docs/scrapers/dojo.rb | 6 +
lib/docs/scrapers/drupal.rb | 9 +
lib/docs/scrapers/electron.rb | 6 +
lib/docs/scrapers/elixir.rb | 6 +
lib/docs/scrapers/ember.rb | 6 +
lib/docs/scrapers/erlang.rb | 6 +
lib/docs/scrapers/eslint.rb | 4 +
lib/docs/scrapers/express.rb | 4 +
lib/docs/scrapers/falcon.rb | 6 +
lib/docs/scrapers/fish.rb | 6 +
lib/docs/scrapers/flow.rb | 4 +
lib/docs/scrapers/git.rb | 6 +
lib/docs/scrapers/gnu/gcc.rb | 7 +
lib/docs/scrapers/gnu/gnu_fortran.rb | 7 +
lib/docs/scrapers/go.rb | 9 +
lib/docs/scrapers/godot.rb | 6 +
lib/docs/scrapers/graphite.rb | 6 +
lib/docs/scrapers/grunt.rb | 4 +
lib/docs/scrapers/handlebars.rb | 4 +
lib/docs/scrapers/haskell.rb | 7 +
lib/docs/scrapers/haxe.rb | 7 +
lib/docs/scrapers/homebrew.rb | 6 +
lib/docs/scrapers/immutable.rb | 4 +
lib/docs/scrapers/influxdata.rb | 7 +
lib/docs/scrapers/jasmine.rb | 6 +
lib/docs/scrapers/jekyll.rb | 6 +
lib/docs/scrapers/jest.rb | 6 +
lib/docs/scrapers/jquery/jquery_core.rb | 4 +
lib/docs/scrapers/jquery/jquery_mobile.rb | 7 +
lib/docs/scrapers/jquery/jquery_ui.rb | 4 +
lib/docs/scrapers/jsdoc.rb | 6 +
lib/docs/scrapers/julia.rb | 6 +
lib/docs/scrapers/knockout.rb | 6 +
lib/docs/scrapers/koa.rb | 4 +
lib/docs/scrapers/kotlin.rb | 6 +
lib/docs/scrapers/laravel.rb | 6 +
lib/docs/scrapers/leaflet.rb | 6 +
lib/docs/scrapers/less.rb | 7 +
lib/docs/scrapers/liquid.rb | 6 +
lib/docs/scrapers/lodash.rb | 6 +
lib/docs/scrapers/love.rb | 6 +
lib/docs/scrapers/lua.rb | 6 +
lib/docs/scrapers/marionette.rb | 4 +
lib/docs/scrapers/matplotlib.rb | 6 +
lib/docs/scrapers/meteor.rb | 6 +
lib/docs/scrapers/mocha.rb | 4 +
lib/docs/scrapers/modernizr.rb | 4 +
lib/docs/scrapers/moment.rb | 6 +
lib/docs/scrapers/mongoose.rb | 7 +
lib/docs/scrapers/rdoc/minitest.rb | 6 +
lib/docs/scrapers/rdoc/rails.rb | 6 +
lib/docs/scrapers/rdoc/ruby.rb | 12 ++
lib/tasks/updates.thor | 286 ++++++++++++++++++++++++++----
85 files changed, 739 insertions(+), 68 deletions(-)
---
diff --git a/Gemfile b/Gemfile
index 31b57064..5b8fae70 100644
--- a/Gemfile
+++ b/Gemfile
@@ -40,6 +40,7 @@ group :docs do
gem 'unix_utils', require: false
gem 'tty-pager', require: false
gem 'net-sftp', '>= 2.1.3.rc2', require: false
+ gem 'terminal-table', require: false
end
group :test do
diff --git a/Gemfile.lock b/Gemfile.lock
index d968a27b..3ed8570b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -101,6 +101,8 @@ GEM
unicode-display_width (~> 1.4.0)
unicode_utils (~> 1.4.0)
strings-ansi (0.1.0)
+ terminal-table (1.8.0)
+ unicode-display_width (~> 1.1, >= 1.1.1)
thin (1.7.2)
daemons (~> 1.0, >= 1.0.9)
eventmachine (~> 1.0, >= 1.0.4)
@@ -153,6 +155,7 @@ DEPENDENCIES
sinatra-contrib
sprockets
sprockets-helpers
+ terminal-table
thin
thor
tty-pager
diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb
index a7e388a8..b124c6db 100644
--- a/lib/docs/core/scraper.rb
+++ b/lib/docs/core/scraper.rb
@@ -132,7 +132,7 @@ module Docs
end
end
- def get_latest_version(&block)
+ def get_latest_version(options, &block)
raise NotImplementedError
end
@@ -147,15 +147,15 @@ module Docs
# 1 -> 2 = outdated
# 1.1 -> 1.2 = outdated
# 1.1.1 -> 1.1.2 = not outdated
- def is_outdated(current_version, latest_version)
- current_parts = current_version.split(/\./).map(&:to_i)
+ def is_outdated(scraper_version, latest_version)
+ scraper_parts = scraper_version.split(/\./).map(&:to_i)
latest_parts = latest_version.split(/\./).map(&:to_i)
# Only check the first two parts, the third part is for patch updates
[0, 1].each do |i|
- break if i >= current_parts.length or i >= latest_parts.length
- return true if latest_parts[i] > current_parts[i]
- return false if latest_parts[i] < current_parts[i]
+ break if i >= scraper_parts.length or i >= latest_parts.length
+ return true if latest_parts[i] > scraper_parts[i]
+ return false if latest_parts[i] < scraper_parts[i]
end
false
@@ -231,38 +231,62 @@ module Docs
{}
end
+ #
# Utility methods for get_latest_version
+ #
- def fetch(url, &block)
- Request.run(url) do |response|
+ def fetch(url, options, &block)
+ headers = {}
+
+ if options.key?(:github_token) and url.start_with?('https://api.github.com/')
+ headers['Authorization'] = "token #{options[:github_token]}"
+ end
+
+ options[:logger].debug("Fetching #{url}")
+
+ Request.run(url, { headers: headers }) do |response|
if response.success?
block.call response.body
else
+ options[:logger].error("Couldn't fetch #{url} (response code #{response.code})")
block.call nil
end
end
end
- def fetch_doc(url, &block)
- fetch(url) do |body|
- parser = Parser.new(body)
- block.call parser.html
+ def fetch_doc(url, options, &block)
+ fetch(url, options) do |body|
+ block.call Nokogiri::HTML.parse body, nil, 'UTF-8'
end
end
- def fetch_json(url, &block)
- fetch(url) do |body|
+ def fetch_json(url, options, &block)
+ fetch(url, options) do |body|
json = JSON.parse(body)
block.call json
end
end
- def get_npm_version(package, &block)
- fetch_json("https://registry.npmjs.com/#{package}";) do |json|
+ def get_npm_version(package, options, &block)
+ fetch_json("https://registry.npmjs.com/#{package}";, options) do |json|
block.call json['dist-tags']['latest']
end
end
+ def get_latest_github_release(owner, repo, options, &block)
+ fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest";, options, &block)
+ end
+
+ def get_github_tags(owner, repo, options, &block)
+ fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags";, options, &block)
+ end
+
+ def get_github_file_contents(owner, repo, path, options, &block)
+ fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}";, options) do |json|
+ block.call(Base64.decode64(json['content']))
+ end
+ end
+
module FixInternalUrlsBehavior
def self.included(base)
base.extend ClassMethods
diff --git a/lib/docs/scrapers/angular.rb b/lib/docs/scrapers/angular.rb
index fa03eb36..059b0e8e 100644
--- a/lib/docs/scrapers/angular.rb
+++ b/lib/docs/scrapers/angular.rb
@@ -155,8 +155,8 @@ module Docs
end
end
- def get_latest_version(&block)
- get_npm_version('@angular/core', &block)
+ def get_latest_version(options, &block)
+ get_npm_version('@angular/core', options, &block)
end
private
diff --git a/lib/docs/scrapers/angularjs.rb b/lib/docs/scrapers/angularjs.rb
index aa74ca1c..b6e18325 100644
--- a/lib/docs/scrapers/angularjs.rb
+++ b/lib/docs/scrapers/angularjs.rb
@@ -70,8 +70,8 @@ module Docs
self.base_url = "https://code.angularjs.org/#{release}/docs/partials/";
end
- def get_latest_version(&block)
- get_npm_version('angular', &block)
+ def get_latest_version(options, &block)
+ get_npm_version('angular', options, &block)
end
end
end
diff --git a/lib/docs/scrapers/ansible.rb b/lib/docs/scrapers/ansible.rb
index 60fb1953..293f74a7 100644
--- a/lib/docs/scrapers/ansible.rb
+++ b/lib/docs/scrapers/ansible.rb
@@ -88,8 +88,8 @@ module Docs
list_of_all_modules.html)
end
- def get_latest_version(&block)
- fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc|
+ def get_latest_version(options, &block)
+ fetch_doc('https://docs.ansible.com/ansible/latest/index.html', options) do |doc|
block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
end
end
diff --git a/lib/docs/scrapers/apache.rb b/lib/docs/scrapers/apache.rb
index 5eca041e..ba0fa340 100644
--- a/lib/docs/scrapers/apache.rb
+++ b/lib/docs/scrapers/apache.rb
@@ -34,8 +34,8 @@ module Docs
Licensed under the Apache License, Version 2.0.
HTML
- def get_latest_version(&block)
- fetch_doc('http://httpd.apache.org/docs/') do |doc|
+ def get_latest_version(options, &block)
+ fetch_doc('http://httpd.apache.org/docs/', options) do |doc|
block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
end
end
diff --git a/lib/docs/scrapers/apache_pig.rb b/lib/docs/scrapers/apache_pig.rb
index 15c477bf..5454140b 100644
--- a/lib/docs/scrapers/apache_pig.rb
+++ b/lib/docs/scrapers/apache_pig.rb
@@ -43,8 +43,8 @@ module Docs
self.base_url = "https://pig.apache.org/docs/r#{release}/";
end
- def get_latest_version(&block)
- fetch_doc('https://pig.apache.org/') do |doc|
+ def get_latest_version(options, &block)
+ fetch_doc('https://pig.apache.org/', options) do |doc|
item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
block.call item.content.strip.sub(/Release /, '')
end
diff --git a/lib/docs/scrapers/async.rb b/lib/docs/scrapers/async.rb
index 930820b4..18e9bbbf 100644
--- a/lib/docs/scrapers/async.rb
+++ b/lib/docs/scrapers/async.rb
@@ -18,8 +18,8 @@ module Docs
Licensed under the MIT License.
HTML
- def get_latest_version(&block)
- fetch_doc('https://caolan.github.io/async/') do |doc|
+ def get_latest_version(options, &block)
+ fetch_doc('https://caolan.github.io/async/', options) do |doc|
version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
block.call version
end
diff --git a/lib/docs/scrapers/babel.rb b/lib/docs/scrapers/babel.rb
index cc8bec6d..675f86be 100644
--- a/lib/docs/scrapers/babel.rb
+++ b/lib/docs/scrapers/babel.rb
@@ -23,8 +23,8 @@ module Docs
'<div></div>'
end
- def get_latest_version(&block)
- fetch_doc('https://babeljs.io/docs/en/') do |doc|
+ def get_latest_version(options, &block)
+ fetch_doc('https://babeljs.io/docs/en/', options) do |doc|
block.call doc.at_css('a[href="/versions"] > h3').content
end
end
diff --git a/lib/docs/scrapers/backbone.rb b/lib/docs/scrapers/backbone.rb
index 2fb7662f..ad6220e5 100644
--- a/lib/docs/scrapers/backbone.rb
+++ b/lib/docs/scrapers/backbone.rb
@@ -21,8 +21,8 @@ module Docs
Licensed under the MIT License.
HTML
- def get_latest_version(&block)
- fetch_doc('https://backbonejs.org/') do |doc|
+ def get_latest_version(options, &block)
+ fetch_doc('https://backbonejs.org/', options) do |doc|
version = doc.at_css('.version').content
block.call version[1...-1]
end
diff --git a/lib/docs/scrapers/bash.rb b/lib/docs/scrapers/bash.rb
index b62868a6..5556f5b9 100644
--- a/lib/docs/scrapers/bash.rb
+++ b/lib/docs/scrapers/bash.rb
@@ -18,8 +18,8 @@ module Docs
Licensed under the GNU Free Documentation License.
HTML
- def get_latest_version(&block)
- fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body|
+ def get_latest_version(options, &block)
+ fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', options) do |body|
version = body.scan(/, Version ([0-9.]+)/)[0][0]
block.call version[0...-1]
end
diff --git a/lib/docs/scrapers/bluebird.rb b/lib/docs/scrapers/bluebird.rb
index 73888004..8a960b87 100644
--- a/lib/docs/scrapers/bluebird.rb
+++ b/lib/docs/scrapers/bluebird.rb
@@ -19,8 +19,8 @@ module Docs
Licensed under the MIT License.
HTML
- def get_latest_version(&block)
- get_npm_version('bluebird', &block)
+ def get_latest_version(options, &block)
+ get_npm_version('bluebird', options, &block)
end
end
end
diff --git a/lib/docs/scrapers/bootstrap.rb b/lib/docs/scrapers/bootstrap.rb
index 7b2406b8..aa0b4cc3 100644
--- a/lib/docs/scrapers/bootstrap.rb
+++ b/lib/docs/scrapers/bootstrap.rb
@@ -34,5 +34,11 @@ module Docs
options[:only] = %w(getting-started/ css/ components/ javascript/)
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://getbootstrap.com/', options) do |doc|
+ block.call doc.at_css('#bd-versions').content.strip[1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/bottle.rb b/lib/docs/scrapers/bottle.rb
index 25ad7f6e..6e4a19a8 100644
--- a/lib/docs/scrapers/bottle.rb
+++ b/lib/docs/scrapers/bottle.rb
@@ -27,5 +27,12 @@ module Docs
self.release = '0.11.7'
self.base_url = "https://bottlepy.org/docs/#{self.version}/";
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://bottlepy.org/docs/stable/', options) do |doc|
+ label = doc.at_css('.sphinxsidebarwrapper > ul > li > b')
+ block.call label.content.sub(/Bottle /, '')
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/bower.rb b/lib/docs/scrapers/bower.rb
index b032f1d3..1102ee75 100644
--- a/lib/docs/scrapers/bower.rb
+++ b/lib/docs/scrapers/bower.rb
@@ -19,5 +19,9 @@ module Docs
© 2018 Bower contributors<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('bower', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/cakephp.rb b/lib/docs/scrapers/cakephp.rb
index 08dbead0..b123ab7a 100644
--- a/lib/docs/scrapers/cakephp.rb
+++ b/lib/docs/scrapers/cakephp.rb
@@ -71,6 +71,12 @@ module Docs
self.base_url = 'https://api.cakephp.org/2.7/'
end
+ def get_latest_version(options, &block)
+ fetch_doc('https://api.cakephp.org/3.7/', options) do |doc|
+ block.call doc.at_css('.version-picker .dropdown-toggle').content.strip
+ end
+ end
+
private
def parse(response)
diff --git a/lib/docs/scrapers/chai.rb b/lib/docs/scrapers/chai.rb
index 9d8aa4d2..422bd5a9 100644
--- a/lib/docs/scrapers/chai.rb
+++ b/lib/docs/scrapers/chai.rb
@@ -23,5 +23,9 @@ module Docs
© 2016 Chai.js Assertion Library<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('chai', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/chef.rb b/lib/docs/scrapers/chef.rb
index 2fd32a83..337d1202 100644
--- a/lib/docs/scrapers/chef.rb
+++ b/lib/docs/scrapers/chef.rb
@@ -47,5 +47,12 @@ module Docs
options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//]
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://docs-archive.chef.io/', options) do |doc|
+ cell = doc.at_css('.main-archives > tr:nth-child(2) > td:nth-child(2)')
+ block.call cell.content.sub(/Chef Client /, '')
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/clojure.rb b/lib/docs/scrapers/clojure.rb
index c6bdcdea..465a4493 100644
--- a/lib/docs/scrapers/clojure.rb
+++ b/lib/docs/scrapers/clojure.rb
@@ -27,5 +27,11 @@ module Docs
self.release = '1.7'
self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('http://clojure.github.io/clojure/index.html', options) do |doc|
+ block.call doc.at_css('#header-version').content[1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/cmake.rb b/lib/docs/scrapers/cmake.rb
index c455e4fd..dde4721c 100644
--- a/lib/docs/scrapers/cmake.rb
+++ b/lib/docs/scrapers/cmake.rb
@@ -59,5 +59,12 @@ module Docs
self.release = '3.5.2'
self.base_url = 'https://cmake.org/cmake/help/v3.5/'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://cmake.org/documentation/', options) do |doc|
+ link = doc.at_css('.entry-content ul > li > strong > a > big')
+ block.call link.content.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/codeception.rb b/lib/docs/scrapers/codeception.rb
index 919f146d..2e28de7f 100644
--- a/lib/docs/scrapers/codeception.rb
+++ b/lib/docs/scrapers/codeception.rb
@@ -18,5 +18,11 @@ module Docs
© 2011 Michael Bodnarchuk and contributors<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://codeception.com/changelog', options) do |doc|
+ block.call doc.at_css('#page > h4').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/codeceptjs.rb b/lib/docs/scrapers/codeceptjs.rb
index 13189340..e3f4fda8 100644
--- a/lib/docs/scrapers/codeceptjs.rb
+++ b/lib/docs/scrapers/codeceptjs.rb
@@ -21,5 +21,9 @@ module Docs
© 2015 DavertMik <davert codegyre com> (http://codegyre.com)<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('codeceptjs', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/codeigniter.rb b/lib/docs/scrapers/codeigniter.rb
index 573f9b8c..864cf700 100644
--- a/lib/docs/scrapers/codeigniter.rb
+++ b/lib/docs/scrapers/codeigniter.rb
@@ -38,5 +38,12 @@ module Docs
version '3' do
self.release = '3.1.8'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://codeigniter.com/user_guide/changelog.html', options) do |doc|
+ header = doc.at_css('#change-log h2')
+ block.call header.content.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/coffeescript.rb b/lib/docs/scrapers/coffeescript.rb
index 23e9557f..d848d208 100644
--- a/lib/docs/scrapers/coffeescript.rb
+++ b/lib/docs/scrapers/coffeescript.rb
@@ -30,5 +30,9 @@ module Docs
options[:container] = '.container'
end
+
+ def get_latest_version(options, &block)
+ get_npm_version('coffeescript', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/cordova.rb b/lib/docs/scrapers/cordova.rb
index f74c72ff..efe8fb03 100644
--- a/lib/docs/scrapers/cordova.rb
+++ b/lib/docs/scrapers/cordova.rb
@@ -42,5 +42,14 @@ module Docs
self.release = '6.5.0'
self.base_url = 'https://cordova.apache.org/docs/en/6.x/'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://cordova.apache.org/docs/en/latest/', options) do |doc|
+ label = doc.at_css('#versionDropdown').content.strip
+ version = label.scan(/([0-9.]+)/)[0][0]
+ version = version[0...-1] if version.end_with?('.')
+ block.call version
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/crystal.rb b/lib/docs/scrapers/crystal.rb
index 29061a1d..e70317f2 100644
--- a/lib/docs/scrapers/crystal.rb
+++ b/lib/docs/scrapers/crystal.rb
@@ -34,5 +34,11 @@ module Docs
HTML
end
}
+
+ def get_latest_version(options, &block)
+ fetch('https://crystal-lang.org/api', options) do |body|
+ block.call body.scan(/Crystal Docs ([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/d.rb b/lib/docs/scrapers/d.rb
index 6126380e..b0adaf31 100644
--- a/lib/docs/scrapers/d.rb
+++ b/lib/docs/scrapers/d.rb
@@ -26,5 +26,11 @@ module Docs
def initial_urls
%w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html)
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://dlang.org/changelog/', options) do |doc|
+ block.call doc.at_css('#content > ul > li:nth-child(2) > a')['id']
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/d3.rb b/lib/docs/scrapers/d3.rb
index 26b27ca5..cfbbafc9 100644
--- a/lib/docs/scrapers/d3.rb
+++ b/lib/docs/scrapers/d3.rb
@@ -58,5 +58,9 @@ module Docs
options[:root_title] = 'D3.js'
options[:only_patterns] = [/\.md\z/]
end
+
+ def get_latest_version(options, &block)
+ get_npm_version('d3', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/dart.rb b/lib/docs/scrapers/dart.rb
index c345c22f..42d20423 100644
--- a/lib/docs/scrapers/dart.rb
+++ b/lib/docs/scrapers/dart.rb
@@ -31,5 +31,12 @@ module Docs
self.release = '1.24.3'
self.base_url = "https://api.dartlang.org/stable/#{release}/";
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://api.dartlang.org/', options) do |doc|
+ label = doc.at_css('footer > span').content.strip
+ block.call label.sub(/Dart /, '')
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/django.rb b/lib/docs/scrapers/django.rb
index 45273540..746c0f40 100644
--- a/lib/docs/scrapers/django.rb
+++ b/lib/docs/scrapers/django.rb
@@ -63,5 +63,11 @@ module Docs
self.release = '1.8.18'
self.base_url = 'https://docs.djangoproject.com/en/1.8/'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://docs.djangoproject.com/', options) do |doc|
+ block.call doc.at_css('#doc-versions > li.current > span > strong').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/docker.rb b/lib/docs/scrapers/docker.rb
index 92494f8a..dd849391 100644
--- a/lib/docs/scrapers/docker.rb
+++ b/lib/docs/scrapers/docker.rb
@@ -137,5 +137,12 @@ module Docs
options[:container] = '#docs'
options[:only_patterns] << /\Aswarm\//
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://docs.docker.com/', options) do |doc|
+ label = doc.at_css('.nav-container button.dropdown-toggle').content.strip
+ block.call label.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/dojo.rb b/lib/docs/scrapers/dojo.rb
index 937ed21a..66dccb6f 100644
--- a/lib/docs/scrapers/dojo.rb
+++ b/lib/docs/scrapers/dojo.rb
@@ -36,6 +36,12 @@ module Docs
urls.map { |url| "<a href='#{url}'>#{url}</a>" }.join
end
+ def get_latest_version(options, &block)
+ fetch_doc('https://dojotoolkit.org/api/', options) do |doc|
+ block.call doc.at_css('#versionSelector > option[selected]').content
+ end
+ end
+
private
def get_url_list(json, set = Set.new)
diff --git a/lib/docs/scrapers/drupal.rb b/lib/docs/scrapers/drupal.rb
index 5710eb36..92da4193 100644
--- a/lib/docs/scrapers/drupal.rb
+++ b/lib/docs/scrapers/drupal.rb
@@ -98,5 +98,14 @@ module Docs
/\A[\w\-\.]+\.php\/7\.x\z/
]
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('http://cgit.drupalcode.org/drupal', options) do |doc|
+ version = doc.at_css('td.form > form > select > option[selected]').content
+ version = version.scan(/([0-9.]+)/)[0][0]
+ version = version[0...-1] if version.end_with?('.')
+ block.call version
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/electron.rb b/lib/docs/scrapers/electron.rb
index 3cb399f0..dd3cf00a 100644
--- a/lib/docs/scrapers/electron.rb
+++ b/lib/docs/scrapers/electron.rb
@@ -22,5 +22,11 @@ module Docs
© 2013–2018 GitHub Inc.<br>
Licensed under the MIT license.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://electronjs.org/docs', options) do |doc|
+ block.call doc.at_css('.docs-version').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb
index 10d5aac1..d5b8dbe6 100644
--- a/lib/docs/scrapers/elixir.rb
+++ b/lib/docs/scrapers/elixir.rb
@@ -97,5 +97,11 @@ module Docs
'https://elixir-lang.org/getting-started/'
]
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://hexdocs.pm/elixir/api-reference.html', options) do |doc|
+ block.call doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/ember.rb b/lib/docs/scrapers/ember.rb
index 3db20c94..24a8817e 100644
--- a/lib/docs/scrapers/ember.rb
+++ b/lib/docs/scrapers/ember.rb
@@ -56,5 +56,11 @@ module Docs
https://emberjs.com/api/ember-data/2.14/classes/DS
)
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://emberjs.com/api/ember/release', options) do |doc|
+ block.call doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/erlang.rb b/lib/docs/scrapers/erlang.rb
index d6aa2a0b..7dcb0fae 100644
--- a/lib/docs/scrapers/erlang.rb
+++ b/lib/docs/scrapers/erlang.rb
@@ -55,5 +55,11 @@ module Docs
version '18' do
self.release = '18.3'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://www.erlang.org/downloads', options) do |doc|
+ block.call doc.at_css('.col-lg-3 > ul > li').content.strip
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/eslint.rb b/lib/docs/scrapers/eslint.rb
index 8b4c9a2e..dac9c283 100644
--- a/lib/docs/scrapers/eslint.rb
+++ b/lib/docs/scrapers/eslint.rb
@@ -20,5 +20,9 @@ module Docs
© JS Foundation and other contributors<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('eslint', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/express.rb b/lib/docs/scrapers/express.rb
index 0fb4ed14..67ba07e8 100644
--- a/lib/docs/scrapers/express.rb
+++ b/lib/docs/scrapers/express.rb
@@ -28,5 +28,9 @@ module Docs
© 2017 StrongLoop, IBM, and other expressjs.com contributors.<br>
Licensed under the Creative Commons Attribution-ShareAlike License v3.0.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('express', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/falcon.rb b/lib/docs/scrapers/falcon.rb
index 5bfd8efc..cd5b70cd 100644
--- a/lib/docs/scrapers/falcon.rb
+++ b/lib/docs/scrapers/falcon.rb
@@ -33,5 +33,11 @@ module Docs
self.release = '1.2.0'
self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/";
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', options) do |doc|
+ block.call doc.at_css('#changelogs ul > li > a').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/fish.rb b/lib/docs/scrapers/fish.rb
index 5ccfa71c..9340961a 100644
--- a/lib/docs/scrapers/fish.rb
+++ b/lib/docs/scrapers/fish.rb
@@ -46,5 +46,11 @@ module Docs
self.release = '2.2.0'
self.base_url = "https://fishshell.com/docs/#{version}/";
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('http://fishshell.com/docs/current/index.html', options) do |doc|
+ block.call doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/flow.rb b/lib/docs/scrapers/flow.rb
index 16ea70dd..546473f7 100644
--- a/lib/docs/scrapers/flow.rb
+++ b/lib/docs/scrapers/flow.rb
@@ -18,5 +18,9 @@ module Docs
© 2013–present Facebook Inc.<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('flow-bin', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/git.rb b/lib/docs/scrapers/git.rb
index 26b2da95..f10473d0 100644
--- a/lib/docs/scrapers/git.rb
+++ b/lib/docs/scrapers/git.rb
@@ -19,5 +19,11 @@ module Docs
© 2005–2018 Linus Torvalds and others<br>
Licensed under the GNU General Public License version 2.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://git-scm.com/', options) do |doc|
+ block.call doc.at_css('.version').content.strip
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/gnu/gcc.rb b/lib/docs/scrapers/gnu/gcc.rb
index be3bb54e..3252dd6d 100644
--- a/lib/docs/scrapers/gnu/gcc.rb
+++ b/lib/docs/scrapers/gnu/gcc.rb
@@ -99,5 +99,12 @@ module Docs
options[:replace_paths] = CPP_PATHS
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc|
+ label = doc.at_css('ul > li > ul > li > a').content.strip
+ block.call label.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/gnu/gnu_fortran.rb b/lib/docs/scrapers/gnu/gnu_fortran.rb
index 2610178e..f72f7d65 100644
--- a/lib/docs/scrapers/gnu/gnu_fortran.rb
+++ b/lib/docs/scrapers/gnu/gnu_fortran.rb
@@ -25,5 +25,12 @@ module Docs
self.release = '4.9.3'
self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/";
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc|
+ label = doc.at_css('ul > li > ul > li > a').content.strip
+ block.call label.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/go.rb b/lib/docs/scrapers/go.rb
index 7b233317..6f8f7a4a 100644
--- a/lib/docs/scrapers/go.rb
+++ b/lib/docs/scrapers/go.rb
@@ -24,6 +24,15 @@ module Docs
Licensed under the Creative Commons Attribution License 3.0.
HTML
+ def get_latest_version(options, &block)
+ fetch_doc('https://golang.org/pkg/', options) do |doc|
+ footer = doc.at_css('#footer').content
+ version = footer.scan(/go([0-9.]+)/)[0][0]
+ version = version[0...-1] if version.end_with?('.')
+ block.call version
+ end
+ end
+
private
def parse(response) # Hook here because Nokogori removes whitespace from textareas
diff --git a/lib/docs/scrapers/godot.rb b/lib/docs/scrapers/godot.rb
index 7e7da9a6..d43782c2 100644
--- a/lib/docs/scrapers/godot.rb
+++ b/lib/docs/scrapers/godot.rb
@@ -37,5 +37,11 @@ module Docs
self.release = '2.1'
self.base_url = "http://docs.godotengine.org/en/#{self.version}/";
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://docs.godotengine.org/', options) do |doc|
+ block.call doc.at_css('.version').content.strip
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/graphite.rb b/lib/docs/scrapers/graphite.rb
index 49ade898..d1d8b9d1 100644
--- a/lib/docs/scrapers/graphite.rb
+++ b/lib/docs/scrapers/graphite.rb
@@ -17,5 +17,11 @@ module Docs
© 2011–2016 The Graphite Project<br>
Licensed under the Apache License, Version 2.0.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', options) do |doc|
+ block.call doc.at_css('#release-notes li > a').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/grunt.rb b/lib/docs/scrapers/grunt.rb
index 2201c043..1e8af9fb 100644
--- a/lib/docs/scrapers/grunt.rb
+++ b/lib/docs/scrapers/grunt.rb
@@ -26,5 +26,9 @@ module Docs
© GruntJS Team<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('grunt-cli', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/handlebars.rb b/lib/docs/scrapers/handlebars.rb
index 22935d21..7df63102 100644
--- a/lib/docs/scrapers/handlebars.rb
+++ b/lib/docs/scrapers/handlebars.rb
@@ -19,5 +19,9 @@ module Docs
© 2011–2017 by Yehuda Katz<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('handlebars', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb
index 442339b3..383e1990 100755
--- a/lib/docs/scrapers/haskell.rb
+++ b/lib/docs/scrapers/haskell.rb
@@ -68,5 +68,12 @@ module Docs
options[:only_patterns] = [/\Alibraries\//]
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/users_guide/', options) do |doc|
+ label = doc.at_css('.related > ul > li:last-child').content
+ block.call label.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/haxe.rb b/lib/docs/scrapers/haxe.rb
index 33f20b93..5a685efc 100644
--- a/lib/docs/scrapers/haxe.rb
+++ b/lib/docs/scrapers/haxe.rb
@@ -66,5 +66,12 @@ module Docs
version 'Python' do
self.base_url = 'https://api.haxe.org/python/'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://api.haxe.org/', options) do |doc|
+ label = doc.at_css('.container.main-content h1 > small').content
+ block.call label.sub(/version /, '')
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb
index fba79ec0..fef1ed05 100644
--- a/lib/docs/scrapers/homebrew.rb
+++ b/lib/docs/scrapers/homebrew.rb
@@ -19,5 +19,11 @@ module Docs
© 2009–present Homebrew contributors<br>
Licensed under the BSD 2-Clause License.
HTML
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('Homebrew', 'brew', options) do |release|
+ block.call release['name']
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/immutable.rb b/lib/docs/scrapers/immutable.rb
index fa7fb81b..342ce107 100644
--- a/lib/docs/scrapers/immutable.rb
+++ b/lib/docs/scrapers/immutable.rb
@@ -54,5 +54,9 @@ module Docs
JS
capybara.html
end
+
+ def get_latest_version(options, &block)
+ get_npm_version('immutable', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/influxdata.rb b/lib/docs/scrapers/influxdata.rb
index 6c83b66b..4fc98c16 100644
--- a/lib/docs/scrapers/influxdata.rb
+++ b/lib/docs/scrapers/influxdata.rb
@@ -46,5 +46,12 @@ module Docs
© 2015 InfluxData, Inc.<br>
Licensed under the MIT license.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://docs.influxdata.com/influxdb/', options) do |doc|
+ label = doc.at_css('.navbar--current-product').content.strip
+ block.call label.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/jasmine.rb b/lib/docs/scrapers/jasmine.rb
index 82f3c9cf..5f38e3d5 100644
--- a/lib/docs/scrapers/jasmine.rb
+++ b/lib/docs/scrapers/jasmine.rb
@@ -17,5 +17,11 @@ module Docs
© 2008–2017 Pivotal Labs<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('jasmine', 'jasmine', options) do |release|
+ block.call release['name']
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/jekyll.rb b/lib/docs/scrapers/jekyll.rb
index 1faaa9de..a6af352f 100644
--- a/lib/docs/scrapers/jekyll.rb
+++ b/lib/docs/scrapers/jekyll.rb
@@ -28,5 +28,11 @@ module Docs
© 2008–2018 Tom Preston-Werner and Jekyll contributors<br>
Licensed under the MIT license.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://jekyllrb.com/docs/', options) do |doc|
+ block.call doc.at_css('.meta a').content[1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/jest.rb b/lib/docs/scrapers/jest.rb
index f4ce944f..71efcf54 100644
--- a/lib/docs/scrapers/jest.rb
+++ b/lib/docs/scrapers/jest.rb
@@ -17,5 +17,11 @@ module Docs
© 2014–present Facebook Inc.<br>
Licensed under the BSD License.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://jestjs.io/docs/en/getting-started', options) do |doc|
+ block.call doc.at_css('header > a > h3').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/jquery/jquery_core.rb b/lib/docs/scrapers/jquery/jquery_core.rb
index 20aca0dc..dad609e7 100644
--- a/lib/docs/scrapers/jquery/jquery_core.rb
+++ b/lib/docs/scrapers/jquery/jquery_core.rb
@@ -22,5 +22,9 @@ module Docs
/Selectors\/odd/i,
/index/i
]
+
+ def get_latest_version(options, &block)
+ get_npm_version('jquery', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/jquery/jquery_mobile.rb b/lib/docs/scrapers/jquery/jquery_mobile.rb
index 8e5abf1c..53b2c624 100644
--- a/lib/docs/scrapers/jquery/jquery_mobile.rb
+++ b/lib/docs/scrapers/jquery/jquery_mobile.rb
@@ -16,5 +16,12 @@ module Docs
options[:fix_urls] = ->(url) do
url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://jquerymobile.com/', options) do |doc|
+ label = doc.at_css('.download-box > .download-option:last-child > span').content
+ block.call label.sub(/Version /, '')
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/jquery/jquery_ui.rb b/lib/docs/scrapers/jquery/jquery_ui.rb
index 0c90fc1a..05c276e1 100644
--- a/lib/docs/scrapers/jquery/jquery_ui.rb
+++ b/lib/docs/scrapers/jquery/jquery_ui.rb
@@ -15,5 +15,9 @@ module Docs
options[:fix_urls] = ->(url) do
url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/'
end
+
+ def get_latest_version(options, &block)
+ get_npm_version('jquery-ui', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/jsdoc.rb b/lib/docs/scrapers/jsdoc.rb
index bb3781ca..39feca71 100644
--- a/lib/docs/scrapers/jsdoc.rb
+++ b/lib/docs/scrapers/jsdoc.rb
@@ -21,5 +21,11 @@ module Docs
© 2011–2017 the contributors to the JSDoc 3 documentation project<br>
Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
HTML
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('jsdoc3', 'jsdoc', options) do |release|
+ block.call release['tag_name']
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/julia.rb b/lib/docs/scrapers/julia.rb
index 5bc16b77..0875835a 100644
--- a/lib/docs/scrapers/julia.rb
+++ b/lib/docs/scrapers/julia.rb
@@ -49,5 +49,11 @@ module Docs
html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html'
end
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('JuliaLang', 'julia', options) do |release|
+ block.call release['tag_name'][1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/knockout.rb b/lib/docs/scrapers/knockout.rb
index 663f7847..60af1540 100644
--- a/lib/docs/scrapers/knockout.rb
+++ b/lib/docs/scrapers/knockout.rb
@@ -33,5 +33,11 @@ module Docs
© Steven Sanderson, the Knockout.js team, and other contributors<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('knockout', 'knockout', options) do |release|
+ block.call release['tag_name'][1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/koa.rb b/lib/docs/scrapers/koa.rb
index 3ce79cac..4d90e30f 100644
--- a/lib/docs/scrapers/koa.rb
+++ b/lib/docs/scrapers/koa.rb
@@ -34,5 +34,9 @@ module Docs
© 2018 Koa contributors<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('koa', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/kotlin.rb b/lib/docs/scrapers/kotlin.rb
index 415393d1..7539212d 100644
--- a/lib/docs/scrapers/kotlin.rb
+++ b/lib/docs/scrapers/kotlin.rb
@@ -28,5 +28,11 @@ module Docs
© 2010–2018 JetBrains s.r.o.<br>
Licensed under the Apache License, Version 2.0.
HTML
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('JetBrains', 'kotlin', options) do |release|
+ block.call release['tag_name'][1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/laravel.rb b/lib/docs/scrapers/laravel.rb
index 5c88ae0f..cdf32732 100644
--- a/lib/docs/scrapers/laravel.rb
+++ b/lib/docs/scrapers/laravel.rb
@@ -133,5 +133,11 @@ module Docs
url
end
end
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('laravel', 'laravel', options) do |release|
+ block.call release['tag_name'][1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/leaflet.rb b/lib/docs/scrapers/leaflet.rb
index c8e2071c..24bc6142 100644
--- a/lib/docs/scrapers/leaflet.rb
+++ b/lib/docs/scrapers/leaflet.rb
@@ -39,5 +39,11 @@ module Docs
self.base_url = "https://leafletjs.com/reference-#{release}.html";
end
+ def get_latest_version(options, &block)
+ fetch_doc('https://leafletjs.com/index.html', options) do |doc|
+ link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first
+ block.call link['href'].scan(/reference-([0-9.]+)\.html/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/less.rb b/lib/docs/scrapers/less.rb
index a0947e1a..00c884eb 100644
--- a/lib/docs/scrapers/less.rb
+++ b/lib/docs/scrapers/less.rb
@@ -21,5 +21,12 @@ module Docs
© 2009–2016 The Core Less Team<br>
Licensed under the Creative Commons Attribution License 3.0.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('http://lesscss.org/features/', options) do |doc|
+ label = doc.at_css('.footer-links > li').content
+ block.call label.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/liquid.rb b/lib/docs/scrapers/liquid.rb
index 9ebc4041..4630b2d1 100644
--- a/lib/docs/scrapers/liquid.rb
+++ b/lib/docs/scrapers/liquid.rb
@@ -19,5 +19,11 @@ module Docs
© 2005, 2006 Tobias Luetke<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_github_tags('Shopify', 'liquid', options) do |tags|
+ block.call tags[0]['name'][1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/lodash.rb b/lib/docs/scrapers/lodash.rb
index 0461f7b7..5488b9ab 100644
--- a/lib/docs/scrapers/lodash.rb
+++ b/lib/docs/scrapers/lodash.rb
@@ -32,5 +32,11 @@ module Docs
self.release = '2.4.2'
self.base_url = "https://lodash.com/docs/#{release}";
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://lodash.com/docs/', options) do |doc|
+ block.call doc.at_css('#version > option[selected]').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/love.rb b/lib/docs/scrapers/love.rb
index 7f23bded..019edbab 100644
--- a/lib/docs/scrapers/love.rb
+++ b/lib/docs/scrapers/love.rb
@@ -39,5 +39,11 @@ module Docs
© 2006–2016 LÖVE Development Team<br>
Licensed under the GNU Free Documentation License, Version 1.3.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://love2d.org/wiki/Version_History', options) do |doc|
+ block.call doc.at_css('#mw-content-text table a').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/lua.rb b/lib/docs/scrapers/lua.rb
index 40a5c007..30af5523 100644
--- a/lib/docs/scrapers/lua.rb
+++ b/lib/docs/scrapers/lua.rb
@@ -26,5 +26,11 @@ module Docs
self.release = '5.1.5'
self.base_url = 'https://www.lua.org/manual/5.1/'
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://www.lua.org/manual/', options) do |doc|
+ block.call doc.at_css('p.menubar > a').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/marionette.rb b/lib/docs/scrapers/marionette.rb
index fea6617f..12de6d0c 100644
--- a/lib/docs/scrapers/marionette.rb
+++ b/lib/docs/scrapers/marionette.rb
@@ -38,5 +38,9 @@ module Docs
html_filters.push 'marionette/entries_v2'
end
+
+ def get_latest_version(options, &block)
+ get_npm_version('backbone.marionette', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/matplotlib.rb b/lib/docs/scrapers/matplotlib.rb
index ddd1f9de..948955a6 100644
--- a/lib/docs/scrapers/matplotlib.rb
+++ b/lib/docs/scrapers/matplotlib.rb
@@ -64,5 +64,11 @@ module Docs
"https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/";
]
end
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('matplotlib', 'matplotlib', options) do |release|
+ block.call release['tag_name'][1..-1]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/meteor.rb b/lib/docs/scrapers/meteor.rb
index b38d5dc2..02b81bc3 100644
--- a/lib/docs/scrapers/meteor.rb
+++ b/lib/docs/scrapers/meteor.rb
@@ -45,5 +45,11 @@ module Docs
self.base_urls = ['https://guide.meteor.com/v1.3/', "https://docs.meteor.com/v#{self.release}/";]
options[:fix_urls] = nil
end
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://docs.meteor.com/#/full/', options) do |doc|
+ block.call doc.at_css('select.version-select > option').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/mocha.rb b/lib/docs/scrapers/mocha.rb
index 8ab9bdc8..6654d754 100644
--- a/lib/docs/scrapers/mocha.rb
+++ b/lib/docs/scrapers/mocha.rb
@@ -18,5 +18,9 @@ module Docs
© 2011–2018 JS Foundation and contributors<br>
Licensed under the Creative Commons Attribution 4.0 International License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('mocha', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/modernizr.rb b/lib/docs/scrapers/modernizr.rb
index 96c82153..93a738bb 100644
--- a/lib/docs/scrapers/modernizr.rb
+++ b/lib/docs/scrapers/modernizr.rb
@@ -15,5 +15,9 @@ module Docs
© 2009–2017 The Modernizr team<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_npm_version('modernizr', options, &block)
+ end
end
end
diff --git a/lib/docs/scrapers/moment.rb b/lib/docs/scrapers/moment.rb
index 88df0d14..9dd27107 100644
--- a/lib/docs/scrapers/moment.rb
+++ b/lib/docs/scrapers/moment.rb
@@ -22,5 +22,11 @@ module Docs
© JS Foundation and other contributors<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('http://momentjs.com/', options) do |doc|
+ block.call doc.at_css('.hero-title > h1 > span').content
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/mongoose.rb b/lib/docs/scrapers/mongoose.rb
index 71ee04d2..2d221990 100644
--- a/lib/docs/scrapers/mongoose.rb
+++ b/lib/docs/scrapers/mongoose.rb
@@ -26,5 +26,12 @@ module Docs
© 2010 LearnBoost<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ fetch_doc('https://mongoosejs.com/docs/', options) do |doc|
+ label = doc.at_css('.pure-menu-link').content.strip
+ block.call label.sub(/Version /, '')
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/rdoc/minitest.rb b/lib/docs/scrapers/rdoc/minitest.rb
index 761da1de..f676010d 100644
--- a/lib/docs/scrapers/rdoc/minitest.rb
+++ b/lib/docs/scrapers/rdoc/minitest.rb
@@ -21,5 +21,11 @@ module Docs
© Ryan Davis, seattle.rb<br>
Licensed under the MIT License.
HTML
+
+ def get_latest_version(options, &block)
+ get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', options) do |contents|
+ block.call contents.scan(/([0-9.]+)/)[0][0]
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb
index 6bdce34a..9cb2ab9b 100644
--- a/lib/docs/scrapers/rdoc/rails.rb
+++ b/lib/docs/scrapers/rdoc/rails.rb
@@ -93,5 +93,11 @@ module Docs
version '4.1' do
self.release = '4.1.16'
end
+
+ def get_latest_version(options, &block)
+ get_latest_github_release('rails', 'rails', options) do |release|
+ block.call release['name']
+ end
+ end
end
end
diff --git a/lib/docs/scrapers/rdoc/ruby.rb b/lib/docs/scrapers/rdoc/ruby.rb
index dd296765..292540db 100644
--- a/lib/docs/scrapers/rdoc/ruby.rb
+++ b/lib/docs/scrapers/rdoc/ruby.rb
@@ -84,5 +84,17 @@ module Docs
version '2.2' do
self.release = '2.2.10'
end
+
+ def get_latest_version(options, &block)
+ get_github_tags('ruby', 'ruby', options) do |tags|
+ tags.each do |tag|
+ version = tag['name'].gsub(/_/, '.')[1..-1]
+ if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2
+ block.call version
+ break
+ end
+ end
+ end
+ end
end
end
diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor
index eb3467f2..f370544c 100644
--- a/lib/tasks/updates.thor
+++ b/lib/tasks/updates.thor
@@ -1,4 +1,12 @@
class UpdatesCLI < Thor
+ # The GitHub user that is allowed to upload reports
+ # TODO: Update this before creating a PR
+ UPLOAD_USER = 'jmerle'
+
+ # The repository to create an issue in when uploading the results
+ # TODO: Update this before creating a PR
+ UPLOAD_REPO = 'jmerle/devdocs'
+
def self.to_s
'Updates'
end
@@ -6,10 +14,14 @@ class UpdatesCLI < Thor
def initialize(*args)
require 'docs'
require 'progress_bar'
+ require 'terminal-table'
+ require 'date'
super
end
- desc 'check [--verbose] [doc]...', 'Check for outdated documentations'
+ desc 'check [--github-token] [--upload] [--verbose] [doc]...', 'Check for outdated documentations'
+ option :github_token, :type => :string
+ option :upload, :type => :boolean
option :verbose, :type => :boolean
def check(*names)
# Convert names to a list of Scraper instances
@@ -19,23 +31,26 @@ class UpdatesCLI < Thor
# Check all documentations for updates when no arguments are given
docs = Docs.all if docs.empty?
- progress_bar = ::ProgressBar.new docs.length
- progress_bar.write
+ opts = {
+ logger: logger
+ }
- results = docs.map do |doc|
- result = check_doc(doc)
- progress_bar.increment!
- result
+ if options.key?(:github_token)
+ opts[:github_token] = options[:github_token]
end
- valid_results = results.select {|result| result.is_a?(Hash)}
+ with_progress_bar do |bar|
+ bar.max = docs.length
+ bar.write
+ end
- up_to_date_results = valid_results.select {|result| !result[:is_outdated]}
- outdated_results = valid_results.select {|result| result[:is_outdated]}
+ results = docs.map do |doc|
+ result = check_doc(doc, opts)
+ with_progress_bar(&:increment!)
+ result
+ end
- log_results('Up-to-date', up_to_date_results) if options[:verbose] and !up_to_date_results.empty?
- logger.info("") if options[:verbose] and !up_to_date_results.empty? and !outdated_results.empty?
- log_results('Outdated', outdated_results) unless outdated_results.empty?
+ process_results(results)
rescue Docs::DocNotFound => error
logger.error(error)
logger.info('Run "thor docs:list" to see the list of docs.')
@@ -43,53 +58,260 @@ class UpdatesCLI < Thor
private
- def check_doc(doc)
+ def check_doc(doc, options)
# Newer scraper versions always come before older scraper versions
- # Therefore, the first item's release value is the latest current scraper version
+ # Therefore, the first item's release value is the latest scraper version
#
# For example, a scraper could scrape 3 versions: 10, 11 and 12
# doc.versions.first would be the scraper for version 12
instance = doc.versions.first.new
- return nil unless instance.class.method_defined?(:options)
-
- current_version = instance.options[:release]
- return nil if current_version.nil?
+ scraper_version = instance.class.method_defined?(:options) ? instance.options[:release] : nil
+ return error_result(doc, '`options[:release]` does not exist') if scraper_version.nil?
logger.debug("Checking #{doc.name}")
- instance.get_latest_version do |latest_version|
+ instance.get_latest_version(options) do |latest_version|
return {
name: doc.name,
- current_version: current_version,
+ scraper_version: scraper_version,
latest_version: latest_version,
- is_outdated: instance.is_outdated(current_version, latest_version)
+ is_outdated: instance.is_outdated(scraper_version, latest_version)
}
end
-
- return nil
rescue NotImplementedError
logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented")
+ error_result(doc, '`get_latest_version` is not implemented')
rescue
logger.error("Error while checking #{doc.name}")
raise
end
- def log_results(label, results)
- logger.info("#{label} documentations (#{results.length}):")
+ def error_result(doc, reason)
+ {
+ name: doc.name,
+ error: reason
+ }
+ end
+
+ def process_results(results)
+ successful_results = results.select {|result| result.key?(:is_outdated)}
+ failed_results = results.select {|result| result.key?(:error)}
+
+ up_to_date_results = successful_results.select {|result| !result[:is_outdated]}
+ outdated_results = successful_results.select {|result| result[:is_outdated]}
+
+ log_results(outdated_results, up_to_date_results, failed_results)
+ upload_results(outdated_results, up_to_date_results, failed_results) if options[:upload]
+ end
+
+ #
+ # Result logging methods
+ #
+
+ def log_results(outdated_results, up_to_date_results, failed_results)
+ log_failed_results(failed_results) unless failed_results.empty?
+ log_successful_results('Up-to-date', up_to_date_results) unless up_to_date_results.empty?
+ log_successful_results('Outdated', outdated_results) unless outdated_results.empty?
+ end
+
+ def log_successful_results(label, results)
+ title = "#{label} documentations (#{results.length})"
+ headings = ['Documentation', 'Scraper version', 'Latest version']
+ rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]}
+
+ table = Terminal::Table.new :title => title, :headings => headings, :rows => rows
+ puts table
+ end
+
+ def log_failed_results(results)
+ title = "Documentations that could not be checked (#{results.length})"
+ headings = %w(Documentation Reason)
+ rows = results.map {|result| [result[:name], result[:error]]}
+
+ table = Terminal::Table.new :title => title, :headings => headings, :rows => rows
+ puts table
+ end
+
+ #
+ # Upload methods
+ #
+
+ def upload_results(outdated_results, up_to_date_results, failed_results)
+ # We can't create issues without a GitHub token
+ unless options.key?(:github_token)
+ logger.error('Please specify a GitHub token with the public_repo permission for devdocs-bot with the
--github-token parameter')
+ return
+ end
+
+ logger.info('Uploading the results to a new GitHub issue')
+
+ logger.info('Checking if the GitHub token belongs to the correct user')
+ github_get('/user') do |user|
+ # Only allow the DevDocs bot to upload reports
+ if user['login'] == UPLOAD_USER
+ issue = results_to_issue(outdated_results, up_to_date_results, failed_results)
+
+ logger.info('Creating a new GitHub issue')
+ github_post("/repos/#{UPLOAD_REPO}/issues", issue) do |created_issue|
+ search_params = {
+ q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}",
+ sort: 'created',
+ order: 'desc'
+ }
+
+ logger.info('Checking if the previous issue is still open')
+ github_get('/search/issues', search_params) do |matching_issues|
+ previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']}
+
+ if previous_issue.nil?
+ logger.info('No previous issue found')
+ log_upload_success(created_issue)
+ else
+ comment = "This report was superseded by ##{created_issue['number']}."
+
+ logger.info('Commenting on the previous issue')
+ github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body:
comment}) do |_|
+ if previous_issue['closed_at'].nil?
+ logger.info('Closing the previous issue')
+ github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state:
'closed'}) do |_|
+ log_upload_success(created_issue)
+ end
+ else
+ logger.info('The previous issue has already been closed')
+ log_upload_success(created_issue)
+ end
+ end
+ end
+ end
+ end
+ else
+ logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified
github token is not for #{UPLOAD_USER}.")
+ end
+ end
+ end
+
+ def results_to_issue(outdated_results, up_to_date_results, failed_results)
+ results = [
+ successful_results_to_markdown('Outdated', outdated_results),
+ successful_results_to_markdown('Up-to-date', up_to_date_results),
+ failed_results_to_markdown(failed_results)
+ ]
+
+ results_str = results.select {|result| !result.nil?}.join("\n\n")
+
+ title = "Documentation versions report for #{Date.today.strftime('%B')} 2019"
+ body = <<-MARKDOWN
+## What is this?
+
+This is an automatically created issue which contains information about the version status of the
documentations available on DevDocs. The results of this report can be used by maintainers when updating
outdated documentations.
+
+Maintainers can close this issue when all documentations are up-to-date. This issue is automatically closed
when the next report is created.
+
+## Results
+
+The #{outdated_results.length + up_to_date_results.length + failed_results.length} documentations are
divided as follows:
+- #{outdated_results.length} that #{outdated_results.length == 1 ? 'is' : 'are'} outdated
+- #{up_to_date_results.length} that #{up_to_date_results.length == 1 ? 'is' : 'are'} up-to-date (patch
updates are ignored)
+- #{failed_results.length} that could not be checked
+ MARKDOWN
+
+ {
+ title: title,
+ body: body.strip + "\n\n" + results_str
+ }
+ end
+
+ def successful_results_to_markdown(label, results)
+ return nil if results.empty?
+
+ title = "#{label} documentations (#{results.length})"
+ headings = ['Documentation', 'Scraper version', 'Latest version']
+ rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]}
+
+ results_to_markdown(title, headings, rows)
+ end
+
+ def failed_results_to_markdown(results)
+ return nil if results.empty?
+
+ title = "Documentations that could not be checked (#{results.length})"
+ headings = %w(Documentation Reason)
+ rows = results.map {|result| [result[:name], result[:error]]}
+
+ results_to_markdown(title, headings, rows)
+ end
+
+ def results_to_markdown(title, headings, rows)
+ "<details>\n<summary>#{title}</summary>\n\n#{create_markdown_table(headings, rows)}\n</details>"
+ end
+
+ def create_markdown_table(headings, rows)
+ header = headings.join(' | ')
+ separator = '-|' * headings.length
+ body = rows.map {|row| row.join(' | ')}
+
+ header + "\n" + separator[0...-1] + "\n" + body.join("\n")
+ end
+
+ def log_upload_success(created_issue)
+ logger.info("Successfully uploaded the results to #{created_issue['html_url']}")
+ end
+
+ #
+ # HTTP utilities
+ #
+
+ def github_get(endpoint, params = {}, &block)
+ github_request(endpoint, {method: :get, params: params}, &block)
+ end
+
+ def github_post(endpoint, params, &block)
+ github_request(endpoint, {method: :post, body: params.to_json}, &block)
+ end
+
+ def github_patch(endpoint, params, &block)
+ github_request(endpoint, {method: :patch, body: params.to_json}, &block)
+ end
- results.each do |result|
- logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}")
+ def github_request(endpoint, opts, &block)
+ url = "https://api.github.com#{endpoint}";
+
+ # GitHub token authentication
+ opts[:headers] = {
+ Authorization: "token #{options[:github_token]}"
+ }
+
+ # GitHub requires the Content-Type to be application/json when a body is passed
+ if opts.key?(:body)
+ opts[:headers]['Content-Type'] = 'application/json'
end
+
+ logger.debug("Making a #{opts[:method]} request to #{url}")
+
+ Docs::Request.run(url, opts) do |response|
+ # response.success? is false if the response code is 201
+ # GitHub returns 201 Created after an issue is created
+ if response.success? || response.code == 201
+ block.call JSON.parse(response.body)
+ else
+ logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})")
+ block.call nil
+ end
+ end
+ end
+
+ # A utility method which ensures no progress bar is shown when stdout is not a tty
+ def with_progress_bar(&block)
+ return unless $stdout.tty?
+ @progress_bar ||= ::ProgressBar.new
+ block.call @progress_bar
end
def logger
@logger ||= Logger.new($stdout).tap do |logger|
logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
- logger.formatter = proc do |severity, datetime, progname, msg|
- prefix = severity != "INFO" ? "[#{severity}] " : ""
- "#{prefix}#{msg}\n"
- end
+ logger.formatter = proc {|severity, datetime, progname, msg| "[#{severity}] #{msg}\n"}
end
end
end
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
