Finish Homebrew scraper

This commit is contained in:
Thibaut Courouble 2017-11-26 18:55:05 -05:00
parent 438fa8e6d0
commit b8c8a42f86
17 changed files with 83 additions and 46 deletions

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 38 KiB

View file

@ -1,7 +1,7 @@
[
[
"2017-11-26",
"New documentations: <a href=\"/bluebird/\">Bluebird</a> and <a href=\"/eslint/\">ESLint</a>"
"New documentations: <a href=\"/bluebird/\">Bluebird</a>, <a href=\"/eslint/\">ESLint</a> and <a href=\"/homebrew/\">Homebrew</a>"
], [
"2017-11-18",
"Added print & PDF stylesheet.\nFeedback welcome on <a href=\"https://twitter.com/DevDocs\" target=\"_blank\" rel=\"noopener\">Twitter</a> and <a href=\"https://github.com/Thibaut/devdocs\" target=\"_blank\" rel=\"noopener\">GitHub</a>."

View file

@ -302,6 +302,11 @@ credits = [
'2005-2016 Haxe Foundation',
'MIT',
'http://haxe.org/foundation/open-source.html'
], [
'Homebrew',
'2009-present Homebrew contributors',
'BSD',
'https://raw.githubusercontent.com/Homebrew/brew/master/LICENSE.txt'
], [
'Immutable.js',
'2014-2016 Facebook, Inc.',

View file

@ -166,3 +166,4 @@
._icon-d:before { background-position: -7rem -2rem; @extend %doc-icon-2; }
._icon-bluebird:before { background-position: -8rem -2rem; @extend %doc-icon-2; }
._icon-eslint:before { background-position: -9rem -2rem; @extend %doc-icon-2; }
._icon-homebrew:before { background-position: 0 -3rem; @extend %doc-icon-2; }

View file

@ -1,9 +0,0 @@
module Docs
class Brew
class CleanHtmlFilter < Filter
def call
doc
end
end
end
end

View file

@ -1,14 +0,0 @@
module Docs
class Brew
class EntriesFilter < Docs::EntriesFilter
def get_name
at_css('h1').content
end
def get_type
name
end
end
end
end

View file

@ -0,0 +1,19 @@
module Docs
class Homebrew
class CleanHtmlFilter < Filter
def call
css('hr')
css('div.highlighter-rouge').each do |node|
lang = node['class'][/language-(\w+)/, 1]
node['data-language'] = lang if lang
node.content = node.content.strip
node.name = 'pre'
node.remove_attribute('class')
end
doc
end
end
end
end

View file

@ -0,0 +1,32 @@
module Docs
class Homebrew
class EntriesFilter < Docs::EntriesFilter
def get_name
name = at_css('h1').content.strip
name.remove! %r{\(.*}
name
end
CONTRIBUTOR_SLUGS = %w(
How-To-Open-a-Homebrew-Pull-Request
Formula-Cookbook
Acceptable-Formulae
Versions
Node-for-Formula-Authors
Python-for-Formula-Authors
Migrating-A-Formula-To-A-Tap
Rename-A-Formula
How-to-Create-and-Maintain-a-Tap
Brew-Test-Bot
Prose-Style-Guidelines)
def get_type
if CONTRIBUTOR_SLUGS.include?(slug)
'Contributors'
else
'Users'
end
end
end
end
end

View file

@ -1,22 +0,0 @@
module Docs
class Brew < UrlScraper
self.name = 'Homebrew'
self.type = 'brew'
self.release = '1.3.6'
self.base_url = 'https://docs.brew.sh'
self.root_path = '/'
self.links = {
home: 'https://brew.sh',
code: 'https://github.com/Homebrew/brew'
}
options[:container] = ->(filter) { filter.root_page? ? '#home' : '#page' }
html_filters.push 'brew/entries', 'brew/clean_html'
options[:attribution] = <<-HTML
Homebrew was created by Max Howell. <br>
Licensed under the BSD 2-Clause License.
HTML
end
end

View file

@ -0,0 +1,24 @@
module Docs
class Homebrew < UrlScraper
self.name = 'Homebrew'
self.type = 'simple'
self.release = '1.3.6'
self.base_url = 'https://docs.brew.sh/'
self.links = {
home: 'https://brew.sh',
code: 'https://github.com/Homebrew/brew'
}
html_filters.push 'homebrew/entries', 'homebrew/clean_html'
options[:container] = ->(filter) { filter.root_page? ? '#home' : '#page' }
options[:skip_patterns] = [/maintainer/i, /core\-contributor/i]
options[:skip] = %w(Kickstarter-Supporters.html)
options[:attribution] = <<-HTML
&copy; 2009&ndash;present Homebrew contributors<br>
Licensed under the BSD 2-Clause License.
HTML
end
end

Binary file not shown.

Before

Width:  |  Height:  |  Size: 672 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 551 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

View file

@ -0,0 +1 @@
https://github.com/Homebrew/brew/tree/master/docs/img