Skip to content

Commit 9ad721e

Browse files
ThibautThibaut
committed
Finish Twig scraper
1 parent 6b0e60f commit 9ad721e

11 files changed

Lines changed: 61 additions & 32 deletions

File tree

‎assets/images/docs.png‎

862 Bytes
Loading

‎assets/images/[email protected]‎

3.3 KB
Loading

‎assets/javascripts/news.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
[
22
[
3+
"2016-09-18",
4+
"New documentation: <a href=\"/twig/\">Twig</a>"
5+
], [
36
"2016-09-05",
47
"New documentations: <a href=\"/fish/\">Fish</a>, <a href=\"/bottle/\">Bottle</a> and <a href=\"/scikit_image/\">scikit-image</a>"
58
], [

‎assets/javascripts/templates/pages/about_tmpl.coffee‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,11 @@ credits = [
524524
'2015 The TensorFlow Authors',
525525
'Apache',
526526
'https://raw.githubusercontent.com/tensorflow/tensorflow/master/LICENSE'
527+
], [
528+
'Twig',
529+
'2009-2016 The Twig Team',
530+
'BSD',
531+
'http://twig.sensiolabs.org/license'
527532
], [
528533
'TypeScript',
529534
'Microsoft and other contributors',

‎assets/stylesheets/application-dark.css.scss‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@
8484
'pages/socketio',
8585
'pages/sphinx',
8686
'pages/sphinx_simple',
87-
'pages/tcl_tk',
8887
'pages/support_tables',
88+
'pages/tcl_tk',
8989
'pages/tensorflow',
9090
'pages/underscore',
9191
'pages/vagrant',

‎assets/stylesheets/global/_icons.scss‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@
118118
._icon-bower:before { background-position: -4rem -6rem; }
119119
._icon-fish:before { background-position: -5rem -6rem; @extend %darkIconFix !optional; }
120120
._icon-scikit_image:before { background-position: -6rem -6rem; }
121+
._icon-twig:before { background-position: -7rem -6rem; }
121122
._icon-bottle:before { background-position: 0 -7rem; }
122123
._icon-docker:before { background-position: -1rem -7rem; }
123124
._icon-cakephp:before { background-position: -2rem -7rem; }

‎lib/docs/filters/twig/clean_html.rb‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,36 @@ module Docs
22
class Twig
33
class CleanHtmlFilter < Filter
44
def call
5+
css('.infobar', '.offline-docs', '.headerlink').remove
56

6-
css('.infobar', '.admonition-note', 'ul.pages', '.offline-docs').remove
7+
css('.builtin-reference', '.admonition-wrapper', 'h1 > code', 'h2 > code', '.body-web', '.reference em').each do |node|
8+
node.before(node.children).remove
9+
end
10+
11+
css('.section').each do |node|
12+
node.first_element_child['id'] = node['id'] if node['id']
13+
node.before(node.children).remove
14+
end
15+
16+
doc.child.remove until doc.child.name == 'h1'
17+
18+
css('.literal-block').each do |node|
19+
pre = node.at_css('.highlight pre') || node.at_css('pre')
20+
pre.content = pre.content
21+
node.replace(pre)
22+
end
23+
24+
css('p.versionadded').each do |node|
25+
node.name = 'div'
26+
end
27+
28+
css('h3:contains("Arguments")').each do |node|
29+
node.name = 'h4'
30+
end
31+
32+
css('.navigation').each do |node|
33+
node['style'] = 'text-align: center'
34+
end
735

836
doc
937
end

‎lib/docs/filters/twig/entries.rb‎

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,32 +2,24 @@ module Docs
22
class Twig
33
class EntriesFilter < Docs::EntriesFilter
44
def get_name
5-
name = at_css('h1').content.strip.tr('¶', '')
6-
name
5+
at_css('h1').content
76
end
87

98
def get_type
10-
if slug.include?('deprecate')
11-
'Deprecated'
12-
elsif slug.include?('extensions')
13-
'Doc\\Extensions'
14-
elsif slug.include?('tags')
15-
'tags'
16-
elsif slug.include?('filters')
17-
'filters'
18-
elsif slug.include?('functions')
19-
'functions'
20-
elsif slug.include?('tests')
21-
'tests'
22-
elsif slug.include?('-operator') || slug.include?('#math') || slug.include?('comparisons')
23-
'operators'
24-
elsif slug.in?('doc/index') || slug.include?('intro') || slug.include?('recipes') || slug.include?('internals') || slug.include?('coding_standards') || slug.include?('installation') || slug.include?('api') || slug.include?('advanced')
25-
'Doc'
26-
elsif slug.include?('templates')
27-
'Doc\\Templates'
9+
if slug.start_with?('filters')
10+
'Filters'
11+
elsif slug.start_with?('functions')
12+
'Functions'
13+
elsif slug.start_with?('tags')
14+
'Tags'
15+
elsif slug.start_with?('tests')
16+
'Tests'
17+
elsif slug.start_with?('extensions')
18+
'Extensions'
19+
else
20+
'Miscellaneous'
2821
end
2922
end
30-
3123
end
3224
end
3325
end

‎lib/docs/scrapers/twig.rb‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
module Docs
22
class Twig < UrlScraper
3-
self.type = 'twig'
3+
self.type = 'sphinx'
4+
self.release = '1.24'
5+
self.base_url = 'http://twig.sensiolabs.org/doc/'
6+
self.root_path = 'index.html'
7+
self.initial_paths = %w(extensions/index.html)
48

59
options[:attribution] = <<-HTML
6-
&copy; 2009&ndash;2016 by SensioLabs<br>
7-
Licensed under the three clause BSD license.
10+
&copy; 2009&ndash;2016 by the Twig Team<br>
11+
Licensed under the three clause BSD license.<br>
12+
The Twig logo is &copy; 2010&ndash;2016 SensioLabs
813
HTML
914

10-
self.release = '1.24.1'
11-
self.base_url = 'http://twig.sensiolabs.org/'
12-
self.root_path = 'documentation'
13-
1415
html_filters.push 'twig/clean_html', 'twig/entries'
1516

1617
options[:container] = 'div.bd > div.content'
17-
options[:skip_patterns] = [/\Aapi/, /\Alicense/]
18-
options[:skip] = %w(doc/deprecated.html doc/advanced_legacy.html)
18+
options[:skip] = %w(deprecated.html advanced_legacy.html)
1919
end
2020
end

‎public/icons/docs/twig/16.png‎

-66 Bytes
Loading

0 commit comments

Comments
 (0)