Simplify file scraper setup; scrape files in the "docs/[slug]" directory

Thibaut · Thibaut · commit bf003669baee · 2018-11-25T16:57:37.000-05:00
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,4 @@ public/fonts
 public/docs/**/*
 !public/docs/docs.json
 !public/docs/**/index.json
+/docs/
diff --git a/lib/docs.rb b/lib/docs.rb
@@ -29,6 +29,7 @@ module Docs
   self.rescue_errors = false
 
   class DocNotFound < NameError; end
+  class SetupError < StandardError; end
 
   def self.all
     Dir["#{root_path}/docs/scrapers/**/*.rb"].
diff --git a/lib/docs/core/doc.rb b/lib/docs/core/doc.rb
@@ -95,6 +95,9 @@ def store_page(store, id)
             false
           end
         end
+      rescue Docs::SetupError => error
+        puts "ERROR: #{error.message}"
+        false
       end
 
       def store_pages(store)
@@ -118,6 +121,9 @@ def store_pages(store)
             false
           end
         end
+      rescue Docs::SetupError => error
+        puts "ERROR: #{error.message}"
+        false
       end
 
       private
diff --git a/lib/docs/core/scrapers/file_scraper.rb b/lib/docs/core/scrapers/file_scraper.rb
@@ -1,28 +1,39 @@
 module Docs
   class FileScraper < Scraper
+    SOURCE_DIRECTORY = File.expand_path '../../../../../docs', __FILE__
+
     Response = Struct.new :body, :url
 
     class << self
-      attr_accessor :dir
-
       def inherited(subclass)
         super
         subclass.base_url = base_url
-        subclass.dir = dir
       end
     end
 
     self.base_url = 'http://localhost/'
 
     html_filters.push 'clean_local_urls'
 
+    def source_directory
+      @source_directory ||= File.join(SOURCE_DIRECTORY, self.class.path)
+    end
+
     private
 
+    def assert_source_directory_exists
+      unless Dir.exists?(source_directory)
+        raise SetupError, "The #{self.class.name} scraper requires the original documentation files to be stored in the \"#{source_directory}\" directory."
+      end
+    end
+
     def request_one(url)
-      Response.new read_file(file_path_for(url)), URL.parse(url)
+      assert_source_directory_exists
+      Response.new read_file(url_to_path(url)), URL.parse(url)
     end
 
     def request_all(urls)
+      assert_source_directory_exists
       queue = [urls].flatten
       until queue.empty?
         result = yield request_one(queue.shift)
@@ -34,12 +45,12 @@ def process_response?(response)
       response.body.present?
     end
 
-    def file_path_for(url)
-      File.join self.class.dir, url.remove(base_url.to_s)
+    def url_to_path(url)
+      url.remove(base_url.to_s)
     end
 
     def read_file(path)
-      File.read(path)
+      File.read(File.join(source_directory, path))
     rescue
       instrument 'warn.doc', msg: "Failed to open file: #{path}"
       nil
diff --git a/lib/docs/scrapers/c.rb b/lib/docs/scrapers/c.rb
@@ -1,7 +1,6 @@
 module Docs
   class C < FileScraper
     self.type = 'c'
-    self.dir = '/Users/Thibaut/DevDocs/Docs/c'
     self.base_url = 'http://en.cppreference.com/w/c/'
     self.root_path = 'header.html'
 
diff --git a/lib/docs/scrapers/cpp.rb b/lib/docs/scrapers/cpp.rb
@@ -3,7 +3,6 @@ class Cpp < FileScraper
     self.name = 'C++'
     self.slug = 'cpp'
     self.type = 'c'
-    self.dir = '/Users/Thibaut/DevDocs/Docs/cpp'
     self.base_url = 'http://en.cppreference.com/w/cpp/'
     self.root_path = 'header.html'
 
diff --git a/lib/docs/scrapers/dart.rb b/lib/docs/scrapers/dart.rb
@@ -24,13 +24,11 @@ class Dart < FileScraper
 
     version '2' do
       self.release = '2.0.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Dart2'
       self.base_url = "https://api.dartlang.org/stable/#{release}/"
     end
 
     version '1' do
       self.release = '1.24.3'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Dart1'
       self.base_url = "https://api.dartlang.org/stable/#{release}/"
     end
   end
diff --git a/lib/docs/scrapers/django.rb b/lib/docs/scrapers/django.rb
@@ -36,37 +36,31 @@ class Django < FileScraper
 
     version '2.1' do
       self.release = '2.1.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Django21'
       self.base_url = 'https://docs.djangoproject.com/en/2.1/'
     end
 
     version '2.0' do
       self.release = '2.0.7'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Django20'
       self.base_url = 'https://docs.djangoproject.com/en/2.0/'
     end
 
     version '1.11' do
       self.release = '1.11.9'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Django111'
       self.base_url = 'https://docs.djangoproject.com/en/1.11/'
     end
 
     version '1.10' do
       self.release = '1.10.8'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Django110'
       self.base_url = 'https://docs.djangoproject.com/en/1.10/'
     end
 
     version '1.9' do
       self.release = '1.9.13'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Django19'
       self.base_url = 'https://docs.djangoproject.com/en/1.9/'
     end
 
     version '1.8' do
       self.release = '1.8.18'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Django18'
       self.base_url = 'https://docs.djangoproject.com/en/1.8/'
     end
   end
diff --git a/lib/docs/scrapers/erlang.rb b/lib/docs/scrapers/erlang.rb
@@ -42,22 +42,18 @@ class Erlang < FileScraper
 
     version '21' do
       self.release = '21.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang21'
     end
 
     version '20' do
       self.release = '20.3'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang20'
     end
 
     version '19' do
       self.release = '19.3'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang19'
     end
 
     version '18' do
       self.release = '18.3'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang18'
     end
   end
 end
diff --git a/lib/docs/scrapers/gnu/gcc.rb b/lib/docs/scrapers/gnu/gcc.rb
@@ -48,61 +48,53 @@ class Gcc < Gnu
 
     version '7' do
       self.release = '7.3.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gcc7'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
     end
 
     version '7 CPP' do
       self.release = '7.3.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp7'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
 
       options[:replace_paths] = CPP_PATHS
     end
 
     version '6' do
       self.release = '6.4.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gcc6'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
 
       options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
     end
 
     version '6 CPP' do
       self.release = '6.4.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp6'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
 
       options[:replace_paths] = CPP_PATHS
     end
 
     version '5' do
       self.release = '5.4.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gcc5'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
 
       options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
     end
 
     version '5 CPP' do
       self.release = '5.4.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp5'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
 
       options[:replace_paths] = CPP_PATHS
     end
 
     version '4' do
       self.release = '4.9.3'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gcc4'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
 
       options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
     end
 
     version '4 CPP' do
       self.release = '4.9.3'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp4'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
 
       options[:replace_paths] = CPP_PATHS
diff --git a/lib/docs/scrapers/gnu/gnu_fortran.rb b/lib/docs/scrapers/gnu/gnu_fortran.rb
@@ -8,25 +8,21 @@ class GnuFortran < Gnu
 
     version '7' do
       self.release = '7.3.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran7'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
     end
 
     version '6' do
       self.release = '6.4.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran6'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
     end
 
     version '5' do
       self.release = '5.4.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran5'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
     end
 
     version '4' do
       self.release = '4.9.3'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran4'
       self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
     end
   end
diff --git a/lib/docs/scrapers/nokogiri2.rb b/lib/docs/scrapers/nokogiri2.rb
@@ -3,7 +3,6 @@ class Nokogiri2 < Rdoc
     self.name = 'Nokogiri'
     self.slug = 'nokogiri'
     self.release = '1.8.1'
-    self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Nokogiri'
 
     html_filters.replace 'rdoc/entries', 'nokogiri2/entries'
 
diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb
@@ -2,7 +2,6 @@ module Docs
   class Numpy < FileScraper
     self.name = 'NumPy'
     self.type = 'sphinx'
-    self.dir = '/Users/Thibaut/DevDocs/Docs/numpy/reference/'
     self.root_path = 'index.html'
     self.links = {
       home: 'http://www.numpy.org/',
diff --git a/lib/docs/scrapers/openjdk.rb b/lib/docs/scrapers/openjdk.rb
@@ -1,11 +1,10 @@
 module Docs
   class Openjdk < FileScraper
+    # Downloaded from packages.debian.org/sid/openjdk-8-doc
+    # Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
     self.name = 'OpenJDK'
     self.type = 'openjdk'
     self.root_path = 'overview-summary.html'
-    # Downloaded from packages.debian.org/sid/openjdk-8-doc
-    # Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
-    self.dir = '/Users/Thibaut/DevDocs/Docs/OpenJDK'
 
     html_filters.insert_after 'internal_urls', 'openjdk/clean_urls'
     html_filters.push 'openjdk/entries', 'openjdk/clean_html'
diff --git a/lib/docs/scrapers/perl.rb b/lib/docs/scrapers/perl.rb
@@ -2,7 +2,6 @@ module Docs
   class Perl < FileScraper
     self.name = 'Perl'
     self.type = 'perl'
-    self.dir = '/Users/Thibaut/DevDocs/Docs/Perl'
     self.root_path = 'index.html'
     self.links = {
       home: 'https://www.perl.org/'
diff --git a/lib/docs/scrapers/php.rb b/lib/docs/scrapers/php.rb
@@ -1,5 +1,7 @@
 module Docs
   class Php < FileScraper
+    # Downloaded from php.net/download-docs.php
+
     include FixInternalUrlsBehavior
 
     self.name = 'PHP'
@@ -23,9 +25,6 @@ class Php < FileScraper
       code: 'https://git.php.net/?p=php-src.git;a=summary'
     }
 
-    # Downloaded from php.net/download-docs.php
-    self.dir = '/Users/Thibaut/DevDocs/Docs/PHP'
-
     html_filters.push 'php/internal_urls', 'php/entries', 'php/clean_html', 'title'
     text_filters.push 'php/fix_urls'
 
diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb
@@ -23,33 +23,29 @@ class Python < FileScraper
       Licensed under the PSF License.
     HTML
 
-    version '3.7' do
+    version '3.7' do # docs.python.org/3.7/download.html
       self.release = '3.7.0'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Python37' # docs.python.org/3.7/download.html
       self.base_url = 'https://docs.python.org/3.7/'
 
       html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
     end
 
-    version '3.6' do
+    version '3.6' do # docs.python.org/3.6/download.html
       self.release = '3.6.6'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Python36' # docs.python.org/3.6/download.html
       self.base_url = 'https://docs.python.org/3.6/'
 
       html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
     end
 
-    version '3.5' do
+    version '3.5' do # docs.python.org/3.5/download.html
       self.release = '3.5.3'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Python35' # docs.python.org/3.5/download.html
       self.base_url = 'https://docs.python.org/3.5/'
 
       html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
     end
 
-    version '2.7' do
+    version '2.7' do # docs.python.org/2.7/download.html
       self.release = '2.7.13'
-      self.dir = '/Users/Thibaut/DevDocs/Docs/Python27' # docs.python.org/2.7/download.html
       self.base_url = 'https://docs.python.org/2.7/'
 
       html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html'
diff --git a/lib/docs/scrapers/rdoc/minitest.rb b/lib/docs/scrapers/rdoc/minitest.rb
@@ -1,9 +1,9 @@
 module Docs
   class Minitest < Rdoc
+     # Run "rake docs" in the gem directory
     self.name = 'Ruby / Minitest'
     self.slug = 'minitest'
     self.release = '5.10.3'
-    self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Minitest' # rake docs
     self.links = {
       code: 'https://github.com/seattlerb/minitest'
     }
diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb
@@ -4,7 +4,6 @@ class Rails < Rdoc
 
     self.name = 'Ruby on Rails'
     self.slug = 'rails'
-    self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Rails'
     self.initial_paths = %w(guides/index.html)
     self.links = {
       home: 'http://rubyonrails.org/',
diff --git a/lib/docs/scrapers/rdoc/ruby.rb b/lib/docs/scrapers/rdoc/ruby.rb
diff --git a/lib/docs/scrapers/sqlite.rb b/lib/docs/scrapers/sqlite.rb
diff --git a/test/lib/docs/core/scrapers/file_scraper_test.rb b/test/lib/docs/core/scrapers/file_scraper_test.rb
