Raise error and stop scraping on 4xx/5xx status code

Thibaut · Thibaut · commit 3eb5ccb7eac7 · 2015-12-13T15:39:00.000-05:00
diff --git a/lib/docs/core/response.rb b/lib/docs/core/response.rb
@@ -4,6 +4,10 @@ def success?
       code == 200
     end
 
+    def error?
+      code != 404 && code >= 400 && code <= 599
+    end
+
     def empty?
       body.empty?
     end
diff --git a/lib/docs/core/scrapers/url_scraper.rb b/lib/docs/core/scrapers/url_scraper.rb
@@ -29,6 +29,10 @@ def request_options
     end
 
     def process_response?(response)
+      if response.error?
+        raise "Error status code (#{response.code}): #{response.url}"
+      end
+
       response.success? && response.html? && base_url.contains?(response.effective_url)
     end
 
diff --git a/test/lib/docs/core/response_test.rb b/test/lib/docs/core/response_test.rb
@@ -29,6 +29,28 @@ class DocsResponseTest < MiniTest::Spec
     end
   end
 
+  describe "#error?" do
+    it "returns false when the code is 200" do
+      options.code = 200
+      refute response.error?
+    end
+
+    it "returns false when the code is 404" do
+      options.code = 404
+      refute response.error?
+    end
+
+    it "returns true when the code is 400" do
+      options.code = 400
+      assert response.error?
+    end
+
+    it "returns true when the code is 500" do
+      options.code = 500
+      assert response.error?
+    end
+  end
+
   describe "#empty?" do
     it "returns true when the body is empty" do
       options.body = ''
diff --git a/test/lib/docs/core/scrapers/url_scraper_test.rb b/test/lib/docs/core/scrapers/url_scraper_test.rb
@@ -89,13 +89,18 @@ class Scraper < Docs::UrlScraper
 
   describe "#process_response?" do
     let :response do
-      OpenStruct.new success?: true, html?: true, effective_url: scraper.root_url
+      OpenStruct.new success?: true, html?: true, effective_url: scraper.root_url, error?: false
     end
 
     let :result do
       scraper.send :process_response?, response
     end
 
+    it "raises when the response is an error" do
+      response.send 'error?=', true
+      assert_raises(RuntimeError) { result }
+    end
+
     it "returns false when the response isn't successful" do
       response.send 'success?=', false
       refute result
