forked from freeCodeCamp/devdocs
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathrequester.rb
More file actions
65 lines (56 loc) · 1.63 KB
/
requester.rb
File metadata and controls
65 lines (56 loc) · 1.63 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
module Docs
class Requester < Typhoeus::Hydra
include Instrumentable
attr_reader :request_options
def self.run(urls, options = {}, &block)
urls = urls.dup
requester = new(options)
requester.on_response(&block) if block
requester.on_response do # cheap hack to ensure root page is processed first
if urls
requester.request(urls)
urls = nil
end
end
requester.request(urls.shift)
requester.run
requester
end
def initialize(options = {})
@request_options = options.extract!(:request_options)[:request_options].try(:dup) || {}
options[:max_concurrency] ||= 20
options[:pipelining] = 0
super
end
def request(urls, options = {}, &block)
requests = [urls].flatten.map do |url|
build_and_queue_request(url, options, &block)
end
requests.length == 1 ? requests.first : requests
end
def queue(request)
request.on_complete(&method(:handle_response))
super
end
def on_response(&block)
@on_response ||= []
@on_response << block if block
@on_response
end
private
def build_and_queue_request(url, options, &block)
request = Request.new(url, request_options.merge(options))
request.on_complete(&block) if block
queue(request)
request
end
def handle_response(response)
instrument 'handle_response.requester', url: response.url do
on_response.each do |callback|
result = callback.call(response)
result.each { |url| request(url) } if result.is_a?(Array)
end
end
end
end
end