forked from crawlbase/proxycrawl-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcrawling_api.py
More file actions
28 lines (25 loc) · 796 Bytes
/
crawling_api.py
File metadata and controls
28 lines (25 loc) · 796 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
try:
# For Python 3.0 and later
from urllib.parse import urlencode, quote_plus
except ImportError:
# Fall back to Python 2's
from urllib import urlencode, quote_plus
from proxycrawl.base_api import BaseAPI
#
# A Python class that acts as wrapper for ProxyCrawl Crawling API.
#
# Read ProxyCrawl API documentation https://proxycrawl.com/docs/crawling-api/
#
# Copyright ProxyCrawl
# Licensed under the Apache License 2.0
#
class CrawlingAPI(BaseAPI):
def get(self, url, options = {}):
options['url'] = url
return self.request(options)
def post(self, url, data, options = {}):
if isinstance(data, dict):
data = urlencode(data)
data = data.encode('utf-8')
options['url'] = url
return self.request(options, data)