forked from opencivicdata/python-legistar-scraper
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathrefresh_fixtures.py
More file actions
56 lines (36 loc) · 1.49 KB
/
refresh_fixtures.py
File metadata and controls
56 lines (36 loc) · 1.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import os
import sys
import lxml
from legistar.bills import LegistarBillScraper
from legistar.events import LegistarEventsScraper
from legistar.people import LegistarPersonScraper
def save_page(page, jurisdiction, outfile):
test_directory = os.path.abspath(os.path.dirname(__file__))
project_directory = os.path.join(test_directory, '..')
with open(os.path.join(project_directory, 'tests', 'fixtures', jurisdiction, outfile), 'wb') as f:
f.write(lxml.html.tostring(page))
def refresh_bills(jurisdiction):
s = LegistarBillScraper()
s.LEGISLATION_URL = 'https://{}.legistar.com/Legislation.aspx'.format(jurisdiction)
page = next(s.searchLegislation('bus'))
save_page(page, jurisdiction, 'bills.html')
def refresh_events(jurisdiction):
s = LegistarEventsScraper()
s.EVENTSPAGE = 'https://{}.legistar.com/Calendar.aspx'.format(jurisdiction)
page = next(s.eventPages('2018-01-01'))
save_page(page, jurisdiction, 'events.html')
def refresh_people(jurisdiction):
s = LegistarPersonScraper()
MEMBERLIST = 'https://{}.legistar.com/People.aspx'.format(jurisdiction)
page = next(s.pages(MEMBERLIST))
save_page(page, jurisdiction, 'people.html')
if __name__ == '__main__':
try:
_, jurisdictions = sys.argv
jurisdictions = jurisdictions.split(',')
except ValueError:
jurisdictions = ('chicago', 'metro', 'nyc')
for j in jurisdictions:
refresh_bills(j)
refresh_events(j)
refresh_people(j)