This repository was archived by the owner on Sep 14, 2019. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 23
Expand file tree
/
Copy pathtasks.py
More file actions
98 lines (75 loc) · 2.41 KB
/
tasks.py
File metadata and controls
98 lines (75 loc) · 2.41 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
import glob
import logging
import sqlalchemy as sa
from invoke import run, task
import app
import aws
import refresh_log
import utils
from refresh_log import AutoapiTableRefreshLog as RefreshLog
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
@task
def requirements(upgrade=True):
cmd = 'pip install -r requirements.txt'
if upgrade:
cmd += ' --upgrade'
run(cmd)
@task
def apify(filename, tablename=None):
try:
filenames = glob.glob(filename, recursive=True)
except TypeError: # recursive glob in Python 3.5+ only
filenames = glob.glob(filename)
if len(filenames) > 1 and tablename:
raise Exception("Can't specify a `tablename` for >1 file")
for filename in filenames:
_tablename = tablename or utils.get_name(filename)
logger.info('Importing {0} to table {1}'.format(filename, _tablename))
try:
utils.drop_table(_tablename)
except sa.exc.OperationalError as e:
logger.debug('DROP TABLE {} failed, may not exist?'.format(
_tablename))
logger.debug(str(e))
try:
utils.load_table(filename, _tablename)
except Exception as e:
logger.error('Failed to load table from file {}'.format(filename))
logger.error(str(e))
logger.info('Finished importing {0}'.format(filename))
@task
def delete_from_bucket(filename, bucket_name=None):
aws.delete_from_bucket(filename, bucket_name)
@task
def copy_to_bucket(filename, bucket_name=None):
aws.copy_to_bucket(filename, bucket_name)
@task
def list_bucket(bucket_name=None):
aws.list_bucket(bucket_name)
@task
def fetch_bucket(bucket_name=None):
aws.fetch_bucket(bucket_name)
@task
def serve():
app.make_app().run(host='0.0.0.0')
@task
def refresh(clear_tables=True):
logger.info('Refresh invoked')
with app.make_app().app_context():
rlog_id = RefreshLog.start()
logger.info('refresh log id {} started'.format(rlog_id))
try:
aws.fetch_bucket(clear_tables=clear_tables)
logger.debug('bucket fetched')
utils.refresh_tables()
logger.info('refresh complete')
refresh_log.stop(rlog_id)
except Exception as e:
refresh_log.stop(rlog_id, err_msg=str(e))
@task
def quick_refresh():
refresh(clear_tables=False)
@task
def clear():
utils.clear_tables()