forked from ml-lab/VisualSearchServer
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathfabfile.py
More file actions
165 lines (142 loc) · 5.38 KB
/
fabfile.py
File metadata and controls
165 lines (142 loc) · 5.38 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
import os,sys,logging,time,shutil
from fabric.state import env
from fabric.api import env,local,run,sudo,put,cd,lcd,puts,task,get,hide
from settings import BUCKET_NAME,DATA_PATH,INDEX_PATH
try:
import inception
except ImportError:
print "could not import main module limited to boostrap actions"
pass
from settings import USER,private_key,HOST
env.user = USER
env.key_filename = private_key
env.hosts = [HOST,]
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='logs/fab.log',
filemode='a')
@task
def notebook():
"""
Run an IPython notebook on an AWS server
"""
from IPython.lib.security import passwd
command = "ipython notebook --ip=0.0.0.0 --certfile=mycert.pem --NotebookApp.password={} --no-browser".format(passwd())
print command
run(command)
@task
def gen_ssl():
run("openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout mycert.key -out mycert.pem")
@task
def setup():
"""
Task for initial set up of AWS instance.
Used AMI modified for Python2.7 https://gist.github.com/AlexJoz/1670baf0b32573ca7923
Following commands show other packages/libraries installed while setting up the AMI
"""
connect()
sudo("chmod 777 /mnt/") # sometimes the first one will fail due to timeout and in any case this is idempotent
sudo("chmod 777 /mnt/")
sudo("add-apt-repository ppa:kirillshkrogalev/ffmpeg-next")
sudo("apt-get update")
sudo("apt-get install -y ffmpeg")
run("git clone https://github.com/AKSHAYUBHAT/VisualSearchServer")
sudo("pip install fabric")
sudo("pip install --upgrade awscli")
sudo("pip install --upgrade fabric")
sudo("pip install --upgrade flask")
sudo("pip install --upgrade ipython")
sudo("pip install --upgrade jupyter")
sudo("apt-get install -y python-scipy")
sudo("apt-get install -y libblas-dev liblapack-dev libatlas-base-dev gfortran")
sudo("pip install --upgrade nearpy")
@task
def connect():
"""
Creates connect.sh for the current host
:return:
"""
fh = open("connect.sh",'w')
fh.write("#!/bin/bash\n"+"ssh -i "+env.key_filename+" "+"ubuntu"+"@"+HOST+"\n")
fh.close()
@task
def server():
"""
start server
"""
local('python server.py')
@task
def demo_fashion():
"""
Start Demo using precomputed index for 450 thousand female fashion images.
"""
local('aws s3api get-object --bucket aub3visualsearch --key "fashion_index.tar.gz" --request-payer requester /mnt/fashion_index.tar.gz')
local('cd /mnt/;tar -zxvf fashion_index.tar.gz')
local('echo "\nDEMO=\'fashion_images\'" >> settings.py')
local('echo "\nINDEX_PATH=\'/mnt/fashion_index/\'" >> settings.py')
local('python server.py &')
local('tail -f logs/server.log')
@task
def demo_nyc():
"""
Start Demo using precomputed index for 26 thousand street view / dashcam style images.
"""
local('aws s3api get-object --bucket aub3visualsearch --key "nyc_index.tar.gz" --request-payer requester /mnt/nyc_index.tar.gz')
local('cd /mnt/;tar -zxvf nyc_index.tar.gz')
local('echo "\nDEMO=\'nyc_images\'" >> settings.py')
local('echo "\nINDEX_PATH=\'/mnt/nyc_index/\'" >> settings.py')
local('python server.py &')
local('tail -f logs/server.log')
@task
def index():
"""
Index images
"""
logging.info("Starting with images present in {} storing index in {}".format(DATA_PATH,INDEX_PATH))
try:
os.mkdir(INDEX_PATH)
except:
print "Could not created {}, if its on /mnt/ have you set correct permissions?".format(INDEX_PATH)
raise ValueError
inception.load_network()
count = 0
start = time.time()
with inception.tf.Session() as sess:
for image_data in inception.get_batch(DATA_PATH):
logging.info("Batch with {} images loaded in {} seconds".format(len(image_data),time.time()-start))
start = time.time()
count += 1
features,files = inception.extract_features(image_data,sess)
logging.info("Batch with {} images processed in {} seconds".format(len(features),time.time()-start))
start = time.time()
inception.store_index(features,files,count,INDEX_PATH)
@task
def clear():
"""
delete logs
"""
local('rm logs/*.log &')
@task
def test():
inception.load_network()
count = 0
start = time.time()
test_images = os.path.join(os.path.dirname(__file__),'tests/images')
test_index = os.path.join(os.path.dirname(__file__), 'tests/index')
try:
shutil.rmtree(test_index)
except:
pass
os.mkdir(test_index)
with inception.tf.Session() as sess:
for image_data in inception.get_batch(test_images,batch_size=2):
# batch size is set to 2 to distinguish between latency associated with first batch
if len(image_data):
print "Batch with {} images loaded in {} seconds".format(len(image_data),time.time()-start)
start = time.time()
count += 1
features,files = inception.extract_features(image_data,sess)
print "Batch with {} images processed in {} seconds".format(len(features),time.time()-start)
start = time.time()
inception.store_index(features,files,count,test_index)