-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdownloader.py
More file actions
executable file
·74 lines (68 loc) · 2.13 KB
/
downloader.py
File metadata and controls
executable file
·74 lines (68 loc) · 2.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#!/usr/bin/env python
import urllib2
import os,sys
import sys, getopt
import threading
def get_args():
url = None
thread_num = 1
usage = sys.argv[0] + ' -u URL -n thread_num'
try:
opts, args = getopt.getopt(sys.argv[1:], "hu:n:", ["help", "url=", "thread_num="])
except getopt.GetoptError:
print usage
sys.exit(1)
if opts:
for op, value in opts:
if op == '-u':
url = value
elif op == '-n':
thread_num = int(value)
else:
print usage
sys.exit(1)
else:
print usage
sys.exit(2)
return url, thread_num
def get_size(url):
opener = urllib2.build_opener()
req = opener.open(url,timeout=3)
meta = req.info()
file_size = int(meta.getheaders("Content-Length")[0])
return file_size
def get_thread_list(size,th_num):
if th_num <= 1:
lst = [[0, size-1]]
else:
step = size / th_num
lst=[[step * i, step * (i + 1) - 1] for i in xrange(0,th_num-1)]
lst.append([step*(i+1), size-1])
#print lst
return lst
def down_file(url,fd,Range_list):
response = urllib2.Request(url)
# response.add_header('Range', 'bytes=%d-%d' % (Range_list[0],Range_list[1]))
response.add_header('Range','bytes={0[0]}-{0[1]}'.format(Range_list))
r = urllib2.urlopen(response,timeout=3)
context = r.read()
fd.seek(Range_list[0])
fd.write(context)
fd.close()
# print fd
def main():
url, thread_num = get_args()
filename=url.split('/')[-1] #'file.download'
filesize = get_size(url)
Range_list = get_thread_list(filesize, thread_num)
fd = []
for i in xrange(thread_num):
fd.append(open(filename,'w+')) #fd must be different! The same fd in multi-threding will be crash in writing
t = threading.Thread(target=down_file,args=(url,fd.pop(),Range_list[i]),)
print "Range_list %d: " % i , Range_list[i]
t.start()
print "Currently, there are %d child threads" % (threading.activeCount() - 1)
t.join()
print "Download %s done!" % filename
if __name__ == '__main__':
main()