Python multi-threaded HTTP Download Implementation example

Test platform Ubuntu 13.04 x86_64 Python 2.7.4

It took nearly two hours, the problem was initially not expected to pass a file object into the thread inside, resulting in the download of the file and the source file MD5 is not the same, wasting a lot of time.

Interested students can be taken to add parameters, improved, you can also add a breakpoint to continue to pass.

Copy the Code code as follows:

#-*-Coding:utf-8-*-
# Author:toughguy
# email:wj0630@gmail.com
# Writing this thing to get a preliminary look at Python's multithreading mechanism
# usually do not write the habit of commenting, this time spent in the code to write a comment is also hope that there is a problem place please correct me, because maybe I did not understand.
# test Platform Ubuntu 13.04 x86_64 Python 2.7.4

Import threading
Import Urllib2
Import Sys

Max_thread = 10
# Initialize Lock
Lock = Threading. Rlock ()

Class Downloader (threading. Thread):
def __init__ (self, URL, start_size, end_size, fobj, buffer):
Self.url = URL
Self.buffer = Buffer
Self.start_size = Start_size
Self.end_size = End_size
Self.fobj = Fobj
Threading. Thread.__init__ (self)

def run (self):
"""
Just a vest.
"""
With Lock:
print ' starting:%s '% self.getname ()
Self._download ()

Def _download (self):
"" "
I'm the one who moved the bricks
" "
req = urllib2. Request (Self.url)
# Add HTTP Header (range) to set the range of downloaded data
req.headers[' range ' = ' bytes=%s-%s '% (self.start_size, Self.end_size)
F = urllib2.urlopen (req)
# initializes the current thread file object offset
offset = self.start_size
While 1:
block = F.R EAD (Self.buffer)
# Exits when the current thread data gets finished
if not block:
with Lock:
print '%s done. '% self.getname ()
Break
# When writing data, of course, lock thread
# using with lock instead of traditional lock.acquire () ... lock.release ()
# requires Python >= 2.5
with Lock:
S Ys.stdout.write ('%s saveing block ... '% self.getname ())
# Set File object offset address
Self.fobj.seek (offset)
# write fetched data
Self.fobj.write (block)
offset = offset + len (block)
Sys.stdout.write (' done.\n ')

def main (URL, thread=3, save_file= ", buffer=1024):
# Maximum number of threads cannot exceed Max_thread
thread = thread if thread <= max_thread else max_thread
# Get the file size
req = Urllib2.urlopen (URL)
size = Int (Req.info (). Getheaders (' content-length ') [0])
# Initialize File object
Fobj = open (Save_file, ' WB ')
# calculates the HTTP Range size responsible for each thread based on the number of threads
Avg_size, pad_size = divmod (size, thread)
plist = []
For i in Xrange (thread):
Start_size = I*avg_size
End_size = start_size + avg_size-1
if i = = thread-1:
# Last thread plus pad_size
End_size = end_size + pad_size + 1
t = Downloader (URL, start_size, end_size, fobj, buffer)
Plist.append (t)

# Start Moving Bricks
For T in Plist:
T.start ()

# Wait for all threads to end
For T in Plist:
T.join ()

# End of course remember to close the file object
Fobj.close ()
print ' Download completed! '

if __name__ = = ' __main__ ':
url = ' Http://192.168.1.2:8082/downloads/10M.zip '
Main (Url=url, thread=10, save_file= ' Test.iso ', buffer=4096)

