Python multithreaded HTTP download implementation sample _python

Test platform Ubuntu 13.04 x86_64 Python 2.7.4

Spent nearly two hours, the problem is mainly at the beginning did not expect to pass a file object into the thread, resulting in downloaded files and source files MD5 different, waste a lot of time.

Interested students can be taken to add parameters, improved, can also be added to the extension of the breakpoint.

Copy Code code as follows:

#-*-Coding:utf-8-*-
# Author:toughguy
# email:wj0630@gmail.com
# Writing this thing to get a sense of how Python's multi-threaded mechanism
# usually do not write the habit of commenting, this time in the code to write a note is also the hope that there is a problem place please correct me, because I may not understand myself.
# test Platform Ubuntu 13.04 x86_64 Python 2.7.4

Import threading
Import Urllib2
Import Sys

Max_thread = 10
# Initialize Lock
Lock = Threading. Rlock ()

Class Downloader (threading. Thread):
def __init__ (self, URL, start_size, end_size, fobj, buffer):
Self.url = URL
Self.buffer = Buffer
Self.start_size = Start_size
Self.end_size = End_size
Self.fobj = Fobj
Threading. Thread.__init__ (self)

def run (self):
"""
Just a vest.
"""
With Lock:
print ' starting:%s '% self.getname ()
Self._download ()

def _download (self):
"""
I'm the one who moved bricks.
"""
req = Urllib2. Request (Self.url)
# Add HTTP Header (range) to set the range of download data
req.headers[' Range ' = ' bytes=%s-%s '% (self.start_size, self.end_size)
F = Urllib2.urlopen (req)
# initializes the current thread file object offset
offset = Self.start_size
While 1:
block = F.read (self.buffer)
# exit after current thread data is fetched
If not block:
With Lock:
print '%s done. '% Self.getname ()
Break
# when it's written like data, of course, lock the thread.
# Use with lock instead of the traditional lock.acquire () ... lock.release ()
# need Python >= 2.5
With Lock:
Sys.stdout.write ('%s saveing block ... '% self.getname ())
# Set File Object offset address
Self.fobj.seek (offset)
# Write to the acquired data
Self.fobj.write (Block)
Offset = offset + len (block)
Sys.stdout.write (' done.\n ')

def main (URL, thread=3, save_file= ', buffer=1024):
# Maximum number of threads cannot exceed Max_thread
thread = thread if thread <= max_thread else max_thread
# Get the size of the file
req = Urllib2.urlopen (URL)
size = Int (Req.info (). Getheaders (' content-length ') [0])
# Initialize File objects
Fobj = open (Save_file, ' WB ')
# Calculate the HTTP Range size that each thread is responsible for based on the number of threads
Avg_size, pad_size = divmod (size, thread)
plist = []
For i in Xrange (thread):
Start_size = I*avg_size
End_size = start_size + avg_size-1
if i = = thread-1:
# Last thread plus pad_size
End_size = end_size + pad_size + 1
t = Downloader (URL, start_size, end_size, fobj, buffer)
Plist.append (t)

# Start Moving Bricks #
For T in Plist:
T.start ()

# Waiting for all threads to end
For T in Plist:
T.join ()

# The end of course remember to close the file object
Fobj.close ()
print ' Download completed! '

if __name__ = = ' __main__ ':
url = ' Http://192.168.1.2:8082/downloads/10M.zip '
Main (Url=url, thread=10, save_file= ' Test.iso ', buffer=4096)