python基於queue和threading實現多線程下載執行個體

本文執行個體講述了python基於queue和threading實現多線程下載的方法，分享給大家供大家參考。具體方法如下：

主代碼如下：

  #download worker   queue_download = Queue.Queue(0)   DOWNLOAD_WORKERS = 20   for i in range(DOWNLOAD_WORKERS):     DownloadWorker(queue_download).start() #start a download worker        for md5 in MD5S:     queue_download.put(md5)   for i in range(DOWNLOAD_WORKERS):     queue_download.put(None) 

其中downloadworkers.py
類繼承 threading.Thread,重載run方法..在__init__中調用threading.Thread.__init__(self),
在run方法中實現耗時的操作

import threading import Queue import md5query import DOM import os,sys class DownloadWorker(threading.Thread):   """"""    def __init__(self, queue):     """Constructor"""     self.__queue = queue     threading.Thread.__init__(self)     def run(self):     while 1:       md5 = self.__queue.get()       if md5 is None:         break #reached end of queue       #this is a time-cost produce       self._down(md5)        print "task:", md5, "finished"    def _down(self, md5):     config = {       'input':sys.stdin,        'output':'./samples',        'location':'xxx',        'has-fn':False,        'options':{'connect.timeout':60, 'timeout':3600},        'log':file('logs.txt', 'w'),      }     print 'download %s...' % (md5)     try:       data = downloadproc(config['location'], config['options'])#我的下載過程       if data:         dom, fileData = md5query.splited(data)         filename = md5         if config['has-fn']:           filename = '%s_%s' % (md5, dom.nodeValue2('xxxxxxx', '').encode('utf-8'))#這是我的下載的方法         f = file(os.path.join(config['output'], filename), 'w')         f.write(fileData)         f.close()          print '%s\tok' % (md5)       else:         print>>config['log'], '%s\t%s' % (md5, 'failed')     except Exception, e:       print>>config['log'], '%s\t%s' % (md5, str(e))

希望本文所述對大家的Python程式設計有所協助。

