Implementation of the Python thread pool "Go"

ImportUrllib2Import TimeImportSocket fromDatetimeImportdatetime fromThread_poolImport*defMain (): Url_list= {"Sina":"http://www.sina.com.cn",               "Sohu":"http://www.sohu.com",               "Yahoo":"http://www.yahoo.com",               "Xiaonei":"http://www.xiaonei.com",               "Qihoo":"http://www.qihoo.com",               "Laohan":"http://www.laohan.org",               "Eyou":"http://www.eyou.com",               "Chinaren":"http://www.chinaren.com",               "Douban":"http://www.douban.com",               "163":"http://www.163.com",               "Daqi":"http://www.daqi.com",               "QQ":"http://www.qq.com",               "baidu_1":"HTTP://WWW.BAIDU.COM/S?WD=ASDFASDF",               "baidu_2":"HTTP://WWW.BAIDU.COM/S?WD=DDDDDDDF",               "Google_1":"Http://www.baidu.com/s?wd=sadfas",               "google_2":"HTTP://WWW.BAIDU.COM/S?WD=SADFLASD",               "Hainei":"http://www.hainei.com",               "Microsoft":"http://www.microsoft.com",               "Wlzuojia":"http://www.wlzuojia.com"}     #using the thread poolSocket.setdefaulttimeout (10)    Print 'Start Testing'WM= Workermanager (50)     forUrl_nameinchUrl_list.keys (): Wm.add_job (Do_get_con, Url_name, Url_list[url_name]) wm.wait_for_complete ()Print 'End Testing' defDo_get_con (url_name,url_link):Try: FD=Urllib2.urlopen (url_link) data=fd.read () F_hand= Open ("/tmp/ttt/%s"% Url_name,"W") f_hand.write (data) f_hand.close ()exceptexception,e:Pass if __name__=="__main__": Main () Thread_pool code (not original, turn from: http:blog.daviesliu.net/2006/10/09/234822/)ImportQueue, threading, sys fromThreadingImportThreadImport TimeImportUrllib#Working threadclassWorker (Thread): Worker_count=0 Timeout= 1def __init__(Self, workQueue, resultqueue, * *kwds): Thread.__init__(Self, * *kwds) Self.id=Worker.worker_count Worker.worker_count+ = 1Self.setdaemon (True) Self.workqueue=WorkQueue Self.resultqueue=resultqueue Self.start ()defRun (self):" "the Get-some-work, do-some-work main loop of worker threads" "         whileTrue:Try: Callable, args, Kwds= Self.workQueue.get (timeout=worker.timeout) Res= Callable (*args, * *Kwds)Print "worker[%2d]:%s"%(self.id, str (res)) Self.resultQueue.put (res )#time.sleep (worker.sleep)            exceptQueue.empty: Break            except :                Print 'worker[%2d]'% Self.id, Sys.exc_info () [: 2]                Raise               classWorkermanager:def __init__(Self, num_of_workers=10, timeout = 2): Self.workqueue=queue.queue () self.resultqueue=queue.queue () self.workers=[] self.timeout=Timeout Self._recruitthreads (num_of_workers)def_recruitthreads (Self, num_of_workers): forIinchRange (num_of_workers): Worker=worker (Self.workqueue, Self.resultqueue) self.workers.append (worker)defWait_for_complete (self):#... then, wait for each of the them to terminate:         whileLen (self.workers): Worker=Self.workers.pop () worker.join ( )ifWorker.isalive () and  notself.workQueue.empty (): Self.workers.append (worker)Print "All jobs is completed."     defAdd_job (self, callable, *args, * *Kwds): Self.workQueue.put ((callable, args, Kwds))defGet_result (self, *args, * *Kwds):returnSelf.resultQueue.get (*args, **kwds)

Implementation of the Python thread pool "Go"