- Import Urllib2
- Import time
- Import socket
- From datetime import datetime
- From Thread_pool Import *
- def main ():
- Url_list = {"Sina":"http://www.sina.com.cn",
- " Sohu":"http://www.sohu.com",
- " Yahoo":"http://www.yahoo.com",
- " xiaonei":"http://www.xiaonei.com",
- " Qihoo":"http://www.qihoo.com",
- " Laohan":"http://www.laohan.org",
- " Eyou":"http://www.eyou.com",
- " Chinaren":"http://www.chinaren.com",
- " Douban":"http://www.douban.com",
- " 163":"http://www.163.com",
- " Daqi":"http://www.daqi.com",
- " QQ":"http://www.qq.com",
- " baidu_1":"Http://www.baidu.com/s?wd=asdfasdf",
- " baidu_2":"Http://www.baidu.com/s?wd=dddddddf",
- " google_1":"Http://www.baidu.com/s?wd=sadfas",
- " google_2":"HTTP://WWW.BAIDU.COM/S?WD=SADFLASD",
- " hainei":"http://www.hainei.com",
- " Microsoft":"http://www.microsoft.com",
- "Wlzuojia":"http://www.wlzuojia.com"}
- #使用线程池
- Socket.setdefaulttimeout (Ten)
- print ' Start testing '
- WM = Workermanager (+)
- For Url_name in url_list.keys ():
- Wm.add_job (Do_get_con, Url_name, Url_list[url_name])
- Wm.wait_for_complete ()
- print ' End testing '
- def do_get_con (Url_name,url_link):
- Try:
- FD = Urllib2.urlopen (Url_link)
- data = Fd.read ()
- F_hand = open ("/tmp/ttt/%s"% Url_name,"W")
- F_hand.write (data)
- F_hand.close ()
- except Exception,e:
- Pass
- if __name__ = = "__main__":
- Main ()
- Thread_pool Code (non-original, turn from: http://blog.daviesliu.net/2006/10/09/234822/)
- Import Queue, threading, Sys
- From threading Import Thread
- Import time
- Import Urllib
- # working thread
- Class Worker (Thread):
- Worker_count = 0
- Timeout = 1
- def __init__ (self , workQueue, Resultqueue, **kwds):
- Thread.__init__ (self , **kwds)
- self.id = Worker.worker_count
- Worker.worker_count + = 1
- Self.setdaemon ( True)
- self.workqueue = WorkQueue
- self.resultqueue = Resultqueue
- Self.start ()
- def run (self ):
- 'the Get-some-work, do-some-work main loop of worker threads '
- while True:
- Try:
- Callable, args, Kwds = self.workQueue.get (timeout=worker.timeout)
- res = callable (*args, **kwds)
- print "worker[%2d]:%s"% (self.id, str (res))
- self.resultQueue.put (RES)
- #time. Sleep (worker.sleep)
- except Queue.empty:
- Break
- except:
- print ' worker[%2d] '% self.id, Sys.exc_info () [:2]
- Raise
- Class Workermanager:
- def __init__ (self , num_of_workers=, timeout = 2):
- self.workqueue = Queue.queue ()
- self.resultqueue = Queue.queue ()
- self.workers = []
- Self.timeout = Timeout
- self._recruitthreads (num_of_workers)
- def _recruitthreads (self , num_of_workers):
- For I in range (num_of_workers):
- Worker = Worker ( self.workqueue, self.resultqueue)
- self.workers.append (worker)
- def wait_for_complete (self ):
- # ... then, wait for each of the them to terminate:
- While Len (self.workers):
- Worker = Self.workers.pop ()
- Worker.join ()
- if Worker.isalive () and not self.workQueue.empty ():
- self.workers.append (worker)
- print "All jobs is completed."
- def add_job (self , callable, *args, **kwds):
- Self.workQueue.put ((callable, args, Kwds))
- def get_result (self , *args, **kwds):
- return self.resultQueue.get (*args, **kwds)
Implementation of the Python thread pool