Implementation of the Python thread pool

2. Import Urllib2
4. Import time
6. Import socket
8. From datetime import datetime
10. From Thread_pool Import *
14. def main ():
16. Url_list = {"Sina":"http://www.sina.com.cn",
18. " Sohu":"http://www.sohu.com",
20. " Yahoo":"http://www.yahoo.com",
22. " xiaonei":"http://www.xiaonei.com",
24. " Qihoo":"http://www.qihoo.com",
26. " Laohan":"http://www.laohan.org",
28. " Eyou":"http://www.eyou.com",
30. " Chinaren":"http://www.chinaren.com",
32. " Douban":"http://www.douban.com",
34. " 163":"http://www.163.com",
36. " Daqi":"http://www.daqi.com",
38. " QQ":"http://www.qq.com",
40. " baidu_1":"Http://www.baidu.com/s?wd=asdfasdf",
42. " baidu_2":"Http://www.baidu.com/s?wd=dddddddf",
44. " google_1":"Http://www.baidu.com/s?wd=sadfas",
46. " google_2":"HTTP://WWW.BAIDU.COM/S?WD=SADFLASD",
48. " hainei":"http://www.hainei.com",
50. " Microsoft":"http://www.microsoft.com",
52. "Wlzuojia":"http://www.wlzuojia.com"}
56. #使用线程池
58. Socket.setdefaulttimeout (Ten)
60. print ' Start testing '
62. WM = Workermanager (+)
64. For Url_name in url_list.keys ():
66. Wm.add_job (Do_get_con, Url_name, Url_list[url_name])
68. Wm.wait_for_complete ()
70. print ' End testing '
74. def do_get_con (Url_name,url_link):
76. Try:
78. FD = Urllib2.urlopen (Url_link)
80. data = Fd.read ()
82. F_hand = open ("/tmp/ttt/%s"% Url_name,"W")
84. F_hand.write (data)
86. F_hand.close ()
88. except Exception,e:
90. Pass
94. if __name__ = = "__main__":
96. Main ()
98. Thread_pool Code (non-original, turn from: http://blog.daviesliu.net/2006/10/09/234822/)
102. Import Queue, threading, Sys
104. From threading Import Thread
106. Import time
108. Import Urllib
112. # working thread
114. Class Worker (Thread):
116. Worker_count = 0
118. Timeout = 1
120. def __init__ (self , workQueue, Resultqueue, **kwds):
122. Thread.__init__ (self , **kwds)
124. self.id = Worker.worker_count
126. Worker.worker_count + = 1
128. Self.setdaemon ( True)
130. self.workqueue = WorkQueue
132. self.resultqueue = Resultqueue
134. Self.start ()
138. def run (self ):
140. 'the Get-some-work, do-some-work main loop of worker threads '
142. while True:
144. Try:
146. Callable, args, Kwds = self.workQueue.get (timeout=worker.timeout)
148. res = callable (*args, **kwds)
150. print "worker[%2d]:%s"% (self.id, str (res))
152. self.resultQueue.put (RES)
154. #time. Sleep (worker.sleep)
156. except Queue.empty:
158. Break
160. except:
162. print ' worker[%2d] '% self.id, Sys.exc_info () [:2]
164. Raise
168. Class Workermanager:
170. def __init__ (self , num_of_workers=, timeout = 2):
172. self.workqueue = Queue.queue ()
174. self.resultqueue = Queue.queue ()
176. self.workers = []
178. Self.timeout = Timeout
180. self._recruitthreads (num_of_workers)
184. def _recruitthreads (self , num_of_workers):
186. For I in range (num_of_workers):
188. Worker = Worker ( self.workqueue, self.resultqueue)
190. self.workers.append (worker)
194. def wait_for_complete (self ):
196. # ... then, wait for each of the them to terminate:
198. While Len (self.workers):
200. Worker = Self.workers.pop ()
202. Worker.join ()
204. if Worker.isalive () and not self.workQueue.empty ():
206. self.workers.append (worker)
208. print "All jobs is completed."
212. def add_job (self , callable, *args, **kwds):
214. Self.workQueue.put ((callable, args, Kwds))
218. def get_result (self , *args, **kwds):
220. return self.resultQueue.get (*args, **kwds)

Implementation of the Python thread pool