Implementation of the Python thread pool

Source: Internet
Author: User

  1. Import Urllib2
  2. Import time
  3. Import socket
  4. From datetime import datetime
  5. From Thread_pool Import *
  6. def main ():
  7. Url_list = {"Sina":"http://www.sina.com.cn",
  8. " Sohu":"http://www.sohu.com",
  9. " Yahoo":"http://www.yahoo.com",
  10. " xiaonei":"http://www.xiaonei.com",
  11. " Qihoo":"http://www.qihoo.com",
  12. " Laohan":"http://www.laohan.org",
  13. " Eyou":"http://www.eyou.com",
  14. " Chinaren":"http://www.chinaren.com",
  15. " Douban":"http://www.douban.com",
  16. " 163":"http://www.163.com",
  17. " Daqi":"http://www.daqi.com",
  18. " QQ":"http://www.qq.com",
  19. " baidu_1":"Http://www.baidu.com/s?wd=asdfasdf",
  20. " baidu_2":"Http://www.baidu.com/s?wd=dddddddf",
  21. " google_1":"Http://www.baidu.com/s?wd=sadfas",
  22. " google_2":"HTTP://WWW.BAIDU.COM/S?WD=SADFLASD",
  23. " hainei":"http://www.hainei.com",
  24. " Microsoft":"http://www.microsoft.com",
  25. "Wlzuojia":"http://www.wlzuojia.com"}
  26. #使用线程池
  27. Socket.setdefaulttimeout (Ten)
  28. print ' Start testing '
  29. WM = Workermanager (+)
  30. For Url_name in url_list.keys ():
  31. Wm.add_job (Do_get_con, Url_name, Url_list[url_name])
  32. Wm.wait_for_complete ()
  33. print ' End testing '
  34. def do_get_con (Url_name,url_link):
  35. Try:
  36. FD = Urllib2.urlopen (Url_link)
  37. data = Fd.read ()
  38. F_hand = open ("/tmp/ttt/%s"% Url_name,"W")
  39. F_hand.write (data)
  40. F_hand.close ()
  41. except Exception,e:
  42. Pass
  43. if __name__ = = "__main__":
  44. Main ()
  45. Thread_pool Code (non-original, turn from: http://blog.daviesliu.net/2006/10/09/234822/)
  46. Import Queue, threading, Sys
  47. From threading Import Thread
  48. Import time
  49. Import Urllib
  50. # working thread
  51. Class Worker (Thread):
  52. Worker_count = 0
  53. Timeout = 1
  54. def __init__ (self , workQueue, Resultqueue, **kwds):
  55. Thread.__init__ (self , **kwds)
  56. self.id = Worker.worker_count
  57. Worker.worker_count + = 1
  58. Self.setdaemon ( True)
  59. self.workqueue = WorkQueue
  60. self.resultqueue = Resultqueue
  61. Self.start ()
  62. def run (self ):
  63. 'the Get-some-work, do-some-work main loop of worker threads '
  64. while True:
  65. Try:
  66. Callable, args, Kwds = self.workQueue.get (timeout=worker.timeout)
  67. res = callable (*args, **kwds)
  68. print "worker[%2d]:%s"% (self.id, str (res))
  69. self.resultQueue.put (RES)
  70. #time. Sleep (worker.sleep)
  71. except Queue.empty:
  72. Break
  73. except:
  74. print ' worker[%2d] '% self.id, Sys.exc_info () [:2]
  75. Raise
  76. Class Workermanager:
  77. def __init__ (self , num_of_workers=, timeout = 2):
  78. self.workqueue = Queue.queue ()
  79. self.resultqueue = Queue.queue ()
  80. self.workers = []
  81. Self.timeout = Timeout
  82. self._recruitthreads (num_of_workers)
  83. def _recruitthreads (self , num_of_workers):
  84. For I in range (num_of_workers):
  85. Worker = Worker ( self.workqueue, self.resultqueue)
  86. self.workers.append (worker)
  87. def wait_for_complete (self ):
  88. # ... then, wait for each of the them to terminate:
  89. While Len (self.workers):
  90. Worker = Self.workers.pop ()
  91. Worker.join ()
  92. if Worker.isalive () and not self.workQueue.empty ():
  93. self.workers.append (worker)
  94. print "All jobs is completed."
  95. def add_job (self , callable, *args, **kwds):
  96. Self.workQueue.put ((callable, args, Kwds))
  97. def get_result (self , *args, **kwds):
  98. return self.resultQueue.get (*args, **kwds)

Implementation of the Python thread pool

Related Article

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.