# !/usr/bin/env python # -*- coding:utf-8 -*- import queue import threading import time import re , Urllib2from bs4 import beautifulsoup class workmanager (object): def __init__ (self, work_num=1000,thread_num=2): self.work_queue = queue.queue () self.threads = [] self.__init_work_queue (Work_num) Self.__init_thread_pool (thread_num) "" " Initialize thread "" def __init_thread_pool (Self,threAd_num): for i in range (Thread_num): self.threads.append (Work ( self.work_queue)) "" " Initialize Work queue "" def __init_work_ Queue (self, jobs_num): for i in range (Jobs_num): Self.add_job (do_job, i,jobs_num) "" " Add a task to the queue "" def add_job (Self, func, *args): self.work_queue.put (func, list (args)) #任务入队, a synchronization mechanism is implemented within the queue "" wait for all threads to finish running "" def wait_allcomplete (self): for item in self.threads: if item.isalive (): Item.join () class work (Threading. Thread): def __init__ (Self, work_queue): threading. Thread.__init__ (self) self.work_queue = Work_queue self.start () def run (self): #死循环 so that the created thread shuts down Under certain conditions exit while True: try: do, args = self.work_queue.get (Block=false) #任务异步出队, a synchronization mechanism is implemented within the queue do (args) Self.work_queue.task_done () #通知系统任务完成 except: break #具体要做的任务 def do_job (args): #time. SleeP (0.1) #模拟处理时间 url = ' http://www.baidu.com ' page = urllib2.urlopen (URL) soup = beautifulsoup (Page.read (), "Html.parser") print soup.title.string,list (args),args[0] #print agrs #print threading.current_thread (), list (args) if __name__ == ' __main__ ': start = time.time () ' Start ' work_manager = workmanager (100, 2) work_manager.wait_allcomplete () end = time.time () print "cost all time: %s " % (End-start)
Python multithreaded operations