#!/usr/bin/python
Import Queue, threading, Sys
From threading Import Thread
Import Time,urllib
Class Worker (Thread):
Worker_count = 0
def __init__ (self, workQueue, resultqueue, timeout = 0, **kwds):
Thread.__init__ (self, **kwds)
Self.id = Worker.worker_count
Worker.worker_count + = 1
Self.setdaemon (True)
Self.workqueue = WorkQueue
Self.resultqueue = Resultqueue
Self.timeout = Timeout
Self.start ()
def run (self):
' The Get-some-work, do-some-work main loop of worker threads '
While True:
Try
Callable, args, Kwds = Self.workQueue.get (timeout=self.timeout)
res = callable (*args, **kwds)
Print "worker[%2d]:%s"% (Self.id, str (res))
Self.resultQueue.put (RES)
Except Queue.empty:
Break
Except:
print ' worker[%2d] '% self.id, Sys.exc_info () [: 2]
Class Workermanager:
def __init__ (self, num_of_workers=10, timeout = 1):
Self.workqueue = Queue.queue ()
Self.resultqueue = Queue.queue ()
Self.workers = []
Self.timeout = Timeout
Self._recruitthreads (Num_of_workers)
def _recruitthreads (self, num_of_workers):
For I in Range (num_of_workers):
Worker = Worker (Self.workqueue, Self.resultqueue, Self.timeout)
Self.workers.append (worker)
def wait_for_complete (self):
# ... then, wait for each of the them to terminate:
While Len (self.workers):
Worker = Self.workers.pop ()
Worker.join ()
If Worker.isalive () and not Self.workQueue.empty ():
Self.workers.append (worker)
Print "All jobs is completed."
def add_job (self, callable, *args, **kwds):
Self.workQueue.put ((callable, args, Kwds))
def get_result (self, *args, **kwds):
Return Self.resultQueue.get (*args, **kwds)
if __name__ = = ' __main__ ':
def test_job (id, sleep = 0.001):
Try
A=urllib.urlopen (' http://www.baidu.com/'). Read ()
Except
print ' [%4d] '% ID, sys.exc_info () [: 2]
Return ID
Import socket
Socket.setdefaulttimeout (10)
print ' Start testing '
WM = Workermanager (2)
For I in range (2):
Wm.add_job (Test_job, I, i*0.001)
Print Wm.get_result (i, i*0.001)
Wm.wait_for_complete ()
print ' End testing '
Personal understanding
1. First instantiate Workermanager (2), when instantiating the Workermanager class, first create 2 queues, one for the execution queue, one for the result queue
2. Perform _recruitthreads, loop 2 times, instantiate the worker class in this function, and biographies the 2 teams that you just created, and insert the ancestor in the workers list, workers this is used to determine if the tasks in the queue are executed.
The worker class inherits the thread, initializes the thread, and invokes the Start method in the thread module, where the Start method has the Run method, which overrides the Run method, Self.start () That is, the run method in the worker class
3. Because the following data is Wm.add_job (Test_job, I, i*0.001) to be added to the queue when performing the task, go to the Get task execution queue, see if the queue is empty, if not empty, execute res = callable (*args , **kwds), and writes the data to the result queue, but due to the first initialization, no tasks in the queue will be stuck in this
4. Perform wm.add_job (Test_job, I, i*0.001), put 2 tasks into the queue you just created, and these 2 tasks are performed in multithreaded
5.wm.wait_for_complete () The first step is to remove the data from the list one at a time, and wait for the thread to exit after the deletion, note the difference between join and IsAlive, and join whether the thread execution timeout or the normal exit return value is None, And IsAlive is to see if the thread is still alive, if the thread is still alive, and the task queue is not empty, or to add the Ganso previously removed from the list, because the thread has not completed, know to judge workers the list of the data is not available in the entire program
But the above procedure in judging the thread is not all executed logic, I think there are writing problems
If more than one task, if you want 10 threads to execute the Test1 function, 10 tasks to execute the TEST2 function what to do, know what, it's almost clear what's going on.
This article is from the "Expect batch Sync data" blog, so be sure to keep this source http://4249964.blog.51cto.com/4239964/1569188
Python Thread pool Learning