Detailed description of the Python Thread Pool

The following is a summary of my experiences with the Python thread pool. for users who have never been familiar with programming languages or have a few programming languages, the Python language is definitely one of the best choices, it is recommended that beginners learn programming from Python first.

 
 

  
  
import Queue, threading, sys   
  
  
from threading import Thread   
  
  
import time,urllib   
  
  
# working thread   
  
  
class Worker(Thread):   
  
  
   worker_count = 0   
  
  
   def __init__( self, workQueue, resultQueue, timeout = 0, **kwds):   
  
  
       Thread.__init__( self, **kwds )   
  
  
       self.id = Worker.worker_count   
  
  
       Worker.worker_count += 1   
  
  
       self.setDaemon( True )   
  
  
       self.workQueue = workQueue   
  
  
       self.resultQueue = resultQueue   
  
  
       self.timeout = timeout   
  
  
       self.start( )   
  
  
   def run( self ):   
  
  
       ''' the get-some-work, do-some-work main loop of worker threads '''   
  
  
       while True:   
  
  
           try:   
  
  
               callable, args, kwds = self.workQueue.get(timeout=self.timeout)   
  
  
               res = callable(*args, **kwds)   
  
  
               print "worker[%2d]: %s" % (self.id, str(res) )   
  
  
               self.resultQueue.put( res )   
  
  
           except Queue.Empty:   
  
  
               break   
  
  
           except :   
  
  
               print 'worker[%2d]' % self.id, sys.exc_info()[:2]   
  
  
                  
  
  
class WorkerManager:   
  
  
   def __init__( self, num_of_workers=10, timeout = 1):   
  
  
       self.workQueue = Queue.Queue()   
  
  
       self.resultQueue = Queue.Queue()   
  
  
       self.workers = []   
  
  
       self.timeout = timeout   
  
  
       self._recruitThreads( num_of_workers )   
  
  
   def _recruitThreads( self, num_of_workers ):   
  
  
       for i in range( num_of_workers ):   
  
  
           worker = Worker( self.workQueue, self.resultQueue, self.timeout )   
  
  
           self.workers.append(worker)   
  
  
   def wait_for_complete( self):   
  
  
       # ...then, wait for each of them to terminate:   
  
  
       while len(self.workers):   
  
  
           worker = self.workers.pop()   
  
  
           worker.join( )   
  
  
           if worker.isAlive() and not self.workQueue.empty():   
  
  
               self.workers.append( worker )   
  
  
       print "All jobs are are completed."   
  
  
   def add_job( self, callable, *args, **kwds ):   
  
  
       self.workQueue.put( (callable, args, kwds) )   
  
  
   def get_result( self, *args, **kwds ):   
  
  
       return self.resultQueue.get( *args, **kwds )  
 
 

The Worker class is a Python thread pool that constantly obtains the tasks to be executed from the workQueue queue, executes the tasks, and writes the results to the resultQueue. WorkQueue and resultQueue are both ready-made and secure, and their internal operations on each thread are mutually exclusive. When the task retrieved from workQueue times out, the thread ends.

WorkerManager initializes the Python thread pool, provides interfaces for adding tasks to the queue and obtaining results, and waits until all tasks are completed. A typical test example is as follows. It uses 10 threads to download the content of a fixed page. Different tasks should be executed in actual applications.

 
 

  
  
def test_job(id, sleep = 0.001 ):   
  
  
   try:   
  
  
       urllib.urlopen('[url]https://www.gmail.com/[/url]').read()   
  
  
   except:   
  
  
       print '[%4d]' % id, sys.exc_info()[:2]   
  
  
   return id   
  
  
def test():   
  
  
   import socket   
  
  
   socket.setdefaulttimeout(10)   
  
  
   print 'start testing'   
  
  
   wm = WorkerManager(10)   
  
  
   for i in range(500):   
  
  
       wm.add_job( test_job, i, i*0.001 )   
  
  
   wm.wait_for_complete()   
  
  
   print 'end testing'  
 
 

1. Introduction to Python system files
2. How to correctly use Python Functions
3. Detailed introduction and analysis of Python build tools
4. Advantages of Python in PythonAndroid
5. How to Use the Python module to parse the configuration file?