A simple Python thread pool framework

Beginner python, the implementation of a simple thread pool framework, in addition to wokers (worker threads) in the thread pools, also creates a separate log thread for the output of the log. The queue is used to communicate between threads.

The code is as follows:

　　

1 #!/usr/bin/env python2 #-*-coding:utf-8-*-3 4 __author__="Pandaychen"5 6 ImportQueue7 ImportSYS8 ImportOS9 ImportThreadingTen Import Time One ImportSignal A  - defhandler (): -     Print "Press CTRL + C to end ...." theSys.exit (1) -  -  - defCall_function (para): +Time.sleep (5) -     returnpara +  A  at defLoggingfun (t_filename,t_logcontent): -LogPath ='./log/' -Curdate = Time.strftime ("%y%m%d") -NewPath ='./log/'+t_filename+'_'+curdate -  -     ifos.path.exists (logpath): in         Pass -     Else: to Os.mkdir (LogPath) +  -     Try: theFilehd = open (NewPath,'A +') *Newcontent ='['+str (Time.strftime ("%y-%m-%d%h:%m:%s"))+']'+t_logcontent+'\ n' $ filehd.writelines (newcontent)Panax Notoginseng filehd.close () -     exceptexception,e: the         Pass +  A classLogthread (Threading. Thread): the     def __init__(self,logqueue,**Kwds): +Threading. Thread.__init__(self,**Kwds) -Self.logqueue =Logqueue $ Self.setdaemon (True) $  -     defRun (self): -          while1: the             #log = Self.logQueue.get (False) -Log =Self.logQueue.get ()Wuyi             iflog: theLoggingfun ("Test", log) -                 Pass Wu             Else: -Loggingfun ("Test","log thread sleep 1s") AboutTime.sleep (1) $  - #encapsulation as a thread class - classWorker (Threading. Thread):#working with work requests -     def __init__(Self, workQueue, Resultqueue,logqueue, threadid,**Kwds): AThreading. Thread.__init__(Self, * *Kwds) + Self.setdaemon (True) theSelf.workqueue =WorkQueue -Self.resultqueue =Resultqueue $Self.logqueue =Logqueue theSelf.threadid =ThreadID the  the     defRun (self): the          while1: -             Try: inCallable, args, Kwds = Self.workQueue.get (False)#Get a task theres = callable (*args, * *Kwds) theStrres ="Thread:"+ STR (SELF.THREADID) +"Done ,"+"args:"+Str (RES) About  the self.logQueue.put (strres) theSelf.resultQueue.put (RES)#put result the             exceptQueue.empty: +                  Break -  the classWorkmanagerpool:#thread pool management, creatingBayi     def __init__(Self, num_of_workers=10): theSelf.workqueue = Queue.queue ()#Request Queue theSelf.resultqueue = Queue.queue ()#queue for output results -Self.logqueue =Queue.queue () -Self.workers = [] the self._recruitthreads (num_of_workers) the  the     def_recruitthreads (Self, num_of_workers): the          forIinchRange (num_of_workers): -Worker = Worker (Self.workqueue, Self.resultqueue,self.logqueue,i)#Create worker Threads the Worker.setdaemon (True) theSelf.workers.append (worker)#join to Thread queue the 94Logthread =Logthread (self.logqueue) the self.workers.append (Logthread) the  the 98     defStart (self): About          forWinchself.workers: - W.start ()101 102     defWait_for_complete (self):103          whileLen (self.workers):104Worker = Self.workers.pop ()#Take a thread out of a pool to process a request the Worker.join ()106             ifWorker.isalive () and  notself.workQueue.empty ():107Self.workers.append (worker)#rejoin the thread pool108                 Print 'All jobs were complete.'109  the 111     defAdd_job (self, callable, *args, * *Kwds): theSelf.workQueue.put ((callable, args, Kwds))#adding requests to the work queue113  the     defGet_result (self, *args, * *Kwds): the         returnSelf.resultQueue.get (*args, * *Kwds) the 117 118 119 defMain (): - signal.signal (signal. SIGINT, Handler)121 signal.signal (signal. SIGTERM, Handler)122 123     Try:124num_of_threads = Int (sys.argv[1]) the     except:126Num_of_threads = 10127Start =time.time () -Workermanagepool =Workmanagerpool (num_of_threads)129         #Print Num_of_threads theURLs = ['http://bbs.qcloud.com'] * 1000131          forIinchURLs: the Workermanagepool.add_job (call_function, i)133 134 Workermanagepool.start ()135 Workermanagepool.wait_for_complete ()136         PrintTime.time ()-Start137 138 if __name__=='__main__':139Main ()

A simple Python thread pool framework