Python custom process pool instance analysis [producer and consumer model problems], python instance analysis

Python custom process pool instance analysis [producer and consumer model problems], python instance analysis

This article analyzes the Python custom process pool. We will share this with you for your reference. The details are as follows:

Code Description:

# Encoding = UTF-8 # author: walker # date: 2014-05-21 # function: the User-Defined Process pool traverses files in the directory from multiprocessing import Process, Queue, Lockimport time, OS # Consumer class Consumer (Process): def _ init _ (self, queue, ioLock): super (Consumer, self ). _ init _ () self. queue = queue self. ioLock = ioLock def run (self): while True: task = self. queue. get () # When no task exists in the queue, the process if isinstance (task, str) and task = 'quit': break; time. sleep (1) # Assume that the task processing takes 1 second self. ioLock. acquire () print (str (OS. getpid () + ''+ task) self. ioLock. release () self. ioLock. acquire () print 'Bye-bye' self. ioLock. release () # Producer def Producer (): queue = Queue () # This queue is a process/thread-safe ioLock = Lock () subNum = 4 # Number of sub-processes workers = build_worker_pool (queue, ioLock, subNum) start_time = time. time () for parent, dirnames, filenames in OS. walk (r 'd: \ test '): for filename in filenames: Queue. put (filename) ioLock. acquire () print ('qsize: '+ str (queue. qsize () ioLock. release () while queue. qsize ()> subNum * 10: # control the number of tasks in the queue time. sleep (1) for worker in workers: queue. put ('quit') for worker in workers: worker. join () ioLock. acquire () print ('done! Time taken :{}'. format (time. time ()-start_time) ioLock. release () # create a process pool def build_worker_pool (queue, ioLock, size): workers = [] for _ in range (size): worker = Consumer (queue, ioLock) worker. start () workers. append (worker) return workersif _ name _ = '_ main _': Producer ()

Ps:

self.ioLock.acquire()...self.ioLock.release()

Available:

with self.ioLock:  ...

.

Another interesting example:

# Encoding = UTF-8 # author: walker # date: 2016-01-06 # function: A fun example of multi-process import OS, sys, timefrom multiprocessing import Poolcur_dir_fullpath = OS. path. dirname (OS. path. abspath (_ file _) g_List = ['a'] # modify the global variable g_Listdef ModifyDict_1 (): global g_List g_List.append ('B ') # modify the global variable g_Listdef ModifyDict_2 (): global g_List g_List.append ('C') # process a def ProcOne (num): print ('procone' + str (num) + ', g_List: '+ repr (g_List) # process all def ProcAll (): pool = Pool (processes = 4) for I in range (1, 20): # ProcOne (I) # pool. apply (ProcOne, (I,) pool. apply_async (ProcOne, (I,) pool. close () pool. join () ModifyDict_1 () # modify the global variable g_Listif _ name _ = '_ main _': ModifyDict_2 () # modify the global variable g_List print ('in main g_List: '+ repr (g_List) ProcAll ()

Running results in Windows 7:

λ python3 demo.pyIn main g_List :['a', 'b', 'c']ProcOne 1, g_List:['a', 'b']ProcOne 2, g_List:['a', 'b']ProcOne 3, g_List:['a', 'b']ProcOne 4, g_List:['a', 'b']ProcOne 5, g_List:['a', 'b']ProcOne 6, g_List:['a', 'b']ProcOne 7, g_List:['a', 'b']ProcOne 8, g_List:['a', 'b']ProcOne 9, g_List:['a', 'b']ProcOne 10, g_List:['a', 'b']ProcOne 11, g_List:['a', 'b']ProcOne 12, g_List:['a', 'b']ProcOne 13, g_List:['a', 'b']ProcOne 14, g_List:['a', 'b']ProcOne 15, g_List:['a', 'b']ProcOne 16, g_List:['a', 'b']ProcOne 17, g_List:['a', 'b']ProcOne 18, g_List:['a', 'b']ProcOne 19, g_List:['a', 'b']

Running results in Ubuntu 14.04:

In main g_List :['a', 'b', 'c']ProcOne 1, g_List:['a', 'b', 'c']ProcOne 2, g_List:['a', 'b', 'c']ProcOne 3, g_List:['a', 'b', 'c']ProcOne 5, g_List:['a', 'b', 'c']ProcOne 4, g_List:['a', 'b', 'c']ProcOne 8, g_List:['a', 'b', 'c']ProcOne 9, g_List:['a', 'b', 'c']ProcOne 7, g_List:['a', 'b', 'c']ProcOne 11, g_List:['a', 'b', 'c']ProcOne 6, g_List:['a', 'b', 'c']ProcOne 12, g_List:['a', 'b', 'c']ProcOne 13, g_List:['a', 'b', 'c']ProcOne 10, g_List:['a', 'b', 'c']ProcOne 14, g_List:['a', 'b', 'c']ProcOne 15, g_List:['a', 'b', 'c']ProcOne 16, g_List:['a', 'b', 'c']ProcOne 17, g_List:['a', 'b', 'c']ProcOne 18, g_List:['a', 'b', 'c']ProcOne 19, g_List:['a', 'b', 'c']

We can see that the second modification in Windows 7 is not successful, but the modification in Ubuntu is successful. According to the uliweb author limodou, the reason is that in Windows, the sub-process is restarted and in Linux, the fork is implemented.