Python custom Process Pool instance analysis "producer, consumer model problem" _python

This example analyzes the Python custom process pool. Share to everyone for your reference, specific as follows:

The code illustrates everything:

#encoding =utf-8 #author: Walker #date: 2014-05-21 #function: Custom process Pool Traverse directory file from multiprocessing import process, Queue, Lo 
    CK import time, Os #消费者 class Consumer (Process): Def __init__ (self, queue, Iolock): Super (Consumer, self). __init__ () Self.queue = Queue Self.iolock = Iolock def run (self): while true:task = Self.queue.get () #队列中无任务时,
      will block process if isinstance (task, str) and task = = ' quit ': break; Time.sleep (1) #假定任务处理需要1秒钟 Self.ioLock.acquire () print (str (os.getpid ()) + ' + task ' Self.ioLock.rele ASE () Self.ioLock.acquire () print ' Bye-bye ' Self.ioLock.release () #生产者 def Producer (): queue = queue () #这个队 The column is process/thread-Safe Iolock = Lock () Subnum = 4 #子进程数量 workers = Build_worker_pool (queue, Iolock, subnum) start_time = time 
      . Time () to parent, dirnames, filenames in Os.walk (R ' D:\test '): for filename in filenames:queue.put (filename)
 Iolock.acquire () print (' qsize: ' + str (queue.qsize ()))     Iolock.release () while queue.qsize () > Subnum *: #控制队列中任务数量 time.sleep (1) for worker in workers : Queue.put (' quit ') for worker in Workers:worker.join () Iolock.acquire () print (' done!
  Time taken: {} '. Format (Time.time ()-start_time)) Iolock.release () #创建进程池 def build_worker_pool (queue, Iolock, size):
  Workers = [] for _ in range (size): worker = Consumer (queue, Iolock) Worker.start () workers.append (worker)

 Return workers if __name__ = = ' __main__ ': Producer ()

Ps:

Self.ioLock.acquire () ...
Self.ioLock.release ()

Available:

With Self.iolock: ...
  

Alternative.

One More fun example:

#encoding =utf-8
#author: Walker
#date: 2016-01-06
#function: A fun example of multiple processes
import OS, sys,
time From multiprocessing import Pool
Cur_dir_fullpath = Os.path.dirname (Os.path.abspath (__file__))
g_list = [' A ']
#修改全局变量g_List
def modifydict_1 ():
  global g_list
  g_list.append (' B ')
#修改全局变量g_List
Def modifydict_2 ():
  global g_list
  g_list.append (' C ')
#处理一个
def procone (num):
  print ('  Procone ' + str (num) + ', g_list: ' + repr (g_list))
#处理所有
def procall ():
  pool = pool (processes = 4)
  for I in range (1):
    #ProcOne (i)
    #pool. Apply (Procone, (i))
    Pool.apply_async (Procone
  , (i)) Pool.close ()
  pool.join modifydict_1 ()
#修改全局变量g_List
if __name__ = = ' __main__ ':
  modifydict_2 ( ) #修改全局变量g_List
  print (' in main g_list: ' + repr (g_list))
  Procall ()

Results of running under Windows7:

Λpython3 demo.py in
main g_list: [' A ', ' B ', ' C ']
procone 1, g_list:[' A ', ' B ']
procone 2, g_list:[' A ', ' B ']
   procone 3, g_list:[' A ', ' B ']
procone 4, g_list:[' A ', ' B ']
procone 5, g_list:[' A ', ' B ']
procone 6, g_list:[' A ', ' B ']
procone 7, g_list:[' A ', ' B ']
Procone 8, g_list:[' A ', ' B ']
procone 9, g_list:[' A ', ' B ']
Procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
Procone, G_li St:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']< C18/>procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']

Results of running under Ubuntu 14.04:

In main g_list: [' A ', ' B ', ' C ']
procone 1, g_list:[' A ', ' B ', ' C ']
Procone 2, g_list:[' A ', ' B ', ' C ']
Procone 3, G_list:[' A ', ' B ', ' C ']
procone 5, g_list:[' A ', ' B ', ' C ']
procone 4, g_list:[' A ', ' B ', ' C ']
Procone 8, G_list: [' A ', ' B ', ' C ']
Procone 9, g_list:[' A ', ' B ', ' C ']
procone 7, g_list:[' A ', ' B ', ' C ']
procone One, g_list:[' a ', ' B ', ' C ']
Procon E 6, g_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone 1  0, g_list:[' a ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone 16, G_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone, G _list:[' A ', ' B ', ' C '

You can see that the second modification was unsuccessful under Windows7, and Ubuntu was successfully modified. According to Uliweb author Limodou, the reason is that Windows is a fully restarted implementation of the child process, Linux is fork implementation.

More information about Python-related content can be viewed in this site: "Python URL Operation tips Summary", "Python Picture Operation tips Summary", "Python data structure and algorithm tutorial", "Python Socket Programming Skills Summary", " Python function Usage Tips Summary, python string manipulation tips, Python introductory and Advanced classic tutorials, and Python file and directory how-to tips

I hope this article will help you with Python programming.