This example analyzes the Python custom process pool. Share to everyone for your reference, specific as follows:
The code illustrates everything:
#encoding =utf-8 #author: Walker #date: 2014-05-21 #function: Custom process Pool Traverse directory file from multiprocessing import process, Queue, Lo
CK import time, Os #消费者 class Consumer (Process): Def __init__ (self, queue, Iolock): Super (Consumer, self). __init__ () Self.queue = Queue Self.iolock = Iolock def run (self): while true:task = Self.queue.get () #队列中无任务时,
will block process if isinstance (task, str) and task = = ' quit ': break; Time.sleep (1) #假定任务处理需要1秒钟 Self.ioLock.acquire () print (str (os.getpid ()) + ' + task ' Self.ioLock.rele ASE () Self.ioLock.acquire () print ' Bye-bye ' Self.ioLock.release () #生产者 def Producer (): queue = queue () #这个队 The column is process/thread-Safe Iolock = Lock () Subnum = 4 #子进程数量 workers = Build_worker_pool (queue, Iolock, subnum) start_time = time
. Time () to parent, dirnames, filenames in Os.walk (R ' D:\test '): for filename in filenames:queue.put (filename)
Iolock.acquire () print (' qsize: ' + str (queue.qsize ())) Iolock.release () while queue.qsize () > Subnum *: #控制队列中任务数量 time.sleep (1) for worker in workers : Queue.put (' quit ') for worker in Workers:worker.join () Iolock.acquire () print (' done!
Time taken: {} '. Format (Time.time ()-start_time)) Iolock.release () #创建进程池 def build_worker_pool (queue, Iolock, size):
Workers = [] for _ in range (size): worker = Consumer (queue, Iolock) Worker.start () workers.append (worker)
Return workers if __name__ = = ' __main__ ': Producer ()
Ps:
Self.ioLock.acquire () ...
Self.ioLock.release ()
Available:
Alternative.
One More fun example:
#encoding =utf-8
#author: Walker
#date: 2016-01-06
#function: A fun example of multiple processes
import OS, sys,
time From multiprocessing import Pool
Cur_dir_fullpath = Os.path.dirname (Os.path.abspath (__file__))
g_list = [' A ']
#修改全局变量g_List
def modifydict_1 ():
global g_list
g_list.append (' B ')
#修改全局变量g_List
Def modifydict_2 ():
global g_list
g_list.append (' C ')
#处理一个
def procone (num):
print (' Procone ' + str (num) + ', g_list: ' + repr (g_list))
#处理所有
def procall ():
pool = pool (processes = 4)
for I in range (1):
#ProcOne (i)
#pool. Apply (Procone, (i))
Pool.apply_async (Procone
, (i)) Pool.close ()
pool.join modifydict_1 ()
#修改全局变量g_List
if __name__ = = ' __main__ ':
modifydict_2 ( ) #修改全局变量g_List
print (' in main g_list: ' + repr (g_list))
Procall ()
Results of running under Windows7:
Λpython3 demo.py in
main g_list: [' A ', ' B ', ' C ']
procone 1, g_list:[' A ', ' B ']
procone 2, g_list:[' A ', ' B ']
procone 3, g_list:[' A ', ' B ']
procone 4, g_list:[' A ', ' B ']
procone 5, g_list:[' A ', ' B ']
procone 6, g_list:[' A ', ' B ']
procone 7, g_list:[' A ', ' B ']
Procone 8, g_list:[' A ', ' B ']
procone 9, g_list:[' A ', ' B ']
Procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
Procone, G_li St:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']< C18/>procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
procone, g_list:[' A ', ' B ']
Results of running under Ubuntu 14.04:
In main g_list: [' A ', ' B ', ' C ']
procone 1, g_list:[' A ', ' B ', ' C ']
Procone 2, g_list:[' A ', ' B ', ' C ']
Procone 3, G_list:[' A ', ' B ', ' C ']
procone 5, g_list:[' A ', ' B ', ' C ']
procone 4, g_list:[' A ', ' B ', ' C ']
Procone 8, G_list: [' A ', ' B ', ' C ']
Procone 9, g_list:[' A ', ' B ', ' C ']
procone 7, g_list:[' A ', ' B ', ' C ']
procone One, g_list:[' a ', ' B ', ' C ']
Procon E 6, g_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone 1 0, g_list:[' a ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone 16, G_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone, g_list:[' A ', ' B ', ' C ']
Procone, G _list:[' A ', ' B ', ' C '
You can see that the second modification was unsuccessful under Windows7, and Ubuntu was successfully modified. According to Uliweb author Limodou, the reason is that Windows is a fully restarted implementation of the child process, Linux is fork implementation.
More information about Python-related content can be viewed in this site: "Python URL Operation tips Summary", "Python Picture Operation tips Summary", "Python data structure and algorithm tutorial", "Python Socket Programming Skills Summary", " Python function Usage Tips Summary, python string manipulation tips, Python introductory and Advanced classic tutorials, and Python file and directory how-to tips
I hope this article will help you with Python programming.