Python3 multi-threaded, multi-process

A thread in Python is a dummy thread, and switching between different threads is resource-intensive because the context of the stored thread is required, and the constant switchover consumes resources.

Python multithreading is suitable for IO-intensive tasks (such as the socket Server network concurrency);
Python multithreading is not suitable for CPU-intensive tasks, mainly CPU-based, such as a large number of mathematical calculations.
So if there are CPU-intensive tasks to do, you can do it through multiple processes (not multithreading).
If the CPU has 8 cores, each core CPU can use 1 processes, each process can be calculated with 1 threads.
There is no need to use the Gil lock between processes because the process is independent and does not share data.
The process can be many, but the 8-core CPU can only operate on 8 tasks at a time.

Multi-process

#测试多进程import Multiprocessingimport Timedef Run (name):    time.sleep (2)    print (' Heelo ', name) if __name__ = = ' __ Main__ ': For    i in range: #起了10个进程        p = multiprocessing. Process (target=run,args= (' msc%s '%i,))        P.start () execution result: Hello Msc1hello msc0hello msc2hello msc3hello Msc5hello Msc4hello Msc6hello Msc7hello Msc8hello msc9

import multiprocessingimport time,threadingdef thread_run (): Print ( Threading.get_ident ()) #get_ident获取当前线程iddef run (name): Time.sleep (2) print (' Hello ', name) T = Threading.        Thread (Target=thread_run,) #在每个进程中又起了1个线程 T.start () if __name__ = = ' __main__ ': For I in range: #起了10个进程 p = multiprocessing. Process (target=run,args= (' msc%s '%i,)) P.start () execution result: Hello Msc013996hello msc214208hello msc113964hello msc314012he Llo Msc615192hello Msc715136hello msc87036hello msc412344hello Msc915332hello msc513616 

From multiprocessing import processimport osdef info (title):    Print (    ' module name: ', __name__)    print (' Parent process: ', Os.getppid ())  #获取父进程的id    print (' Process ID: ', os.getpid ())   #获取自身的id    Print ("\ n") def f (name):    info (' \033[31;1mfunction f\033[0m ')    print (' Hello ', name) if __name__ = = ' __main__ ':    info (' \033[32;1mmain process line\033[0m ')  # #直接调用函数    # p = Process (target=f, args= (' Bob ',))    # P.start ()    # P.join () Execution result: main process linemodule name: __main__parent process:7172   #父进程就是python
Process id:14880       #这个子进程就是python的代码程序

##每个进程都会有一个父进程。

From multiprocessing import processimport osdef info (title):    Print (    ' module name: ', __name__)    print (' Parent process: ', Os.getppid ())  #获取父进程的id    print (' Process ID: ', os.getpid ())   #获取自身的id    Print ("\ n") def f (name):    info (' \033[31;1mcalled from child process function f\033[0m ')    print (' Hello ', name) if __name__ = = ' __main__ ':    info (' \033[32;1mmain process line\033[0m ')    p = Process (target=f, args= (' MSc ',))    #设置子进程    p.start ()   #启动子进程    # p.join () Execution result: main process linemodule Name: __main__parent process: 1136    #主进程pycharmprocess id:14684       #子进程python代码called from child process function Fmodule name: __mp_main__ Parent process:14684   #主进程python代码 (Child of 1136) process id:15884 sub-       15884## of child processes in #python代码 (main process 14684) Each process has a master process (parent process) Hello MSC

Inter-process communication

The data between the default processes is not shared, and if it is certain that the reciprocal visits can be achieved through the queue, the queue is used in the same way as the queue in the thread, but the queue in the thread can only be used between threads.

 thread import Queueimport threadingdef f (): Q.put ([42,none, ' Heelo ']) if __name__ = = ' __main__ ': q = queue. Queue () p = Threading. Thread (target=f,) P.start () print (Q.get ()) P.join () execution Result: [None, ' Heelo ']## put the data through a child thread and then get out of the content on the main thread, indicating that the data between the threads can be shared. 

import queuefrom multiprocessing import processdef F (): Q.put ([66,None, ' Hello ']) #这里的q属于主进程if __name__ = = ' __main__ ': q = queue.    The Queue () #主进程起的q p = Process (Target=f,) # # Defines the child process in the main process, and if the child process is started in the main process, the memory between the main and child processes is independent.    # # Because of memory independence, the child process P is unable to access Q in the main process Def f (). P.start () print (Q.get ()) P.join () execution Result: Process Process-1:traceback (most recent call last): File "C:\Python35\lib\m  ultiprocessing\process.py ", line 249, in _bootstrap Self.run () File" C:\Python35\lib\multiprocessing\process.py ", line     Self._target, in Run (*self._args, **self._kwargs) File "D:\Program Files (x86) \python\day31\test.py", line 7, in F Q.put ([66,none, ' hello ']) #这里的q属于主进程NameError: Name ' Q ' is not defined# #可以看到已经报错 because the child process cannot access the main process's Q 

Import queuefrom multiprocessing import processdef F (QQ): Qq.put ([66,none, ' hello ']) if __name__ = = ' __main__ ': q = qu Eue. Queue () p = Process (target=f,args= (q,)) #将父进程q传给子进程 P.start () print (Q.get ()) P.join () execution Result: Traceback (most rec ENT call last): File "D:/program Files (x86)/python/day31/test.py", Line <module> p.start () file "C:\Pyt hon35\lib\multiprocessing\process.py ", line, in Start Self._popen = Self._popen (self) File" C:\Python35\lib\multip rocessing\context.py ", line 212, in _popen return _default_context.get_context (). Process._popen (process_obj) File "C:\Python35\lib\multiprocessing\context.py", line 313, _popen return Popen (proces s_obj) File "C:\Python35\lib\multiprocessing\popen_spawn_win32.py", line $, in __init__ reduction.dump (process_obj, t O_child) File "C:\Python35\lib\multiprocessing\reduction.py", line-a, in dump Forkingpickler (file, protocol). Dump (obj ) Typeerror:can ' t pickle _thread.lock objects## This is because we will thread theQ to another process, which is not possible, the thread belongs only to the current process and cannot be passed to other processes. # # If you want to pass Q to the child process, you must pass the process Q in instead of thread Q.

from multiprocessing import process,queue# #大写的Queue是进程队列; queue is thread queuing # # Uppercase queue needs to import def f (QQ) from multiprocessing: Qq.put ([66,none, ' hello ']) if __name__ = = ' __main__ ': q = Queue () p = proces S (target=f,args= (q,)) #将父进程q传给子进程 P.start () print (Q.get ()) #父进程去get子进程的内容 p.join () execution Result: [All, None, ' Hello ']# #父进程可 The contents of the get sub-process are put in, and on the surface it feels like two processes share the data. "
 has now implemented inter-process communication. The parent process will pass Q to the child process, in fact, a copy of the Q to the child process, at this time the child process more than a Q process queue, 
 But why the parent process can get the child process put in the data, because the current two processes in the memory space is still independent, but the child process put the data The pickle serialization is placed in an intermediate position in memory, and the parent process takes the data from that middle position (not the data taken from the child process). 
 Therefore, interprocess communication is not a shared data, but a passing of data. 
 ' 

Data between processes can also be piped to the from multiprocessing import process, Pipedef F (conn):    conn.send ([All, None, ' Hello from Child1 '] )  #发送数据给parent_conn    conn.close ()    #发完数据需要关闭if __name__ = = ' __main__ ':    parent_conn, child_conn = Pipe (    # # # Build pipeline. The build generates two return objects, which correspond to the phones on both ends and are connected through a pipe line.    # # Two objects were handed to two variables.    p = Process (Target=f, args= (Child_conn,))   #child_conn需要传给对端, for send data to Parent_conn    P.start ()    Print (PARENT_CONN.RECV ())  #parent_conn在这端, used for recv data    p.join () execution results: [All, None, ' Hello from Child1 ']

From multiprocessing import Process, Pipedef F (conn):    conn.send ([All, None, ' Hello from Child1 '])    conn.send ([ , None, ' Hello from Child2 '])  #发送两次数据    conn.close () if __name__ = = ' __main__ ':    parent_conn, child_conn = Pipe ()    p = Process (Target=f, args= (Child_conn,))    P.start ()    print (PARENT_CONN.RECV ())      P.join () Execution result: [None, ' Hello from child1 ']## can see that the end only received a single data

from multiprocessing import Process, Pipedef F (conn): Conn.send ([, None, ' Hel Lo from Child1 ']) conn.send ([All, None, ' Hello from Child2 ']) conn.close () if __name__ = = ' __main__ ': parent_conn , Child_conn = Pipe () p = Process (Target=f, args= (Child_conn,)) P.start () print (PARENT_CONN.RECV ()) Print (Pare NT_CONN.RECV ()) #第二次接收数据 p.join () execution Result: [None, ' Hello from Child1 '][66, none, ' Hello from child2 ']# #对端发送几次, this end will need to receive a few Times 

from multiprocessing import Process, Pipedef F (conn): Conn.send ([, None, ' Hel Lo from Child1 ']) conn.send ([All, None, ' Hello from Child2 ']) #发送两次数据 conn.close () if __name__ = = ' __main__ ': pare Nt_conn, Child_conn = Pipe () p = Process (Target=f, args= (Child_conn,)) P.start () print (PARENT_CONN.RECV ()) PRI NT (PARENT_CONN.RECV ()) print (PARENT_CONN.RECV ()) #对端发送两次, this paragraph receives three p.join () execution results: [Five, None, ' Hello from Child1 '][66, N One, ' Hello from child2 ']## program card Master, unless the peer sends the data at once. 

From multiprocessing import Process, Pipedef F (conn):    conn.send ([All, None, ' Hello from Child1 '])    conn.send ([66, None, ' Hello from Child2 '])  #发送两次数据    print (CONN.RECV ()) #接收数据    conn.close () if __name__ = = ' __main__ ':    Parent_conn, Child_conn = Pipe ()    p = Process (Target=f, args= (Child_conn,))    P.start ()    print (Parent_ CONN.RECV ())    print (PARENT_CONN.RECV ())    parent_conn.send ("Data from Parent_conn")   #发送数据    P.join () execution Result: [Child1, none, ' Hello from '][66, none, ' Hello from child2 ']data from parent_conn# #通过管道实现了相互发送接收数据 (data transfer implemented)

Inter-process data interaction and sharing

From multiprocessing import Process, Managerimport osdef f (d, L, N): d[1] = ' 1 ' #放入key和value到空字典中 d[' 2 '] = 2 d[0    . [] = None l.append (n) #将每个进程的n值放入列表中; n values are different for each process.  Print (l) if __name__ = = ' __main__ ': With manager () as manager: #做一个别名 when the manager is equivalent to manager () d = manager.dict ()        #生成一个可在多个进程之间传递和共享的字典 L = manager.list (range (5)) #生成一个可在多个进程之间传递和共享的列表; Generates 5 data in a list by range (5) p_list = [] For I in range: #生成10个进程 p = Process (Target=f, args= (d, L, i)) #将字典和列表传给每个进程, each process can be modified p. Start () P_list.append (P) # put each process into an empty list for res in P_list:res.join () print (d) #所 After the process has finished printing the dictionary print (L) #所有进程都执行完毕后打印列表执行结果: #列表生成的时候自动加入了0-4 of these 5 numbers; then each process adds its respective n values to the list [0, 1, 2, 3, 4, 2][0, 1, 2, 3, 4, 2, 3][0, 1, 2, 3, 4, 2, 3, 4][0, 1, 2, 3, 4, 2, 3, 4, 1][0, 1, 2, 3, 4, 2, 3, 4, 1, 0][0, 1, 2, 3, 4, 2, 3, 4 [0, 1, 2, 3, 4, 2, 3, 4, 1, 0, 5, 6] [0, 1, 2, 3, 4, 2, 3, 4, 1, 0, 5, 6, 7] [0, 1, 2, 3, 4, 2, 3, 4, 1, 0, 5, 6, 7, 8][0, 1, 2, 3, 4, 2, 3, 4, 1, 0, 5, 6, 7, 8, 9] #第十个进程把每个进程添加的n值都加入到列表 {0.25:none, 1: ' 1 ', ' 2 ': 2} #最后打印的字典 [0, 1, 2, 3, 4, 2, 3, 4, 1, 0, 5, 6, 7, 8, 9] #最后打印的列表Process finished with exit code 0

From multiprocessing import Process, Managerimport osdef f (d, L): d[os.getpid ()] = Os.getpid () l.append (Os.getpid ())        Print (l) if __name__ = = ' __main__ ': With Manager () as Manager:d = Manager.dict () #对字典做个调整, PID is also added to the dictionary            L = manager.list (range (5)) p_list = [] for i in range: p = Process (Target=f, args= (d, L)) P.start () P_list.append (p) for RES in P_list:res.join () print (d) PRI  NT (L) execution results: [0, 1, 2, 3, 4, 2240][0, 1, 2, 3, 4, 2240, 10152][0, 1, 2, 3, 4, 2240, 10152, 10408][0, 1, 2, 3, 4, 2240, 10152, 10408, 6312][0, 1, 2, 3, 4, 2240, 10152, 10408, 6312, 17156][0, 1, 2, 3, 4, 2240, 10152, 10408, 6312, 17156, 6184][0, 1, 2, 3, 4, 2240, 10152, 10408, 6312, 17156, 6184, 16168][0, 1, 2, 3, 4, 2240, 10152, 10408, 6312, 17156, 6184, 16168, 11384] [0, 1, 2, 3, 4, 2240, 10152, 10408, 6312, 17156, 6184, 16168, 11384, 15976] [0, 1, 2, 3, 4, 2240, 10152, 10408, 6312, 17156, 6184, 16168, 11384, 15976, 16532]{2240:2240, 10,152:10,152, 10,408:10,408, 6,312:6,312, 17,156:17,156, 6,184:6,184, 16,168:16,168, 11,384:11,384, 1597 6:15976, 16532:16532}[0, 1, 2, 3, 4, 2240, 10152, 10408, 6312, 17156, 6184, 16168, 11384, 15976, 16532]# #现在我们看到可以实现进程间的数 Sharing, modification, and delivery. # #Manager () comes with a lock that controls the process of modifying data between processes at the same time; # #字典和列表的数据不是一份, but because of 10 processes, there are 10 dictionaries and 10 lists. Once each process is modified, it is copied to other processes, and other processes can modify the latest data so that the data is not altered.

Process synchronization

There's a lock inside the process.

From multiprocessing import Process, Lock   #从multiprocessing导入Lock这个锁def f (L, i):    l.acquire ()     #获取修改数据的锁    print (' Hello World ', i)    l.release ()     #释放锁if __name__ = = ' __main__ ':    lock = Lock ()   #实例锁 for    num In range:   #生成10个进程        Process (target=f, args= (lock, Num)). Start () #执行子进程并传入参数给子进程执行结果: Hello World 1hello World 4hello World 0hello World 3hello World 2hello World 5hello World 6hello World 8hello World 7hello World 9## can see altogether 10 Process is not sequential, indicating which process is not to be executed first. ‘‘‘
The data between processes is independent, why do we have to lock again, this is because all processes use the same screen to output data;
For example, we now output the data is Hello World x, in the process of output it is very likely that one process has not finished output (such as only the output of Hello wo), the other Process execution output (may be seen on the screen Hello Wohello world0201 phenomenon).
So it takes a lock to control that only one process output data to the screen at a time.
‘‘‘

Process Pool

Performing a multi-process, the child process will copy a complete data from the main process, one, 10 processes may not feel, but if there are 100 or 1000, or even more of the time the overhead is particularly large, it will be obvious that the multi-process implementation of the lag phenomenon.

A process pool can set how many processes can run on the CPU at the same time.

From multiprocessing import Process, pool# imported from multiprocessing poolimport time,osdef Foo (i): Time.sleep (2) print ("in P Rocess ", Os.getpid ()) #打印进程id return i + 100def Bar (ARG): print ('-->exec done: ', arg) if __name__ = = ' __main__ ': #  #这行代码用途是如果主动执行该代码的 the. py file, the code below the code can be executed, if the. PY module is imported into another module, and the. PY module is executed from another module, the code below the line will not be executed.    There are times when you can use this method for testing, and write some test code under that line of code: Pool = Pool (5) #同时只能放入5个进程 for me in range: #创建10个进程, but because of the pool limit, only 5 processes that are placed in the process pool are executed (), others are suspended, and if two of the processes in the process pool are executed, the        In 2 processes. # Pool.apply_async (Func=foo, args= (i,), Callback=bar) pool.apply (Func=foo, args= (i,)) #pool. Apply to put the process into the pool PRI NT (' End ') #执行完毕 Pool.close () #允许pool中的进程关闭 (Close must be in front of the join, you can understand that close is equivalent to a switch bar) Pool.join () # Before the process in the process pool finishes executing and then close, if the note Then the program shuts down directly. Execution result: In process 13616in process 10912in process 12472in process 15180in process 12404in process 13616in Process 10912in PR Ocess 12472in Process 15180in process 12404end# #可以看到通过串行的方式将结果打印出来, this is because we are using pool.apply. The pool.apply is executed serially.

from multiprocessing import Process, Poolimport time,osdef Foo (i): Time.sleep ( 2) Print ("In Process", os.getpid ()) return i + 100def Bar (ARG): Print ('-->exec-Done: ', arg) if __name__ = = ' __ma In__ ': Pool = Pool (5) for I in range: Pool.apply_async (Func=foo, args= (i)) # # # Use Pool.apply_async can parallel the print (' End ') pool.close () # pool.join () comment out the execution result: end## only executed the print (' End ') code, the result of the other process is not seen, because the other process has not completed, the main process P Ool.close () is done, and all other processes are not executed after close. # # to be closed after other processes have finished executing, you must use Pool.join () 

From  multiprocessing import process, Poolimport time,osdef Foo (i):    time.sleep (2)    print ("In Process", Os.getpid ())    return i + 100def Bar (ARG):    print ('-->exec done: ', arg) if __name__ = = ' __main__ ':    pool = Pool (5)    For I in range:        pool.apply_async (Func=foo, args= (i,))    print (' End ')    pool.close ()    pool.join ()  Execution Result: Endin process 14756in Process 14596in process 10836in process 12536in process 12904in process 14756in process 14596in Process 10836in Process 12536in process 12904# #从执行结果来看, 5 5 are printed out.

Callback

From multiprocessing import process, Poolimport time,osdef Foo (i): Time.sleep (2) print ("In Process", Os.getpid ())    return i + 100def Bar (ARG): print ('-->exec done: ', arg,os.getpid ()) If __name__ = = ' __main__ ': Pool = Pool (5)        Print ("Main process:", Os.getpid ()) #打印主进程id for I in range: Pool.apply_async (Func=foo, args= (i,), Callback=bar)        # #callback叫做回调, the Callback=bar executes after the Func=foo is executed (callbacks are executed after each process has finished executing).        The # # callback can be used to do some follow-up when the code is finished, such as after the command has been viewed, backed up by a callback, or after performing any action, log, etc. # # backups, write logs, etc. can also be performed in sub-processes, but why use callbacks!        This is because if you use a child process, there are 10 child processes to connect to the database 10 times, and the use of the callback is the main process to connect to the database, so only connect once, so write can greatly improve operational efficiency.    # #通过主进程建立数据库的连接的话, because only one connection can be made to the database in the same process, the connection is not repeated even if the process is callback multiple times, because the database restricts the maximum number of connections to the same process, which is set by the database. Print (' End ') pool.close () Pool.join () execution Result: main process: 14340 #主进程是 14340endin process 13936-->exec done:100 14340 #可 To see that the callback is called by the main thread in process 15348-->exec done:101 14340in process 10160-->exec done:102 14340in process 11612-->ex EC done:103 14340in Process 14836-->exec done:104 14340in Process 13936-->exec done:105 14340in process 15348-->exec done:106 14340in process 10　　 160-->exec done:107 14340in Process 11612-->exec done:108 14340in process 14836-->exec done:109 14340

Article according to Code Veterans share blog, a little bit out, multithreading and process let me headache for three days, thank the great God sharing experience, let me less detours.

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

　　

Python3 multi-threaded, multi-process