Multithreading, process, and coprocessor usage

Import requests
Import re
From lxml import etree
Import OS
Import Random
Import threading
Import time
Import datetime
Import Hashlib
Import multiprocessing
From string import punctuation
From gevent Import Monkey
Import Gevent

Monkey.patch_socket ()
Class Dou:
def __init__ (self):
Self.headers = {
' Referer ': ' http://www.doutula.com/',
' User-agent ': ' mozilla/5.0 (Windows NT 10.0; Win64; x64) applewebkit/537.36 (khtml, like Gecko) chrome/66.0.3359.117 safari/537.36 '
}
def filters (Self,data):
STRs = ' '
For I in data:
If I not in punctuation:
Strs+=i
Return Strs.strip ()

def get_source (Self,url,headers=1):
If headers==1:
Source = Requests.get (url,headers=self.headers,timeout=5). Content
return source
Else
Source = Requests.get (URL, headers=headers,timeout=5). Content
return source
def make_dir (self,dir_name):
Dir_name = Self.filters (dir_name)
Dir_name = ' f:\\27\\dou\\ ' + dir_name
If Os.path.isdir (dir_name):
Print (Dir_name, ' already exists ')
Return Dir_name
Else
Os.mkdir (Dir_name)
Print (Dir_name, ' Start creation ')
Return Dir_name

def save_pic (Self,data):
Dir_name = data[0]
Pic_name= Data[1]
Pic_url= Data[2]
Pic_type= Data[3]
Try
Pic_name = Self.filters (pic_name)
Pic_path = dir_name+ ' \ \ ' +pic_name+pic_type
Pic_source = Self.get_source (Pic_url)
OP = open (Pic_path, ' WB ')
Op.write (Pic_source)
Op.close ()
Except
Pass

def begin_by_page (self,page):
Try
Source = Self.get_source (' http://www.doutula.com/article/list/?page= ' +str (page)). Decode (' UTF8 ')
Dir_name_lists = etree. HTML (source). XPath ('//*[@id = ' home ']/div/div[2]/a/div[1]/text () ')
Pic_lists = etree. HTML (source). XPath ('//*[@id = ' home ']/div/div[2]/a/@href ')
Print (len (pic_lists))
For I in range (len (pic_lists)):
Print (Dir_name_lists[i])

Dir_name = Self.make_dir (Dir_name_lists[i])

Pic_href = Pic_lists[i]
Show_source = Self.get_source (pic_href). Decode (' UTF8 ')
PIC_SRC = etree. HTML (Show_source). XPath ('/html/body/div[2]/div[1]/div/div[2]/li/div[3]/div/table/tbody/tr[1]/td/a/img/@src ')
Pic_name = etree. HTML (Show_source). XPath ('/html/body/div[2]/div[1]/div/div[2]/li/div[3]/div/table/tbody/tr[1]/td/a/img/@alt ')
# print (len (PIC_SRC), Len (pic_name), PIC_SRC)
Ths_lists = []
For II in range (Len (PIC_SRC)):
# Print (ii)
Pic_type = pic_src[ii][-4:]
Pic_names = Pic_name[ii]
Pic_url = Pic_src[ii]
# Print (Dir_name,pic_type)
Xie = Gevent.spawn (Self.save_pic, (Dir_name, Pic_names, Pic_url, Pic_type))
Xie.start ()
Ths_lists.append (Xie)
# ths = Threading. Thread (Target=self.save_pic, args= (Dir_name, Pic_names, Pic_url, Pic_type,))
# Ths.start ()
# ths_lists.append (THS)
# self.save_pic (Dir_name, Pic_names, Pic_url, Pic_type)

Print (ths_lists)
For ths in ths_lists:
Ths.join ()
Except
Print (' problem generated, problem page ', page ')

if __name__ = = ' __main__ ':
Thread_lists = []
def run ():
Dou = Dou ()
For page in range (50,81):
While True:
If Len (thread_lists) <8:
# th = Threading. Thread (target=dou.begin_by_page,args= (page,))
th = multiprocessing. Process (target=dou.begin_by_page,args= (page,))
Th.start ()
Thread_lists.append (TH)
Break
Else
Time.sleep (3)
Print (thread_lists)
Print (' Process pool is full ')
For ths in thread_lists:
If not ths.is_alive ():
Thread_lists.remove (THS)
For ths in thread_lists:
Ths.join ()

Def custom_run ():
Dou = Dou ()
For page in range (1, 11):
Dou.begin_by_page (page)


Start_time = Datetime.datetime.now ()
Run ()
Over_time = Datetime.datetime.now ()
The_time = Over_time-start_time
Print (The_time)


#多线程10页时间: 0:01:04.329867
#啥也不加10页的时间: 0:05:48.150459
#多进程10页0:01:23.375896
#多进程加线程 0:01:19.268139

This code for the crawl bucket Graph network Code, now open the process to add the association, the number of co-operation is not controlled, each save a picture to generate a co-formed, the process opened 8, because I this computer is eight core, the thread of code first commented out, during the problem, the program run completed can not automatically end, Finally find the reason is because the request is stuck, resulting in the process has been active, then the progress will not end, the entire program will not end, in the request to add a timeout, you can solve the problem

Multithreading, process, and coprocessor usage