Multithreading, process, and coprocessor usage

Source: Internet
Author: User

Import requests
Import re
From lxml import etree
Import OS
Import Random
Import threading
Import time
Import datetime
Import Hashlib
Import multiprocessing
From string import punctuation
From gevent Import Monkey
Import Gevent

Monkey.patch_socket ()
Class Dou:
def __init__ (self):
Self.headers = {
' Referer ': ' http://www.doutula.com/',
' User-agent ': ' mozilla/5.0 (Windows NT 10.0; Win64; x64) applewebkit/537.36 (khtml, like Gecko) chrome/66.0.3359.117 safari/537.36 '
}
def filters (Self,data):
STRs = ' '
For I in data:
If I not in punctuation:
Strs+=i
Return Strs.strip ()

def get_source (Self,url,headers=1):
If headers==1:
Source = Requests.get (url,headers=self.headers,timeout=5). Content
return source
Else
Source = Requests.get (URL, headers=headers,timeout=5). Content
return source
def make_dir (self,dir_name):
Dir_name = Self.filters (dir_name)
Dir_name = ' f:\\27\\dou\\ ' + dir_name
If Os.path.isdir (dir_name):
Print (Dir_name, ' already exists ')
Return Dir_name
Else
Os.mkdir (Dir_name)
Print (Dir_name, ' Start creation ')
Return Dir_name

def save_pic (Self,data):
Dir_name = data[0]
Pic_name= Data[1]
Pic_url= Data[2]
Pic_type= Data[3]
Try
Pic_name = Self.filters (pic_name)
Pic_path = dir_name+ ' \ \ ' +pic_name+pic_type
Pic_source = Self.get_source (Pic_url)
OP = open (Pic_path, ' WB ')
Op.write (Pic_source)
Op.close ()
Except
Pass

def begin_by_page (self,page):
Try
Source = Self.get_source (' http://www.doutula.com/article/list/?page= ' +str (page)). Decode (' UTF8 ')
Dir_name_lists = etree. HTML (source). XPath ('//*[@id = ' home ']/div/div[2]/a/div[1]/text () ')
Pic_lists = etree. HTML (source). XPath ('//*[@id = ' home ']/div/div[2]/a/@href ')
Print (len (pic_lists))
For I in range (len (pic_lists)):
Print (Dir_name_lists[i])

Dir_name = Self.make_dir (Dir_name_lists[i])

Pic_href = Pic_lists[i]
Show_source = Self.get_source (pic_href). Decode (' UTF8 ')
PIC_SRC = etree. HTML (Show_source). XPath ('/html/body/div[2]/div[1]/div/div[2]/li/div[3]/div/table/tbody/tr[1]/td/a/img/@src ')
Pic_name = etree. HTML (Show_source). XPath ('/html/body/div[2]/div[1]/div/div[2]/li/div[3]/div/table/tbody/tr[1]/td/a/img/@alt ')
# print (len (PIC_SRC), Len (pic_name), PIC_SRC)
Ths_lists = []
For II in range (Len (PIC_SRC)):
# Print (ii)
Pic_type = pic_src[ii][-4:]
Pic_names = Pic_name[ii]
Pic_url = Pic_src[ii]
# Print (Dir_name,pic_type)
Xie = Gevent.spawn (Self.save_pic, (Dir_name, Pic_names, Pic_url, Pic_type))
Xie.start ()
Ths_lists.append (Xie)
# ths = Threading. Thread (Target=self.save_pic, args= (Dir_name, Pic_names, Pic_url, Pic_type,))
# Ths.start ()
# ths_lists.append (THS)
# self.save_pic (Dir_name, Pic_names, Pic_url, Pic_type)

Print (ths_lists)
For ths in ths_lists:
Ths.join ()
Except
Print (' problem generated, problem page ', page ')

if __name__ = = ' __main__ ':
Thread_lists = []
def run ():
Dou = Dou ()
For page in range (50,81):
While True:
If Len (thread_lists) <8:
# th = Threading. Thread (target=dou.begin_by_page,args= (page,))
th = multiprocessing. Process (target=dou.begin_by_page,args= (page,))
Th.start ()
Thread_lists.append (TH)
Break
Else
Time.sleep (3)
Print (thread_lists)
Print (' Process pool is full ')
For ths in thread_lists:
If not ths.is_alive ():
Thread_lists.remove (THS)
For ths in thread_lists:
Ths.join ()

Def custom_run ():
Dou = Dou ()
For page in range (1, 11):
Dou.begin_by_page (page)


Start_time = Datetime.datetime.now ()
Run ()
Over_time = Datetime.datetime.now ()
The_time = Over_time-start_time
Print (The_time)


#多线程10页时间: 0:01:04.329867
#啥也不加10页的时间: 0:05:48.150459
#多进程10页0:01:23.375896
#多进程加线程 0:01:19.268139

This code for the crawl bucket Graph network Code, now open the process to add the association, the number of co-operation is not controlled, each save a picture to generate a co-formed, the process opened 8, because I this computer is eight core, the thread of code first commented out, during the problem, the program run completed can not automatically end, Finally find the reason is because the request is stuck, resulting in the process has been active, then the progress will not end, the entire program will not end, in the request to add a timeout, you can solve the problem

Multithreading, process, and coprocessor usage

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.