[Windward Rain] Process + Ctrip

#-*-coding=utf-8-*-ImportRequests fromMultiprocessingImportProcessImportgevent fromGeventImportmonkey; Monkey.patch_all ()Importsys Reload (SYS) sys.setdefaultencoding ('UTF8')  defFetch (URL):Try: S=requests. Session () R= S.get (url,timeout=1)#Crawl the page here    exceptexception,e:Printereturn "'     defProcess_start (url_list): Tasks= []       forUrlinchurl_list:tasks.append (Gevent.spawn (Fetch,url)) Gevent.joinall (tasks)#use the coprocessor to perform  defTask_start (Filepath,flag = 100000):#start a process per 10W URLWith open (filepath,'R') as Reader:#reads a URL from a given fileURL =reader.readline (). Strip () Url_list= []#This list is used to hold the co-taski = 0#counter, which records how many URLs are added to the thread queue         whileurl!="': I+ = 1url_list.append (URL)#each time the URL is read out, the URL is added to the queue            ifi = = Flag:#a certain number of URLs starts a process and executesp = Process (target=process_start,args=(Url_list,)) P.start () url_list= []#Resetting the URL queuei = 0#Reset CounterURL =reader.readline (). Strip ()ifUrl_list not[]:#If there is a URL remaining in the task queue after exiting the loopp = Process (target=process_start,args= (url_list,))#Put all the remaining URLs in the last process to executeP.start ()if __name__=='__main__': Task_start ('./testdata.txt')#read the specified file

[Windward Rain] Process + Ctrip