Python crawl Baidu URL

1 #!/usr/bin/env python2 #-*-coding:utf-8-*-3 #@Date: 2017-08-29 18:38:234 #@Author: Enderzhou ([email protected])5 #@Link: http://www.cnblogs.com/enderzhou/6 #@Version: $Id $7 8 ImportRequests9 ImportSYSTen  fromQueueImportQueue One ImportThreading A  fromBs4ImportBeautifulSoup as BS - ImportRe -  the #default crawl Baidu 76 page search result URL, call format Python.exe this file name. PY search keywords, such as keywords with special symbols enclosed in quotation marks.  - #The crawl result has a TXT document output. At present, Baidu has not been able to promote the link, the follow-up may be improved. In addition, the same Web site will be added to the same path does not pass the parameter URL filtering.  - #https://www.baidu.com/s?wd=ichunqiu&pn=10 - #WD parameter for search content keyword PN parameter control page The second page is 10 new 10 The maximum page parameter is 750 or 76 pages.  +  -headers = {'user-agent':'mozilla/5.0 (Windows NT 10.0; WOW64) applewebkit/537.36 (khtml, like Gecko) chrome/60.0.3112.78 safari/537.36',} +  A classBaiduspider (Threading. Thread): at     def __init__(self,queue): -Threading. Thread.__init__(self) -Self._queue =Queue -  -     defRun (self): -          while  notself._queue.empty (): inURL =Self._queue.get () -              Try: to self.spider (URL) +              exceptException as E: -                  #Print e the                  Pass *          $     defSpider (self,url):Panax NotoginsengR = Requests.get (url=url,headers=headers) -Soup = BS (r.content,'Html.parser') theUrllist = Soup.find_all (name='a', attrs={'Data-click': Re.compile (('.')),'class': None,'Data-is-main-url': None}) +          forIinchurllist: AL = Requests.get (url=i['href'],headers=headers) the             ifL.status_code = = 200: +ll = L.url.split ('/') -LLL = ll[0]+'//'+ll[2]+'\ n' $                 #can change whether the primary domain name is displayed as required $Sys.stdout.write (lll+l.url+'\ n') -F1 = open ('Out_para.txt','A +') -F1.write (l.url+'\ n') the f1.close () -With open ('Out_index.txt') as F:Wuyi                     iflll not inchf.read (): theF2 = open ('Out_index.txt','A +') - f2.write (LLL) Wu f2.close () -  About defMain (keyword): $Queue =Queue () -      forIinchRange (0,760,10): -L ='https://www.baidu.com/s?wd='+keyword+'&pn='+Str (i) -         #Print L A queue.put (L) +Threads = [] theThread_count = 5 -      forIinchRange (thread_count): $ threads.append (Baiduspider (queue)) the      forTinchThreads: the T.start () the      forTinchThreads: the T.join () -  in if __name__=='__main__': the     ifLen (SYS.ARGV)! = 2: the         Print 'Enter:python%s keyword'%Sys.argv[0] AboutSys.exit (-1) the     Else: theF1 = open ('Out_para.txt','W') the f1.close () +F2 = open ('Out_index.txt','W') - f2.close () theMain (sys.argv[1])

Python crawl Baidu URL