"Reprint" Python2 crawler crawls all HTML files for a path

1 #-*-coding:utf-8-*-2 ImportRe3 ImportUrllib24 5  fromCollectionsImportdeque6 7 #to save a file's suffix8suffix='. html'9 #Regular expressions for extracting article titlesTenRex_title=r'<title> (. *?) </title>' One #extract the regular expression of the required link ARex_url=r'/jdbc/(. +?). HTML' - #The seed URL, which starts crawling from this URL -Base_url='http://www.yiibai.com/jdbc/' the  -  - #Save the retrieved text as an HTML file - defsavehtml (file_name,file_content): + #Note The Windows file naming disabled, such as/ -With Open (File_name.replace ('/','_') +suffix,"WB") as F: + #write files with bytes instead of STR, so to transcode A f.write (file_content) at #Get article title - defGetTitle (file_content): -Linkre=Re.search (rex_title,file_content) -     if(Linkre): -         Print('get the article topic:'+linkre.group (1)) -         returnLinkre.group (1) in   - #two data structures, queues and collections used by crawlers toQueue=deque () +Visited=set () - #Initialize seed link the queue.append (Base_url) *Count=0 $  Panax Notoginseng  whileQueue: -Url=queue.popleft ()#team first element out of the team theVisited |= {URL}#Mark as visited +   A     Print('already crawled:'+ STR (count) +'Crawling <---'+URL) theCount + = 1 +urlop=urllib2.urlopen (URL) - #working with HTML links only $     if 'HTML'  not inchUrlop.headers.getheader ('Content-type'): $         Continue - #Avoid program exception abort -     Try: theData=Urlop.read () -Title=gettitle (data). Decode ('Utf-8');Wuyi     #Save File the savehtml (title,data) -     except: Wu         Continue -   About #The regular expression extracts all the links in the page and determines if they have been accessed and then joins the queue to be crawled $Linkre =re.compile (Rex_url) -      forSub_linkinchLinkre.findall (data): -sub_url=base_url+sub_link+SUFFIX; - #has been visited, no longer processed A         ifSub_urlinchvisited: +             Pass the         Else: -         #Set Access $Visited |={Sub_url} the         #Join Queue the queue.append (Sub_url) the             Print('Join the quene--->'+sub_url)

Http://blog.csdn.net/wangshihui512/article/details/51100605#python

22 and 49 lines with the original source if you add the instead can not be executed may be related to the encoding

"Reprint" Python2 crawler crawls all HTML files for a path