Python's path-a reptile example

urlcontroller.py

Import bscontrollerfrom urllib Import Requestclass Spidermain (object): Def __init__ (self): Self.header = {' user- Agent ': ' mozilla/5.0 (X11; Linux x86_64) applewebkit/537.11 (khtml, like Gecko) chrome/23.0.1271.64 safari/537.11 ', ' Accept ': ' text/htm l,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 ', ' accept-charset ': ' iso-8859-1,utf-8;q=0.7,*;q=0.3 ' , ' accept-encoding ': ' None ', ' accept-language ': ' en-us,en;q=0.8 ', ' Connection ':            ' Keep-alive '} self.bsmanage = Bscontroller.bsmanage () def getUrl (Self,rooturl): For I in range (1,500): url = rooturl+ '%s '%i+ '. html ' req = Request.              Request (URL) for h in Self.header:req.add_header (H, Self.header[h]) Try:              html = Request.urlopen (req). Read () # print (HTML) self.bsManage.getPageUrl (html,i) Req.close () except request. Urlerror as E: If Hasattr (E, ' code '): Print (' Error code: ', E.code) elif hasattr (E, ' reason '): Print (' Reason: ', E.reason) if __name__== ' __main__ ': rooturl = ' http://www.meitulu.com/item/' obj_root = Spidermain () Obj_root.geturl (Rooturl)

bscontroller.py

From BS4 import beautifulsoupfrom urllib import requestimport osclass bsmanage:def __init__ (self): Self.pageurl = ' http://www.meitulu.com/item/' Self.header = {' user-agent ': ' mozilla/5.0 (X11; Linux x86_64) applewebkit/537.11 (khtml, like Gecko) chrome/23.0.1271.64 safari/537.11 ', ' Accept ': ' Text/html,a            pplication/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 ', ' accept-charset ': ' iso-8859-1,utf-8;q=0.7,*;q=0.3 ',    ' accept-encoding ': ' None ', ' accept-language ': ' en-us,en;q=0.8 ', ' Connection ': ' Keep-alive '} # HTML is the HTML that gets to the Web page # I represents i_x.html def getpageurl (self,html,i): Soup = beautifulsoup (html, ' Html.parser ', fro         m_encoding= ' Utf-8 ') # gets to the last connection Lasturl = Soup.find_all (' div ', {' id ': ' pages '}) [0].find_all (' a ') [ -2][' href '] # print (HTML) # print (Lasturl) # Gets the number to the last page if I < 10:len = 1 Elif i&lt ; 100:len = 2 Elif i<1000:len = 3 Elif I<10000:len = 4 lastpage = Int (lasturl[29+len:-5]) # Create picture file Clamp if not os.path.exists (' img '): Os.mkdir (' img ') path = ' img/%s '%i if not os.path.exists (p ATH): Os.mkdir (PATH) # Crawl first page because URL format is different # Gets the connection to the desired picture array links = soup.find_all (' img ', cl ass_= ' content_img ') for link in links:name = str (link[' src ']) [ -21:] data = Request.url Open (link[' src '). Read () img = open (' img/%s/'%i + name, ' wb+ ') img.write (data) I Mg.close () # print ('%d has climbed '%i) # str = self.pageurl + '%s '%i + '. html ' # print (str) # each page LastPage a small page for J in Range (2,lastpage+1): # re-stitching the URL to get to the next page URL url = self.pageurl + '%s_%s        '% (i,j) + '. html ' self.saveimgwithurl (url,i) print ('%d already crawled '%i) def saveimgwithurl (self,url,i): req = Request. RequesT (URL) for h in Self.header:req.add_header (H, self.header[h]) try:html = Request.url             Open (req). Read () soup = beautifulsoup (html, ' Html.parser ', from_encoding= ' Utf-8 ') # Gets the connection to the desired picture array Links = soup.find_all (' img ', class_= ' content_img ') for link in links:name = str (lin k[' src ']) [ -21:] data = request.urlopen (link[' src '). Read () img = open (' img/%s/'% i + name, ' wb+ ') img.write (data) img.close () except request. Urlerror as E:if hasattr (E, ' code '): Print (' Error code: ', E.code) elif hasattr (E, ' r Eason '): Print (' Reason: ', E.reason)

Python's path-a reptile example