python 爬圖

標籤：mkdir   head   tool   swp   parse   url   http   tieba   res   

 

利用bs庫進行爬取，在下載html時，使用代理user_agent來下載，並且下載次數是2次，當第一次下載失敗後，並且http狀態代碼是500-600之間，然後會重新下載一次

soup = BeautifulSoup(html, "html.parser")
當前頁面時html的
噹噹前頁面時html5時

soup = BeautifulSoup(html, "html5lib")

#-*- coding:utf-8 -*-import reimport urllibimport urllib2import lxml.htmlimport itertoolsimport osfrom bs4 import BeautifulSoupdef download(url,user_agent=‘wswp‘,num_try = 2):    print ‘Downloading:‘,url    headers = {‘User_agent‘:user_agent}    request = urllib2.Request(url,headers=headers)    try:        html = urllib2.urlopen(request).read()    except urllib2.URLError as e:        print ‘Download error‘,e.reason        html = None        if num_try > 0:            if hasattr(e,‘code‘) and 500 <= e.code <600:                return download(url,user_agent,num_try-1)    return htmldef download_picture(url,path,name):    if not os.path.isdir(path):        os.mkdir(path)    f = open(path+‘/‘ + name + ‘.jpg‘, ‘wb‘)    f.write(download(url))    f.close()    def bs_scraper(html):    soup = BeautifulSoup(html, "html.parser")    results = soup.find_all(name=‘img‘,attrs={‘class‘:‘BDE_Image‘})    tt = 0    for each in results:        src = each.get(‘src‘)        print src        download_picture(src,‘/picture‘,str(tt))        tt = tt + 1url = ‘https://tieba.baidu.com/p/4693368072‘html = download(url)bs_scraper(html)

 

python 爬圖