python爬蟲學習(2)__抓取糗百段子，與存入mysql資料庫

標籤：

import pymysqlimport requestsfrom bs4 import BeautifulSoup
#pymysql連結資料庫conn=pymysql.connect(host=‘127.0.1‘,unix_socket=‘/tmp/mysql.sock‘,user=‘root‘,passwd=‘19950311‘,db=‘mysql‘)cur=conn.cursor()cur.execute("USE scraping") 
#儲存段子標題，內容def store(title,content):    cur.execute("insert into pages(title,content) values(\"%s\",\"%s\")",(title,content))    cur.connection.commit()global linksclass QiuShi(object):    def __init__(self,start_url):        self.url=start_url    def crawing(self):        try:            html=requests.get(self.url,‘lxml‘)            return html.content        except  ConnectionError as e:            return ‘‘    def extract(self,htmlContent):        if len(htmlContent)>0:            bsobj=BeautifulSoup(htmlContent,‘lxml‘)            #print bsobj            jokes=bsobj.findAll(‘div‘,{‘class‘:‘article block untagged mb15‘})            for j in jokes:                text=j.find(‘h2‘).text                content=j.find(‘div‘,{‘class‘:‘content‘}).string                if text != None and content != None:                     # print text,content，資料庫編碼為utf-8                     store(text.encode(‘utf-8‘),content.encode(‘utf-8‘))                     print text.encode(‘utf-8‘),content.encode(‘utf-8‘)                     print ‘------------------------------------------------------------------------------‘        else:            print ‘‘    def main(self):        text=self.crawing()        self.extract(text)try:    qiushi=QiuShi(‘http://www.qiushibaike.com/‘)    qiushi.main()finally:
#關閉cursor,connection    cur.close()    conn.close()

 

python爬蟲學習(2)__抓取糗百段子，與存入mysql資料庫