分享一個簡單的python+mysql網路資料抓取

標籤：try   src   臨時   exe   creat   上傳   mat   網站   爬取   

　　最近學習python網路爬蟲，所以自己寫了一個簡單的程式練練手（呵呵。。）。我使用的環境是python3.6和mysql8.0，抓取目標網站為百度熱點（http://top.baidu.com/）。我只抓取了即時熱點內容，其他欄目應該類似。代碼中有兩個變數SECONDS_PER_CRAWL和CRAWL_PER_UPDATE_TO_DB，前者為抓取頻率，後者為抓取多少次寫一次資料庫，可自由設定。我抓取的資料內容是熱點資訊，連結，關注人數和時間。其在記憶體中存放的結構為dict{tuple(熱點資訊，連結):list[tuple(關注人數，時間)...]...}，資料庫儲存是將熱點資訊和連結放到hotnews表中，關注人數和對應時間放到直接以熱點資訊為表名的表內（因為熱點資訊可能長時間存在但是隨時間變化關注度會有變化）。下面可以看下資料庫存放範例

代碼比較簡單我就暫時沒有上傳github直接貼到下面供大家參考:

 1 # -*- coding: UTF-8 -*- 2  3 from bs4 import BeautifulSoup 4 import threading 5 import requests 6 import pymysql 7 import string 8 import time 9 import sys10 import re11 12 13 #爬取頻率，單位：秒14 SECONDS_PER_CRAWL = 1015 16 #更新到資料庫頻率，單位：爬取次數17 CRAWL_PER_UPDATE_TO_DB = 118 19 #爬取目標網站url20 CRAWL_TARGET_URL = ‘http://top.baidu.com/‘21 22 class DataProducer(threading.Thread):23 24     #臨時存放爬取結果25     NewsDict = {}26 27     def __init__(self):28         threading.Thread.__init__(self)29         db = pymysql.connect(host="localhost", user="root", port=3306, passwd=None, db="crawler", charset="utf8")30         cursor = db.cursor()31         sql = """CREATE TABLE IF NOT EXISTS HOTNEWS (32               INFORMATION VARCHAR(64) NOT NULL,33               HYPERLINK      VARCHAR(256),34               PRIMARY KEY (INFORMATION) );"""35         cursor.execute(sql)36         db.close()37 38     def run(self):39         print("DataProducer Thread start!")40         crawl_data(self.NewsDict)41         print("DataProducer Thread exit!")42 43 44 def crawl_data(nd):45     count = 0;46     while 1:47         req = requests.get(url=CRAWL_TARGET_URL)48         req.encoding = req.apparent_encoding49         bf = BeautifulSoup(req.text, "html.parser")50         texts = bf.find_all(‘ul‘, id="hot-list", class_="list")51         bfs = BeautifulSoup(str(texts), "html.parser")52         spans = bfs.find_all(‘span‘, class_ = re.compile("icon-fall|icon-rise|icon-fair"))53         lis = bfs.find_all(‘a‘, class_ = "list-title")54         for i in range(10):55             vtup = (spans[i].get_text(), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))56             ktup = (lis[i].get(‘title‘), lis[i].get(‘href‘))57             if ktup in nd.keys():58                 nd[ktup].append(vtup)59             else:60                 nd[ktup] = [vtup]61         count = count+162         if count%CRAWL_PER_UPDATE_TO_DB == 0:63             update_to_db(nd)64             nd.clear()65         time.sleep(SECONDS_PER_CRAWL)66 67 def update_to_db(nd):68     db = pymysql.connect(host="localhost", user="root", port=3306, passwd=None, db="crawler", charset="utf8")69     cursor = db.cursor()70     for k in nd.keys():71         #將熱點插入至主表72         sql1 = "REPLACE INTO HOTNEWS (INFORMATION, HYPERLINK) VALUES ( ‘%s‘, ‘%s‘);"73         #熱點建立其分表74         sql2 = "CREATE TABLE IF NOT EXISTS `%s`( NUMBEROFPEOPLE INT NOT NULL, OCCURTIME DATETIME, PRIMARY KEY (OCCURTIME) );"75         try:76             cursor.execute(sql1%(k[0], k[1]))77             cursor.execute(sql2%(k[0]))78             db.commit()79         except:80             db.rollback()81         #將每個熱點資料插入到對應熱點分表82         for e in nd[k]:83             insert_sql = "INSERT INTO %s (NUMBEROFPEOPLE, OCCURTIME) VALUES (%d, ‘%s‘);"84             try:85                 cursor.execute(insert_sql%(k[0], int(e[0]), e[1]))86             except:87                 db.rollback()88     db.close()89 90 if __name__ == ‘__main__‘:91     t = DataProducer()92     t.start()93     t.join()

 

分享一個簡單的python+mysql網路資料抓取