python 東方財富網&百度股票資料定向爬蟲 執行個體

功能：

1、 擷取 上交所 深 交所的股票資訊

2、 輸出儲存到檔案中

技術路線： requests -beatiful soup - re

候選：資料網站選擇

1、靜態網站，資訊靜態存在HTML頁面中，非js 代碼產生

2、 F12 ， 原始碼查看

多找資訊源

方法：

1、 從東方財富網擷取 股票列表資訊

2、 根據股票列表逐個到百度股票 擷取個股資訊

3、 將結果儲存到檔案中

import requestsfrom bs4 import BeautifulSoupimport tracebackimport redef getHTMLText(url):    try:        r = requests.get(url)        r.raise_for_status()        r.encoding = r.apparent_encoding        return r.text    except:        return ""def getStockList(lst, stockURL):    html = getHTMLText(stockURL)    soup = BeautifulSoup(html, 'html.parser')     a = soup.find_all('a')    for i in a:        try:            href = i.attrs['href']            lst.append(re.findall(r"[s][hz]\d{6}", href)[0])        except:            continuedef getStockInfo(lst, stockURL, fpath):    for stock in lst:        url = stockURL + stock + ".html"        html = getHTMLText(url)        try:            if html=="":                continue            infoDict = {}            soup = BeautifulSoup(html, 'html.parser')            stockInfo = soup.find('div',attrs={'class':'stock-bets'})            name = stockInfo.find_all(attrs={'class':'bets-name'})[0]            infoDict.update({'股票名稱': name.text.split()[0]})            keyList = stockInfo.find_all('dt')            valueList = stockInfo.find_all('dd')            for i in range(len(keyList)):                key = keyList[i].text                val = valueList[i].text                infoDict[key] = val            with open(fpath, 'a', encoding='utf-8') as f:                f.write( str(infoDict) + '\n' )        except:            traceback.print_exc()            continuedef main():    stock_list_url = 'http://quote.eastmoney.com/stocklist.html'    stock_info_url = 'https://gupiao.baidu.com/stock/'    output_file = 'D:/BaiduStockInfo.txt'    slist=[]    getStockList(slist, stock_list_url)    getStockInfo(slist, stock_info_url, output_file)main()

改進代碼 —-1、添加進度條，增加使用者體驗
2、

import requestsfrom bs4 import BeautifulSoupimport tracebackimport redef getHTMLText(url, code="utf-8"):    try:        r = requests.get(url)        r.raise_for_status()        r.encoding = code        return r.text    except:        return ""def getStockList(lst, stockURL):    html = getHTMLText(stockURL, "GB2312")    soup = BeautifulSoup(html, 'html.parser')     a = soup.find_all('a')    for i in a:        try:            href = i.attrs['href']            lst.append(re.findall(r"[s][hz]\d{6}", href)[0])        except:            continuedef getStockInfo(lst, stockURL, fpath):    count = 0    for stock in lst:        url = stockURL + stock + ".html"        html = getHTMLText(url)        try:            if html=="":                continue            infoDict = {}            soup = BeautifulSoup(html, 'html.parser')            stockInfo = soup.find('div',attrs={'class':'stock-bets'})            name = stockInfo.find_all(attrs={'class':'bets-name'})[0]            infoDict.update({'股票名稱': name.text.split()[0]})            keyList = stockInfo.find_all('dt')            valueList = stockInfo.find_all('dd')            for i in range(len(keyList)):                key = keyList[i].text                val = valueList[i].text                infoDict[key] = val            with open(fpath, 'a', encoding='utf-8') as f:                f.write( str(infoDict) + '\n' )                count = count + 1                print("\r當前進度: {:.2f}%".format(count*100/len(lst)),end="")        except:            count = count + 1            print("\r當前進度: {:.2f}%".format(count*100/len(lst)),end="")            continuedef main():    stock_list_url = 'http://quote.eastmoney.com/stocklist.html'    stock_info_url = 'https://gupiao.baidu.com/stock/'    output_file = 'D:/BaiduStockInfo.txt'    slist=[]    getStockList(slist, stock_list_url)    getStockInfo(slist, stock_info_url, output_file)main()