python 爬蟲入門

標籤：指定   open   htm   入門   %s   編碼   script   res   pytho   

import requestsimport re# TODO 下載 每一個小說的首頁url# TODO 大迴圈# 1. 下載小說首頁novel_url = ‘http://www.jingcaiyuedu.com/book/15205/list.html‘response = requests.get(novel_url)# 處理字元編碼 顯式的指定，response.encoding = ‘utf-8‘html = response.text # 字串# print(html)# 2. 提取 章節url 非貪婪匹配title = re.findall(r‘<meta name="keywords" content="《(.*?)》‘,html)[0]# print(title)# id = list dl 有兩個dl = re.findall(r‘<dl id="list">.*?</dl>‘,html)[1]# print(dl)chapter_info_list = re.findall(r‘<a.*?href="(.*?)".*?>(.*?)</a>‘,dl)# print(chapter_info_list)# 資料持久化 寫入txtfb = open(‘%s.txt‘%title,‘w‘,encoding=‘utf-8‘)# 3. 迴圈的去訪問每個章節，提取內容for chapter_info in chapter_info_list:    chapter_url = chapter_info[0]    chapter_title = chapter_info[1]    # 處理 相對url    if ‘http‘ not in chapter_url:        chapter_url = ‘http://www.jingcaiyuedu.com%s‘ % chapter_url    # 下載章節頁面    chapter_response = requests.get(chapter_url)    chapter_response.encoding = "utf-8"    chapter_html = chapter_response.text    # print(chapter_response.text)    # 提取內容    chapter_content = re.findall(r‘<script>a1\(\);</script>(.*?)<script>a2\(\);</script>‘,chapter_html)[0]    # 清洗資料，把多餘的字元處理掉    chapter_content = chapter_content.replace(‘ ‘,‘‘)    chapter_content = chapter_content.replace(‘<br/>‘,‘‘)    chapter_content = chapter_content.replace(‘<br>‘,‘‘)    chapter_content = chapter_content.replace(‘&nbsp;‘,‘‘)    # print(chapter_content)    # 寫入檔案    fb.write(chapter_title)    fb.write(‘\n‘)    fb.write(chapter_content)    fb.write(‘\n‘)    # chapter_response.close()    print(chapter_url)    # exit()

 

python 爬蟲入門