python爬蟲：編寫多進程爬蟲學習筆記__python

# -*- coding: utf-8 -*-"""Created on Sat Oct 22 21:01:23 2016@author: hhxsym"""import requestsimport jsonimport osimport pymongoimport timefrom bs4 import BeautifulSoupfrom multiprocessing import Pool  #進程調用的包inpath="C:\\Users\\hhxsym\\Desktop\\課程群Python爬蟲"inpath = unicode(inpath , "utf8")os.chdir(inpath)  #不做編碼轉換後，中文路徑無法開啟，更改#串連資料庫client =pymongo.MongoClient('localhost', 27017) #串連資料庫sense = client['sense'] #建立資料庫url_list = sense['url_list'] #建立資料庫表def get_city_urls():    url = 'http://www.senseluxury.com'    with open('city.html') as f:   #本地讀取        response = f.read() #直接讀取到文本    soup = BeautifulSoup(response, 'lxml')    urls = soup.select('#destination_nav > div > div > div > dl.dl-list > dt > a') #CSS結構類型，注意空格    return [url.get("href") for url in urls]def get_page_list(city, page=1):    now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))    url = 'http://www.senseluxury.com/destinations_list/%s' % city.split('/')[-1]    payload = {'page':page, 'callback':'jsonp'}    responses = requests.get(url,payload) #請求網頁，獲得響應的內容,requests.get(url地址，關鍵字url參數)    #print responses.url    print responses.status_code    #print responses.text[6:-1] #列印json格式的“字串” (1)    wb_data = json.loads(responses.text[6:-1]) #將字串轉換為python的字典 (2)    print type(responses.text), type(wb_data) #對比兩種類型 (1)(2)對比    #print json.dumps(wb_data, encoding='utf-8', ensure_ascii=False) #json.dumps方法，變換成中文列印    #通過迴圈擷取資料    for i in wb_data['val']['data']:        title = i['title']        url = 'http://www.senseluxury.com'+i['url'] #資料拼接，獲得我們想要的資料        id = i['id']        server=i['server'].replace(' ',' ').split()        memo = i['memo']        price = i['price']        address = i['address']        subject =i['subject']        data = {'title':title, 'id':id, 'server':server,'memo':memo,                'prie':price, 'address':address, 'subject':subject,                'create_time':now}        url_list.insert_one(data) #插入資料（字典）        # 註：產生的資料列表中 _id是自動產生的        #print title, url        print dataif __name__=='__main__':    #get_page_list(1)    #print get_city_urls()    #get_page_list('http://www.senseluxury.com/destinations/2', page=1)    city_urls = get_city_urls()    print city_urls    pool = Pool(processes=4) #設定進程數量    pool.map(get_page_list, city_urls) #pool.map(函數名稱，迭代對象)    pool.close()  # 等待進程池中所有進程執行結束之後再關閉    pool.join()   #關閉之後要計入它，作用：防止主程式在子進程結束前關閉# 網頁json類型的查看：瀏覽器 -> 右鍵 檢查 -> network -> XHR ->頁面觸發(跳轉頁面) -> name勾選-> Response ->查看是否出現json格式字串# http://jsoneditoronline.org/  線上格式化網站，查看接送嵌套格式