python爬蟲selenium爬取開開貸黑名單

第一次用selenium爬取黑名單資料，但是不夠自動化，頁面總長和每頁有多少條記錄都是手動設定變數添加的，很不智能。

這次代碼改進了一下內容：

（1）把頁碼有關的資訊切出來，自動擷取頁數

（2）尋找每頁有多少記錄

（3）利用兩個list儲存資料，更好維護

（4）利用css_selector擷取資料，並且改了

（5）寫成了函數，更加規範

（6）拋出異常

（7）timeout的問題，原來設定了30，後來timeout拋出了異常，改為120

題外話：selenium很方便，最大的好處是解決了動態網頁的問題，雖然本題不是動態網頁，但是相對速度也慢些，爬取378條資料需要超過400秒。

import time,csvimport tracebackfrom selenium import webdriverfrom selenium.webdriver.common.keys import Keysurl_whole='http://www.kaikaidai.com/Lend/Black.aspx'# 載入所有頁面def parsePage():  #設定驅動瀏覽器s  browser=webdriver.Chrome()  #設定響應  browser.set_page_load_timeout(120)  #擷取網址  browser.get(url_whole)  #找多少頁  page_info=browser.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > div > table > tbody > tr > td:nth-child(1)')  #  把有關頁碼的資訊切出來，第1頁/共38頁/每頁10條/共378條  pages=page_info.text.split('/')[1]  pages=int(pages[1:3])  #遍曆每一頁  list_data = []  for page in range(pages):    #自動讀取頁數，設定頁數    elem=browser.find_element_by_name('rpMessage')    elem.send_keys(page)    elem.send_keys(Keys.RETURN)    #找每頁有多少記錄    records=browser.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main').find_elements_by_class_name('hmd_ytab')    #page_datas = loadRecords(records)    idx = 1    for record in records:      idx +=1      try:        #利用css_selector擷取資料        name = record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(1) > td:nth-child(3) > a').text        hid=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(2) > td:nth-child(2)').text        email=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(1) > td:nth-child(5)').text        homenumber=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(2) > td:nth-child(4)').text        numofloan=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(1) > td:nth-child(7)').text        numofkai=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(2) > td:nth-child(6)').text        address=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(3) > td:nth-child(2)').text        mobilephone=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(3) > td:nth-child(4)').text        daysofloan=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(3) > td:nth-child(6)').text        companyname=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(4) > td:nth-child(2)').text        totalamount=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(4) > td:nth-child(6)').text        companyaddress=record.find_element_by_css_selector('#form1 > div:nth-child(11) > div > div.jklb_bkd > div.main > table:nth-child('+str(idx)+') > tbody > tr:nth-child(5) > td:nth-child(3)').text        data = []        data.append(name)        data.append(hid)        data.append(email)        data.append(homenumber)        data.append(numofkai)        data.append(numofloan)        data.append(address)        data.append(mobilephone)        data.append(daysofloan)        data.append(companyname)        data.append(companyaddress)        data.append(totalamount)        list_data.append(data)      except:        traceback.print_exc()        #print(record.text)  print(len(list_data))  return list_data# 寫入csv檔案def writeCsv(list_data):  filePath = 'C:\\Users\\Desktop\\pywork\\kkd\\kkd.csv'  #title = ['name','hid','email','homenumber','numofkai','numofloan','address','mobilephone','daysofloan','companyname','companyaddress','totalamount']  with open(filePath,"w+",newline="") as csvfile:    writer = csv.writer(csvfile)    #先寫入columns_name    #writer.writerow(title)    #寫入多行用writerows    writer.writerows(list_data)def main():  list_data = parsePage()  writeCsv(list_data)if __name__ == "__main__":  main()

資料結果是正確的，涉及隱私，這裡不貼了