python爬蟲實戰（一）——即時擷取代理ip，python爬蟲

python爬蟲實戰（一）——即時擷取代理ip，python爬蟲

在爬蟲學習的過程中，維護一個自己的代理池是非常重要的。

詳情看代碼：

　　1.運行環境 python3.x，需求庫：bs4，requests

　　2.即時抓取西刺-國內高匿代理中前3頁的代理ip（可根據需求自由修改）

　　3.多線程對抓取的代理進行驗證並儲存驗證後的代理ip

#-*-coding:utf8-*-import re,threading,requests,timeimport urllib.requestfrom bs4 import BeautifulSoup as BSrawProxyList = []checkedProxyList = []targets = []headers = { 'User-Agent': r'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36', 'Connection': 'keep-alive' }for i in range(1,4): target = r"http://www.xicidaili.com/nn/%d" %i targets.append(target) #print (targets)#擷取代理的類class ProxyGet(threading.Thread): def __init__(self,target): threading.Thread.__init__(self) self.target =target def getProxy(self): print ("目標網站："+self.target) r = requests.get(self.target,headers =headers) page = r.text soup = BS(page,"lxml") #這裡的class_用的是"Searching by CSS class""，BS文檔中有詳細介紹 tr_list = soup.find_all("tr", class_= "odd") for i in range(len(tr_list)): row = [] #.stripped_strings 方法返回去除前後空白的Python的string對象. for text in tr_list[i].stripped_strings: row.append(text) #row = ['58.208.16.141','808','江蘇蘇州','高匿','HTTP,......] ip =row[0] port = row[1] agent = row[4].lower() addr =agent+ "://" + ip + ":" + port proxy = [ip, port, agent, addr] rawProxyList.append(proxy) def run(self): self.getProxy()#檢驗代理類class ProxyCheck(threading.Thread): def __init__(self,proxyList): threading.Thread.__init__(self) self.proxyList = proxyList self.timeout =2 self.testUrl = "https://www.baidu.com/" def checkProxy(self): for proxy in self.proxyList: proxies = {} if proxy[2] =="http": proxies['http'] = proxy[3] else: proxies['https'] = proxy[3] t1 =time.time() try: r = requests.get(self.testUrl, headers=headers, proxies=proxies, timeout=self.timeout) time_used = time.time() - t1 if r: checkedProxyList.append((proxy[0],proxy[1],proxy[2],proxy[3],time_used)) else: continue except Exception as e: continue def run(self): self.checkProxy() print("hello")if __name__ =="__main__": getThreads = [] checkedThreads = []# 對每個目標網站開啟一個線程負責抓取代理for i in range(len(targets)): t= ProxyGet(targets[i]) getThreads.append(t)for i in range(len(getThreads)): getThreads[i].start()for i in range(len(getThreads)): getThreads[i].join()print ('.'*10+"總共抓取了%s個代理" %len(rawProxyList) +'.'*10)#開啟20個線程負責校正，將抓取到的代理分成20份，每個線程校正一份for i in range(10): n =len(rawProxyList)/10 #print (str(int(n * i))+ ":" +str(int(n * (i+1)))) t = ProxyCheck(rawProxyList[int(n * i):int(n * (i+1))]) checkedThreads.append(t)for i in range(len(checkedThreads)): checkedThreads[i].start()for i in range(len(checkedThreads)): checkedThreads[i].join()print ('.'*10+"總共有%s個代理通過校正" %len(checkedProxyList) +'.'*10 )#持久化f = open("proxy_list.txt",'w+')for checked_proxy in sorted(checkedProxyList): print ("checked proxy is: %s\t%s" %(checked_proxy[3],checked_proxy[4]) ) f.write("%s:%s\t%s\t%s\t%s\n" % (checked_proxy[0], checked_proxy[1], checked_proxy[2], checked_proxy[3], checked_proxy[4]))f.close()View Code