為自己的爬蟲更換代理和HTML頭部

標籤：模組   字串   []   os x   __init__   end   linux   3.0   代理   

import requests
import re
import random
import time

class download():

def __init__(self):

self.iplist = [] ##初始化一個list用來存放我們擷取到的IP
html = requests.get("http://haoip.cc/tiqu.htm") ##不解釋咯
iplistn = re.findall(r‘r/>(.*?)<b‘, html.text, re.S) ##表示從html.text中擷取所有r/><b中的內容，re.S的意思是包括匹配包括分行符號，findall返回的是個list哦！
for ip in iplistn:
i = re.sub(‘\n‘, ‘‘, ip) ##re.sub 是re模組替換的方法，這兒表示將\n替換為空白
self.iplist.append(i.strip()) ##添加到我們上面初始化的list裡面

self.user_agent_list = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]

def get(self, url, timeout, proxy=None, num_retries=6): ##給函數一個預設參數proxy為空白
UA = random.choice(self.user_agent_list) ##從self.user_agent_list中隨機取出一個字串
headers = {‘User-Agent‘: UA} ##構造成一個完整的User-Agent （UA代表的是上面隨機取出來的字串哦）

if proxy == None: ##當代理為空白時，不使用代理擷取response（別忘了response啥哦！之前說過了！！）
try:
return requests.get(url, headers=headers, timeout=timeout)##這樣伺服器就會以為我們是真的瀏覽器了
except:##如過上面的代碼執行報錯則執行下面的代碼

if num_retries > 0: ##num_retries是我們限定的重試次數
time.sleep(10) ##延遲十秒
print(u‘擷取網頁出錯，10S後將擷取倒數第：‘, num_retries, u‘次‘)
return self.get(url, timeout, num_retries-1) ##調用自身 並將次數減1
else:
print(u‘開始使用代理‘)
time.sleep(10)
IP = ‘‘.join(str(random.choice(self.iplist)).strip()) ##下面有解釋哦
proxy = {‘http‘: IP}
return self.get(url, timeout, proxy,) ##代理不為空白的時候

else: ##當代理不為空白
try:
IP = ‘‘.join(str(random.choice(self.iplist)).strip()) ##將從self.iplist中擷取的字串處理成我們需要的格式（處理了些什麼自己看哦，這是基礎呢）
proxy = {‘http‘: IP} ##構造成一個代理
return requests.get(url, headers=headers, proxies=proxy, timeout=timeout) ##使用代理擷取response
except:

if num_retries > 0:
time.sleep(10)
IP = ‘‘.join(str(random.choice(self.iplist)).strip())
proxy = {‘http‘: IP}
print(u‘正在更換代理，10S後將重新擷取倒數第‘, num_retries, u‘次‘)
print(u‘當前代理是：‘, proxy)
return self.get(url, timeout, proxy, num_retries - 1)
else:
print(u‘代理也不好使了！取消代理‘)
return self.get(url, 3)

request = download() ##

為自己的爬蟲更換代理和HTML頭部