Python實現爬蟲設定代理IP和偽裝成瀏覽器的方法分享

這篇文章主要介紹了關於Python實現爬蟲設定代理IP和偽裝成瀏覽器的方法分享，有著一定的參考價值，現在分享給大家，有需要的朋友可以參考一下

1.python爬蟲瀏覽器偽裝

#匯入urllib.request模組import urllib.request#佈建要求頭headers=("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0")#建立一個openeropener=urllib.request.build_opener()#將headers添加到opener中opener.addheaders=[headers]#將opener安裝為全域urllib.request.install_opener(opener)#用urlopen開啟網頁data=urllib.request.urlopen(url).read().decode('utf-8','ignore')

2.設定代理

#定義代理ipproxy_addr="122.241.72.191:808"#設定代理proxy=urllib.request.ProxyHandle({'http':proxy_addr})#建立一個openeropener=urllib.request.build_opener(proxy,urllib.request.HTTPHandle)#將opener安裝為全域urllib.request.install_opener(opener)#用urlopen開啟網頁data=urllib.request.urlopen(url).read().decode('utf-8','ignore')

3.同時設定用代理和類比瀏覽器訪問

#定義代理ipproxy_addr="122.241.72.191:808"#建立一個請求req=urllib.request.Request(url)#添加headersreq.add_header("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)#設定代理proxy=urllib.request.ProxyHandle("http":proxy_addr)#建立一個openeropener=urllib.request.build_opener(proxy,urllib.request.HTTPHandle)#將opener安裝為全域urllib.request.install_opener(opener)#用urlopen開啟網頁data=urllib.request.urlopen(req).read().decode('utf-8','ignore')

4.在要求標頭中添加多個資訊

import urllib.requestpage_headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0",       "Host":"www.baidu.com",       "Cookie":"xxxxxxxx"       }req=urllib.request.Request(url,headers=page_headers)data=urllib.request.urlopen(req).read().decode('utf-8','ignore')

5.添加post請求參數

import urllib.requestimport urllib.parse#設定post參數page_data=urllib.parse.urlencode([                  ('pn',page_num),                  ('kd',keywords)                  ])#設定headerspage_headers={     'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0',     'Connection':'keep-alive',     'Host':'www.lagou.com',     'Origin':'https://www.lagou.com',     'Cookie':'JSESSIONID=ABAAABAABEEAAJA8F28C00A88DC4D771796BB5C6FFA2DDA; user_trace_token=20170715131136-d58c1f22f6434e9992fc0b35819a572b',     'Accept':'application/json, text/javascript, */*; q=0.01',     'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',     'Referer':'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98?labelWords=&fromSearch=true&suginput=',     'X-Anit-Forge-Token':'None',     'X-Requested-With':'XMLHttpRequest'     }#開啟網頁req=urllib.request.Request(url,headers=page_headers)data=urllib.request.urlopen(req,data=page_data.encode('utf-8')).read().decode('utf-8')

6.利用phantomjs類比瀏覽器請求

#1.下載phantomjs安裝到本地,並設定環境變數from selenium import webdriverbs=webdriver.PhantomJS()#開啟urlbs.get(url)#擷取網頁源碼url_data=bs.page_source#將瀏覽到的網頁儲存為圖片bs.get_screenshot_as_file(filename)

7.phantomjs設定user-agent和cookie

from selenium import webdriverfrom selenium.webdriver.common.desired_capabilities import DesiredCapabilitiesdcap = dict(DesiredCapabilities.PHANTOMJS)dcap["phantomjs.page.settings.userAgent"] = ("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0")bs = webdriver.PhantomJS(desired_capabilities=dcap)bs.get(url)#刪除cookiebs.delete_all_cookies()#設定cookie#cookie格式：在瀏覽器cookie中查看，一個cookie需要包含以下參數，domain、name、value、pathcookie={  'domain':'.www.baidu.com', #注意前面有.  'name':'xxxx',   'value':'xxxx',   'path':'xxxx'  }#向phantomjs中添加cookiebs.add_cookie(cookie)

8.利用web_driver工具

#1.下載web_driver工具（如chromdriver.exe）及對應的瀏覽器#2.將chromdriver.exe放到某個目錄，如c:\chromdriver.exefrom selenium import webdriverdriver=webdriver.Chrome(executable_path="C:\chromdriver.exe")#開啟urldriver.get(url)