標籤:automatic 需要 spro base final pytho login sha .so
米撲代理,全球領導的代理品牌,專註代理行業近十年,提供開放、私密、獨享代理,並可免費試用
米撲代理官網:https://proxy.mimvp.com
本文樣本,是結合米撲代理的私密、獨享、開放代理,專門研發的樣本,
支援 http、https的無密碼、白名單ip、密碼授權三種類型
本部落格轉自米撲部落格:Python + Selenium + Firefox 使用代理 auth 的使用者名稱密碼授權
樣本中,用的外掛程式 xpi 請到米撲代理官網,或米撲官方 github 下載
本文,直接給出完整的代碼,都經過嚴格驗證通過,具體請見注釋
Python + Firefox + 外掛程式(closeproxy.xpi)
其中,closeproxy.xpi檔案,需要Google、Bing搜下都能搜到
完整的測試代碼如下:
from selenium import webdriverfrom selenium.webdriver.firefox.firefox_binary import FirefoxBinaryfrom selenium.webdriver.common.proxy import *from pyvirtualdisplay import Displayfrom base64 import b64encodeproxy = { "host": "123.57.78.100", "port": "12345", "user": "username", "pass": "password"}profile = webdriver.FirefoxProfile()# add new headerprofile.add_extension("modify_headers-0.7.1.1-fx.xpi")profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx")profile.set_preference("modifyheaders.config.active", True)profile.set_preference("modifyheaders.headers.count", 1)profile.set_preference("modifyheaders.headers.action0", "Add")profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip")profile.set_preference("modifyheaders.headers.value0", "yes")profile.set_preference("modifyheaders.headers.enabled0", True)# add proxyprofile.set_preference(‘network.proxy.type‘, 1)profile.set_preference(‘network.proxy.http‘, proxy[‘host‘])profile.set_preference(‘network.proxy.http_port‘, int(proxy[‘port‘]))profile.set_preference(‘network.proxy.no_proxies_on‘, ‘localhost, 127.0.0.1‘)#profile.set_preference("network.proxy.username", ‘aaaaa‘)#profile.set_preference("network.proxy.password", ‘bbbbb‘)# Proxy auto loginprofile.add_extension(‘closeproxy.xpi‘)credentials = ‘{user}:{pass}‘.format(**proxy)credentials = b64encode(credentials.encode(‘ascii‘)).decode(‘utf-8‘)profile.set_preference(‘extensions.closeproxyauth.authtoken‘, credentials)profile.update_preferences()driver = webdriver.Firefox(profile)driver.get("https://proxy.mimvp.com/ip.php")print driver.page_sourcedriver.quit()
米撲代理出品:完整、驗證、權威的樣本
#!/usr/bin/env python# -*- coding:utf-8 -*-## Selenium + Firefox 支援 http、https## 米撲代理樣本:# https://proxy.mimvp.com/demo2.php# # 米撲代理購買:# https://proxy.mimvp.com# # mimvp.com# 2017-01-08# Python + Selenium + Firefox 設定密碼時,需要使用到兩個外掛程式:# 外掛程式1: modify_headers-0.7.1.1-fx.xpi# :https://github.com/mimvp/mimvp-proxy-demo## 方式2: close_proxy_authentication-1.1.xpi# :https://github.com/mimvp/mimvp-proxy-demo# # 本樣本由米撲代理原創,測試代理程式來自於米撲代理# 密碼授權和白名單ip設定,請見米撲代理 - 測試人員中樞:https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteipfrom selenium import webdriverfrom selenium.webdriver.firefox.firefox_binary import FirefoxBinaryfrom selenium.webdriver.common.proxy import *from pyvirtualdisplay import Display# from xvfbwrapper import Xvfbimport bs4, osfrom base64 import b64encodeimport sysreload(sys)sys.setdefaultencoding(‘utf8‘)## webdriver + firefox (不使用代理,爬取網頁)def spider_url_firefox(url): browser = None display = None try: display = Display(visible=0, size=(800, 600)) display.start() browser = webdriver.Firefox() # 開啟 FireFox 瀏覽器 browser.get(url) content = browser.page_source print("content: " + str(content)) finally: if browser: browser.quit() if display: display.stop()## webdriver + firefox + proxy + whiteip (無密碼,或白名單ip授權)## 米撲代理:https://proxy.mimvp.comdef spider_url_firefox_by_whiteip(url): browser = None display = None ## 白名單ip,請見米撲代理測試人員中樞: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip mimvp_proxy = { ‘ip‘ : ‘140.143.62.84‘, # ip ‘port_https‘ : 19480, # http, https ‘port_socks‘ : 19481, # socks5 ‘username‘ : ‘mimvp-user‘, ‘password‘ : ‘mimvp-pass‘ } try: display = Display(visible=0, size=(800, 600)) display.start() profile = webdriver.FirefoxProfile() # add proxy profile.set_preference(‘network.proxy.type‘, 1) # ProxyType.MANUAL = 1 if url.startswith("http://"): profile.set_preference(‘network.proxy.http‘, mimvp_proxy[‘ip‘]) profile.set_preference(‘network.proxy.http_port‘, mimvp_proxy[‘port_https‘]) # 訪問http網站 elif url.startswith("https://"): profile.set_preference(‘network.proxy.ssl‘, mimvp_proxy[‘ip‘]) profile.set_preference(‘network.proxy.ssl_port‘, mimvp_proxy[‘port_https‘]) # 訪問https網站 else: profile.set_preference(‘network.proxy.socks‘, mimvp_proxy[‘ip‘]) profile.set_preference(‘network.proxy.socks_port‘, mimvp_proxy[‘port_socks‘]) profile.set_preference(‘network.proxy.ftp‘, mimvp_proxy[‘ip‘]) profile.set_preference(‘network.proxy.ftp_port‘, mimvp_proxy[‘port_https‘]) profile.set_preference(‘network.proxy.no_proxies_on‘, ‘localhost,127.0.0.1‘) ## 不存在此用法,不能這麼設定使用者名稱密碼 (捨棄)# profile.set_preference("network.proxy.username", ‘mimvp-guest‘)# profile.set_preference("network.proxy.password", ‘welcome2mimvp‘) profile.update_preferences() browser = webdriver.Firefox(profile) # 開啟 FireFox 瀏覽器 browser.get(url) content = browser.page_source print("content: " + str(content)) finally: if browser: browser.quit() if display: display.stop()## webdriver + firefox + proxy + https (https密碼授權)## 米撲代理:https://proxy.mimvp.comdef spider_url_firefox_by_proxy(url): browser = None display = None ## 授權密碼,請見米撲代理測試人員中樞: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip mimvp_proxy = { ‘ip‘ : ‘140.143.62.84‘, # ip ‘port_https‘ : 19480, # http, https ‘port_socks‘ : 19481, # socks5 ‘username‘ : ‘mimvp-user‘, ‘password‘ : ‘mimvp-pass‘ } try: display = Display(visible=0, size=(800, 600)) display.start() profile = webdriver.FirefoxProfile() # add new header profile.add_extension("modify_headers-0.7.1.1-fx.xpi") profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx") profile.set_preference("modifyheaders.config.active", True) profile.set_preference("modifyheaders.headers.count", 1) profile.set_preference("modifyheaders.headers.action0", "Add") profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip") profile.set_preference("modifyheaders.headers.value0", "yes") profile.set_preference("modifyheaders.headers.enabled0", True) # add proxy profile.set_preference(‘network.proxy.type‘, 1) # ProxyType.MANUAL = 1 if url.startswith("http://"): profile.set_preference(‘network.proxy.http‘, mimvp_proxy[‘ip‘]) profile.set_preference(‘network.proxy.http_port‘, mimvp_proxy[‘port_https‘]) # 訪問http網站 elif url.startswith("https://"): profile.set_preference(‘network.proxy.ssl‘, mimvp_proxy[‘ip‘]) profile.set_preference(‘network.proxy.ssl_port‘, mimvp_proxy[‘port_https‘]) # 訪問https網站 # Proxy auto login (自動填寫密碼,進行代理授權) profile.add_extension(‘close_proxy_authentication-1.1.xpi‘) credentials = ‘{username}:{password}‘.format(username=mimvp_proxy[‘username‘], password=mimvp_proxy[‘password‘]) # auth credentials = b64encode(credentials.encode(‘ascii‘)).decode(‘utf-8‘) profile.set_preference(‘extensions.closeproxyauth.authtoken‘, credentials) profile.update_preferences() browser = webdriver.Firefox(profile) # 開啟 FireFox 瀏覽器 browser.get(url) content = browser.page_source print("content: " + str(content)) finally: if browser: browser.quit() if display: display.stop()## webdriver + firefox + proxy + socks (socks密碼授權)## 米撲代理:https://proxy.mimvp.comdef spider_url_firefox_by_socks(url): browser = None display = None ## 授權密碼,請見米撲代理測試人員中樞: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip mimvp_proxy = { ‘ip‘ : ‘140.143.62.84‘, # ip ‘port_https‘ : 19480, # http, https ‘port_socks‘ : 19481, # socks5 ‘username‘ : ‘mimvp-user‘, ‘password‘ : ‘mimvp-pass‘ } proxy_config = Proxy({ ‘proxyType‘ : ProxyType.MANUAL, # 1 ‘httpProxy‘ : mimvp_proxy[‘ip‘] + ":" + str(mimvp_proxy[‘port_https‘]), ‘sslProxy‘ : mimvp_proxy[‘ip‘] + ":" + str(mimvp_proxy[‘port_https‘]), ‘socksProxy‘ : mimvp_proxy[‘ip‘] + ":" + str(mimvp_proxy[‘port_socks‘]), ‘ftpProxy‘ : mimvp_proxy[‘ip‘] + ":" + str(mimvp_proxy[‘port_https‘]), ‘noProxy‘ : ‘localhost,127.0.0.1‘, ‘socksUsername‘ : mimvp_proxy[‘username‘], ‘socksPassword‘ : mimvp_proxy[‘password‘], }) try: display = Display(visible=0, size=(800, 600)) display.start() profile = webdriver.FirefoxProfile() # add new header profile.add_extension("modify_headers-0.7.1.1-fx.xpi") profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx") profile.set_preference("modifyheaders.config.active", True) profile.set_preference("modifyheaders.headers.count", 1) profile.set_preference("modifyheaders.headers.action0", "Add") profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip") profile.set_preference("modifyheaders.headers.value0", "yes") profile.set_preference("modifyheaders.headers.enabled0", True) # auto save auth profile.set_preference("signon.autologin.proxy", ‘true‘) profile.set_preference("network.websocket.enabled", ‘false‘) profile.set_preference(‘network.proxy.share_proxy_settings‘, ‘false‘) profile.set_preference(‘network.automatic-ntlm-auth.allow-proxies‘, ‘false‘) profile.set_preference(‘network.auth.use-sspi‘, ‘false‘) profile.update_preferences() browser = webdriver.Firefox(proxy=proxy_config, firefox_profile=profile) # 開啟 FireFox 瀏覽器 browser.get(url) content = browser.page_source print("content: " + str(content)) finally: if browser: browser.quit() if display: display.stop()if __name__ == ‘__main__‘: url = ‘https://ip.cn‘ url = ‘https://mimvp.com‘ url = ‘https://proxy.mimvp.com/ip.php‘ # 不使用代理,爬取網頁,成功 spider_url_firefox(url) # 代理無密碼,或設定白名單ip,成功 spider_url_firefox_by_whiteip(url) # http, https 密碼授權,成功 spider_url_firefox_by_proxy(url) # socks5 密碼授權,失敗 (仍然是本機ip請求的,不是代理ip請求) spider_url_firefox_by_socks(url)
Python + Selenium + Firefox 使用代理 auth 的使用者名稱密碼授權