Rice flapping agent, the global leader of the agent brand, focus on the agency industry for nearly a decade, to provide open, private, exclusive agent, and free trial
The agent of Rice Flutter official website: https://proxy.mimvp.com
This article is an example of a private, exclusive, open proxy, specially developed for the M-topology agent,
support for HTTP, HTTPS, no password, white list IP, password authorization three types
This blog to from the rice Flutter blog: Python + Selenium + Firefox Use proxy auth user name password authorization
example, with the plug-in XPI please go to the official website of the meter flapping agent, or a meter flapping the government github download
This article, directly give the complete code, have been strictly verified through, see the note
Python + Firefox + plugins (CLOSEPROXY.XPI)
Which,closeproxy.xpi files , need Google, Bing search can search
The complete test code is as follows:
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.proxy import *
from pyvirtualdisplay import Display
from base64 import b64encode
proxy = {
"host": "123.57.78.100",
"port": "12345",
"user": "username",
"pass": "password"
}
profile = webdriver.FirefoxProfile()
# add new header
profile.add_extension("modify_headers-0.7.1.1-fx.xpi")
profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
profile.set_preference("modifyheaders.config.active", True)
profile.set_preference("modifyheaders.headers.count", 1)
profile.set_preference("modifyheaders.headers.action0", "Add")
profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip")
profile.set_preference("modifyheaders.headers.value0", "yes")
profile.set_preference("modifyheaders.headers.enabled0", True)
# add proxy
profile.set_preference(‘network.proxy.type‘, 1)
profile.set_preference(‘network.proxy.http‘, proxy[‘host‘])
profile.set_preference(‘network.proxy.http_port‘, int(proxy[‘port‘]))
profile.set_preference(‘network.proxy.no_proxies_on‘, ‘localhost, 127.0.0.1‘)
#profile.set_preference("network.proxy.username", ‘aaaaa‘)
#profile.set_preference("network.proxy.password", ‘bbbbb‘)
# Proxy auto login
profile.add_extension(‘closeproxy.xpi‘)
credentials = ‘{user}:{pass}‘.format(**proxy)
credentials = b64encode(credentials.encode(‘ascii‘)).decode(‘utf-8‘)
profile.set_preference(‘extensions.closeproxyauth.authtoken‘, credentials)
profile.update_preferences()
driver = webdriver.Firefox(profile)
driver.get("https://proxy.mimvp.com/ip.php")
print driver.page_source
driver.quit()
M-Topology Agent: Complete, validated, authoritative examples
#! / usr / bin / env python
#-*-coding: utf-8-*-
##
# Selenium + Firefox supports http, https
##
# mipu Proxy example:
# https://proxy.mimvp.com/demo2.php
##
# mipu Proxy buy:
# https://proxy.mimvp.com
##
# mimvp.com
# 2017-01-08
# Python + Selenium + Firefox When setting a password, you need to use two plugins:
# Plug-in 1: modify_headers-0.7.1.1-fx.xpi
#: Https://github.com/mimvp/mimvp-proxy-demo
##
# Method 2: close_proxy_authentication-1.1.xpi
#: Https://github.com/mimvp/mimvp-proxy-demo
##
# This example was originally created by Mipu agent, the test agent is from Mipu agent
# For password authorization and whitelist ip settings, please see Mipu Agent-Member Center: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.proxy import *
from pyvirtualdisplay import Display
# from xvfbwrapper import Xvfb
import bs4, os
from base64 import b64encode
import sys
reload (sys)
sys.setdefaultencoding (‘utf8’)
## webdriver + firefox (do not use proxy, crawl web pages)
def spider_url_firefox (url):
browser = None
display = None
try:
display = Display (visible = 0, size = (800, 600))
display.start ()
browser = webdriver.Firefox () # Open the FireFox browser
browser.get (url)
content = browser.page_source
print ("content:" + str (content))
finally:
if browser: browser.quit ()
if display: display.stop ()
## webdriver + firefox + proxy + whiteip (no password, or whitelist ip authorization)
## mipu Proxy: https://proxy.mimvp.com
def spider_url_firefox_by_whiteip (url):
browser = None
display = None
## Whitelist ip, please see Mipu Agent Member Center: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
mimvp_proxy = {
‘Ip’: ‘140.143.62.84’, # ip
‘Port_https’: 19480, # http, https
‘Port_socks’: 19481, # socks5
‘Username’: ‘mimvp-user’,
‘Password’: ‘mimvp-pass’
}
try:
display = Display (visible = 0, size = (800, 600))
display.start ()
profile = webdriver.FirefoxProfile ()
# add proxy
profile.set_preference (‘network.proxy.type’, 1) # ProxyType.MANUAL = 1
if url.startswith ("http: //"):
profile.set_preference (‘network.proxy.http‘, mimvp_proxy [‘ip‘])
profile.set_preference (‘network.proxy.http_port’, mimvp_proxy [‘port_https‘]) # visit http site
elif url.startswith ("https: //"):
profile.set_preference (‘network.proxy.ssl‘, mimvp_proxy [‘ip‘])
profile.set_preference (‘network.proxy.ssl_port’, mimvp_proxy [‘port_https‘]) # visit https
else:
profile.set_preference (‘network.proxy.socks’, mimvp_proxy [‘ip‘])
profile.set_preference (‘network.proxy.socks_port’, mimvp_proxy [‘port_socks‘])
profile.set_preference (‘network.proxy.ftp’, mimvp_proxy [‘ip‘])
profile.set_preference (‘network.proxy.ftp_port’, mimvp_proxy [‘port_https‘])
profile.set_preference (‘network.proxy.no_proxies_on’, ‘localhost, 127.0.0.1’)
## This usage does not exist, you cannot set the username and password this way (discard)
# profile.set_preference ("network.proxy.username", ‘mimvp-guest’)
# profile.set_preference ("network.proxy.password", ‘welcome2mimvp’)
profile.update_preferences ()
browser = webdriver.Firefox (profile) # Open the FireFox browser
browser.get (url)
content = browser.page_source
print ("content:" + str (content))
finally:
if browser: browser.quit ()
if display: display.stop ()
## webdriver + firefox + proxy + https (https password authorization)
## mipu Proxy: https://proxy.mimvp.com
def spider_url_firefox_by_proxy (url):
browser = None
display = None
## For authorization password, please see Mipu Agent Member Center: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
mimvp_proxy = {
‘Ip’: ‘140.143.62.84’, # ip
‘Port_https’: 19480, # http, https
‘Port_socks’: 19481, # socks5
‘Username’: ‘mimvp-user’,
‘Password’: ‘mimvp-pass’
}
try:
display = Display (visible = 0, size = (800, 600))
display.start ()
profile = webdriver.FirefoxProfile ()
# add new header
profile.add_extension ("modify_headers-0.7.1.1-fx.xpi")
profile.set_preference ("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
profile.set_preference ("modifyheaders.config.active", True)
profile.set_preference ("modifyheaders.headers.count", 1)
profile.set_preference ("modifyheaders.headers.action0", "Add")
profile.set_preference ("modifyheaders.headers.name0", "Proxy-Switch-Ip")
profile.set_preference ("modifyheaders.headers.value0", "yes")
profile.set_preference ("modifyheaders.headers.enabled0", True)
# add proxy
profile.set_preference (‘network.proxy.type’, 1) # ProxyType.MANUAL = 1if url.startswith ("http: //"):
profile.set_preference (‘network.proxy.http‘, mimvp_proxy [‘ip‘])
profile.set_preference (‘network.proxy.http_port’, mimvp_proxy [‘port_https‘]) # visit http site
elif url.startswith ("https: //"):
profile.set_preference (‘network.proxy.ssl‘, mimvp_proxy [‘ip‘])
profile.set_preference (‘network.proxy.ssl_port’, mimvp_proxy [‘port_https‘]) # visit https
# Proxy auto login (Automatically fill in the password for proxy authorization)
profile.add_extension (‘close_proxy_authentication-1.1.xpi’)
credentials = ‘{username}: {password}‘. format (username = mimvp_proxy [‘username’], password = mimvp_proxy [‘password’]) # auth
credentials = b64encode (credentials.encode (‘ascii‘)). decode (‘utf-8‘)
profile.set_preference (‘extensions.closeproxyauth.authtoken’, credentials)
profile.update_preferences ()
browser = webdriver.Firefox (profile) # Open the FireFox browser
browser.get (url)
content = browser.page_source
print ("content:" + str (content))
finally:
if browser: browser.quit ()
if display: display.stop ()
## webdriver + firefox + proxy + socks (socks password authorization)
## mipu Proxy: https://proxy.mimvp.com
def spider_url_firefox_by_socks (url):
browser = None
display = None
## For authorization password, please see Mipu Agent Member Center: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
mimvp_proxy = {
‘Ip’: ‘140.143.62.84’, # ip
‘Port_https’: 19480, # http, https
‘Port_socks’: 19481, # socks5
‘Username’: ‘mimvp-user’,
‘Password’: ‘mimvp-pass’
}
proxy_config = Proxy ({
‘ProxyType’: ProxyType.MANUAL, # 1
‘HttpProxy’: mimvp_proxy [‘ip‘] + ":" + str (mimvp_proxy [‘port_https‘]),
‘SslProxy’: mimvp_proxy [‘ip‘] + ":" + str (mimvp_proxy [‘port_https‘]),
‘SocksProxy’: mimvp_proxy [‘ip‘] + ":" + str (mimvp_proxy [‘port_socks‘]),
‘FtpProxy’: mimvp_proxy [‘ip‘] + ":" + str (mimvp_proxy [‘port_https‘]),
‘NoProxy’: ‘localhost, 127.0.0.1’,
‘SocksUsername’: mimvp_proxy [‘username’],
‘SocksPassword’: mimvp_proxy [‘password’],
})
try:
display = Display (visible = 0, size = (800, 600))
display.start ()
profile = webdriver.FirefoxProfile ()
# add new header
profile.add_extension ("modify_headers-0.7.1.1-fx.xpi")
profile.set_preference ("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
profile.set_preference ("modifyheaders.config.active", True)
profile.set_preference ("modifyheaders.headers.count", 1)
profile.set_preference ("modifyheaders.headers.action0", "Add")
profile.set_preference ("modifyheaders.headers.name0", "Proxy-Switch-Ip")
profile.set_preference ("modifyheaders.headers.value0", "yes")
profile.set_preference ("modifyheaders.headers.enabled0", True)
# auto save auth
profile.set_preference ("signon.autologin.proxy", ‘true‘)
profile.set_preference ("network.websocket.enabled", ‘false‘)
profile.set_preference (‘network.proxy.share_proxy_settings’, ‘false’)
profile.set_preference (‘network.automatic-ntlm-auth.allow-proxies’, ‘false’)
profile.set_preference (‘network.auth.use-sspi’, ‘false’)
profile.update_preferences ()
browser = webdriver.Firefox (proxy = proxy_config, firefox_profile = profile) # Open the FireFox browser
browser.get (url)
content = browser.page_source
print ("content:" + str (content))
finally:
if browser: browser.quit ()
if display: display.stop ()
if __name__ == ‘__main__’:
url = ‘https://ip.cn’
url = ‘https://mimvp.com’
url = ‘https://proxy.mimvp.com/ip.php’
# Crawl webpage without proxy, success
spider_url_firefox (url)
# Proxy without password, or set whitelist ip, success
spider_url_firefox_by_whiteip (url)
# http, https password authorization, success
spider_url_firefox_by_proxy (url)
# socks5 Password authorization, failed (still requested by local ip, not proxy ip request)
spider_url_firefox_by_socks (url)
Python + Selenium + Firefox uses proxy auth user name password authorization