Python + Selenium + Firefox uses proxy auth user name password authorization

Rice flapping agent, the global leader of the agent brand, focus on the agency industry for nearly a decade, to provide open, private, exclusive agent, and free trial

The agent of Rice Flutter official website: https://proxy.mimvp.com

This article is an example of a private, exclusive, open proxy, specially developed for the M-topology agent,

support for HTTP, HTTPS, no password, white list IP, password authorization three types

This blog to from the rice Flutter blog: Python + Selenium + Firefox Use proxy auth user name password authorization

example, with the plug-in XPI please go to the official website of the meter flapping agent, or a meter flapping the government github download

This article, directly give the complete code, have been strictly verified through, see the note

Python + Firefox + plugins (CLOSEPROXY.XPI)

Which,closeproxy.xpi files , need Google, Bing search can search

The complete test code is as follows:

from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.proxy import *
from pyvirtualdisplay import Display
from base64 import b64encode


proxy = {
    "host": "123.57.78.100",
    "port": "12345",
    "user": "username",
    "pass": "password"
}

profile = webdriver.FirefoxProfile()

# add new header
profile.add_extension("modify_headers-0.7.1.1-fx.xpi")
profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
profile.set_preference("modifyheaders.config.active", True)
profile.set_preference("modifyheaders.headers.count", 1)
profile.set_preference("modifyheaders.headers.action0", "Add")
profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip")
profile.set_preference("modifyheaders.headers.value0", "yes")
profile.set_preference("modifyheaders.headers.enabled0", True)

# add proxy
profile.set_preference(‘network.proxy.type‘, 1)
profile.set_preference(‘network.proxy.http‘, proxy[‘host‘])
profile.set_preference(‘network.proxy.http_port‘, int(proxy[‘port‘]))
profile.set_preference(‘network.proxy.no_proxies_on‘, ‘localhost, 127.0.0.1‘)
#profile.set_preference("network.proxy.username", ‘aaaaa‘)
#profile.set_preference("network.proxy.password", ‘bbbbb‘)

# Proxy auto login
profile.add_extension(‘closeproxy.xpi‘)
credentials = ‘{user}:{pass}‘.format(**proxy)
credentials = b64encode(credentials.encode(‘ascii‘)).decode(‘utf-8‘)
profile.set_preference(‘extensions.closeproxyauth.authtoken‘, credentials)

profile.update_preferences()

driver = webdriver.Firefox(profile)
driver.get("https://proxy.mimvp.com/ip.php")
print driver.page_source

driver.quit()

M-Topology Agent: Complete, validated, authoritative examples

#! / usr / bin / env python
#-*-coding: utf-8-*-
##
# Selenium + Firefox supports http, https
##
# mipu Proxy example:
# https://proxy.mimvp.com/demo2.php
##
# mipu Proxy buy:
# https://proxy.mimvp.com
##
# mimvp.com
# 2017-01-08

# Python + Selenium + Firefox When setting a password, you need to use two plugins:
# Plug-in 1: modify_headers-0.7.1.1-fx.xpi
#: Https://github.com/mimvp/mimvp-proxy-demo
##
# Method 2: close_proxy_authentication-1.1.xpi
#: Https://github.com/mimvp/mimvp-proxy-demo
##
# This example was originally created by Mipu agent, the test agent is from Mipu agent
# For password authorization and whitelist ip settings, please see Mipu Agent-Member Center: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip


from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.proxy import *
from pyvirtualdisplay import Display
# from xvfbwrapper import Xvfb

import bs4, os
from base64 import b64encode

import sys
reload (sys)
sys.setdefaultencoding (‘utf8’)


## webdriver + firefox (do not use proxy, crawl web pages)
def spider_url_firefox (url):
    browser = None
    display = None
    try:
        display = Display (visible = 0, size = (800, 600))
        display.start ()
        browser = webdriver.Firefox () # Open the FireFox browser
        browser.get (url)
        content = browser.page_source
        print ("content:" + str (content))
    finally:
        if browser: browser.quit ()
        if display: display.stop ()


## webdriver + firefox + proxy + whiteip (no password, or whitelist ip authorization)
## mipu Proxy: https://proxy.mimvp.com
def spider_url_firefox_by_whiteip (url):
    browser = None
    display = None
    
    ## Whitelist ip, please see Mipu Agent Member Center: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
    mimvp_proxy = {
                    ‘Ip’: ‘140.143.62.84’, # ip
                    ‘Port_https’: 19480, # http, https
                    ‘Port_socks’: 19481, # socks5
                    ‘Username’: ‘mimvp-user’,
                    ‘Password’: ‘mimvp-pass’
                  }
    
    try:
        display = Display (visible = 0, size = (800, 600))
        display.start ()
        
        profile = webdriver.FirefoxProfile ()
        
        # add proxy
        profile.set_preference (‘network.proxy.type’, 1) # ProxyType.MANUAL = 1
        if url.startswith ("http: //"):
            profile.set_preference (‘network.proxy.http‘, mimvp_proxy [‘ip‘])
            profile.set_preference (‘network.proxy.http_port’, mimvp_proxy [‘port_https‘]) # visit http site
        elif url.startswith ("https: //"):
            profile.set_preference (‘network.proxy.ssl‘, mimvp_proxy [‘ip‘])
            profile.set_preference (‘network.proxy.ssl_port’, mimvp_proxy [‘port_https‘]) # visit https
        else:
            profile.set_preference (‘network.proxy.socks’, mimvp_proxy [‘ip‘])
            profile.set_preference (‘network.proxy.socks_port’, mimvp_proxy [‘port_socks‘])
            profile.set_preference (‘network.proxy.ftp’, mimvp_proxy [‘ip‘])
            profile.set_preference (‘network.proxy.ftp_port’, mimvp_proxy [‘port_https‘])
            profile.set_preference (‘network.proxy.no_proxies_on’, ‘localhost, 127.0.0.1’)
        
        ## This usage does not exist, you cannot set the username and password this way (discard)
# profile.set_preference ("network.proxy.username", ‘mimvp-guest’)
# profile.set_preference ("network.proxy.password", ‘welcome2mimvp’)
    
        profile.update_preferences ()
        
        browser = webdriver.Firefox (profile) # Open the FireFox browser
        browser.get (url)
        content = browser.page_source
        print ("content:" + str (content))
    finally:
        if browser: browser.quit ()
        if display: display.stop ()


## webdriver + firefox + proxy + https (https password authorization)
## mipu Proxy: https://proxy.mimvp.com
def spider_url_firefox_by_proxy (url):
    browser = None
    display = None
    
    ## For authorization password, please see Mipu Agent Member Center: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
    mimvp_proxy = {
                    ‘Ip’: ‘140.143.62.84’, # ip
                    ‘Port_https’: 19480, # http, https
                    ‘Port_socks’: 19481, # socks5
                    ‘Username’: ‘mimvp-user’,
                    ‘Password’: ‘mimvp-pass’
                  }

    try:
        display = Display (visible = 0, size = (800, 600))
        display.start ()
        
        profile = webdriver.FirefoxProfile ()
        
        # add new header
        profile.add_extension ("modify_headers-0.7.1.1-fx.xpi")
        profile.set_preference ("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
        profile.set_preference ("modifyheaders.config.active", True)
        profile.set_preference ("modifyheaders.headers.count", 1)
        profile.set_preference ("modifyheaders.headers.action0", "Add")
        profile.set_preference ("modifyheaders.headers.name0", "Proxy-Switch-Ip")
        profile.set_preference ("modifyheaders.headers.value0", "yes")
        profile.set_preference ("modifyheaders.headers.enabled0", True)

        # add proxy
        profile.set_preference (‘network.proxy.type’, 1) # ProxyType.MANUAL = 1if url.startswith ("http: //"):
            profile.set_preference (‘network.proxy.http‘, mimvp_proxy [‘ip‘])
            profile.set_preference (‘network.proxy.http_port’, mimvp_proxy [‘port_https‘]) # visit http site
        elif url.startswith ("https: //"):
            profile.set_preference (‘network.proxy.ssl‘, mimvp_proxy [‘ip‘])
            profile.set_preference (‘network.proxy.ssl_port’, mimvp_proxy [‘port_https‘]) # visit https
 
        # Proxy auto login (Automatically fill in the password for proxy authorization)
        profile.add_extension (‘close_proxy_authentication-1.1.xpi’)
        credentials = ‘{username}: {password}‘. format (username = mimvp_proxy [‘username’], password = mimvp_proxy [‘password’]) # auth
        credentials = b64encode (credentials.encode (‘ascii‘)). decode (‘utf-8‘)
        profile.set_preference (‘extensions.closeproxyauth.authtoken’, credentials)

        profile.update_preferences ()
        
        browser = webdriver.Firefox (profile) # Open the FireFox browser
        browser.get (url)
        content = browser.page_source
        print ("content:" + str (content))
    finally:
        if browser: browser.quit ()
        if display: display.stop ()


## webdriver + firefox + proxy + socks (socks password authorization)
## mipu Proxy: https://proxy.mimvp.com
def spider_url_firefox_by_socks (url):
    browser = None
    display = None
    
    ## For authorization password, please see Mipu Agent Member Center: https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
    mimvp_proxy = {
                    ‘Ip’: ‘140.143.62.84’, # ip
                    ‘Port_https’: 19480, # http, https
                    ‘Port_socks’: 19481, # socks5
                    ‘Username’: ‘mimvp-user’,
                    ‘Password’: ‘mimvp-pass’
                  }

    proxy_config = Proxy ({
                    ‘ProxyType’: ProxyType.MANUAL, # 1
                    ‘HttpProxy’: mimvp_proxy [‘ip‘] + ":" + str (mimvp_proxy [‘port_https‘]),
                    ‘SslProxy’: mimvp_proxy [‘ip‘] + ":" + str (mimvp_proxy [‘port_https‘]),
                    ‘SocksProxy’: mimvp_proxy [‘ip‘] + ":" + str (mimvp_proxy [‘port_socks‘]),
                    ‘FtpProxy’: mimvp_proxy [‘ip‘] + ":" + str (mimvp_proxy [‘port_https‘]),
                    ‘NoProxy’: ‘localhost, 127.0.0.1’,
                    ‘SocksUsername’: mimvp_proxy [‘username’],
                    ‘SocksPassword’: mimvp_proxy [‘password’],
                  })
    
    try:
        display = Display (visible = 0, size = (800, 600))
        display.start ()
        
        profile = webdriver.FirefoxProfile ()
        
        # add new header
        profile.add_extension ("modify_headers-0.7.1.1-fx.xpi")
        profile.set_preference ("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
        profile.set_preference ("modifyheaders.config.active", True)
        profile.set_preference ("modifyheaders.headers.count", 1)
        profile.set_preference ("modifyheaders.headers.action0", "Add")
        profile.set_preference ("modifyheaders.headers.name0", "Proxy-Switch-Ip")
        profile.set_preference ("modifyheaders.headers.value0", "yes")
        profile.set_preference ("modifyheaders.headers.enabled0", True)
        
        # auto save auth
        profile.set_preference ("signon.autologin.proxy", ‘true‘)
        profile.set_preference ("network.websocket.enabled", ‘false‘)
        profile.set_preference (‘network.proxy.share_proxy_settings’, ‘false’)
        profile.set_preference (‘network.automatic-ntlm-auth.allow-proxies’, ‘false’)
        profile.set_preference (‘network.auth.use-sspi’, ‘false’)
        profile.update_preferences ()
        
        browser = webdriver.Firefox (proxy = proxy_config, firefox_profile = profile) # Open the FireFox browser
        browser.get (url)
        content = browser.page_source
        print ("content:" + str (content))
    finally:
        if browser: browser.quit ()
        if display: display.stop ()


if __name__ == ‘__main__’:
    url = ‘https://ip.cn’
    url = ‘https://mimvp.com’
    url = ‘https://proxy.mimvp.com/ip.php’
    
    # Crawl webpage without proxy, success
    spider_url_firefox (url)
    
    # Proxy without password, or set whitelist ip, success
    spider_url_firefox_by_whiteip (url)
    
    # http, https password authorization, success
    spider_url_firefox_by_proxy (url)

    # socks5 Password authorization, failed (still requested by local ip, not proxy ip request)
    spider_url_firefox_by_socks (url)

Python + Selenium + Firefox uses proxy auth user name password authorization

Alibaba Cloud Hot Products
Elastic Compute Service (ECS)	Dedicated Host (DDH)	ApsaraDB RDS for MySQL (RDS)	ApsaraDB for PolarDB(PolarDB)	AnalyticDB for PostgreSQL (ADB for PG)
AnalyticDB for MySQL(ADB for MySQL)	Data Transmission Service (DTS)	Server Load Balancer (SLB)	Global Accelerator (GA)	Cloud Enterprise Network (CEN)
Object Storage Service (OSS)	Content Delivery Network (CDN)	Short Message Service (SMS)	Container Service for Kubernetes (ACK)	Data Lake Analytics (DLA)
ApsaraDB for Redis (Redis)	ApsaraDB for MongoDB (MongoDB)	NAT Gateway	VPN Gateway	Cloud Firewall
Anti-DDoS	Web Application Firewall (WAF)	Log Service	DataWorks	MaxCompute
Elastic MapReduce (EMR)	Elasticsearch