These two days to see the Python proxy IP implementation, finally finalized, haha, look at the code
From BS4 import beautifulsoup import requests import random import urllib2 def get_ip_list (URL, headers): Web_data = R Equests.get (URL, headers=headers) soup = BeautifulSoup (web_data.text, ' lxml ') ips = Soup.find_all (' tr ') Ip_lis t = [] for I in range (1, Len (IPS)): Ip_info = ips[i] TDs = Ip_info.find_all (' TD ') Ip_list.appe nd (Tds[1].text + ': ' + tds[2].text) return ip_list def get_random_ip (ip_list): Proxy_list = [] for IP in ip_l
Ist:proxy_list.append (' http://' + IP) proxy_ip = random.choice (proxy_list) proxies = {' http ': proxy_ip} return proxies If __name__ = = ' __main__ ': url = ' http://www.xicidaili.com/nn/' headers = {' User-agent ': ' mozilla/5.0 (Windows NT 6.1; Win64; x64) applewebkit/537.36 (khtml, like Gecko) chrome/53.0.2785.143 safari/537.36 '} ip_list = get_ip_list (URL, Heade rs=headers) proxies = get_random_ip (ip_list) print (proxies) Enable_proxy = True Proxy_handler = Urllib2. Proxyhandler (proxies) Null_proxy_handler = Urllib2. Proxyhandler ({}) if Enable_proxy:opener = Urllib2.build_opener (proxy_handler) Else:opener = Urllib2.build_opener
(Null_proxy_handler) Urllib2.install_opener (opener) response = Urllib2.urlopen (' http://www.baidu.com ', timeout=10) Print (Response.read ())