Python automatically gets the proxy list and crawls the site through the proxy

This script implements the following functions:

1: Automatically get the latest available agent information from a proxy website: IP address, port, protocol type (web site for every minute call this book made a limit)

2: Automatically fill in the agent information and carry out the capture of the website information

Note: X

The code is as follows

Copy Code

#-*-Coding:utf8-*- ''' ''' Import Urllib2 Import Urllib Import Cookielib Import Hashlib Import re Import time Import JSON From Pip._vendor.distlib._backport.tarfile import Turead Class Spide: def __init__ (Self,proxy_ip,proxy_type,proxy_port,use_proxy=false): print ' Using the proxy info: ', proxy_ip Self.proxy = Urllib2. Proxyhandler ({proxy_type:proxy_ip+ ":" +proxy_port}) Self.usercode = "" Self.userid = "" SELF.CJ = Cookielib. Lwpcookiejar (); Self.opener = Urllib2.build_opener (urllib2. Httpcookieprocessor (SELF.CJ)); If Use_proxy: Self.opener = Urllib2.build_opener (self.proxy) Urllib2.install_opener (Self.opener); #获取代理列表 def get_proxy (self): Proxy_info_json = "" #first get the proxy info from Try Reqrequest_proxy = Urllib2. Request (' Http://gXiXmXmXeXpXrXoXxXy.com/api/getProxy '); Reqrequest_proxy.add_header (' Accept ', ' */* '); Reqrequest_proxy.add_header (' accept-language ', ' zh-cn,zh;q=0.8 '); Reqrequest_proxy.add_header (' user-agent ', ' mozilla/5.0 (Windows NT 6.1; WOW64) applewebkit/537.36 (khtml, like Gecko) chrome/28.0.1500.72 safari/537.36 '); Reqrequest_proxy.add_header (' Content-type ', ' application/x-www-form-urlencoded '); Proxy_info = Urllib2.urlopen (reqrequest_proxy). read (); Print Proxy_info Proxy_info_json = Json.loads (proxy_info) return_str=proxy_info_json[' protocol ']+ ":" +proxy_info_json[' IP ']+proxy_info_json[' Port '] Except Exception,e: print ' Proxy have problem ' #print proxy_info_json[' protocol '] #print proxy_info_json[' IP '] #print proxy_info_json[' Port '] Return Proxy_info_json #print Proxy_info def chrome (self): Try Reqrequest = Urllib2. Request (' http://www.503error.com '); Reqrequest.add_header (' Accept ', ' */* '); Reqrequest.add_header (' accept-language ', ' zh-cn,zh;q=0.8 '); Reqrequest.add_header (' user-agent ', ' mozilla/5.0 (Windows NT 6.1; WOW64) applewebkit/537.36 (khtml, like Gecko) chrome/28.0.1500.72 safari/537.36 '); Reqrequest.add_header (' Content-type ', ' application/x-www-form-urlencoded '); Content = Urllib2.urlopen (reqrequest). read (); Except Exception,e: print ' Oppps ' print ' Done ' if __name__ = = "__main__": For the count in range (100): print ' ################################: ', Count print ' geting the new proxy info: ' Test = spide (proxy_ip= ' test ', proxy_type= ' http ', proxy_port= ' 3128 ', use_proxy=false) Proxy_list = Test.get_proxy () #print proxy_list print ' Start to Chrome ' Spide1 = spide (proxy_ip=proxy_list[' IP '],proxy_type=proxy_list[' protocol '],proxy_port=proxy_list[' Port '],use_ Proxy=true) Spide1.chrome () Time.sleep (5)