scrapy實現ip代理池

標籤：root   lis   middle   speed   title   process   pid   connect   header   

首先需要在ip代理的網站爬取有用的ip，儲存到資料庫中

import requestsfrom scrapy.selector import Selectorimport pymysqlconn = pymysql.connect(host = ‘127.0.0.1‘, user = ‘root‘ ,passwd = ‘root‘,db = ‘mysql18_text‘,charset = ‘utf8‘)cursor = conn.cursor()def crawl_ips():    #爬取xici的免費ip代理    agent = ‘Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0‘    header = {        ‘User-Agent‘:agent    }    for i in range(1,3458):        reas = requests.get(‘http://www.xicidaili.com/nn/‘,headers = header)        Selectora = Selector(reas)        all_trs = Selectora.xpath(‘//table[@id="ip_list"]/tr‘)        ip_list = []        for tr in all_trs[1:]:            spend_str = tr.xpath(‘./td/div[@class="bar"]/@title‘).extract()[0] ##提取速度            if spend_str:                speed = float(spend_str.split(‘秒‘)[0])                all_text = tr.xpath(‘./td/text()‘).extract()                ip = all_text[0]                port = all_text[1]                proxy_type = all_text[5]                ip_list.append((ip,port,speed,proxy_type))        for ip_info in ip_list:            cursor.execute(                """insert project_ip(ip,port,speed,proxy_type) VALUES(‘{0}‘,‘{1}‘,‘{2}‘,‘HTTP‘)""".format(                    ip_info[0],ip_info[1],ip_info[2]                )            )            conn.commit()        print(ip_list)                crawl_ips()conn.close()cursor.close()

隨機在資料庫中擷取一個ip的代碼

class GetIP(object):        def delete_ip(self,ip):        #從資料庫中刪除無效的ip        delete_sql = """            delete from project_ip where ip=‘{0}‘        """.format(ip)        cursor.execute(delete_sql)        conn.commit()        return True            def judge_ip(self,ip,port):        #判斷一個ip是否可用        http_url = ‘http://www.baidu.com‘        proxy_url = ‘https://{0}:{1}‘.format(ip,port)                try:            proxy_dict = {                ‘http‘:proxy_url,            }            requests.get(http_url,proxies = proxy_dict)            return True        except Exception as e:            print("ip出現異常")            #出現異常後就把這個ip給刪除掉            self.delete_ip(ip)            return False        else:            code = response.status_code            if code>=200 and code<300:                print(‘effective ip‘)                return True            else:                print(‘invalid‘)                self.delete_ip(ip)                return False        def get_random_ip(self):        #從資料庫中隨機擷取到一個可用的ip        random_sql = """            SELECT ip,port FROM project_ip            ORDER BY RAND()            LIMIT 1        """        result = cursor.execute(random_sql)                for ip_info in cursor.fetchall():            ip = ip_info[0]            port = ip_info[1]            judge_re = self.judge_ip(ip,port)
　　　　　　　if judge_re:#如果返回True
　　　　　　　　  return "http://‘{0}‘:‘{1}‘".format(ip,port)
　　　　　　　else:
　　　　　　　　　 return get_random_ip()

Middleware動態設定ip代理

class RandomProxyMiddleware(object):    def process_request(self,request,spider):        get_ip = GetIP()#這裡需要匯入那個函數        request.meta[‘proxy‘] = get_ip.get_random_ip()

 

scrapy實現ip代理池