scrapy爬取西刺網站ip

標籤：collect   imp   domain   parse   raw   pid   init   import   pip   

# scrapy爬取西刺網站ip# -*- coding: utf-8 -*-import scrapyfrom xici.items import XiciItemclass XicispiderSpider(scrapy.Spider):    name = "xicispider"    allowed_domains = ["www.xicidaili.com/nn"]    start_urls = [‘http://www.xicidaili.com/nn/‘]    def parse(self, response):        item = XiciItem()        for each in response.css(‘#ip_list tr‘):            ip = each.css(‘td:nth-child(2)::text‘).extract_first()            port = each.css(‘td:nth-child(3)::text‘).extract_first()            if ip:                ip_port = ip + ‘:‘ + port                item[‘ip_port‘] = ip_port                yield item

import pymongoclass XiciPipeline(object):    collection_name = ‘scrapy_items‘    def __init__(self, mongo_uri, mongo_db):        self.mongo_uri = mongo_uri        self.mongo_db = mongo_db    #這裡的from經常拼錯啊    @classmethod    def from_crawler(cls, crawler):        return cls(            mongo_uri=crawler.settings.get(‘MONGO_URI‘),            mongo_db=crawler.settings.get(‘MONGO_DB‘)        )    def open_spider(self, spider):        self.client = pymongo.MongoClient(self.mongo_uri)        self.db = self.client[self.mongo_db]    def close_spider(self, spider):        self.client.close()    def process_item(self, item, spider):        self.db[self.collection_name].insert(dict(item))        return item

 

scrapy爬取西刺網站ip