python爬蟲爬取人人車（二手車）、利用padas、matplotlib組建圖表

標籤：arc   callback   soup   enc   text   selector   字串   pos   nat   

該程式主要為了抓取人人車賣車資訊，包括車系、車型號、購車日期、賣車價格、行駛路程、首付價格等等資訊。話不多說直接代碼。

車系py檔案

# -*- coding: utf-8 -*-import refrom urllib.request import urlopenfrom scrapy.http import Request# from urllib.request import Requestfrom bs4 import BeautifulSoupfrom lxml import etreeimport pymongoimport scrapyfrom scrapy.selector import HtmlXPathSelectorclient = pymongo.MongoClient(host="127.0.0.1")db = client.renrenchecollection = db.Carclass          #表名classificationimport redis        #匯入redis資料庫r = redis.Redis(host=‘127.0.0.1‘, port=6379, db=0)class renrencheSpider(scrapy.Spider):    name = "Carinfo1"    allowed_domains = ["renrenche.com"]   #允許訪問的域    start_urls = [        "https://www.renrenche.com/bj/ershouche/"    ]    #每爬完一個網頁會回調parse方法    def parse(self, response):        hxs = HtmlXPathSelector(response)        hx = hxs.select(‘//div[@class="brand-more-content"]/div[@class="brand-section brand-section-1"]/p[@class="bl"]/span[@class="bn"]/a‘)        for secItem in hx:            url = secItem.select("@href").extract()            c = "https://www.renrenche.com"+url[0]            name = secItem.select("text()").extract()            classid =self.insertMongo(name,None)            print(c)            print(name)            request = Request(c,callback=lambda response,pid=str(classid):self.parse_subclass(response,pid))            yield request    def parse_subclass(self, response,pid):        # print(response.body.decode(‘utf-8‘))        hxs = HtmlXPathSelector(response)        hx = hxs.select(‘//ul[@id="filter_series"]/li[@class=""]/a‘)        for secItem in hx:            urls = secItem.select("@href").extract()            url = "https://www.renrenche.com" + urls[0]            name = secItem.select("text()").extract()            print(url)            print(name)            classid = self.insertMongo(name,pid)            self.pushRedis(classid,url,pid)    def insertMongo(self,classname,pid):        classid = collection.insert({‘classname‘:classname,‘pid‘:pid})        return classid    def pushRedis(self,classid,url,pid,):        carurl = ‘%s,%s,%s‘ %(classid,url,pid)        r.lpush(‘carurl‘,carurl)

　　賣車各種資訊py檔案

# -*- coding: utf-8 -*-import refrom urllib.request import urlopenfrom scrapy.http import Requestimport pymongoimport scrapyfrom time import sleepfrom scrapy.selector import HtmlXPathSelectorclient = pymongo.MongoClient(host="127.0.0.1")db = client.renrenchecollection = db.Carinfoimport redis  # 匯入redis資料庫r = redis.Redis(host=‘127.0.0.1‘, port=6379, db=0)class renrencheSpider(scrapy.Spider):    name = "Carinfo2"    allowed_domains = ["renrenche.com"]    dict = {}    start_urls = []    def __init__(self):  # 定義一個方法        a = r.lrange(‘carurl‘, 0, -1)        for item in a:            novelurl = bytes.decode(item)            arr = novelurl.split(‘,‘)  # 分割字串            renrencheSpider.start_urls.append(arr[1])            pid = arr[0]            url = arr[1]            self.dict[url] = {"pid":pid,"num":0}    def parse(self, response):        classInfo = self.dict[response.url]        pid = classInfo[‘pid‘]        num = classInfo[‘num‘]        # print(self.dict)        if num>3:            return None        hxs = HtmlXPathSelector(response)        hx = hxs.select(‘//ul[@class="row-fluid list-row js-car-list"]‘)        s=""        for secItem in hx:            hx1 = secItem.select(‘//li[@class="span6 list-item car-item"]/a[@rrc-event-param="search"]/h3‘)            name = hx1.select("text()").extract()            a = "型號："+name[0]            s +=a+"\n"            # classid = collection.insert({‘carinfo‘: a, ‘pid‘: pid})            # print(a)        for secItem in hx:            hx2 = secItem.select(‘//div[@class="mileage"]/span[@class="basic"]‘)            name = hx2.select("text()").extract()            b = "購車年份/公裡數："+name[0]+"/"+name[1]            s +=b+"\n"            # print(b)        for secItem in hx:            hx3 = secItem.select(‘//div[@class="tags-box"]/div[@class="price"]‘)            name = hx3.select("text()").extract()            c = str(name[0])            c = c.strip()            c = "賣車價格："+c+"萬"            s +=c+"\n"            # print(c)        for secItem in hx:            hx4 = secItem.select(‘//div[@class="down-payment"]/div[@class="m-l"]‘)            name = hx4.select("text()").extract()            d = "首付："+name[0]+"萬"            s +=d+"\n"            # print(d)        print(s)        classid = self.insertMongo(s, pid)    def insertMongo(self, classname, pid):        classid = collection.insert({‘classname‘: classname, ‘pid‘: pid})        return classid        # self.pushRedis(classid, pid)    #     print(‘-----------遞迴--------------‘)    #    #     hxs = HtmlXPathSelector(response)    #     hx = hxs.select(‘//li[@class="lbf-pagination-item"]/a[@class="lbf-pagination-next "]‘)    #     urls = hx.select("@href").extract()    #     d = "https:" + urls[0]    #     classInfo[‘num‘] +=1    #     self.dict[d] = classInfo    #     print(d)    #     request = Request(d, callback=self.parse)    #     yield request    #     print(‘--------end--------------‘)    #    # def pushRedis(self, classid, c, pid):    #     novelnameurl = ‘%s,%s,%s‘ % (classid, c, pid)    #     r.lpush(‘novelnameurl‘, novelnameurl)

　　

python爬蟲爬取人人車（二手車）、利用padas、matplotlib組建圖表