python Scrapy google trends

標籤：from   for   coding   tar   geo   git   dss   lock   elf   

# -*- coding: utf-8 -*-import scrapy,jsonfrom urllib import parseclass GoogleTrendsSpider(scrapy.Spider):    name = ‘google_trends‘    allowed_domains = [‘google.com‘]    #擷取token地址    GENERAL_URL = ‘https://trends.google.com/trends/api/explore?{}‘    #擷取關鍵字csv地址    INTEREST_OVER_TIME_URL = ‘https://trends.google.com/trends/api/widgetdata/multiline/csv?{}‘    #開啟useragent和代理中介軟體    custom_settings = {        ‘DOWNLOADER_MIDDLEWARES‘ : {            ‘blockchain.middlewares.RandomUserAgent‘: 390,            ‘blockchain.middlewares.RandomProxy‘: 544,        },        # ‘COOKIES_ENABLED‘ : False        ‘DOWNLOAD_DELAY‘:1    }        def start_requests(self):        ‘‘‘            從頁面中擷取的csv下載url參數            ‘keyword‘: ‘關鍵字‘,             ‘time‘: ‘now 7-d‘,             ‘geo‘: ‘‘        ‘‘‘        req = {            ‘comparisonItem‘:[{‘keyword‘: ‘關鍵字‘, ‘time‘: ‘now 7-d‘, ‘geo‘: ‘‘}],            ‘category‘: 0        }        req = json.dumps(req).encode(‘utf-8‘)        token_payload = {            ‘hl‘: b‘en-US‘,            ‘tz‘: b‘-480‘,            ‘req‘: req,            ‘property‘: b‘‘,        }        body = parse.urlencode(token_payload)        url = self.GENERAL_URL.format(body)        reqs.append(scrapy.Request(url=url,callback=self.parse_token,meta={‘item‘:{‘coin_id‘:row.id}}))        return reqs    def parse_token(self,response):        ‘‘‘            解析結果，擷取token        ‘‘‘        bodyObj = json.loads(response.body.decode(‘utf-8‘)[4:])        for row in bodyObj[‘widgets‘]:            if row[‘id‘] == ‘TIMESERIES‘:                token = row[‘token‘]                request = row[‘request‘]                params = {                    ‘tz‘: ‘-480‘,                    ‘req‘: json.dumps(request),                    ‘token‘:token                }                body = parse.urlencode(params)                url = self.INTEREST_OVER_TIME_URL.format(body)                yield scrapy.Request(url=url,callback=self.parse_row,meta={‘item‘:response.meta[‘item‘]})    def parse_row(self, response):        ‘‘‘        解析csv        ‘‘‘        bodytext = response.body.decode(‘utf-8‘)        print(bodytext)

　　抄自：pytrends

python Scrapy google trends