Python Spider 抓取今日頭條街拍美圖

標籤：windows   on()   lse   美圖   沒有   use   not   with open   image   

"""抓取今日頭條街拍美圖"""import osimport timeimport requestsfrom hashlib import md5class SpiderToutiao(object):    def __init__(self):        # 指定下載目錄        self.download_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "download")        # 通過分析請求, 我們發現了需要的地址如下, 且分頁是通過 offset + 20 控制的        self.url = "https://www.toutiao.com/search_content/"                    "?offset={0}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=3&from=gallery"        # 構造要求標頭, 偽裝成 Ajax 請求        self.headers = {            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "                          "Chrome/66.0.3359.139 Safari/537.36",            "Referer": "https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D",            "X-Requested-With": "XMLHttpRequest"        }    def handler(self, offset=0):        while True:            response = requests.get(self.url.format(offset), headers=self.headers)            if response.status_code == 200:                print("INFO -> Current URL: <%s>" % response.url)                json_data = response.json().get("data")                # 開始解析資料                if json_data:                    for item in json_data:                        _title = item.get("title")                        _imgDict = item.get("image_list")                        # 修正一下URL, 預設給的圖片地址是小圖, 我們要高清大圖                        _imgList = [str("http:" + _.get("url")).replace("list", "large") for _ in _imgDict]                        # 建立儲存目錄                        _downloadDir = os.path.join(self.download_dir, _title)                        if not os.path.exists(_downloadDir):                            os.makedirs(_downloadDir)                        # 下載並儲存檔案                        for img in _imgList:                            r = requests.get(img)                            _file = os.path.join(_downloadDir, md5(r.content).hexdigest() + ".jpg")                            if not os.path.exists(_file):                                with open(_file, "wb") as f:                                    f.write(r.content)                            else:                                print("INFO -> ig <%s>" % _file)                # 說明沒有資料了, 程式退出                else:                    break                # 分頁自增                offset += 20                # 間隔時間                time.sleep(.9)            else:                print(response.reason)                exit(999)if __name__ == "__main__":    spider = SpiderToutiao()    spider.handler()

 

Python Spider 抓取今日頭條街拍美圖