Python 爬蟲執行個體（11）—— 爬蟲 蘇寧易購

標籤：lin   set   爬蟲   backup   target   cep   app   driver   添加   

# coding:utf-8import jsonimport redisimport timeimport requestssession = requests.session()import logging.handlersimport pickleimport sysimport reimport datetimefrom bs4 import BeautifulSoupfrom selenium import webdriverimport osimport sysreload(sys)sys.setdefaultencoding(‘utf8‘)r =redis.Redis(host="123.56.74.190",port=6379,password="ZBHRwlb1608")import platformsysStr = platform.system()if sysStr =="Windows":    LOG_FILE_check = ‘C:\\log\\wlb\\crawler\\cic.log‘else:    LOG_FILE_check = ‘/log/wlb/crawler/cic.log‘handler = logging.handlers.RotatingFileHandler(LOG_FILE_check, maxBytes=128 * 1024 * 1024,backupCount=10)  # 執行個體化handler  200M 最多十個檔案fmt = ‘\n‘ + ‘%(asctime)s - %(filename)s:%(lineno)s  - %(message)s‘formatter = logging.Formatter(fmt)  # 執行個體化formatterhandler.setFormatter(formatter)  # 為handler添加formatterlogger = logging.getLogger(‘check‘)  # 擷取名為tst的loggerlogger.addHandler(handler)  # 為logger添加handlerlogger.setLevel(logging.DEBUG)def spider():    chromedriver = "C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe"    os.environ["webdriver.chrome.driver"] = chromedriver    browser = webdriver.Chrome(chromedriver)    # 設定瀏覽器需要開啟的url    url = "https://www.suning.com/"    browser.get(url)    time.sleep(5)    browser.find_element_by_id("searchKeywords").send_keys(u‘手機‘)    time.sleep(2)    for i in range(1,100):        browser.find_element_by_name("index1_none_search_ss1").click()        browser.find_element_by_id("nextPage").click()        result = browser.page_source        soup = BeautifulSoup(result,‘html.parser‘)        result_ul = soup.find_all(‘div‘,attrs={"id":"filter-results"})[0]        result_list = result_ul.find_all(‘div‘,attrs={"class":"li-bg"})        print len(result_list)        print result_list[1]        # for item in result_list:        #     print item        #     print "==" * 30        #        # time.sleep(500)        for item in result_list:            item = str(item).replace(‘\n‘,‘‘).replace(‘\r‘,‘‘).replace(‘\t‘,‘‘)            print "==" * 30            print item            try:                sold_price = re.findall(‘pricefn="priceCenterShow"><i>¥</i>(.*?)<i>.*?</i></span>‘,item)[0]            except:                sold_price = re.findall(‘<i>¥</i>(.*?)<i>.*?</i></span>‘,item)[0]            try:                item_name = re.findall(‘<i class=".*?" style=".*?"></i>(.*?)</b></a>‘,item)[0]            except:                item_name = re.findall(‘target="_blank" title="(.*?)"><i class=‘,item)[0]            try:                item_url = re.findall(‘class=".*?" href="(.*?)" name‘,item)[0]            except:                item_url = re.findall(‘<a class=".*?" href="(.*?)" id=‘, item)[0]            try:                item_desc = re.findall(‘<span><i></i><em>(.*?)</em><b></b></span>‘,item)[0]            except:                item_desc = re.findall(‘<em>(.*?)</em>‘, item)[0]            print item_url            print item_name            print sold_price            print item_desc    time.sleep(500)spider()

 

Python 爬蟲執行個體（11）—— 爬蟲 蘇寧易購