Python-爬蟲-針對有frame架構的頁面

標籤：img   ide   turn   tin   頁面   code   color   splay   listdir   

有的頁面會使用frame 架構，使用Selenium + PhantomJS 後並不會載入iframe 架構中的網頁內容。iframe 架構相當於在頁面中又載入了一個頁面，需要使用Selenium 的 switch_to.frame() 方法載入（官網給的方法是switch_to_frame()，但是IDE提醒使用前面的方法替代該方法）。

比如：

        driver.switch_to.frame(‘g_iframe‘)

        html = driver.page_source

然後結合BeautifulSoup擷取網頁中資訊。

這次我們爬取http://music.163.com/#/artist/album?id=101988&limit=120&offset=0頁面中的專輯資訊，比如，圖片、網址及專輯名字。

"""http://music.163.com/#/artist/album?id=101988&limit=120&offset=0爬取上述網址中的專輯資訊"""from selenium import webdriverfrom urllib.request import urlretrieveimport osfrom bs4 import BeautifulSoupclass DownloadInfo():    def __init__(self):        self.url = ‘http://music.163.com/#/artist/album?id=101988&limit=120&offset=0‘        self.basePath = os.path.dirname(__file__)    def makedir(self, name):        path = os.path.join(self.basePath, name)        isExist = os.path.exists(path)        if not isExist:            os.makedirs(path)            print(‘The file is created now.‘)        else:            print(‘The file existed.‘)        #切換到該目錄下        os.chdir(path)        return path    def connect(self, url):        driver = webdriver.PhantomJS()        driver.get(url)        print(‘success‘)        return driver    def getFileNames(self, path):        pic_names = os.listdir(path)        return pic_names    def getInfo(self):        driver = self.connect(self.url)        driver.switch_to.frame(‘g_iframe‘)        path = self.makedir(‘Infos‘)        pic_names = self.getFileNames(path)        imgs = driver.find_elements_by_xpath("//div[@class=‘u-cover u-cover-alb3‘]/img")        titles = driver.find_elements_by_xpath("//li/p[@class=‘dec dec-1 f-thide2 f-pre‘]/a")        dates = driver.find_elements_by_xpath("//span[@class=‘s-fc3‘]")        count = 0        for img in imgs:            album_name = titles[count].text            count += 1            photo_name = album_name.replace(‘/‘, ‘‘) + ‘.jpg‘            print(photo_name)            if photo_name in pic_names:                print(‘圖片已下載。‘)            else:                urlretrieve(img.get_attribute(‘src‘), photo_name)        for title in titles:            print(title.text)        for date in dates:            print(date.text)"""    def getInfo(self):        driver = self.connect(self.url)        driver.switch_to.frame(‘g_iframe‘)        html = driver.page_source        path = self.makedir(‘Infos‘)        pic_names  = self.getFileNames(path)        all_li = BeautifulSoup(html, ‘lxml‘).find(id=‘m-song-module‘).find_all(‘li‘)        for li in all_li:            album_img = li.find(‘img‘)[‘src‘]            album_name = li.find(‘p‘, class_=‘dec‘)[‘title‘]            album_date = li.find(‘span‘, class_=‘s-fc3‘).get_text()            print(album_img)            print(album_name)            print(album_date)            photo_name = album_name.replace(‘/‘, ‘‘) + ‘.jpg‘            if photo_name in pic_names:                print(‘圖片已下載。‘)            else:                urlretrieve(album_img, photo_name)"""if __name__ == ‘__main__‘:    obj = DownloadInfo()    obj.getInfo()

View Code

 

Python-爬蟲-針對有frame架構的頁面