Selenium 實現爬蟲_網路爬蟲

1下載

selenium-server-standalone-2.41.0.jar

chromedriver_win32.zip

       IEDriverServer_x64_2.42.0.zip

    

2設定環境

1)解壓chromedriver_win32.zip，把chromedriver.exe拷貝至C:/ selenium/chrome/

2)解壓IEDriverServer_x64_2.42.0.zip把IEDriverServer.exe拷貝至C:/ selenium/ie/

3)ie驅動的路徑加入至環境變數PATH

 

3 程式碼範例

 建java工程，添加jar包selenium-server-standalone-2.41.0.jar。

3.1 baidu

import org.openqa.selenium.By; import org.openqa.selenium.NoSuchElementException; import org.openqa.selenium.StaleElementReferenceException; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebDriverException; import org.openqa.selenium.WebElement; import org.openqa.selenium.ie.InternetExplorerDriver; import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeOptions; import org.openqa.selenium.firefox.FirefoxDriver; import org.openqa.selenium.support.ui.ExpectedCondition; import org.openqa.selenium.support.ui.ExpectedConditions; import org.openqa.selenium.support.ui.WebDriverWait;

privatestaticvoid example() { //for firefox //WebDriver driver = new FirefoxDriver(); //for chrome System.setProperty("webdriver.chrome.driver","C:/ selenium/chrome/chromedriver.exe"); WebDriver driver =newChromeDriver(); //for IE //WebDriver driver = new InternetExplorerDriver(); WebDriverWait w =newWebDriverWait(driver,10); driver.get("http://www.baidu.com/");  

// 等價於 driver.navigate().to("http://www.baidu.com/"); w.until(ExpectedConditions.visibilityOfElementLocated(By.id("kw1"))); w.until(ExpectedConditions.elementToBeClickable(By.id("su1"))); // Find the text input element by its name WebElement element = driver.findElement(By.id("kw1")); element.sendKeys("liaoxiangui"); element.submit();  

//WebDriver會自動的找element對應的form，並提交. 等價於：  

//driver.findElement(By.id("su1")).click(); System.out.println("Page title is: "+ driver.getTitle()); // baidu's search is rendered dynamically with JavaScript. // Wait for the page to load, timeout after 10 seconds w.until(newExpectedCondition<Boolean>(){publicBoolean apply(WebDriver d){return d.getTitle().toLowerCase().startsWith("liaoxiangui");}}); System.out.println("Page title is: "+ driver.getTitle()); try{Thread.sleep(10000);}catch(Exception e){} //Close the browser driver.quit(); }

3.2 一號店

privatestaticvoid yihaodian() { System.setProperty("webdriver.chrome.driver","C:/work/research/bijia/selenium/chromedriver_win32/chromedriver.exe"); ChromeOptions options =newChromeOptions(); //options.addArguments("--disable-images"); WebDriver driver =newChromeDriver(options); try { driver.get("http://www.yhd.com/ctg/s2/c21289-0-60761/b/a-s1-v0-p15-price-d0-f0-m1-rt0-pid-mid0-k/#page=1&sort=1"); Boolean first =true; int products=0; WebDriverWait ww =newWebDriverWait(driver,10); while(true) { System.out.println("processing filter page:"+driver.getCurrentUrl()); if(first) { first=false; ww.until(ExpectedConditions.presenceOfElementLocated(By.id("startShopping"))); WebElement s = driver.findElement(By.id("selectProvince")); s.click(); s = driver.findElement(By.id("p_13")); s.click(); s = driver.findElement(By.id("startShopping")); s.click(); } ww.until(ExpectedConditions.presenceOfAllElementsLocatedBy(By.cssSelector("#itemSearchList > li")));

//視窗最大化。 必須使得那個按鈕在螢幕上可見，否則會拋異常? //driver.manage().window().maximize();

final int liCounter = driver.findElements(By.cssSelector("#itemSearchList > li")).size(); for(int kk=0;kk<liCounter;kk++) { try { String cssStr ="#itemSearchList > li:nth-of-type("+(kk+1)+")"; WebElement li = driver.findElement(By.cssSelector(cssStr));; products++; String t; try{ ww.until(ExpectedConditions.visibilityOf(li.findElement(By.cssSelector(".owner > a")))); t = li.findElement(By.cssSelector(".owner > a")).getAttribute("title"); System.out.println("vendor name="+t); } catch(NoSuchElementException e) { System.out.println("vendor=自營"); } ww.until(ExpectedConditions.visibilityOf(li.findElement(By.className("electrical_item_box")))); t = li.findElement(By.className("electrical_item_box")).getAttribute("comproid"); System.out.println("selfid="+t); ww.until(ExpectedConditions.visibilityOf(li.findElement(By.cssSelector("div > .search_prod_img > img")))); t = li.findElement(By.cssSelector("div > .search_prod_img > img")).getAttribute("src"); System.out.println("pic url="+t); ww.until(ExpectedConditions.visibilityOf(li.findElement(By.cssSelector(".title > .title")))); t = li.findElement(By.cssSelector(".title > .title")).getText(); System.out.println("title="+t); t = li.findElement(By.cssSelector(".title > .title")).getAttribute("href"); System.out.println("detailed url="+t); ww.until(ExpectedConditions.visibilityOf(li.findElement(By.className("color_red")))); t = li.findElement(By.className("color_red")).getText(); System.out.println("price="+t); } catch(StaleElementReferenceException ex)//see http://docs.seleniumhq.org/exceptions/stale_element_reference.jsp { kk--; System.out.println("stale element. retry to get it."); } //break; } WebElement ne =null; while(true) { try { ww.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("#rankOpDiv"))); //如果找不到指定的element，selenium的做法是拋異常。 ne = driver.findElement(By.cssSelector("#rankOpDiv .select_page_btn a.next"));//throw NoSuchElementException ww.until(ExpectedConditions.elementToBeClickable(By.cssSelector("#rankOpDiv a.next"))); ne.click(); } catch(StaleElementReferenceException ex) { System.out.println("retry going to next page."); continue; } catch(NoSuchElementException e) { System.out.println("this category end!"); } break; } if(ne ==null) break; } System.out.println("get "+products+" products."); } catch(Exception ex) { System.out.println(ex); } //Close the browser driver.quit(); }