使用HtmlAgilityPack批量抓取網頁資料

標籤：ext   request   ring   validate   軟體   取數   lin   迴圈   back   

【轉】使用HtmlAgilityPack批量抓取網頁資料

相關軟體點擊下載

登入的處理。因為有些網頁資料需要登陸後才能提取。這裡要使用ieHTTPHeaders來提取登入時的提交資訊。

抓取網頁

 HtmlAgilityPack.HtmlDocument htmlDoc;

            if (!string.IsNullOrEmpty(登入URL))
            {
                htmlDoc = htmlWeb.Load(登入URL, 提交的使用者驗證資訊, 擷取資料的網頁URL);
            }
            else
            {
                htmlDoc = htmlWeb.Load(擷取資料的網頁URL);
            }
        

 

 

 ArrayList list = new ArrayList();
            list.add("//table/tr[1]/td");
            list.add("//table/tr[2]/td");
            //擷取迴圈的節點的xpath，比如：//table/tr
            HtmlNodeCollection repeatNodes = htmlDoc.DocumentNode.SelectNodes("//table/tr");

            //迴圈節點
            foreach (HtmlNode node in repeatNodes)
            {
                //迴圈擷取資料
                foreach (string dataPath in list)
                {

                    HtmlNode dataNode = node.SelectSingleNode(list);
                    if (dataNode != null)
                    {
                        string text = dataNode.InnerText;
                    }

                }
            }

如果出現亂碼，調整編碼集為gb2312或者是utf-8

htmlWeb.DefaultEncoding = System.Text.Encoding.GetEncoding(strEncode);

-------------------------------------------------------------------------------------------

using System;

using System.Collections.Generic;

using System.Text;

using Microsoft.VisualStudio.TestTools.WebTesting;

using HtmlAgilityPack;

publicclassWebTest1Coded : WebTest

{

publicoverrideIEnumerator<WebTestRequest> GetRequestEnumerator()

{

WebTestRequest request1 = newWebTestRequest("http://www.microsoft.com/");

request1.ValidateResponse += newEventHandler<ValidationEventArgs>(request1_ValidateResponse);

yieldreturn request1;

}

void request1_ValidateResponse(object sender, ValidationEventArgs e)

{

//load the response body string as an HtmlAgilityPack.HtmlDocument

HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

doc.LoadHtml(e.Response.BodyString);

//locate the "Nav" element

HtmlNode navNode = doc.GetElementbyId("Nav");

//pick the first <li> element

HtmlNode firstNavItemNode = navNode.SelectSingleNode(".//li");

//validate the first list item in the Nav element says "Windows"

e.IsValid = firstNavItemNode.InnerText == "Windows";

}

}

使用HtmlAgilityPack批量抓取網頁資料