C#擷取網頁資訊核心方法（入門一）

標籤：

目錄：資訊採集入門系列目錄

 下面記錄的是我自己整理的C#請求頁面核心類，主要有如下幾個方法

1.HttpWebRequest Get請求獲得頁面html

2.HttpWebRequest Post請求獲得頁面html

3.類比登入獲得cookie內容

4.類比登入獲得cookie字串

5.代理的設定

6.利用webbrowser 擷取js產生的頁面

7.為webbrowser設定cookie，類比登入

8.每個方法的使用demo+demo下載

 

HttpWebRequest Get請求獲得頁面html

        /// <summary>        /// get請求獲得頁面的html        /// </summary>        /// <param name="url">需要擷取的url</param>        /// <param name="proxy">代理，沒有設定為null,不然每次去讀代理造成請求很慢</param>        /// <param name="cookie">該網站所需要的cookie</param>        /// <param name="timeout">逾時時間</param>        /// <returns>頁面請求後的html</returns>        public static string Crawl(string url, WebProxy proxy, CookieContainer cookie, int timeout = 10000)        {            string result = string.Empty;            HttpWebRequest request = null;            WebResponse response = null;            StreamReader streamReader = null;            try            {                request = (HttpWebRequest)HttpWebRequest.Create(url);                request.Proxy = proxy;                request.Timeout = timeout;                request.AllowAutoRedirect = true;                request.CookieContainer = cookie;                response = (HttpWebResponse)request.GetResponse();                streamReader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);                result = streamReader.ReadToEnd();            }            catch (Exception ex)            {                throw ex;            }            finally            {                if (request != null)                {                    request.Abort();                }                if (response != null)                {                    response.Close();                }                if (streamReader != null)                {                    streamReader.Dispose();                }            }            return result;        }

 

HttpWebRequest Post請求獲得頁面html

        /// <summary>        /// post請求獲得頁面        /// </summary>        /// <param name="url">需要擷取的url</param>        /// <param name="postdata">post的資料字串，如id=1&name=test</param>        /// <param name="proxy">代理</param>        /// <param name="cookie">coolie</param>        /// <param name="timeout">逾時</param>        /// <returns></returns>        public static string Crawl(string url, string postdata,WebProxy proxy, CookieContainer cookie, int timeout = 10000)        {            string result = string.Empty;            HttpWebRequest request = null;            WebResponse response = null;            StreamReader streamReader = null;            try            {                request = (HttpWebRequest)HttpWebRequest.Create(url);                request.Proxy = proxy;                request.Timeout = timeout;                request.AllowAutoRedirect = true;                request.CookieContainer = cookie;                byte[] bs = Encoding.ASCII.GetBytes(postdata);                string responseData = String.Empty;                request.Method = "POST";                request.ContentType = "application/x-www-form-urlencoded";                request.ContentLength = bs.Length;                using (Stream reqStream = request.GetRequestStream())                {                    reqStream.Write(bs, 0, bs.Length);                    reqStream.Close();                }                response = (HttpWebResponse)request.GetResponse();                streamReader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);                result = streamReader.ReadToEnd();            }            catch (Exception ex)            {                throw ex;            }            finally            {                if (request != null)                {                    request.Abort();                }                if (response != null)                {                    response.Close();                }                if (streamReader != null)                {                    streamReader.Dispose();                }            }            return result;        }

 

類比登入獲得cookie內容

        /// <summary>        ///根據類比請求頁面獲得cookie        /// </summary>        /// <param name="url">類比的url</param>        /// <returns>cookie</returns>        public static CookieContainer GetCookie(string url, WebProxy proxy, int timeout = 10000)        {            HttpWebRequest request = null;            HttpWebResponse response = null;            try            {                CookieContainer cc = new CookieContainer();                request = (HttpWebRequest)HttpWebRequest.Create(url);                request.Proxy = proxy;                request.Timeout = timeout;                request.AllowAutoRedirect = true;                request.CookieContainer = cc;                response = (HttpWebResponse)request.GetResponse();                response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);                return cc;            }            catch (Exception ex)            {                throw ex;            }            finally            {                if (request != null)                {                    request.Abort();                }                if (response != null)                {                    response.Close();                }            }        }

 

類比登入獲得cookie字串

        /// <summary>        /// 獲得cookie字串，webbrowser可以使用        /// </summary>        /// <param name="url"></param>        /// <param name="proxy"></param>        /// <param name="timeout"></param>        /// <returns></returns>        public static string GetCookieString(string url, WebProxy proxy, int timeout = 10000)        {            HttpWebRequest request = null;            HttpWebResponse response = null;            try            {                CookieContainer cc = new CookieContainer();                request = (HttpWebRequest)HttpWebRequest.Create(url);                request.Proxy = proxy;                request.Timeout = timeout;                request.AllowAutoRedirect = true;                request.CookieContainer = cc;                response = (HttpWebResponse)request.GetResponse();                response.Cookies = request.CookieContainer.GetCookies(request.RequestUri);                string strcrook = request.CookieContainer.GetCookieHeader(request.RequestUri);                return strcrook;            }            catch (Exception ex)            {                throw ex;            }            finally            {                if (request != null)                {                    request.Abort();                }                if (response != null)                {                    response.Close();                }            }        }

 

代理的設定

       /// <summary>        /// 建立代理        /// </summary>        /// <param name="port">代理連接埠</param>        /// <param name="user">使用者名稱</param>        /// <param name="password">密碼</param>        /// <returns></returns>        public static WebProxy CreatePorxy(string port, string user, string password)        {            WebProxy proxy = new WebProxy();             proxy.Address = new Uri(port);             proxy.Credentials = new NetworkCredential(user, password);             return proxy;        }

 

利用webbrowser 擷取js產生的頁面

為webbrowser設定cookie，類比登入

 

C#擷取網頁資訊核心方法（入門一）