)asp.net c#如何採集需要登入的頁面？

原文地址：http://www.cnblogs.com/subendong/archive/2013/01/31/2886568.html

首先說明：程式碼片段是從網路擷取，然後自己修改。我想好的東西應該拿來分享。

先說下原理：當我們採集頁面的時候，如果被採集的網站需要登入才能採集。不管是基於Cookie還是基於Session，我們都會首先發送一個Http要求標頭，這個Http要求標頭裡面就包含了網站需要的Cookie資訊。當網站接收到發送過來的Http要求標頭時，會從Http要求標頭擷取相關的Cookie或者Session資訊，然後由程式來處理，決定你是否有許可權訪問當前頁面。

好了，原理搞清楚了，就好辦了。我們所要做的僅僅是在採集的時候（或者說HttpWebRequest提交資料的時候），將Cookie資訊放入Http要求標頭裡面就可以了。

在這裡我提供2種方法。

第一種，直接將Cookie資訊放入HttpWebRequest的CookieContainer裡。看代碼：

protected void Page_Load(object sender, EventArgs e)        {            //設定Cookie，存入Hashtable            Hashtable ht = new Hashtable();            ht.Add("username", "youraccount");            ht.Add("id", "yourid");            this.Collect(ht);        }        public void Collect(Hashtable ht)        {            string content = string.Empty;            string url = "http://www.ibest100.com/需要登入後才能採集的頁面";            string host = "http://www.ibest100.com";            try            {                //擷取提交的位元組                byte[] bs = Encoding.UTF8.GetBytes(content);                //設定提交的相關參數                HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url);                req.Method = "POST";                req.ContentType = "application/json;charset=utf-8";                req.ContentLength = bs.Length;                //將Cookie放入CookieContainer，然後再將CookieContainer添加到HttpWebRequest                CookieContainer cc = new CookieContainer();                cc.Add(new Uri(host), new Cookie("username", ht["username"].ToString()));                cc.Add(new Uri(host), new Cookie("id", ht["id"].ToString()));                req.CookieContainer = cc;                //提交請求資料                Stream reqStream = req.GetRequestStream();                reqStream.Write(bs, 0, bs.Length);                reqStream.Close();                //接收返回的頁面，必須的，不能省略                WebResponse wr = req.GetResponse();                System.IO.Stream respStream = wr.GetResponseStream();                System.IO.StreamReader reader = new System.IO.StreamReader(respStream, System.Text.Encoding.GetEncoding("utf-8"));                string t = reader.ReadToEnd();                System.Web.HttpContext.Current.Response.Write(t);                wr.Close();            }            catch (Exception ex)            {                System.Web.HttpContext.Current.Response.Write("異常在getPostRespone:" + ex.Source + ":" + ex.Message);            }        }

第二種，每次開啟採集程式時，需要先到被採集的網站類比登入一次，擷取CookieContainer，然後再採集。看代碼：

protected void Page_Load(object sender, EventArgs e)        {            try            {                CookieContainer cookieContainer = new CookieContainer();                string formatString = "username={0}&password={1}";//***************                string postString = string.Format(formatString, "youradminaccount", "yourpassword");                //將提交的字串資料轉換成位元組數組                byte[] postData = Encoding.UTF8.GetBytes(postString);                //設定提交的相關參數                string URI = "http://www.ibest100.com/登入頁面";//***************                HttpWebRequest request = WebRequest.Create(URI) as HttpWebRequest;                request.Method = "POST";                request.KeepAlive = false;                request.ContentType = "application/x-www-form-urlencoded";                request.CookieContainer = cookieContainer;                request.ContentLength = postData.Length;                // 提交請求資料                System.IO.Stream outputStream = request.GetRequestStream();                outputStream.Write(postData, 0, postData.Length);                outputStream.Close();                //接收返回的頁面，必須的，不能省略                HttpWebResponse response = request.GetResponse() as HttpWebResponse;                System.IO.Stream responseStream = response.GetResponseStream();                System.IO.StreamReader reader = new System.IO.StreamReader(responseStream, Encoding.GetEncoding("gb2312"));                string srcString = reader.ReadToEnd();                //開啟您要訪問的頁面                URI = "http://www.ibest100.com/需要登入後才能採集的頁面";//***************                request = WebRequest.Create(URI) as HttpWebRequest;                request.Method = "GET";                request.KeepAlive = false;                request.CookieContainer = cookieContainer;                // 接收返回的頁面                response = request.GetResponse() as HttpWebResponse;                responseStream = response.GetResponseStream();                reader = new System.IO.StreamReader(responseStream, Encoding.GetEncoding("gb2312"));                srcString = reader.ReadToEnd();                //輸出擷取的頁面或者處理                Response.Write(srcString);            }            catch (WebException we)            {                string msg = we.Message;                Response.Write(msg);            }        }

 

也許有人會問，如果對方登入的時候要驗證碼怎麼辦？那你就用第一種方式吧，只不過需要你分析對方的Cookie。

應用範圍：採集資料、論壇發帖、部落格發文。