擷取URL網頁資訊

標籤：iter   設定   agent   save   log   sha   ext   toe   oid   

 

 

        static string GetHtml(string url)        {string strHTML = "";            WebClient myWebClient = new WebClient();            Stream myStream = myWebClient.OpenRead(url);            StreamReader sr = new StreamReader(myStream, System.Text.Encoding.GetEncoding("utf-8"));            strHTML = sr.ReadToEnd();            myStream.Close();            return strHTML;        }        static bool GetHtml(string url)      {                HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);            myHttpWebRequest.Timeout = 20 * 1000; //連線逾時            myHttpWebRequest.Accept = "*/*";            myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0;)";            HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();            Stream stream = myHttpWebResponse.GetResponseStream();            StreamReader sr =               new StreamReader(stream, Encoding.GetEncoding("utf-8"));            var html = sr.ReadToEnd();           
        }

 

 

擷取當前Cookies儲存後擷取整站資訊

static void Main(string[] args)        {                       Console.WriteLine("確保掛載VPN後。。。斷行符號鍵繼續！");            Console.ReadKey();            string url = "http://www.sciencedirect.com/science/journal/00118486/61/6";            string indata = "aa=zhuye";            CookieContainer myCookieContainer =                new CookieContainer();            //建立一個cookiecontainer來存放cookie集合            HttpWebRequest myHttpWebRequest =                (HttpWebRequest)WebRequest.Create(url);            //建立一個httpwebrequest            myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";            myHttpWebRequest.ContentLength = indata.Length;            myHttpWebRequest.Method = "post";            myHttpWebRequest.CookieContainer = myCookieContainer;            //設定httpwebrequest的cookiecontainer為            //剛才建立的那個mycookiecontainer            Stream myRequestStream = myHttpWebRequest.GetRequestStream();            StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312"));            myStreamWriter.Write(indata);            //把資料寫入httpwebrequest的request流            myStreamWriter.Close();            myRequestStream.Close();            //關閉開啟對象            HttpWebResponse myHttpWebResponse =                (HttpWebResponse)myHttpWebRequest.GetResponse();            //建立一個httpwebresponse            myHttpWebResponse.Cookies = myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);                      Console.WriteLine("擷取cookies成功！。。。關閉VPN後任意鍵繼續");            Console.ReadKey();            var htmlData = SaveCook(myHttpWebRequest, prevUrl, myCookieContainer, myHttpWebResponse);                                           }        }

 

 public static string SaveCook(HttpWebRequest myHttpWebRequest, string url, CookieContainer myCookieContainer, HttpWebResponse myHttpWebResponse)        {            //拿到了cookie，再進行請求就能直接讀取到登入後的內容了            myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);            myHttpWebRequest.CookieContainer = myCookieContainer;//*                                                                 //剛才那個cookiecontainer已經存有了cookie,把它附加到             myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();            myHttpWebResponse.Cookies =                myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);            Stream myresponsestream = myHttpWebResponse.GetResponseStream();            StreamReader mystreamreader =               new StreamReader(myresponsestream, Encoding.GetEncoding("gb2312"));            var html = mystreamreader.ReadToEnd();            //把資料從httpwebresponse的response流中讀出            mystreamreader.Close();            myresponsestream.Close();            return html;        }

 

擷取URL網頁資訊