Three types of asp.net web page capture source code implementation (asp.net collection, read source code)

Statement 1 is recommended /// <summary> /// use HttpWebRequest to obtain the webpage source code // It is very effective for webpages with BOM, no matter what the encoding is, you can correctly identify /// </summary> /// <param name = "url"> webpage address "</param> /// <returns> to return the webpage source file </returns> public static string GetHtmlSource2 (string url) {// processing content string html = ""; HttpWebRequest request = (HttpWebRequest) WebRequest. create (url); request. accept = "*/*"; // Accept any file request. userAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2 ;. net clr 1.1.4322) "; // simulate using IE to browse http://www.52mvc.com Request. allowAutoRedirect = true; // whether 302 is allowed // request. cookieContainer = new CookieContainer (); // cookie container, request. referer = url; // reference of the current page HttpWebResponse response = (HttpWebResponse) request. getResponse (); Stream stream = response. getResponseStream (); StreamReader reader = new StreamReader (stream, Encoding. default); html = reader. readToEnd (); stream. close (); return html;} Format 2 using System; using Sys Tem. collections. generic; using System. linq; using System. web; using System. IO; using System. text; using System. net; namespace MySql {public class GetHttpData {public static string GetHttpData2 (string Url) {string sException = null; string sRslt = null; WebResponse oWebRps = null; WebRequest oWebRqst = WebRequest. create (Url); oWebRqst. timeout = 50000; try {oWebRps = oWebRqst. getResponse ();} catch (W EbException e) {sException = e. Message. ToString () ;}catch (Exception e) {sException = e. ToString () ;}finally {if (oWebRps! = Null) {StreamReader oStreamRd = new StreamReader (oWebRps. getResponseStream (), Encoding. getEncoding ("UTF-8"); sRslt = oStreamRd. readToEnd (); oStreamRd. close (); oWebRps. close () ;}} return sRslt ;}} write 3 public static string getHtml (string url, params string [] charSets) // url is the address of the website to be accessed, charSet is the encoding of the target webpage. if the input is null or "", the code of the webpage will be automatically analyzed {try {string charSet = null; if (charSets. length = 1) {charSet = c HarSets [0];} WebClient myWebClient = new WebClient (); // create a WebClient instance myWebClient // Note: // some webpages may not be available, there are various causes, such as cookie and encoding problems. // This is the case for specific analysis. For example, adding a cookie to the header // webclient. headers. add ("Cookie", cookie); // This may require some overload methods. You can write it as needed // obtain or set the network creden。 used to authenticate requests to Internet resources. MyWebClient. credentials = CredentialCache. defaultCredentials; // if the server wants to verify the user name, password // NetworkCredential mycred = new NetworkCredential (struser, strpassword); // myWebClient. credentials = mycred; // download data from the resource and return a byte array. (Add @ because the URL contains the "/" symbol.) byte [] myDataBuffer = myWebClient. downloadData (url); string strWebData = Encoding. default. getString (myDataBuffer); // obtain the character encoding description of the webpage. Match charSetMatch = Regex. match (strWebData, "<meta ([^ <] *) charset = ([^ <] *) \" ", RegexOptions. ignoreCase | RegexOptions. multiline); string webCharSet = charSetMatch. groups [2]. value; if (charSet = null | charSet = "") charSet = webCharSet; if (charSet! = Null & charSet! = "" & Encoding. GetEncoding (charSet )! = Encoding. default) {strWebData = Encoding. getEncoding (charSet ). getString (myDataBuffer);} else {strWebData = Encoding. getEncoding ("UTF-8 "). getString (myDataBuffer);} return strWebData;} catch (Exception e) {return "";}}