Asp.net three methods to capture web page source code

Method 1 is recommended
 Copy codeThe Code is as follows:
/// <Summary>

/// Use HttpWebRequest to obtain the webpage source code
/// Effective for webpages with BOM, which can be correctly identified by any code
/// </Summary>
/// <Param name = "url"> webpage address "</param>
/// <Returns> return the webpage source file </returns>
Public static string GetHtmlSource2 (string url)
{
// Process content
String html = "";
HttpWebRequest request = (HttpWebRequest) WebRequest. Create (url );
Request. Accept = "*/*"; // Accept any file
Request. UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2;. net clr 1.1.4322)"; // simulate using IE to browse the http://www.52mvc.com
Request. AllowAutoRedirect = true; // whether to allow 302
// Request. CookieContainer = new CookieContainer (); // cookie container,
Request. Referer = url; // reference of the current page

 
HttpWebResponse response = (HttpWebResponse) request. GetResponse ();
Stream stream = response. GetResponseStream ();
StreamReader reader = new StreamReader (stream, Encoding. Default );
Html = reader. ReadToEnd ();
Stream. Close ();

 
Return html;
}
 

Method 2

Copy codeThe Code is as follows:
Using System;
Using System. Collections. Generic;
Using System. Linq;
Using System. Web;
Using System. IO;
Using System. Text;
Using System. Net;

Namespace MySql
{
Public class GetHttpData
{
Public static string GetHttpData2 (string Url)
{
String sException = null;
String sRslt = null;
WebResponse oWebRps = null;
WebRequest oWebRqst = WebRequest. Create (Url );
OWebRqst. Timeout = 50000;
Try
{

OWebRps = oWebRqst. GetResponse ();

}
Catch (WebException e)
{
SException = e. Message. ToString ();
}
Catch (Exception e)
{
SException = e. ToString ();

}
Finally
{
If (oWebRps! = Null)
{

StreamReader oStreamRd = new StreamReader (oWebRps. GetResponseStream (), Encoding. GetEncoding ("UTF-8 "));
SRslt = oStreamRd. ReadToEnd ();
OStreamRd. Close ();
OWebRps. Close ();
}
}

Return sRslt;
}

}
}

Method 3
Copy codeThe Code is as follows:
Public static string getHtml (string url, params string [] charSets) // The url is the address of the website to be accessed, and charSet is the encoding of the target webpage. If the input is null or "", then the code of the webpage is automatically analyzed.
{
Try
{
String charSet = null;
If (charSets. Length = 1 ){
CharSet = charSets [0];
}
WebClient myWebClient = new WebClient (); // create a WebClient instance myWebClient
// Note the following:
// Some webpages may not be available, for various reasons such as cookie and Encoding Problems
// This requires specific problem analysis, such as adding a cookie to the header
// Webclient. Headers. Add ("Cookie", cookie );
// Some overload methods may be required. Write as needed

 
// Obtain or set the network creden。 used to authenticate requests to Internet resources.
MyWebClient. Credentials = CredentialCache. DefaultCredentials;
// If the server needs to verify the user name and password
// NetworkCredential mycred = new NetworkCredential (struser, strpassword );
// MyWebClient. Credentials = mycred;
// Download data from the resource and return a byte array. (Add @ because there is a "/" symbol in the middle of the URL)
Byte [] myDataBuffer = myWebClient. DownloadData (url );
String strWebData = Encoding. Default. GetString (myDataBuffer );

 
// Obtain the character encoding description of the webpage
Match charSetMatch = Regex. match (strWebData, "<meta ([^ <] *) charset = ([^ <] *) \" ", RegexOptions. ignoreCase | RegexOptions. multiline );
String webCharSet = charSetMatch. Groups [2]. Value;
If (charSet = null | charSet = "")
CharSet = webCharSet;

 
If (charSet! = Null & charSet! = "" & Encoding. GetEncoding (charSet )! = Encoding. Default)
{
StrWebData = Encoding. GetEncoding (charSet). GetString (myDataBuffer );
}
Else {
StrWebData = Encoding. GetEncoding ("UTF-8"). GetString (myDataBuffer );
}
Return strWebData;
}
Catch (Exception e) {return "";}
}