asp.net crawl Web source three methods of implementation _ practical skills

Method 1 Comparison Recommendation

Copy Code code as follows:

<summary>

Using HttpWebRequest to get the source page
For pages with a BOM is very effective, no matter what the code can be correctly identified
</summary>
<param name= "url" > Web address "</param>
<returns> return page source file </returns>
public static string GetHtmlSource2 (string url)
{
Handling content
String html = "";
HttpWebRequest request = (HttpWebRequest) webrequest.create (URL);
Request. Accept = "*/*"; Accept any File
Request. useragent = "mozilla/4.0" (compatible; MSIE 6.0; Windows NT 5.2. NET CLR 1.1.4322) "; Analog using IE in browse http://www.52mvc.com
Request. AllowAutoRedirect = true;//Whether 302 is allowed
Request. Cookiecontainer = new Cookiecontainer ();//cookie container,
Request. Referer = URL; References to the current page


HttpWebResponse response = (HttpWebResponse) request. GetResponse ();
Stream stream = Response. GetResponseStream ();
StreamReader reader = new StreamReader (stream, Encoding.default);
HTML = reader. ReadToEnd ();
Stream. Close ();


return HTML;
}

Method 2

Copy Code code as follows:

Using System;
Using System.Collections.Generic;
Using System.Linq;
Using System.Web;
Using System.IO;
Using System.Text;
Using System.Net;

Namespace MYSQL
{
    public class Gethttpdata
    {
         public static string GetHttpData2 (String Url)
         {
            string sexception = null;
            string srslt = null;
            WebResponse owebrps = null;
            WebRequest owebrqst = WebRequest.Create (Url );
            owebrqst.timeout = 50000;
            Try
             {

Owebrps = Owebrqst.getresponse ();

}
catch (WebException E)
{
Sexception = E.message.tostring ();
}
catch (Exception e)
{
Sexception = E.tostring ();

}
Finally
{
if (Owebrps!= null)
{

StreamReader OSTREAMRD = new StreamReader (Owebrps.getresponsestream (), encoding.getencoding ("Utf-8"));
Srslt = Ostreamrd.readtoend ();
Ostreamrd.close ();
Owebrps.close ();
}
}

return srslt;
}

}
}

Method 3

Copy Code code as follows:

public static string gethtml (string url, params string [] charsets)//url is the address of the Web site to be accessed, CharSet is the encoding of the destination page, if the incoming is null or "", That automatically analyzes the encoding of the Web page.
{
Try
{
string charSet = null;
if (charsets.length = = 1) {
CharSet = Charsets[0];
}
WebClient mywebclient = new WebClient (); Create WebClient instance Mywebclient
To be aware of:
Some Web pages may not come down, for a variety of reasons such as the need for cookies, coding problems and so on
It's a matter of specific analysis, like adding cookies to the head.
WebClient. Headers.add ("Cookie", cookie);
This may require some overloaded methods. Write as you need to.


Gets or sets the network credentials used to authenticate requests to Internet resources.
Mywebclient.credentials = CredentialCache.DefaultCredentials;
If the server wants to authenticate the user name, the password
NetworkCredential mycred = new NetworkCredential (struser, strpassword);
Mywebclient.credentials = mycred;
Downloads data from a resource and returns a byte array. (add @ Because there is a "/" symbol in the middle of the URL)
byte[] Mydatabuffer = mywebclient.downloaddata (URL);
String strwebdata = Encoding.Default.GetString (Mydatabuffer);


Get Web page character encoding description information
Match Charsetmatch = Regex.match (Strwebdata, "<meta" ([^<]*) charset= ([^<]*) \ "", Regexoptions.ignorecase | Regexoptions.multiline);
String webcharset = Charsetmatch.groups[2]. Value;
if (CharSet = null | | charSet = = "")
CharSet = Webcharset;


if (CharSet!= null && charSet!= "" && encoding.getencoding (charSet)!= Encoding.default)
{
Strwebdata = encoding.getencoding (CharSet). GetString (Mydatabuffer);
}
else {
Strwebdata = encoding.getencoding ("Utf-8"). GetString (Mydatabuffer);
}
return strwebdata;
}
catch (Exception e) {return "";}
}