General Solution: http://www.cnblogs.com/kissdodog/archive/2013/02/28/2936950.html
Special handling: If the request page Contentencoding=gzip
Get ContentEncoding
Static void Getch (string url)
{
WebRequest rebrequest = webrequest.create (URL);
HttpWebResponse web = (httpwebresponse) Rebrequest.getresponse ();
String chart = web. CharacterSet;
String conending = web. ContentEncoding;
String contenttype = web. ContentType;
Console.WriteLine (chart);
Console.WriteLine (conending);
Console.WriteLine (contenttype);
}
1. To the HttpWebRequest object, add the following header:
request. Headers.add ("accept-encoding", "gzip");
2. Decode the received stream:
Private StringGetresponsebody (HttpWebResponse response)
{
StringResponsebody= String. Empty;
If(Response. Contentencoding.tolower (). Contains ("Gzip")) {
Using(GZipStream stream= NewGZipStream (
Response. GetResponseStream (), compressionmode.decompress))
{
Using(Streamreaderreader= NewStreamReader (Stream))
{
Responsebody=Reader. ReadToEnd ();
}
}
}
Else If(Response. Contentencoding.tolower (). Contains ("Deflate"))
{
Using(Deflatestream Stream= NewDeflatestream (
Response. GetResponseStream (), compressionmode.decompress))
{
Using(StreamReader Reader=
NewStreamReader (Stream, Encoding.UTF8))
{
Responsebody= reader. ReadToEnd ();
}
}
}
else
{
using= {
using (StreamReader Reader =
new StreamReader (stream, Encoding.UTF8))
{
responsebody = reader. ReadToEnd ();
}
}
}
return Responsebody;
} /span>
Parsing: HTMLDocument doc = new HTMLDocument ();
string html = WC. Downloadstring ("agenthome/-i31-j310-kw/");
Doc. Loadhtml (responsebody);
Htmlnode node = doc. Documentnode.selectsinglenode ("/html/body/div[1]");
Console.WriteLine (node. InnerText);
Console.WriteLine (node. InnerHtml);
Console.WriteLine (node. Name);
Reference http://www.csharpwin.com/csharpspace/13345r5893.shtml
HTML parsing tool Htmlagilitypack
Reference http://zhoufoxcn.blog.51cto.com/792419/595344/
Htmlagilitypack class Library parsing HTML