The followingCodeIs from a Web CrawlerProgramIt is useful and recorded.
Code
Using System;
Using System. collections;
Using System. Collections. Generic;
Using System. reflection;
Using System. IO;
Using System. net;
Using System. text;
Namespace Mycsstudy
{
Class Program
{
/// <Summary>
/// Simple Web Crawler
/// </Summary>
/// <Param name = "url"> </param>
/// <Param name = "charset"> The encoding can be empty. </Param>
/// <Returns> </returns>
Public Static String Fetch ( String URL, String Charset)
{
Encoding encoding;
Httpwebrequest request;
Httpwebresponse response = Null ;
Stream resstream = Null ;
Streamreader SR = Null ;
String Result = String . Empty;
Try
{
Request = (Httpwebrequest) httpwebrequest. Create (URL );
Response = (Httpwebresponse) request. getresponse ();
Resstream = Response. getresponsestream ();
If ( ! String . Isnullorempty (charset ))
{
Encoding = Encoding. getencoding (charset );
}
Else If ( ! String . Isnullorempty (response. characterset ))
{
Encoding = Encoding. getencoding (response. characterset );
}
Else
{
Encoding = Encoding. default;
}
Sr = New Streamreader (resstream, encoding );
Result = Sr. readtoend ();
}
// Catch (exception ex)
// {
// Throw ex;
// }
Finally
{
If (SR ! = Null )
{
Sr. Close ();
}
If (Resstream ! = Null )
{
Resstream. Close ();
}
If (Response ! = Null )
{
Response. Close ();
}
}
Return Result;
}
Static Void Main ( String [] ARGs)
{
String Website = @" Http://www.google.cn " ; // The URL must contain the protocol.
String Strhtml = Fetch (website, Null );
Console. Write (strhtml );
Console. Readline ();
}
}
}