A scheme to crawl the source code of a specified URL from the Internet (C #)

Call Mode:

#region test to get a remote Web page

Getpagecode GPC = new Getpagecode ();

Gpc. Url= "http://ppcode.com/";

Gpc. proxystate=1;//use proxy server, 0 is not used, set to 1 after the following proxy settings to work

Gpc. Proxyaddress= "http://proxyName.com";//Proxy server address

Gpc. proxyport= "80";/the port of the proxy server

Gpc. proxyaccount= "proxy";//Proxy Server account

Gpc. proxypassword= "password";//proxy Server password

Gpc. Proxydomain= "BQC";//proxy Server domain

Gpc. outfilepath=filepath;//sets the path of the output file, if not set, returns a string

Gpc. GetSource ()//processing

String TEMPERR=GPC. notemessage;//If something goes wrong, this will prompt

String TEMPCODE=GPC. String returned by outstring;//

#endregion

Class Code:

Using System;

Using System.Collections;

Using System.ComponentModel;

Using System.Data;

Using System.Drawing;

Using System.IO;

Using System.Net;

Using System.Text;

Using System.Web;





Namespace test.com

{

<summary>

Function: Get the source of the URL page on the Internet

Created: 2004-03-22

Author: rexsp Msn:yubo@x263.net

</summary>

public class Getpagecode

{

#region Private variables

<summary>

Web page URL Address

</summary>

private string Url=null;

<summary>

Whether to use a code server: 0 Use a proxy server without using 1

</summary>

private int proxystate=0;

<summary>

Proxy server Address

</summary>

private string Proxyaddress=null;

<summary>

Proxy Server Port

</summary>

private string Proxyport=null;

<summary>

Proxy Server user Name

</summary>

private string Proxyaccount=null;

<summary>

Proxy Server password

</summary>

private string Proxypassword=null;

<summary>

Proxy Server domain

</summary>

private string Proxydomain=null;

<summary>

Output file path

</summary>

private string Outfilepath=null;

<summary>

The output string

</summary>

private string Outstring=null;

<summary>

Hint Information

</summary>

private string Notemessage;



#endregion



#region Public Properties

<summary>

URL address to read

</summary>

public string Url

{

Get{return URL;}

Set{url=value;}

}

<summary>

Whether to use proxy server flags

</summary>

public int Proxystate

{

Get{return Proxystate;}

Set{proxystate=value;}

}

<summary>

Proxy server Address

</summary>

public string proxyaddress

{

Get{return proxyaddress;}

Set{proxyaddress=value;}

}

<summary>

Proxy Server Port

</summary>

public string ProxyPort

{

Get{return ProxyPort;}

Set{proxyport=value;}

}

<summary>

Proxy Server account

</summary>

public string Proxyaccount

{

Get{return Proxyaccount;}

Set{proxyaccount=value;}

}

<summary>

Proxy Server password

</summary>

public string Proxypassword

{

Get{return Proxypassword;}

Set{proxypassword=value;}

}

<summary>

Proxy Server domain

</summary>

public string Proxydomain

{

Get{return Proxydomain;}

Set{proxydomain=value;}

}

<summary>

Output file path

</summary>

public string Outfilepath

{

Get{return Outfilepath;}

Set{outfilepath=value;}

}

<summary>

The returned string

</summary>

public string outstring

{

Get{return outstring;}



}

<summary>

Back to Tips

</summary>

public string Notemessage

{

Get{return Notemessage;}



}



#endregion



#region Constructors

Public Getpagecode ()

{

}

#endregion



#region Public methods

<summary>

Reads the specified URL address and saves it to the specified file

</summary>

public void GetSource ()

{

WebRequest request = WebRequest.Create (This.url);

Working with proxy servers

if (this.proxystate==1)

{

Default Read 80-port data

if (this.proxyport==null)

This. proxyport= "80";



WebProxy myproxy=new WebProxy ();

myproxy = (webproxy) request. Proxy;

myproxy.address = new Uri (this. proxyaddress+ ":" +this. ProxyPort);

Myproxy.credentials = new NetworkCredential (This.proxyaccount, This.proxypassword, this. Proxydomain);

Request. Proxy = myproxy;

}

Try



{

Request Service

WebResponse response = Request. GetResponse ();

return information

Stream Resstream = Response. GetResponseStream ();

StreamReader sr = new StreamReader (Resstream, System.Text.Encoding.Default);

String Tempcode= Sr. ReadToEnd ();

Resstream.close ();

Sr. Close ();



If the output file path is empty, the resulting content is assigned to the Outstring property

if (this.outfilepath==null)

{

This.outstring=tempcode;

}

Else

{



FileInfo fi = new FileInfo (This.outfilepath);

If there's a file, kill it first.

if (FI. Exists)

Fi. Delete ();



StreamWriter sw = new StreamWriter (This.outfilepath,true,encoding.default);

Sw. Write (Tempcode);

Sw. Flush ();

Sw. Close ();

}

}

Catch

{

This.notemessage= "error, please check the network connectivity;";

}





}

#endregion



}

}