A scheme to crawl the source code of a specified URL from the Internet (C #)
Last Update:2017-02-28
Source: Internet
Author: User
Call Mode:
#region test to get a remote Web page
Getpagecode GPC = new Getpagecode ();
Gpc. Url= "http://ppcode.com/";
Gpc. proxystate=1;//use proxy server, 0 is not used, set to 1 after the following proxy settings to work
Gpc. Proxyaddress= "http://proxyName.com";//Proxy server address
Gpc. proxyport= "80";/the port of the proxy server
Gpc. proxyaccount= "proxy";//Proxy Server account
Gpc. proxypassword= "password";//proxy Server password
Gpc. Proxydomain= "BQC";//proxy Server domain
Gpc. outfilepath=filepath;//sets the path of the output file, if not set, returns a string
Gpc. GetSource ()//processing
String TEMPERR=GPC. notemessage;//If something goes wrong, this will prompt
String TEMPCODE=GPC. String returned by outstring;//
#endregion
Class Code:
Using System;
Using System.Collections;
Using System.ComponentModel;
Using System.Data;
Using System.Drawing;
Using System.IO;
Using System.Net;
Using System.Text;
Using System.Web;
Namespace test.com
{
<summary>
Function: Get the source of the URL page on the Internet
Created: 2004-03-22
Author: rexsp Msn:yubo@x263.net
</summary>
public class Getpagecode
{
#region Private variables
<summary>
Web page URL Address
</summary>
private string Url=null;
<summary>
Whether to use a code server: 0 Use a proxy server without using 1
</summary>
private int proxystate=0;
<summary>
Proxy server Address
</summary>
private string Proxyaddress=null;
<summary>
Proxy Server Port
</summary>
private string Proxyport=null;
<summary>
Proxy Server user Name
</summary>
private string Proxyaccount=null;
<summary>
Proxy Server password
</summary>
private string Proxypassword=null;
<summary>
Proxy Server domain
</summary>
private string Proxydomain=null;
<summary>
Output file path
</summary>
private string Outfilepath=null;
<summary>
The output string
</summary>
private string Outstring=null;
<summary>
Hint Information
</summary>
private string Notemessage;
#endregion
#region Public Properties
<summary>
URL address to read
</summary>
public string Url
{
Get{return URL;}
Set{url=value;}
}
<summary>
Whether to use proxy server flags
</summary>
public int Proxystate
{
Get{return Proxystate;}
Set{proxystate=value;}
}
<summary>
Proxy server Address
</summary>
public string proxyaddress
{
Get{return proxyaddress;}
Set{proxyaddress=value;}
}
<summary>
Proxy Server Port
</summary>
public string ProxyPort
{
Get{return ProxyPort;}
Set{proxyport=value;}
}
<summary>
Proxy Server account
</summary>
public string Proxyaccount
{
Get{return Proxyaccount;}
Set{proxyaccount=value;}
}
<summary>
Proxy Server password
</summary>
public string Proxypassword
{
Get{return Proxypassword;}
Set{proxypassword=value;}
}
<summary>
Proxy Server domain
</summary>
public string Proxydomain
{
Get{return Proxydomain;}
Set{proxydomain=value;}
}
<summary>
Output file path
</summary>
public string Outfilepath
{
Get{return Outfilepath;}
Set{outfilepath=value;}
}
<summary>
The returned string
</summary>
public string outstring
{
Get{return outstring;}
}
<summary>
Back to Tips
</summary>
public string Notemessage
{
Get{return Notemessage;}
}
#endregion
#region Constructors
Public Getpagecode ()
{
}
#endregion
#region Public methods
<summary>
Reads the specified URL address and saves it to the specified file
</summary>
public void GetSource ()
{
WebRequest request = WebRequest.Create (This.url);
Working with proxy servers
if (this.proxystate==1)
{
Default Read 80-port data
if (this.proxyport==null)
This. proxyport= "80";
WebProxy myproxy=new WebProxy ();
myproxy = (webproxy) request. Proxy;
myproxy.address = new Uri (this. proxyaddress+ ":" +this. ProxyPort);
Myproxy.credentials = new NetworkCredential (This.proxyaccount, This.proxypassword, this. Proxydomain);
Request. Proxy = myproxy;
}
Try
{
Request Service
WebResponse response = Request. GetResponse ();
return information
Stream Resstream = Response. GetResponseStream ();
StreamReader sr = new StreamReader (Resstream, System.Text.Encoding.Default);
String Tempcode= Sr. ReadToEnd ();
Resstream.close ();
Sr. Close ();
If the output file path is empty, the resulting content is assigned to the Outstring property
if (this.outfilepath==null)
{
This.outstring=tempcode;
}
Else
{
FileInfo fi = new FileInfo (This.outfilepath);
If there's a file, kill it first.
if (FI. Exists)
Fi. Delete ();
StreamWriter sw = new StreamWriter (This.outfilepath,true,encoding.default);
Sw. Write (Tempcode);
Sw. Flush ();
Sw. Close ();
}
}
Catch
{
This.notemessage= "error, please check the network connectivity;";
}
}
#endregion
}
}