Number of connections, proxies, automatic redirects, gzip requests, and other settings for HttpWebRequest to improve efficiency

Set four:

[Csharp]
Webrequest. ServicePoint. Expect100Continue = false;
// Whether to use Nagle without increasing efficiency
Webrequest. ServicePoint. UseNagleAlgorithm = false;
// Maximum number of connections
Webrequest. ServicePoint. ConnectionLimit = 65500;
// Whether data is buffered with false to improve efficiency
Webrequest. AllowWriteStreamBuffering = false;

 

Proxy usage: All proxies I use do not require accounts and passwords, because if it is not free, it is our own and there are restrictions on access IP addresses.

[Csharp]
If (proxy! = Null)
{
Model. proxy = proxy;
WebProxy myProxy = new WebProxy (proxy. ProxyInfo, false );
MyProxy. Credentials = new NetworkCredential ("","");
Webrequest. Proxy = myProxy;
}
Else
{
Webrequest. Proxy = GlobalProxySelection. GetEmptyWebProxy ();
}
 

Webrequest. Headers. Add (HttpRequestHeader. AcceptEncoding, "gzip, deflate ");

With gzip, if the other party supports gzip, the traffic can be reduced by 4/5. Originally, the size of KB is compressed by gzip, which reduces the bandwidth pressure and increases the speed.

It is worth noting that for gzip, the returned value must be decompressed by gzip.

 

[Csharp]
If (response. ContentEncoding. ToLower (). Contains ("gzip "))
{
Using (GZipStream stream = new GZipStream (response. GetResponseStream (), CompressionMode. Decompress ))
{
Using (StreamReader reader = new StreamReader (stream, Encoding. UTF8 ))
{
 
Model.html = reader. ReadToEnd ();
}
}
}
 

Another point is that the CPU is consumed when gzip is decompressed, so note the following:

 

Appendix Completion Method

 

 

[Csharp]
/// <Summary>
/// Get
/// </Summary>
/// <Param name = "webUrl"> </param>
/// <Returns> </returns>
Public Model. WebRequestReturnModel WebRequestGetHtmlByGet (string webUrl, CookieContainer cookieContainer, Encoding encoding, string refer, Model. Proxy proxy)
{
Model. WebRequestReturnModel model = new Model. WebRequestReturnModel ();
HttpWebRequest webrequest = null;
Try
{
Webrequest = WebRequest. Create (webUrl) as HttpWebRequest;

Webrequest. ServicePoint. Expect100Continue = false;
Webrequest. ServicePoint. UseNagleAlgorithm = false;
Webrequest. ServicePoint. ConnectionLimit = 65500;
Webrequest. AllowWriteStreamBuffering = false;
If (proxy! = Null)
{
Model. proxy = proxy;
WebProxy myProxy = new WebProxy (proxy. ProxyInfo, false );
MyProxy. Credentials = new NetworkCredential ("","");
Webrequest. Proxy = myProxy;
}
Else
{
Webrequest. Proxy = GlobalProxySelection. GetEmptyWebProxy ();
}
// Set other herader
Webrequest = WebRequestSetHeaders (webrequest );
If (! String. IsNullOrEmpty (refer ))
{
Webrequest. Referer = refer;
}
Else
{
Webrequest. Referer = webUrl;
}
Webrequest. Headers. Add (HttpRequestHeader. AcceptEncoding, "gzip, deflate ");
Webrequest. CookieContainer = cookieContainer;
Webrequest. Timeout = 5000;
Webrequest. UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2;. net clr 1.0.3705 ;)";
Webrequest. Accept = "*/*";
Webrequest. KeepAlive = true;
Webrequest. Headers. Add ("Accept-Language", "zh-cn, en-us; q = 0.5 ");
 
Using (HttpWebResponse response = (HttpWebResponse) webrequest. GetResponse ())
{
 
If (response. ContentEncoding. ToLower (). Contains ("gzip "))
{
Using (GZipStream stream = new GZipStream (response. GetResponseStream (), CompressionMode. Decompress ))
{
Using (StreamReader reader = new StreamReader (stream, Encoding. UTF8 ))
{
 
Model.html = reader. ReadToEnd ();
}
}
}
Else if (response. ContentEncoding. ToLower (). Contains ("deflate "))
{
Using (DeflateStream stream = new DeflateStream (response. GetResponseStream (), CompressionMode. Decompress ))
{
Using (StreamReader reader = new StreamReader (stream, Encoding. UTF8 ))
{
 
Model.html = reader. ReadToEnd ();
}
 
}
}
Else
{
Using (Stream stream = response. GetResponseStream ())
{
Using (StreamReader reader = new StreamReader (stream, Encoding. UTF8 ))
{
 
Model.html = reader. ReadToEnd ();
}
}
}
}
}
Catch (Exception ex)
{
Model. exception = ex;
Model.html = string. Empty;
 
}
Model. cookieContainer = cookieContainer;
Return model;
}