Classes used to obtain web pages [go]

This was tested and used stably. Considering uniform error handling, no catch errors were found in the class, and all network errors were captured during use to determine retry or termination. Supports get and post, custom encoding, and cookie, but not file upload.

 

 

 

 

Imports System. Net
Imports System. IO

Public Class HttpDriver

Public Function GetPage (ByVal url As String, Optional ByRef postPara As String = "", Optional ByRef encType As String = "GB2312") As String
Return GetPage (url, postPara, Nothing, False, encType)
End Function

Public Function GetPage (ByVal url As String, ByRef postPara As System. Collections. Hashtable, Optional ByRef encType As String = "GB2312") As String
Return GetPage (url, ColToStr (postPara), encType)
End Function

Public Function GetPage (ByVal url As String, ByRef postPara As String, ByRef cookies As CookieCollection, ByVal hasCookie As Boolean, Optional ByRef encType As String = "GB2312 ", optional ByRef refer As String = "") As String
If (url. StartsWith ("http: //") = False) Then
Url = "http: //" & url
End If
Dim hRqst As HttpWebRequest = HttpWebRequest. Create (url)
If (hasCookie = True AndAlso (Not cookies Is Nothing) Then
HRqst. CookieContainer = New CookieContainer
HRqst. CookieContainer. Add (cookies)
End If

HRqst. ContentType = "application/x-www-form-urlencoded"
HRqst. Headers. Add ("Accept-Language", "zh-cn ")
Dim streamData As Stream
Dim bt () As Byte
If (postPara = "") Then
HRqst. Method = "GET"
Else
HRqst. Method = "POST"
HRqst. AllowWriteStreamBuffering = True
Bt = System. Text. Encoding. ASCII. GetBytes (postPara)
HRqst. ContentLength = bt. Length
HRqst. UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0 )"
HRqst. Referer = refer
HRqst. KeepAlive = False
HRqst. Timeout = 20000
StreamData = hRqst. GetRequestStream ()
StreamData. Write (bt, 0, bt. Length)
StreamData. Close ()
End If
Dim hRsp As HttpWebResponse
HRsp = hRqst. GetResponse ()
StreamData = hRsp. GetResponseStream ()
If (hasCookie = True AndAlso (Not hRsp. Cookies Is Nothing) Then
Cookies. Add (hRsp. Cookies)
End If
If (encType = "") Then
EncType = "GB2312"
End If
Dim readStream As New IO. StreamReader (streamData, System. Text. Encoding. GetEncoding (encType ))
GetPage = readStream. ReadToEnd ()
StreamData. Close ()
End Function

Public Function GetPage (ByVal url As String, ByRef postPara As System. Collections. Hashtable, ByRef cookies As CookieCollection, ByVal hasCookie As Boolean, Optional ByRef encType As String = "GB2312") As String
Return GetPage (url, ColToStr (postPara), cookies, True, encType)
End Function

Public Function GetPage (ByVal url As String, ByRef cookies As CookieCollection, ByVal hasCookie As Boolean, Optional ByRef encType As String = "GB2312") As String
Return GetPage (url, "", cookies, True, encType)
End Function

// This function is used to convert the form item set to a string
Public Shared Function ColToStr (ByRef ht As System. Collections. Hashtable, Optional ByRef encType As String = "GB2312") As String
Dim str As String
Dim para As DictionaryEntry
For Each para In ht
Str & = System. Web. HttpUtility. UrlEncode (CType (para. Key, String), Text. Encoding. GetEncoding (encType ))
Str & = "="
Str & = System. Web. HttpUtility. UrlEncode (CType (para. Value, String), Text. Encoding. GetEncoding (encType ))
Str & = "&"
Next
Str = str. Substring (0, str. Length-1)
Return str
End Function

End Class

 

 

 

If you need to support cookies and refer, you can use the following class to use the httpdriver

 

// This class is used to access pages containing cookies
Imports System. IO

Public Class UserAgent
Private m_cookies As New System. Net. CookieCollection
Private refer As String
Private hd As New HttpDriver

Public Function GetPage (ByVal url As String, Optional ByRef postPara As String = "", Optional ByRef encType As String = "GB2312") As String
GetPage = hd. GetPage (url, postPara, m_cookies, True, encType, refer)
Refer = url
End Function

Public Function GetPage (ByVal url As String, ByRef postPara As Hashtable, Optional ByRef encType As String = "GB2312") As String
Return GetPage (url, hd. ColToStr (postPara), encType)
End Function

Public Function SetCookie (ByVal cookies As System. Net. CookieCollection)
M_cookies = cookies
End Function

Public Function GetCookie () As System. Net. CookieCollection
Return m_cookies
End Function
End Class

 

 

 

Simple get Functions

 

Public Function GetPage (ByVal url As String) As String
Dim hRqst As HttpWebRequest = HttpWebRequest. Create (url)

HRqst. ContentType = "application/x-www-form-urlencoded"
HRqst. Method = "GET"
Dim streamData As Stream

Dim hRsp As HttpWebResponse = hRqst. GetResponse ()
StreamData = hRsp. GetResponseStream ()

Dim readStream As New IO. StreamReader (streamData, System. Text. Encoding. GetEncoding ("GB2312 "))
GetPage = readStream. ReadToEnd ()
StreamData. Close ()
End Function

 

Get Method

 

Dim ua as New UserAgent
Dim content as String
Dim url as String
Url = "www.sina.com.cn"
Content = ua. GetPage (url)
Url = "sohu.com"
Content = ua. GetPage (url)

 

 

Post method (the page and parameter names are examples ). The advantage of UserAgent is that the cookie can be stored transparently. You can use the following methods to save the login information after logging on.
Dim ua as New UserAgent

 

 

Dim ua as New UserAgent
Dim content as String
Dim url as String
Dim ht as New HashTable

Url = "mail.sina.com.cn"
Ht. Add ("username", "username ")
Ht. Add ("password", "password ")
Content = ua. GetPage (url, ht)
Url = "mail.sina.com.cn/default.htm"
Content = ua. GetPage (url)