The way the Go Language crawls Web pages via HTTP _golang

The example in this article describes how the go language crawls Web pages through HTTP. Share to everyone for your reference. The implementation methods are as follows:

Copy Code code as follows:

Package Main
Import (
"FMT"
"Log"
"Net/http"
"Net/url"
"Io/ioutil"
)
Specify Proxy IP
Func Gettransportfieldurl (proxy_addr *string) (Transport *http. Transport) {
Url_i: = URL. url{}
Url_proxy, _: = Url_i.parse (*PROXY_ADDR)
Transport = &http. Transport{proxy:http. Proxyurl (Url_proxy)}
Return
}
Get HTTP proxy address from environment variable $http_proxy or $http_proxy
Func gettransportfromenvironment () (Transport *http. Transport) {
Transport = &http. Transport{proxy:http. Proxyfromenvironment}
Return
}
Func fetch (URL, proxy_addr *string) (HTML string) {
Transport: = Gettransportfieldurl (PROXY_ADDR)
Client: = &http. Client{transport:transport}
Req, Err: = http. Newrequest ("Get", *url, nil)
If Err!= nil {
Log. Fatal (Err. Error ())
}
RESP, err: = client. Do (req)
If Err!= nil {
Log. Fatal (Err. Error ())
}
If resp. StatusCode = 200 {
Robots, err: = Ioutil. ReadAll (resp. body);
Resp. Body.close ()
If Err!= nil {
Log. Fatal (Err. Error ())
}
html = string (robots);
} else {
html = ""
}
Return
}
Func Main () {
PROXY_ADDR: = "http://183.221.250.137:80/"
URL: = "Http://www.baidu.com/s?wd=ip"
HTML: = Fetch (&url, &PROXY_ADDR)
Fmt. PRINTLN (HTML)
}

I hope this article will help you with your go language program.