Go language implements crawler to collect associative words

/****************************************************

Go Language Implementation Crawler _ Lenovo Word Consulting: qq:1465376564

Written by Huanggo Python training course

Python Beijing Weekend Course

Https://github.com/pythonpeixun/article/blob/master/beijing_weekend.md

Python

Shanghai Weekend Course

Https://github.com/pythonpeixun/article/blob/master/shanghai_weekend.md

*****************************************************/

Package Main

Import (

"FMT"

"Io/ioutil"

"Net/http"

"RegExp"

)

Define a new data type

Type Spider struct {

URL string

Header map[string]string

}

Ways to define Spiders

Func (keyword Spider) get_html_header () string {

Client: = &http. client{}

req, err: = http. Newrequest ("GET", Keyword.url, nil)

If Err! = Nil {

}

for key, value: = Range Keyword.header {

req. Header.add (key, value)

}

resp, err: = client. Do (req)

If Err! = Nil {

}

Defer resp. Body.close ()

body, err: = Ioutil. ReadAll (resp. Body)

If Err! = Nil {

}

return string (body)

}

Func Main () {

Header: = map[string]string{"Host": "sug.so.360.cn",

"Referer": "http://www.so.com/",

"DNT": "1",

"user-agent": "mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) applewebkit/537.36 (khtml, like Gecko) chrome/31.0.1650.63 safari/537.36 ",

"Cookie": "__huid=104rl%2b0hjg2yltbarbppiz2w7htblrv43getlevtbdiei%3d",

}

Keyword: = "Technology"

URL: = "http://sug.so.360.cn/suggest?callback=suggest_so&encodein=utf-8&encodeout=utf-8& format=json&fields=word,obdata&word= "+ keyword

Spider: = &spider{url, header}

HTML: = Spider.get_html_header ()

RP1: = RegExp. Mustcompile ("{(. *?)}")

find_txt: = Rp1. Findallstring (HTML,-1)

FMT. Println (Find_txt)

}

Go language implements crawler to collect associative words