Breadth First Mate anonymous function crawl

Package Main

Import (

"FMT"

"Log"

"Net/http"

"OS"

"Golang.org/x/net/html"

)

VAR (

str string = "https://docs.hacknode.org/gopl-zh/"

)

Creatfile is a func ti make infomation in file

Func Creatfile (BT []byte) {

F, err: = OS. OpenFile ("F:/mygo/src/practice/url.txt", OS. O_create|os. O_append, 0666)

If err! = Nil {

Log. Fatal (ERR)

}

Defer F.close ()

Res, err: = F.write ([]byte (BT))

If err! = Nil {

Log. Fatal (ERR)

}

if res = = 0 {

Fmt. Println ("no One")

}

}

Geturlinfomation is a Func get URL infomation

Func geturlinfomation (URL string) []string {

RESP, err: = http. Get (URL)

If err! = Nil {

Log. Fatal (ERR)

}

Defer resp. Body.close ()

If resp. StatusCode! = http. Statusok {//Judging status

Log. Fatal ("Can ' t Connect")

}

Start node processing

Doc, err: = HTML. Parse (resp. Body)

If err! = Nil {

Log. Fatal (ERR)

}

var links []string

Foronenode: = Func (n *html. Node) {

anonymous functions, single node processing, why use anonymous functions, explained below

If N.type = = html. Elementnode && N.data = = "a" {//Determine if node, node is <a> tag

For _, A: = Range n.attr {//Traverse Tag Properties

If A.key! = "href" {

Continue

}

Link, err: = resp. Request.URL.Parse (A.val)//parse out the URL address, that is, the property content

If err! = Nil {

Log. Fatal (ERR)

}

Links = Append (links, link. String ())

}

}

}

Foreachnode (Doc, Foronenode, nil)//traversal start

Return links

}

Foreachnode is breadth-first traversal

Func foreachnode (n *html. Node, Pre, post func (n *html. Node)) {

If pre! = Nil {

Pre (N)

}

For c: = N.firstchild; c! = Nil; c = c.nextsibling {

Foreachnode (c, pre, POST)

}

If post! = nil {

Post (N)

}

}

Func Main () {

BT: = Geturlinfomation ("https://docs.hacknode.org/gopl-zh/")

For _, T: = Range BT {

T + = "\ n"

Creatfile ([]byte (t))

}

Creatfile (BT)

}

Note: Why anonymous functions are used

1. In the body of the function, you need to retain the values taken, if you return using the return function, the function will also use a variable of type *http.respone, when the recursive call, the parameters are very complex

2. If using the above situation, in the breadth-first traversal, the internal use of recursive calls, in the receipt of return values, it will be very troublesome, each call a function, each function returned a value, temporarily did not think how to receive, if someone saw this article, can give some advice, I thank you very much