Use of Go language crawler goquery and grequests

/* Download Tool */package main import ("FMT"//go language version of jquery "Github.com/puerkitobio/goquery" "OS" "s Ync "Strings"//go language version of Request "Github.com/levigross/grequests" "Time" "StrConv") VAR wg sy nc. Waitgroup Func Main () {now: = time.          Now () Initalurls: = []string{' http://www.zngirls.com/girl/18071/album/',} for _, url: = Range initalurls { Doc, Err: = Goquery. NewDocument (URL) if err! = Nil {fmt. Errorf ("Download error:% #v", err) OS. Exit ( -1)} doc. Find (". Igalleryli_link"). Each (func (i int, S *goquery. Selection) {src, exists: = S.find ("img"). Attr ("src") fmt. Printf ("Start download album Image:%v\n", SRC) if (exists) {WG. ADD (1) Go func (src string) {defer WG.                      Done ()//download image//trytimes: = map[int]int N: = 0 S: = STrings. Replace (SRC, "cover/", "", 1) SS: = Strings. Split (S, "/") FM: = Strings.                        Join (Ss[:len (ss)-1], "/") SF0: = fm + "/%d.jpg" SFN: = fm + "/%03d.jpg"  for {//continuous download S: = "" if n = = 0 {s = fmt. Sprintf (SF0, N)} else {s = fmt. Sprintf (SFN, N)} fmt. Printf ("Ready to Download:%v\n", s) res, _: = Grequests. Get (S, &grequests. requestoptions{//struct can give a value to a specified type, not necessarily a value headers:map[string]strin g{"Referer": "Http://www.zngirls.com", "user-agent": "Moz illa/5.0 (Windows NT 6.1; WOW64) applewebkit/537.36 (khtml, like Gecko) chrome/50.0.2661.102 Safari/537.36 "}})//conditions need to be modified, if no picture, return is hotlinking, picture 4kb if Res. StatusCode! = + FMT. Printf ("Download failed, exit album download:%s\n", SRC) Break}/ /Picture may be the site, return the hotlinking picture (around 4kb) Length: = Res. Header.get ("Content-length") slen,_: = StrConv. Atoi (length) if Slen < 4100{FMT.                            Printf ("Download failed, exit album download:%s\n", SRC) Break} Index: = strings. Index (S, "gallery") if index = =-1 {fmt.                            Errorf ("Invalid address, unable to find gallery keyword, parse failed:%s\n", SRC) return} SS2: = Strings. Split (String (s[index:]), "/") dirname: = Strings.              Join (Ss2[:len (SS2)-1], "/")            If _, Err: = OS. Stat (dirname); Err! = Nil {fmt. Printf ("Create download folder:%s\n", dirname) os. Mkdirall (DirName, 0666)} filename: = strings. Join (SS2, "/") Res. Downloadtofile (filename) fmt.              PRINTF ("Successfully downloaded image to:%s\n", filename) n++}} (SRC) }})} WG. Wait ()//4m bandwidth download, need 16m36s, total size 202m,10 folders, 560 files FMT. Printf ("Download task completed, time consuming:% #v \ n", when. Now ().   Sub (now)}

The use of the Go Language crawler goquery and grequests