Go implementation crawl href link

This is a creation in Article, where the information may have evolved or changed.

The first crawl was implemented with a regular expression, and then crawled out of the unnecessary links, and now instead of the Goquery implementation:

Judgeurl Project Judgeurl.gopackage Judgeurlimport ("strings") func Isurl (str string) bool {if strings. Hasprefix (str, "#") | | Strings. Hasprefix (str, "//") | | Strings. Hassuffix (str, ". exe") | | Strings. Hassuffix (str, ": void (0);") {return false} else if strings. Hasprefix (str, "{") && strings. Hassuffix (str, "}") {return false} else if strings. Equalfold (str, "javascript:;") {return false} else {return True}return true}func samepathurl (preUrl string, url string, Mark int) (Newurl string) {last: = Strings. LastIndex (PREURL, "/") if last = = 6 {newurl = preUrl + URL} else {if Mark = = 1 {newurl = preurl[:last] + URL} else {Newpreu RL: = Preurl[:last]newlast: = Strings. LastIndex (Newpreurl, "/") Newurl = Newpreurl[:newlast] + url}}return Newurl}

Weburls_spider Project Main.gopackage mainimport ("FMT" "OS" "Strings" "Github.com/puerkitobio/goquery" test/ Judgeurl ") var urlmap map[string]bool//Prevent duplicate links from entering the dead loop, but getting too many links may be memory overrun func fetch (URL string, Count int) {if count > 1 {//Set The fixed crawl depth is 1 pages return}body, err: = Goquery. NewDocument (URL) if err! = Nil {return}body. Find ("a"). Each (func (i int, aa *goquery. Selection) {href, isexist: = AA. Attr ("href") if isexist = = true {href = strings. Trimspace (HREF) if len (href) > 2 && judgeurl.isurl (href) {if _, OK: = Urlmap[href]; OK = = False {fmt. Println ("Previous URL:", href) if strings. Hasprefix (HREF, "/") | | Strings. Hasprefix (HREF, "./") {href = Judgeurl.samepathurl (url, href, 1)} else if strings. Hasprefix (href, "... /") {href = Judgeurl.samepathurl (url, href, 2)}fmt. Println ("Modified URL:", href) urlmap[href] = truefetch (href, count+1)}}})}func writevalues (outfile string) error {file, ERR: = OS. Create (outfile) if err! = Nil {fmt. Printf ("Failed to create%s file!") ", outfile) return Err}defer file. Close () for k, _: = Range UrlMap {file. WriteString (k + "\ n")}return Nil}func Main () {UrlMap = make (Map[string]bool, 1000000) Fetch ("http://www.hao123.com/", 0) Writevalues ("Urls.dat")}