Go exercise: Web crawler

This is a creation in Article, where the information may have evolved or changed.

Code:

Saferun Lock Setting URL visited

Pass

For i:=0; I < Len (URLs); I++{<-sub_quit}

Parent go thread waits for child go thread to end

Package Mainimport ("FMT") type Fetcher interface {//Fetch returns the body content of the URL and places the URL found on this page into a slice. Fetch (URL string) (body string, URLs []string, err error)}var Lockx = Make (chan int,1) func Saferun (f func ()) {&LT;-LOCKXF ( ) Lockx <-1}var visited Map[string]bool = Make (Map[string]bool)//Crawl use Fetcher to crawl the page recursively from a URL until the maximum depth is reached. Func Crawl (URL string, depth int, fetcher fetcher, quit Chan int) {//TODO: Crawl URL in parallel.    TODO: Do not repeat crawl pages. Neither of the following is true: Defer func () {quit<-1} () If depth <= 0 | | Visited[url] {return}body, URLs, err: = Fetcher. Fetch (URL) saferun (func () {visited[url]=true}) if err! = Nil {fmt. PRINTLN (Err) return}fmt. Printf ("Found:%s%q\n", url, body) Sub_quit: = Make (chan Int,len (URLs)) for _, U: = Range URLs {Crawl (U, depth-1, fetcher,s Ub_quit)}for i:=0; I < Len (URLs); I++{<-sub_quit}return}func Main () {lockx<-1quit: = Make (chan int,1) Crawl ("http://golang.org/", 4, Fetcher,quit) Fmt. Println (<-quit)}//Fakefetcher is a fetcher that returns several results. Type Fakefetcher MAP[STRIng]*fakeresulttype fakeresult struct {body Stringurls []string}func (f fakefetcher) Fetch (URL string) (string, []string, Error) {If res, OK: = F[url]; OK {return res.body, Res.urls, Nil}return "", Nil, FMT. Errorf ("Not Found:%s", url)}//Fetcher is a populated fakefetcher. var fetcher = fakefetcher{"http://golang.org/": &fakeresult{"the Go programming Language", []string{]/HTTP/ golang.org/pkg/"," http://golang.org/cmd/",},}," http://golang.org/pkg/": &fakeresult{" Packages ", []string{] http://golang.org/"," http://golang.org/cmd/"," http://golang.org/pkg/fmt/"," http://golang.org/pkg/os/",},}," http://golang.org/pkg/fmt/": &fakeresult{" Package Fmt ", []string{" http://golang.org/"," http://golang.org/pkg/ ",},}," http://golang.org/pkg/os/": &fakeresult{" Package OS ", []string{" http://golang.org/"," http://golang.org /pkg/",},},}