Golang Beginner's goroutine---web crawler

Source: Internet
Author: User
This is a creation in Article, where the information may have evolved or changed.

Go Tour Practice HTTPS://TOUR.GO-ZH.ORG/CONCURRENCY/10

Package Mainimport ("FMT" "Sync" "Time") type Fetcher interface {//Fetch returns the body content of the URL and places the URL found on this page into a slice. Fetch (URL string) (body string, URLs []string, err Error)}//Safecounter is safe for concurrent use. Type safeurlmap struct {v map[string]boolmux sync. Mutex}func (c *safeurlmap) Put (key string) {C.mux.lock () C.v[key] = Truec.mux.Unlock ()}func (c *safeurlmap) Contains (key s  Tring) bool {c.mux.lock () defer C.mux.unlock () _, OK: = C.v[key]return ok}type Resp struct {URL stringbody string}var urlmap *safeurlmap = &safeurlmap{v:make (map[string]bool)}//Crawl uses Fetcher to crawl pages recursively from a URL until the maximum depth is reached. Func Crawl (URL string, depth int, fetcher fetcher, ch Chan Resp) {if depth <= 0 {return}urlmap.put (URL) body, URLs, err : = Fetcher. Fetch (URL) if err! = Nil {fmt. PRINTLN (Err) return}ch <-Resp{url:url, Body:body}for _, U: = Range URLs {if urlmap.contains (u) {fmt. PRINTF ("has processed:%s\n", u) continue}go Crawl (U, depth-1, Fetcher, ch)}return}func main () {ch: = make (chan Resp) go Cr Awl ("Http://golang.org/", 4, Fetcher, ch) Boom: = time. After (3 * time. Second) for {select {case r: = <-ch:fmt. Printf ("Found:%s%q\n", R.url, r.body) boom = time. After (3 * time. Second) Case <-boom:fmt. Printf ("Time out\n") return}}}//Fakefetcher is a fetcher that returns several results. Type Fakefetcher map[string]*fakeresulttype fakeresult struct {body Stringurls []string}func (f fakefetcher) Fetch (URL St Ring) (String, []string, error) {If res, OK: = F[url]; OK {return res.body, Res.urls, Nil}return "", Nil, FMT. Errorf ("Not Found:%s", url)}//Fetcher is a populated fakefetcher. var fetcher = fakefetcher{"http://golang.org/": &fakeresult{"the Go programming Language", []string{]/HTTP/ golang.org/pkg/"," http://golang.org/cmd/",},}," http://golang.org/pkg/": &fakeresult{" Packages ", []string{] http://golang.org/"," http://golang.org/cmd/"," http://golang.org/pkg/fmt/"," http://golang.org/pkg/os/",},}," http://golang.org/pkg/fmt/": &fakeresult{" Package Fmt ", []string{" http://golang.org/"," http://golang.org/pkg/ ",},}," HTTP://GOLANG.Org/pkg/os/": &fakeresult{" Package OS ", []string{" http://golang.org/"," http://golang.org/pkg/",},},} 

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.