廣度配合匿名爬取（改進）

package main

import (

    "fmt"

    "io/ioutil"

    "log"

    "net/http"

    "os"

    "regexp"

    "strings"

    "sync"

    "golang.org/x/net/html"

)

//timer

var (

    ground sync.WaitGroup

    str    string = "https://docs.hacknode.org/gopl-zh/"

)

func init() {

    defer func() {

        if err := recover(); err != nil {

            log.Fatal("recover error is :", err)

        }

    }()

}

//CreatFile is func to get infomation

func CreatFile(bt []byte, i string) {

    f, err := os.OpenFile("F:/MyGo/src/waitground_user/url"+i+".txt", os.O_CREATE|os.O_APPEND, 0666)

    if err != nil {

        log.Fatal(err)

    }

    defer f.Close()

    _, err = f.Write([]byte(bt))

    if err != nil {

        log.Fatal(err)

    }

}

//GetURLInfomation is get url infomation

func GetURLInfomation(URL string, ch chan int) (bt []byte) {

    resp, err := http.Get(URL)

    if err != nil {

        log.Fatal(err)

    }

    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {

        log.Fatal("Can't connect")

    }

    bt, err = ioutil.ReadAll(resp.Body)

    if err != nil {

        log.Fatal(err)

    }

    ch <- 1

    return bt

}

//GetURLInfomationAdress is a func get URL infomation

func GetURLInfomationAdress(URL string) []string {

    resp, err := http.Get(URL)

    if err != nil {

        log.Fatal(err)

    }

    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {

        log.Fatal("Can't connect:", URL)

    }

    //開始節點處理

    doc, err := html.Parse(resp.Body)

    if err != nil {

        log.Fatal(err)

    }

    var links []string

    ForOneNode := func(n *html.Node) { //單次節點處理

        if n.Type == html.ElementNode && n.Data == "a" {

            for _, a := range n.Attr {

                if a.Key != "href" {

                    continue

                }

                link, err := resp.Request.URL.Parse(a.Val)

                if err != nil {

                    log.Fatal(err)

                }

                if CheckURL(link.String(), links) {

                    links = append(links, link.String()) //這條語句可以改成並行擷取URL地址內容

                }

            }

        }

    }

    ForEachNode(doc, ForOneNode, nil)

    return links

}

//ForEachNode is 廣度優先遍曆

func ForEachNode(n *html.Node, pre, post func(n *html.Node)) {

    if pre != nil {

        pre(n)

    }

    for c := n.FirstChild; c != nil; c = c.NextSibling {

        ForEachNode(c, pre, post)

    }

    if post != nil {

        post(n)

    }

}

//Regex檢查

func checkRegexp(cont string, reg string, style int) (result interface{}) {

    check := regexp.MustCompile(reg)

    switch style {

    case 0:

        result = check.FindString(cont)

    case 1:

        result = check.FindAllString(cont, -1)

    default:

        result = check.FindAll([]byte(cont), -1)

    }

    return

}

//CheckURL is check the repeated fields

func CheckURL(link string, links []string) bool {

    bl := true

    for _, str := range links {

        if str == link {

            bl = false

            break

        }

    }

    return bl

}

func main() {

    bt := GetURLInfomationAdress(str) //第一次運行，擷取所有的標籤連結地址

    fmt.Println("first finish")

    // fmt.Println(bt[1])

    // f := GetURLInfomation(bt[1])

    // CreatFile(f, strconv.Itoa(1))

    ch := make(chan int)

    for _, t := range bt {

        t := t

        go func() { //遍曆所有地址，擷取地址內容

            fname := strings.Split(t, "/")

            ff := fname[len(fname)-1]

            fmt.Println("地址：", t)

            fmt.Println(ff)

            ft := strings.Split(ff, ".")

            fft := ft[0]

            fmt.Println(fft) //進行地址處理，取頁面名當做檔案名稱

            p := GetURLInfomation(t, ch)

            CreatFile([]byte(p), fft)

        }()

    }

    for range ch {

        <-ch

    }

}