Golang: Using channel (pipeline) technology multithreading Download pictures

This is a creation in Article, where the information may have evolved or changed.

Use Channel (pipeline) technology to download pictures in multiple threads.

Package Mainimport ("Io/ioutil" "Log" "Net/http" "OS" "RegExp"//"StrConv" "Strings" "Sync") var urlist = [...] string{"http://stock.591hx.com/article/2014-12-03/0000850005s.shtml"}var album Chan stringvar w Sync. Waitgroupvar dir stringfunc main () {dir = "tmp_chenjo/" ERR: = OS. Mkdir (dir, 0777) if err! = Nil {isexist: = os. Isexist (Err) log.  Println (isexist)}album = Make (Chan string, max) for _, V: = Range Urlist {w.add (1) Go Getalbum (v) w.wait ()}}func getalbum (url String) {data: = GETURL (URL) Body: = string (data)//
Part: = RegExp. Mustcompile (") Match: = part. Findallstringsubmatch (Body,-1) for _, V: = Range Match {if m, _: = RegExp. Matchstring ('. */hnimg/201412/03/.*\.jpg ', v[1]);!m {continue}//println (V[1]) album <-V[1]w.add (1) Go GetItem ()}w. Done ()}func GetItem () {URL: = <-albumprintln (URL) defer func () {ret: = recover () if ret! = Nil {log. PRINTLN (ret) w.done ()} else {W.done ()}} ()//data: = GetUrl (URL)//if len (data) > {//body: = string (data)//part: = Regex P.mustcompile (' bigimgsrc= ' (. *) ')//match: = part. Findallstringsubmatch (Body,-1)//for _, V: = range match {str: = strings. Split (URL, "/") Length: = Len (str) Source: = GetUrl (URL) Name: = Str[length-1]file, err: = OS. Create (dir + name) If Err! = Nil {panic (err)}size, err: = file. Write (source) defer file. Close () if err! = Nil {panic (err)}log. PRINTLN (size)//}//}}func GetUrl (URL string) []byte {ret, err: = http. Get (URL) if err! = Nil {log. Println (URL) Status: = map[string]string{}status["status"] = "status[", "url"] = urlpanic (status)}body: = Ret. BOdydata, _: = Ioutil. ReadAll (body) return Data}

Console output:

G:/share/golang/golang.exe [G:/share/golang]

2015/01/18 21:54:29 True

Http://stock.591hx.com/images/hnimg/201412/03/64/13418266510200941552.jpg

Http://stock.591hx.com/images/hnimg/201412/03/7/10886141709285175583.jpg

Http://stock.591hx.com/images/hnimg/201412/03/0/5391574706574741364.jpg

Http://stock.591hx.com/images/hnimg/201412/03/1/5405780767459854941.jpg

Http://stock.591hx.com/images/hnimg/201412/03/16/13722698761317688276.jpg

Http://stock.591hx.com/images/hnimg/201412/03/7/16853951343108680551.jpg

Http://stock.591hx.com/images/hnimg/201412/03/50/17680852843413447062.jpg

Http://stock.591hx.com/images/hnimg/201412/03/34/14366548421421579970.jpg

Http://stock.591hx.com/images/hnimg/201412/03/4/17141924098089490820.jpg

2015/01/18 21:54:30 29968

2015/01/18 21:54:30 28697

2015/01/18 21:54:30 51586

2015/01/18 21:54:30 53031

2015/01/18 21:54:30 31234

2015/01/18 21:54:31 47618

2015/01/18 21:54:31 54649

2015/01/18 21:54:31 40422

2015/01/18 21:54:31 155527

Success: Process exit code 0.