Go implementation grab Sister pictures

This is a creation in Article, where the information may have evolved or changed.

1: Learning practiced hand
2: I also see http://www.oschina.net/code/snippet_2362573_48180 this article found sister figure this site structure is relatively simple, to achieve easy
3: The code also has a lot of optimization of the place, afraid of the boss to see only go home to engage
4: Welcome correction

Directly on the code

 PackageMainImport("bytes""FMT""Io/ioutil""Net/http""OS""RegExp""StrConv")Const(PATHstring="~/meizi" //File Store pathStartURLstring="Http://www.mzitu.com/model" //Sister Diagram Module List page URLCONCURRENCYint=3 //Concurrent Downloads)var(C1Chan string //Channel: Responsible for StartURL, later can be made from the command parameters read OhC2Chan string //Channel: Responsible for transmitting all module URLsC3Chan[]string //Channel: Responsible for transmitting ImgurlC4Chan int //Channel: Responsible for transmitting the download completion status of each pictureC5Chan int //Channel: Responsible for transmitting the current number of downloads)funcInit () {c1 = Make(Chan string,1) C2 = Make(Chan string, -) C3 = Make(Chan[]string, +) C4 = Make(Chan int,3) C5 = Make(Chan int,Ten)GoCgetlist ()GoCdownload ()}funcMain () {C1 <-StartURLGoCgetmodel () num: =0 forCount: =Rangec5 {num = num + countfmt. Println ("Downloaded:", num)}}//Scheduler, pull all modulesfuncCgetmodel () {modelpage: = GetPage (&LT;-C1) forI: =1; I <= modelpage; i++ {modelurl: = StartURL +"/page/"+ StrConv. Itoa (i) C2 <-Modelurl}}//Scheduler pull all the image URL, there can actually be a layer of morefuncCgetlist () {k: =0TMP: = Make([]string,3) forModelurl: =RangeC2 {imglists: = GetList (Modelurl) for_, Imglist: =Rangeimglists {imgpage: = GetPage (imglist) forJ: =1; J <= Imgpage; J + + {imgurl: = Imglist +"/"+ StrConv. Itoa (j)ifK < CONCURRENCY {tmp[k] = imgurlk++}Else{C3 <-TMPK =0}}}ifK! =0{C3 <-TMPK =0}}}//Scheduler, download picturesfuncCdownload () { forImgurls: =RangeC3 {if Len(Imgurls) >0{ for_, Imgurl: =RangeImgurls {Go func() {Download (imgurl) C4 <-1} ()}num: =0 forK: =Rangec4 {num = num + kifnum = =Len(Imgurls) {c5 <-num Break}}}}}//Picture ListfuncGetList (URLstring) (l []string) {Reg, _: = Regexp.compile (' ) s: = Reg. Findallstringsubmatch (HTML, $)if Len(s) <2{FMT. Println ("Get Failed") OS. Exit( -1)}page, _ = StrConv. Atoi (s[Len(s)-1][1])return}//Download ImagesfuncDownload (URLstring) {Reg, _: = Regexp.compile (' <p><a href= ' http:\/\/www.mzitu.com/.*? ' ></a></p> `) Reg1, _: = Regexp.compile (' http:\/\/pic\.dofay\.com/(. *) '_, HTML, _: = gethtml (URL) iterms: = Reg. Findallstringsubmatch (HTML, -) for_, Iterm: =Rangeiterms {imgurl: = Iterm[1]imgpath: = Reg1. Findallstringsubmatch (Imgurl, -) imgpaths: = bytes. Split ([]byte(Imgpath[0][1]), []byte("/") Path: = Path +"/" //+ iterm[2]Imgresponse, _: = http. Get (Imgurl)deferImgResponse.Body.Close () Imgbyte, _: = Ioutil. ReadAll (imgresponse.body) pInfo, Perr: = OS. Stat (PATH)ifPerr! =Nil|| Pinfo.isdir () = =false{errdir: = os. Mkdir (Path, OS. Modeperm)ifErrdir! =Nil{FMT. PRINTLN (errdir) OS. Exit( -1)}}FN: = Path +"/"+string(imgpaths[Len(imgpaths)-1] _, Ferr: = OS. Stat (FN)varFH *os. FileifFerr! =Nil{FH, _ = os. Create (FN)}Else{FH, _ = os. Open (FN)}deferFh. Close () FH. Write (Imgbyte)}}