Golang Implementing Task Distribution processing

This is a creation in Article, where the information may have evolved or changed.

Package Mainimport ("Flag" "FMT" "OS" "Log" "Net/http" "Io/ioutil" "Github.com/bitly/go-simplejson" "encoding/csv" "IO" Time "" Sync "" Net/url "" StrConv "" Errors ") var (concurrency inttimeout intinfile stringoutfile string) var usage = ' usage:%s [Options]  Options is:-c concurrency number of request to PREFORM-T timeout request timeout-i infile Input file-o outfile Output File ' func main () {flag. Usage = func () {fmt. fprintf (OS. Stderr, usage, OS. Args[0])}flag. Intvar (&concurrency, "C", "Ten," ") flag. Intvar (&timeout, "T", "," ") flag. Stringvar (&infile, "I", "", "") flag. Stringvar (&outfile, "O", "", "") flag. Parse () _, Err: = OS. Stat (infile) if err! = Nil {log. Fatalln ("Error:", err)}f, err: = OS. Create (outfile) if err! = Nil {log. Fatalln ("Error:", err)}defer f.close () var lock sync. MUTEXW: = &worker{concurrency:concurrency,timeout:timeout,infile:infile,lw:&lockwriter{m:lock,writer:f,} ,}w.run ()}type lockwriter struct {m sync. Mutexwriter io. Writer}func (LW Lockwriter) write (b []byte) (n int, err error) {Lw.m.lock () defer lw.m.unlock () return Lw.writer.Write (b)}type carinfo struct {carno stringecode Stringvcode string} Type Worker struct {concurrency inttimeout intinfile stringjobs Chan *carinfolw *lockwriter}func (W *worker) Run () {var wg sync. Waitgroupwg.add (w.concurrency + 1) w.jobs = Make (chan *carinfo, w.concurrency) go func () {w.loadjobs () WG. Done ()} ()//concurrency number for i: = 1; I <= w.concurrency; i++ {go func (n int) {w.dowork (n) WG. Done ()} (i)}WG. Wait ()}func (w *worker) loadjobs () {fin, err: = OS. Open (w.infile) if err! = Nil {log. Fatalln ("File Open failed,error:", err.) Error ())}defer fin. Close () Reader: = csv. Newreader (Fin) for {row, err: = Reader. Read () if err! = Nil {if err! = Io. EOF {log. PRINTLN ("File read error:", err)}break}time. Sleep (time. Duration (1) * time. Second) w.jobs <-&carinfo{carno:row[3], vcode:row[4], Ecode:row[5]}}close (w.jobs)}func (w *Worker) doWork (num int) {total: = 0log. Println ("Worker:", num) uriparams: = URL. Values{}uriparaMs. ADD ("Openudid", "3122DCF3-3A2A-34E9-8DA5-E9DDE29579A4") Uriparams.add ("AppID", "1") Uriparams.add ("Cartype", "02") Uriparams.add ("OS", "Android") Uriparams.add ("AppVersion", "6.6.6") Uriparams.add ("Prefetch", "1") Uriparams.add (" Reqfrom "," 1 ") Uriparams.add (" Secret "," UBJUFDL9KZSDUQIVN4WB063QI4ES3MZHVWVT ") httpClient: = &http. Client{timeout:time. Duration (w.timeout) * time. Second,}for carinfo: = Range w.jobs {log. Println (Carinfo.carno, "|", Carinfo.vcode, "|", Carinfo.ecode) Uriparams.set ("Carno", Carinfo.carno) Uriparams.set (" Vcode ", Carinfo.vcode) Uriparams.set (" Ecode ", Carinfo.ecode) URI: = Fmt. Sprintf ("http://xxx.cn/common_prefix?%s", Uriparams.encode ()) fmt. Println (URI) code, ERR: = W.dorequest (httpClient, URI) if err! = Nil {log. Println ("URL request error:", err)}switch code {case 203, 9999:w.writeresult (carinfo)}total++}log. Println ("worker[", num, "] Total:")}func (w *worker) dorequest (client *http. Client, URI string) (int, error) {req, err: = http. Newrequest ("GET", Uri, nil) if ERR! = Nil {log. Println ("Get new request failed!") Return-1, Err}resp, err: = client. Do (req) if err! = Nil {log. Println ("Error:", err) return-1, Err}defer resp. Body.close () result, err: = Ioutil. ReadAll (resp. Body) If err! = Nil {fmt. Println ("Read respose body failed") return-1, Err}log. Println ("Resonse:", string (Result)) if Resp. StatusCode! = $ {log. Println ("Status code:", resp.) StatusCode) return-1, errors. New ("Stats code Error:" + StrConv. Itoa (resp. StatusCode))}js, err: = Simplejson. Newjson (Result) if err! = nil {return-1, err}code, err: = JS. Get ("code"). Int () return code, Nil}func (w *worker) Writeresult (Carinfo *carinfo) {line: = FMT. Sprintf ("%s,%s,%s\n", Carinfo.carno, Carinfo.vcode, Carinfo.ecode) w.lw.write ([]byte (line))}