Golang crawl tick data and generate CSV file

This is a creation in Article, where the information may have evolved or changed. "Package Mainimport (" Encoding/json "" Encoding/csv "" FMT "" Net/http "" Net/url "" StrConv "" Strings "" Time "" OS "" bytes ") Func call (param URL. Values, res interface{}) Error {req, _: = http. Newrequest (http. Methodpost, "https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false&isSchoolJob=0", strings. Newreader (param. Encode ())) req. Header.set ("Origin", "https://www.lagou.com") req. Header.set ("X-anit-forge-code", "0") req. Header.set ("Accept-language", "zh-cn,zh;q=0.8,en;q=0.6,zh-tw;q=0.4") req. Header.set ("X-requested-with", "XMLHttpRequest") req. Header.set ("accept-encoding", "gzip, deflate, BR") req. Header.set ("Connection", "keep-alive") req. Header.set ("Pragma", "No-cache") req. Header.set ("User-agent", "mozilla/5.0" (Macintosh; Intel Mac OS X 10_12_6) applewebkit/537.36 (khtml, like Gecko) chrome/61.0.3163.100 safari/537.36 ") req. Header.set ("Content-type", "application/x-www-form-urlencoded; Charset=utf-8 ") req. Header.set ("Accept", "Application/json, Text/javascript, */*; q=0.01 ") req. Header.set ("Cache-control", "No-cache") req. Header.set ("Referer", "https://www.lagou.com/jobs/list_go?city=%E5%85%A8%E5%9B%BD&cl=false&fromSearch= True&labelwords=&suginput= ") req. Header.set ("X-anit-forge-token", "None") resp, err: = http. Defaultclient.do (req) If err! = Nil {return Err}defer resp. Body.close () if res! = Nil {d: = json. Newdecoder (resp. Body) D.decode (res)}return nil}//because the pull-hook has only 30 pages. So, this place is directly written dead 30. Const PAGESIZE = 30func Lagou (language String) (res [][]string) {param: = URL. Values{}param. Set ("First", "true") param. Set ("KD", language) Workinfos: = Make ([]workinfo, 0, N) for I: = 1; I <= pagesize; i++ {fmt. Printf ("Now is%d page \ n", i) param. Set ("PN", StrConv. Itoa (i) RET: = Lagouret{}err: = Call (param, &ret) if err! = Nil {fmt. Println ("ERR:", err) return}if ret. Code! = 0 {err = fmt. Errorf ("%s", ret.) MSG) Return}workinfos = append (Workinfos, ret. Content.PositionResult.Result ...) Param. Del ("First") param. Set ("First", "false") time. Sleep (Ten * time. Second)}return convertworkinfo2csv (Workinfos)}func convertworkinfo2csv (Infos []workinfo) (res [][]string) {title: = [] string{"Working Years", "Education level", "salary", "Job title", "Job label", "Location", "Company Name", "Company type", "Financing Situation", "Company label"}res = Make ([][]string, 0) res = append (re S, title) tmp: = Make ([]string, Ten) for _, V: = range infos {tmp = []string{v.workyear, v.education, V.salary, V.positionnam E, Strings. Join (V.positionlables, ":"), V.city + "" +strings. Join (V.businesszones, ":"), V.companyfullname, V.industryfield, V.financestage, strings. Join (V.companylabellist, ":")}res = append (res, tmp)}return}type Error struct {code int ' JSON: "Code" ' MSG string ' JSON: "MS G "'}type lagouret struct {errorcontent struct {hrmap map[string]hrinfo ' JSON:" Hrinfomap "' positionresult struct {Result [ ]workinfo ' JSON: "Result" '} ' JSON: "Positionresult" '} ' JSON: "Content" '}type hrinfo struct {cantalk bool ' JSON: "Cantalk" ' Phone string ' JSON: ' Phone ' ' Portrait string ' json: ' Portrait ' ' Positionname string ' json: ' Positionname ' ' realname string ' JSON: "Realname"' Receiveemail string ' json: ' Receiveemail ' ' UserLevel string ' json: ' UserLevel ' '}type workinfo struct {workyear string ' JSON: "Workyear" ' Education string ' JSON: "Education" ' Jobnature string ' JSON: "Jobnature" ' CompanyID int ' json: "CompanyID "' Positionname string ' JSON:" Positionname "' PositionID int ' json:" PositionID "' Createtime string ' JSON:" Createtime "' City string ' JSON: ' City ' ' CompanyLogo string ' json: ' CompanyLogo ' ' Industryfield string ' json: ' Industryfield ' Positionadvantage string ' JSON: ' Positionadvantage ' ' Salary string ' json: ' Salary ' ' companysize string ' JSON: ' Companysize "' Approve int ' json:" Approve "' score int ' JSON:" Score "' Companyshortname string ' JSON:" Companyshortname "' Positionlables []string ' JSON: "Positionlables" ' industrylables []string ' JSON: ' industrylables ' ' PublisherID int ' JSON: "PublisherID" ' Financestage string ' JSON: "Financestage" ' companylabellist []string ' JSON: ' companylabellist ' ' District String ' JSON: ' District ' ' Businesszones []string ' JSON: ' Businesszones ' ' Formatcreatetime string ' jSon: "Formatcreatetime" ' AdWord int ' json: "AdWord" ' Companyfullname string ' JSON: "Companyfullname" ' imstate string ' json : "Imstate" ' Lastlogin Int64 ' JSON: "Lastlogin" ' Explain interface{} ' JSON: "Explain" ' Plus interface{} ' JSON: "Plus" ' pcshow int ' JSON: "Pcshow" ' appshow int ' json: "Appshow" ' Deliver int ' JSON: "Deliver" ' Gradedescription interface{} ' JSON: " Gradedescription "' Promotionscoreexplain interface{} ' JSON:" Promotionscoreexplain "' Firsttype string ' JSON:" Firsttype "' Secondtype string ' JSON:" Secondtype "' isschooljob int ' json:" Isschooljob "'}func writecsv2file (result [] String, fileName string) {csvbuf: = new (bytes. Buffer) Writter: = csv. Newwriter (CSVBUF) writter. Writeall (Result) writter. Flush () F, err: = OS. Create (fileName) if err! = Nil {fmt. PRINTLN (Err) return}defer f.close () f.write (Csvbuf.bytes ())}func main () {//The parameter of this place is something related to the position you want to check Writecsv2file (Lagou ( "Go"), "Lagou.csv")} "378 reads