Python crawler-grab mobile app data

Grab Super Curriculum topic data.

#!/usr/local/bin/python2.7#-*-Coding:utf8-*-"" "Super Curriculum Topic Crawl" "" Import urllib2from cookielib import cookiejarimport JSON ' Read JSON data ' Def fetch_data (json_data): data = json_data[' data '] Timestamplong = data[' Timestamplong '] Messag EBO = data[' Messagebos '] topiclist = [] for each in messagebo:topicdict = {} if Each.get (' content ', F            alse): topicdict[' content ' = each[' content '] topicdict[' schoolname '] = each[' Schoolname '] topicdict[' messageId ' = each[' messageId '] topicdict[' gender '] = each[' Studentbo ' [' Gender '] topic dict[' time ' = each[' issuetime '] print each[' schoolname '],each[' content '] topiclist.append (topicdict    Return Timestamplong, topiclist "load More" def load (timestamp, headers, url): headers[' content-length '] = ' 159 ' LoadData = ' timestamp=%s&phonebrand=meizu&platform=1&gendertype=-1&topicid=19&phoneversion= 16&selecttype=3&channel=mxmarket&phonemodel=m040&versionnumber=7.2.1& '% timestamp req = urllib2. Request (URL, loaddata, headers) Loadresult = Opener.open (req). Read () LoginStatus = Json.loads (Loadresult). Get (' Statu        S ', False) if loginStatus = = 1:print ' Load successful! '  Timestamp, topiclist = Fetch_data (Json.loads (Loadresult)) load (timestamp, headers, url) else:print ' load Fail ' Print Loadresult return False loginurl = ' http://120.55.151.61/V2/StudentSkip/loginCheckV4.action ' top Icurl = ' http://120.55.151.61/V2/Treehole/Message/getMessageByTopicIdV3.action ' headers = {' Content-type ': ' application/x-www-form-urlencoded; Charset=utf-8 ', ' user-agent ': ' dalvik/1.6.0 (Linux; U Android 4.1.1; M040 build/jro03h) ', ' Host ': ' 120.55.151.61 ', ' Connection ': ' keep-alive ', ' accept-encoding ': ' gzip ', ' content-' Length ': ' 207 ',} '---login section---' ' Logindata = ' phonebrand=meizu&platform=1&devicecode=868033014919494& Account=fcf030e1f2f6341c1c93be5bbc422a3d&phoneversion=16&password=a55b48bb75c79200379d82a18c5f47d6&channel= mxmarket&phonemodel=m040&versionnumber=7.2.1& ' Cookiejar = Cookiejar () opener = Urllib2.build_opener ( Urllib2. Httpcookieprocessor (Cookiejar)) req = Urllib2. Request (loginurl, Logindata, headers) Loginresult = Opener.open (req). Read () LoginStatus = Json.loads (Loginresult). Get ( ' Data ', False) if Loginresult:print ' login successful! ' Else:print ' login fail ' Print Loginresult '---get the topic---' Topicdata = ' TIMESTAMP=0&PHONEBRAND=MEIZU&PLATF orm=1&gendertype=-1&topicid=19&phoneversion=16&selecttype=3&channel=mxmarket& phonemodel=m040&versionnumber=7.2.1& ' headers[' content-length '] = ' 147 ' topicrequest = urllib2. Request (Topicurl, Topicdata, headers) topichtml = Opener.open (topicrequest). Read () Topicjson = Json.loads (topichtml)    Topicstatus = Topicjson.get (' status ', False) print topicjsonif topicstatus = = 1:print ' fetch topic success! ' TimEstamp, topiclist = Fetch_data (topicjson) data = Load (timestamp, headers, topicurl) if Data:timestamp, topic List = fetch_data (data)

