[Python] Initial page crawl--Grab the Shanghai and Shenzhen stock market trading Billboard data

[Python] captures the billboard data of the Shanghai and Shenzhen Stock Exchange

Run under Python 3.5.0

Did not do automatically establish files folder, you need to manually set up Files folder in the Py file directory after running

#Coding=utf-8ImportgzipImportHttp.cookiejarImporturllib.requestImportUrllib.parseImportJSONImportOSImport TimeImportdatetimedefGetopener (head):#deal with the CookiesCJ =Http.cookiejar.CookieJar () Pro=Urllib.request.HTTPCookieProcessor (CJ) Opener=urllib.request.build_opener (Pro) header= []     forKey, ValueinchHead.items (): Elem=(key, value) Header.append (elem) opener.addheaders=Headerreturnopenerdefungzip (data):Try:#try to unzip        Print('extracting .....') Data=gzip.decompress (data)Print('Decompression Complete!')    except:        Print('uncompressed without decompression')    returnDatadefWriteFile (fname,data): filename= R'files/'+fname+'. txt'    ifos.path.exists (filename): Message='file +'+filename +'already exists, skipped'    Else: Message='file +'+filename +'not present, new'F=open (filename,'W') f.write (data) f.close ()Print('File:'+fname+'processing is complete. ')" "Read Fetch data start date if the date does not exist, start reading from 10th before reading if it exists from within the file date read to today" "Header= {    'Connection':'keep-alive',    'Accept':'*/*',    'Accept-language':'zh-cn,zh;q=0.8',    'user-agent':'mozilla/5.0 (Windows NT 6.2) applewebkit/537.36 (khtml, like Gecko) chrome/40.0.2214.111 safari/537.36',    'accept-encoding':'gzip, deflate',    'Host':"',    'Referer':"'}shurl='http://query.sse.com.cn/infodisplay/showTradePublicFile.do?dateTx='#2015-09-28Szurl = ['Http://www.szse.cn/szseWeb/common/szse/files/text/jy/jy',#150923.txt         'Http://www.szse.cn/szseWeb/common/szse/files/text/smeTxt/gk/sme_jy',#150708.txt         'Http://www.szse.cn/szseWeb/common/szse/files/text/nmTxt/gk/nm_jy']#150902.txtStartFileName = R'StartDay.txt'Endday=Datetime.datetime.now ()ifos.path.exists (startfilename):Print('Date profile exists, start reading') F=open (StartFileName,'RT') s=f.readline () f.close ( )ifs!="':        Print('will be from date:'+s+'start reading') Timearray= Time.strptime (S,"%y%m%d") TimeStamp=Int (time.mktime (timearray)) Fromday=Datetime.datetime.utcfromtimestamp (TimeStamp)Else:        Print('The date profile is empty and will be read from the date before 10th') Fromday= Endday-datetime.timedelta (days = 10)Else:    Print('The date profile does not exist and will begin reading from the date before 10th') Fromday= Endday-datetime.timedelta (days = 10) Endday= Endday + Datetime.timedelta (days = 1) whileFromday.strftime ("%y%m%d")!=endday.strftime ("%y%m%d"):    Print(Fromday.strftime ("%y%m%d"))        " "cycle above date grab Shanghai, Shen Certificate, small and medium-sized trading billboard data if the content is not empty file does not exist write file" "    #grab Shanghai Billboard dataurl = shurl + fromday.strftime ("%y-%m-%d")    Print('read SSE billboard \ n'+URL) header['Host'] ='query.sse.com.cn'header['Referer'] ='http://www.sse.com.cn/disclosure/diclosure/public/'    Try: Opener=Getopener (header) Op=opener.open (URL) data=op.read () data=Data.decode () jsondata=json.loads (data) Outdata="'        if(jsondata['filecontents']!="'):             forInfoinchjsondata['filecontents']: Outdata= outdata+ info+'\ n'WriteFile (Fromday.strftime ("%y-%m-%d")+'_ Shanghai', Outdata)except:        Print(Fromday.strftime ("%y-%m-%d")+'Skip over')        #grasping deep certificate, small and medium-sized trading billboard dataI=1 forwr.inchSzurl:if(i==1): Name='Deep Proof'        elif(i==2): Name='SME Board'        Else: Name='Gem Board'URL= URL + fromday.strftime ("%y%m%d")+'. txt'        Print('Read'+name+'Billboard \ n'+URL) header['Host'] ='www.szse.cn'header['Referer'] ='http://www.szse.cn'        Try: Opener=Getopener (header) Op=opener.open (URL) data=op.read () data=ungzip (data) data= Data.decode ('GBK') WriteFile (Fromday.strftime ("%y-%m-%d")+'_'+name,data)except:            Print(Fromday.strftime ("%y-%m-%d")+'Skip over') I=i+1Fromday= Fromday + Datetime.timedelta (days = 1)#the last update date is the current datePrint('set up the latest date') Fromday= Fromday-datetime.timedelta (days = 1) F=open (StartFileName,'W') F.write (Fromday.strftime ("%y%m%d") ) F.close ()Print('Read Complete')

[Python] Preliminary page crawl-grab the Shanghai and Shenzhen stock market trading Billboard data