自動下載並儲存部落格 Python指令碼

謝了一個自動下載指定人的部落格的指令碼

這個指令碼是用來下載csdn部落格的

同樣的方法可以下載一般其他網站的部落格，如sina

 

有時頁面訪問會被拒絕，重新運行即可

這種程式是在分析了指定網站，我在這兒是csdn，之後編寫出的

會牽涉到網頁的編碼問題，有時程式運行會因此終止

我自己的部落格已經下載忘了

只是下載網頁

使用網頁分析後可以整理出文章，那樣更實用

#<br />#blogdownloader_csdn.py<br />#@Author:onezeros@yahoo.cn ||Zhijie Lee<br />#I didnot realize the image_download function<br />#but it's not very difficult,so that you can do it by yourself</p><p>#cmd usage:blogdownloader_csdn.py blogname "full directory path"</p><p>#further extension :classify the articles to<br />#"原創""轉載" etc.according to csdn </p><p>import os<br />import sys<br />import unicodedata<br />import urllib.request</p><p>#globle var<br />username=sys.argv[1]<br />#username='onezeros'<br />#total number of pages<br />total_num=0<br />dst_urls=[]<br />#use data as the file name<br />dst_title=[]</p><p>###########################################<br />#function to find urls of articals<br />#it's neccessory to verify whether the url exits<br />def url_finder(url_directory,firstpage=False):<br /> global username,total_num,dst_urls,dst_title<br /> url_f=urllib.request.urlopen(url_directory)<br /> print("open url "+url_directory+" successfully/n")<br /> url_front='/'+username+'/archive/'<br /> for line in url_f.readlines():<br /> lin=line.decode('utf-8')<br /> pos_front=lin.find(url_front)<br /> if(pos_front!=-1 ):<br /> pos_post=len('2009/12/13/4998191.aspx')+len(url_front)+pos_front<br /> if(lin[pos_post]=='#'):<br /> dst_urls.append('http://blog.csdn.net'+lin[pos_front:pos_post])<br /> s=lin[pos_front+len(url_front):pos_post-len('.aspx')]<br /> s=s.replace('/','-')<br /> print(s)<br /> dst_title.append(s)<br /> if(firstpage==True):<br /> pos=lin.find('第1頁')<br /> if(pos!=-1):<br /> pt=lin.find('頁',pos+5)<br /> total_num=int(lin[pos+5:pt])<br /> if(firstpage==True and total_num==0):<br /> print("something wrong when retriving imformation from first page")<br /> sys.exit(1)<br /> url_f.close()<br /> return</p><p>################################################<br />def main():<br />#creat directory<br /> global username,total_num,dst_urls,dst_title<br /> if (False==os.path.isdir(sys.argv[2])):<br /> print("the dirctory does not exit, do you want to creat it?/n")<br /> while(1):<br /> a=input('type y for yes or n for no/n')<br /> if(a=='y' or a=='Y'):<br /> os.makedirs(sys.argv[2])<br /> print("successed to make dir "+sys.argv[2])<br /> break<br /> elif(a=='n' or a=='N'):<br /> sys.exit(0)<br /> os.chdir(sys.argv[2])<br /> #os.chdir('d:/test')<br /> #deal with first page<br /> url_finder('http://blog.csdn.net/'+username,True)<br /> #generate urls of every directory page</p><p> print(total_num)<br /> for pagenum in range(2,total_num+1): #connot be excuted<br /> url='http://blog.csdn.net/'+username+'/default.aspx?PageNumber='+str(pagenum)<br /> print("openning "+url)<br /> url_finder(url,False)</p><p> #abstract content of the article page<br /> #!!!in csdn's sourcefile ,only users' articals has lebel'<p>'</p><p> #download the page only<br /> if(len(dst_urls)!=len(dst_title)):<br /> print('error happened')<br /> sys.exit(0)<br /> #create and store the files<br /> for i in range(len(dst_urls)):<br /> url_f=urllib.request.urlopen(dst_urls[i])<br /> f=open(dst_title[i]+'.aspx','w')<br /> for lin in url_f.readlines():<br /> lin=lin.decode()<br /> f.write(lin)<br /> f.close()<br /> print("Save "+dst_title[i]+'.aspx successfully')<br /> print("/nDone/n");</p><p>if __name__ == '__main__':<br /> main()</p><p>