Python's approach to reading a newspaper program by parsing web pages

The example described in this article can be realized based on Python view picture newspaper "reference message" and the day's picture newspaper automatically downloaded to the local for viewing functions, the implementation code is as follows:

# coding=gbkimport Urllib2import socketimport reimport timeimport os# timeout in seconds#timeout = 10#socket.setdefaultti Meout (timeout) timeout = 10urllib2.socket.setdefaulttimeout (timeout) Home_url = "Http://www.hqck.net" home_page = "" Try  : Home_page_context = Urllib2.urlopen (home_url) home_page = Home_page_context.read () print "Read home page finishd." Print "-------------------------------------------------" except Urllib2. Urlerror,e:print E.code exit () Except:print E.code exit () Reg_str = R '.+' News_url_reg = Re.compile (reg_str) today_cankao_news = News_url_reg.findall (home_page) If Len (today_cankao_news) = = 0:  Print "Cannot find today ' s news!" Exit () My_news = today_cankao_news[0]print "Latest News link =" + my_newsprinturl_s = My_news.find ("/arc/") Url_e = My_news . Find (". html") Url_e = url_e + 5print "Link index = [" + str (url_s) + "," + str (url_e) + "]" my_news = my_news[url_s:url_e]p Rint "Part URL =" + My_newsfull_news_url = home_url + my_newsprint "Full URL =" + Full_news_urlprintimage_folder = "e:\\ New_folder\\ "if (os.path.exists (image_folder) = = False): Os.makedirs (image_folder) today_num = Time.strftime ('%y-%m-% d ', Time.localtime (Time.time ())) Image_folder = Image_folder + today_num + "\ \" if (os.path.exists (image_folder) = = False) : Os.makedirs (image_folder) print "News image folder =" + Image_folderprintcontext_uri = Full_news_url[0:-5]first_page_u RL = Context_uri + ". html" Try:first_page_context = Urllib2.urlopen (first_page_url) first_page = First_page_context.read () except Urllib2. Httperror, E:print E.code exit () Tot_page_index = First_page.find ("total") Tot_page_index = Tot_page_indextmp_str = First_pag e[tot_page_index:tot_page_index+10]end_s = tmp_str.find ("page") Page_num = tmp_str[2:end_s]print Page_numpage_count = int (page_num) print "Total" + Page_num + "pages:" Printpage_index = 1DOWNLOAD_SUC = Truewhile Page_index <= page_count:p   Age_url = Context_uri if page_index > 1:page_url = Page_url + "_" + str (page_index) Page_url = Page_url + ". html" Print "News page link =" + Page_url Try:news_img_page_context = Urllib2.urlopen (page_url) except URLLIB2. Urlerror,e:print E.reason download_suc = False Break news_img_page = News_img_page_context.read () #f = open ( "E:\\page.html", "W") #f. Write (News_img_page) #f. Close () Reg_str = R ' http://image\s+jpg ' Image_reg = Re.compile (reg_st r) Image_results = Image_reg.findall (news_img_page) If Len (image_results) = = 0:print "Cannot find news page" + STR (p    Age_index) + "!" DOWNLOAD_SUC = False Break Image_url = image_results[0] print "News image url =" + Image_url News_image_context =  Urllib2.urlopen (image_url) image_name = Image_folder + "Page_" + str (page_index) + ". jpg" IMGF = open (image_name, ' WB ')       print "Getting image ..." Try:while true:date = News_image_context.read (1024*10) if not date:break Imgf.write (date) imgf.close () Except:download_suc = False print "Save image" + str (page_index) + "Faile    d! " Print "Unexpected error:" + sys.exc_info () [0] + sys.exc_info () [1] else:print "Save image" + str (page_index) + "suc    ceed! " Print Page_index = Page_index + 1if download_suc = = true:print "News Download succeed! Path = \ "" + str (image_folder) + "\" "Print" Enjoy it! ^^ "Else:print" News Download failed! "

