#!/usr/bin/python#encoding=utf-8import urllib2 import urllib import re import thread import time class spider (object): def __init__ (self): self.page = 1 self.pages = [] Self.enable = false def start (self): self.enable = True page = self.page thread.start_new_thread (self. LoadPage, ()) while self.enable: if self.pages: nowpage = self.pages[0] del self.pages[0] self. ShowPage (nowpage,page) page += 1 def loadpage (self): While self.enable: if len ( Self.pages) <2: try: &Nbsp; mypage = self. GetPage (str (self.page)) self.pages.append (MyPage) self.page += 1 except: print "Unable to link embarrassing encyclopedia" else: time.sleep (1) def showpage (self,nowpage,page): for item in nowpage: print "the %d page"% (page) print item[0],item[1] myinput = raw_input () if myInput == "Q" : self.enable = False break def getpage (self,page): url = "/http m.qiushibaike.com/hot/page/" + page user_agent = ' Mozilla/4.0 (compatible; &NBSP;MSIE&NBSP;5.5;&NBSP;WINDOWS&NBSP;NT) ' headers = {"User-agent": User_agent} req = urllib2. Request (url,headers = headers) res = Urllib2.urlopen (req) mypage = res.read () unicodepage = mypage.decode ("Utf-8") # Find all class= "content" div tags #re. S is any matching pattern, that is. Can match line break myitems = re.findall (' <div.*? class= "Content". *?title= "(. *?)" > (. *?) </div> ', Unicodepage,re. S) items = [] for i in myItems: items.append ([i[0],i[1]]) return items#- Entrance to---------- program ----------- print u "" " --------------------------------------- Program: Embarrassing reptile language:python 2.7 operation: Enter q quit reading embarrassing encyclopedia function: Press ENTER to browse today's embarrassing hot ------------ --------------------------- "" print u ' Please press ENTER to view today's embarrassing content: ' raw_input (' ') my_spider = spider () My_spider. Start ()
Python crawler A second program