運用python抓取部落格園首頁的所有資料，而且定時持續抓取新公布的內容存入mongodb中

標籤：arch   handle   append   windows   else   esc   lib   PFS   lag   

原文地址：運用python抓取部落格園首頁的所有資料，而且定時持續抓取新公布的內容存入mongodb中

依賴包：1.jieba2.pymongo3.HTMLParser# -*- coding: utf-8 -*-"""@author: jiangfuqiang"""from HTMLParser import  HTMLParserimport reimport timefrom datetime import  dateimport pymongoimport urllib2import sysimport tracebackimport jiebadefault_encoding = ‘utf-8‘if sys.getdefaultencoding() != default_encoding:    reload(sys)    sys.setdefaultencoding(default_encoding)isExist = Falseclass FetchCnblog(HTMLParser):    def __init__(self, id):        HTMLParser.__init__(self)        self.result = []        self.data = {}        self.isTitleLink = False        self.id = id        self.isSummary = False        self.isPostItem = False        self.isArticleView = False    def handle_data(self, data):        if self.isTitleLink and self.isPostItem:            self.data[‘title‘] = data            self.isTitleLink = False        elif self.isSummary and self.isPostItem:            data = data.strip()            if data:                self.data[‘desc‘] = data    def handle_starttag(self, tag, attrs):        if tag == ‘a‘:            for key, value in attrs:                if key == ‘class‘:                    if value == ‘titlelnk‘:                        self.isTitleLink = True                    elif value == ‘gray‘ and self.isArticleView:                        self.isArticleView = False                        for key, value in attrs:                            if key == ‘href‘:                                self.data[‘readmoreLink‘] = value                                reg = ‘d+‘                                result = re.search(reg,value)                                self.isPostItem = False                                if result:                                    self.data[‘id‘] = int(result.group())                                else:                                    self.data = {}                                    return                                if self.data[‘id‘] <= self.id:                                    self.data = {}                                    isExist = True                                    return                                else:                                    self.data[‘srouce‘] = "www.cnblogs.com"                                    self.data[‘source_key‘] = ‘cnblogs‘                                    self.data[‘fetchTime‘] = str(date.today())                                    self.data[‘keyword‘] = ",".join(jieba.cut(self.data[‘title‘]))                                    self.result.append(self.data)                                    self.data = {}        elif tag == ‘p‘:            for key, value in attrs:                if key == ‘class‘ and value == ‘post_item_summary‘:                    self.isSummary = True        elif tag == ‘img‘:            for key, value in attrs:                if key == ‘class‘ and value == ‘pfs‘:                    for key, value in attrs:                        if key == ‘src‘:                            self.data[‘imgSrc‘] = value        elif tag == ‘div‘:            for key, value in attrs:                if key == ‘class‘ and value == ‘post_item_foot‘:                    self.isSummary = False                elif key == ‘class‘ and value == ‘post_item‘:                    self.isPostItem = True        elif tag == ‘span‘:            for key , value in attrs:                if key == ‘class‘ and value == ‘article_view‘:                    self.isArticleView = True    def getResult(self):        return self.resultif __name__ == "__main__":    con = pymongo.Connection(‘localhost‘, 27017)    db = con.blog    fetchblog = db.fetch_blog    record = db.record    url = "http://www.cnblogs.com/sitehome/p/%d"    count = 1    flag = False    headers={             ‘User-Agent‘:‘Mozilla/5.0 （Windows； U； Windows NT 6.1； en-US。 rv：1.9.1.6） Gecko/20091201 Firefox/3.5.6‘}    reco = record.find_one({"type":‘cnblogs‘})    id = 0    if reco:        id = reco[‘maxId‘]    while isExist == False:        try:            req = urllib2.Request(url%count,headers=headers)            request = urllib2.urlopen(req)            data = request.read()            fj = FetchCnblog(id)            fj.feed(data)            result = fj.getResult()            if len(result) < 1:                isExist = True            else:                if flag == False:                    flag = True                    dic = result[0]                    id = int(dic[‘id‘])                    record.update({"type":‘cnblogs‘},{"$set":{‘maxId‘:id}},True,False)                result.reverse()                for doc in result:                    fetchblog.insert(doc)                print "page is %d"%count                count += 1                time.sleep(5)        except Exception, e:            traceback.print_exc()            print "parse error",e程式假設在linux,mac下運行。在可在crontab -e中設定定時任務，假設在windows運行，則自己再在程式裡加個定時器就可以

運用python抓取部落格園首頁的所有資料，而且定時持續抓取新公布的內容存入mongodb中