Python爬蟲(二)

標籤：spider   awl   open   pen   __init__   nbsp   title   begin   font   

爬取電影吧一個文章裡的所有樓主發言：

# python2# -*- coding: utf-8 -*-import urllib2import stringimport reclass Baidu_Spider:    feature_pattern = re.compile(r‘id="post_content.*?>\s+(.*?)</div>‘, re.S)    replaceList = [(‘&#39;‘, ‘\‘‘), (‘&quot;‘, ‘\"‘)]    def __init__(self, url):        self.url = url + ‘?see_lz=1‘    def crawl_tieba_lz(self):        begin_page = urllib2.urlopen(self.url).read()        self.print_page_title(begin_page)        count = self.get_page_count(begin_page)        self.handle_data(count)    def handle_data(self, count):        f = open(‘tieba_lz.txt‘, ‘w+‘)        for i in range(count):            url = self.url + ‘&pn=‘ + str(i+1)            hint = ‘第‘ + str(i+1) + ‘頁‘            print ‘正在下載%s: %s‘ % (hint, url)            page = urllib2.urlopen(url).read()            features = re.findall(self.feature_pattern, page)            print hint + ‘下載完成‘            print ‘共有%d條記錄‘ % len(features)            f.write(hint + ‘:\n‘)            for feature in features:                feature = self.handle_record(feature)                print feature                f.write(feature + ‘\n\n‘)        f.close()        print ‘done‘    def handle_record(self, record):        record = re.sub(r‘(<|</)br>‘, ‘\n‘, record)        record = re.sub(r‘<.*?>‘, ‘‘, record)        for item in self.replaceList:            record = record.replace(item[0], item[1])        return record    def get_page_count(self, page):        result = re.search(r‘class="red">(\d+?)</span>‘, page, re.S)        if result:            count = int(result.group(1))            print ‘一共%d頁‘ % count        else:            count = 0;            print ‘無法擷取頁數‘        return count    def print_page_title(self, page):        result = re.search(r‘<h1.*?>(.*?)</h1>‘, page, re.S)        if result:            title = result.group(1)            print ‘標題: %s‘ % title        else:            print ‘無法擷取標題‘spider = Baidu_Spider(‘http://tieba.baidu.com/p/4082863285‘)spider.crawl_tieba_lz()

 

Python爬蟲(二)