Crawlers are a program that automatically captures Internet information. Its value is internet data for me. Using the crawled data, you can do a lot of things, such as data statistics and comparison. you can use the crawled data to make an app in a certain aspect; you can also use the crawled data to create a news reader. Introduction
Crawlers are a program that automatically captures Internet information. Its value is internet data for me. Using the crawled data, you can do a lot of things, such as data statistics and comparison. you can use the crawled data to make an app in a certain aspect; you can also use the crawled data to create a news reader.
Crawler architecture
1) URL manager
2) webpage download tool
3) web analyzer
4) crawler caller
5) Use of value data
Crawler implementation
1) scheduler implementation
# coding:utf-8import url_managerimport html_downloaderimport html_parserimport html_outputerimport url_managerclass SpiderMain(object): def __init__(self): self.urls = url_manager.UrlManager() self.downloader = html_downloader.HtmlDownloader() self.parser = html_parser.HtmlParser() self.outputer = html_outputer.HtmlOutputer() def craw(self, root_url): count = 1 self.urls.add_new_url(root_url) while self.urls.has_new_url(): try: new_url = self.urls.get_new_url() print "craw %d : %s" % (count, new_url) html_cont = self.downloader.download(new_url) new_urls, new_data = self.parser.parse(new_url, html_cont) self.urls.add_new_urls(new_urls) self.outputer.collect_data(new_data) if count == 1000: break count = count + 1 except: print "craw failed" self.outputer.output_html()if __name__ == "__main__": root_url = "http://baike.baidu.com/view/21087.htm" obj_spider = SpiderMain() obj_spider.craw(root_url)
2) URL manager implementation
class UrlManager(object): def __init__(self): self.new_urls = set() self.old_urls = set() def add_new_url(self, url): if url is None: return if url not in self.new_urls and url not in self.old_urls: self.new_urls.add(url) def add_new_urls(self, urls): if urls is None or len(urls) == 0: return for url in urls: self.add_new_url(url) def has_new_url(self): return len(self.new_urls) != 0 def get_new_url(self): new_url = self.new_urls.pop() self.old_urls.add(new_url) return new_url
3) URL Download implementation
import urllib2class HtmlDownloader(object): def download(self, url): if url is None: return None response = urllib2.urlopen(url) if response.getcode() != 200: return None return response.read()
4) URL parser implementation
from bs4 import BeautifulSoupimport reimport urlparseclass HtmlParser(object): def _get_new_urls(self, page_url, soup): new_urls = set() links = soup.find_all('a', href=re.compile(r"/view/\d+\.htm")) for link in links: new_url = link['href'] new_full_url = urlparse.urljoin(page_url, new_url) new_urls.add(new_full_url) return new_urls def _get_new_data(self, page_url, soup): res_data = {} res_data['url'] = page_url title_node = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find("h1") res_data['title'] = title_node.get_text() summary_node = soup.find('p', class_="lemma-summary") res_data['summary'] = summary_node.get_text() return res_data def parse(self, page_url, html_cont): if page_url is None or html_cont is None: return soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8') new_urls = self._get_new_urls(page_url, soup) new_data = self._get_new_data(page_url, soup) return new_urls, new_data
5) value data output display
# coding:utf-8class HtmlOutputer(object): def __init__(self): self.datas = [] def collect_data(self, data): if data is None: return self.datas.append(data) def output_html(self): fout = open('output.html', 'w') fout.write("") fout.write("
") fout.write("") fout.write("
") for data in self.datas: fout.write("
") fout.write("
| %s | " % data['url']) fout.write("
%s | " % data['title'].encode('utf-8')) fout.write("
%s | " % data['summary'].encode('utf-8')) fout.write("
") fout.write("
") fout.write("") fout.write("") fout.close()
Run
This crawler crawls 1000 static webpages related to Python keywords in Baidu Encyclopedia. it mainly extracts keywords and abstract information about the data in the webpage, the crawled information is stored as an HTML file, and then accessed using a browser.