A simple spider collection program based on scrapy implementation

In this paper, a simple spider collection program based on scrapy implementation is described. Share to everyone for your reference. Specific as follows:

# Standard Python Library imports# 3rd party importsfrom scrapy.contrib.spiders import Crawlspider, Rulefrom Scrapy.contri B.LINKEXTRACTORS.SGML Import sgmllinkextractorfrom scrapy.selector import htmlxpathselector# My importsfrom Poetry_ Analysis.items Import poetryanalysisitemhtml_file_name = R '. +\.html ' class Poetryparser (object): "" "provides common par  Sing method for poems formatted this one specific. "" "Date_pattern = R ' (\d{2} \w{3,9} \d{4}) ' Def parse_poem (self, response): HxS = htmlxpathselector (response) ITE m = Poetryanalysisitem () # All poetry text was in pre tags text = hxs.select ('//pre/text () '). Extract () item[' text ' ] = '. Join (text) item[' url '] = Response.url # Head/title contains title-a poem by author Title_text = Hxs.sele CT ('//head/title/text () '). Extract () [0] item[' title ', item[' author '] = title_text.split ('-') item[' author '] = Item [' Author '].replace (' A poem by ', ") for key in [' title ', ' Author ']: Item[key] = Item[key].Strip () item[' Date ' = Hxs.select ("//p[@class = ' small ']/text ()"). Re (Date_pattern) return Itemclass Poetryspider (Crawl   Spider, poetryparser): name = ' example.com_poetry ' allowed_domains = [' www.example.com '] root_path = ' someuser/poetry/' Start_urls = [' http://www.example.com/someuser/poetry/recent/', ' Http://www.example.com/someuser/poetry/less_rec       ent/'] rules = [Rule (Sgmllinkextractor (allow=[start_urls[0] + html_file_name]), callback= ' Parse_poem '), Rule (Sgmllinkextractor (allow=[start_urls[1] + html_file_name]), callback= ' Parse_poem ')

Hopefully this article will help you with Python programming.

