Python implements the method of running Scrapy in the thread

The examples in this paper describe how Python implements scrapy in a thread. Share to everyone for your reference. Specific as follows:

If you want to call Scrapy in a well-written program, you can use the following code to let Scrapy run in a thread.

"" "Code to run Scrapy crawler in a thread-works on scrapy 0.8" "Import threading, Queuefrom twisted.internet import REAC Torfrom scrapy.xlib.pydispatch Import dispatcherfrom scrapy.core.manager import Scrapymanagerfrom scrapy.core.engine Import scrapyenginefrom scrapy.core import Signalsclass crawlerthread (threading. Thread): def __init__ (self): threading. Thread.__init__ (self) self.running = False def run (self): self.running = True scrapymanager.configure (control_re Actor=false) Scrapymanager.start () Reactor.run (Installsignalhandlers=false) def crawl (self, *args): If not self. Running:raise RuntimeError ("Crawlerthread not Running") self._call_and_block_until_signal (signals.spider_closed, \ Scrapymanager.crawl, *args) def stop (self): Reactor.callfromthread (scrapyengine.stop) def _call_and_block_unti L_signal (self, signal, F, *a, **kw): Q = queue.queue () def unblock (): Q.put (None) dispatcher.connect (unblock , signal=signal) reactor.Callfromthread (f, *a, **kw) Q.get () # Usage example Below:import osos.environ.setdefault (' scrapy_settings_module ', ' MYP Roject.settings ') from Scrapy.xlib.pydispatch import dispatcherfrom scrapy.core import signalsfrom scrapy.conf Import Settingsfrom scrapy.crawler import crawlerthreadsettings.overrides[' log_enabled '] = False # avoid LOG noisedef item_ Passed (item): print "Just scraped item:", Itemdispatcher.connect (item_passed, signal=signals.item_passed) crawler = Crawlerthread () print "Starting crawler thread ..." Crawler.start () print "Crawling somedomain.com ..." crawler.crawl (' somedomain.com) # blocking Callprint "crawling anotherdomain.com ..." crawler.crawl (' anotherdomain.com ') # Blocking Callprint "Stopping crawler thread ..." Crawler.stop ()

Hopefully this article will help you with Python programming.

