Scrapy, a fast high-level web crawling & scraping framework for Python.
Update docs, add tests, remove FIXME comment
Eugenio Lacuesta40449fa0eb70
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 915833c54..7db623cf4 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -78,7 +78,8 @@ following methods:If it raises an exception, Scrapy won't bother calling any other spidermiddleware :meth:`process_spider_input` and will call the request- errback. The output of the errback is chained back in the other+ errback if there is one, otherwise it will start the :meth:`process_spider_exception`+ chain. The output of the errback is chained back in the otherdirection for :meth:`process_spider_output` to process it, or:meth:`process_spider_exception` if it raised an exception.diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex ee1e95a0c..d7fe721fb 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -135,7 +135,6 @@ class Scraper(object):return self.spidermw.scrape_response(self.call_spider, request_result, request, spider)else:- # FIXME: don't ignore errors in spider middlewaredfd = self.call_spider(request_result, request, spider)return dfd.addErrback(self._log_download_errors, request_result, request, spider)diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 54756f2ff..832fd3330 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -87,7 +87,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):def test_process_spider_exception_return_none(self):- class ProcessSpiderOutputExceptionReturnNoneMiddleware:+ class ProcessSpiderExceptionReturnNoneMiddleware:def process_spider_exception(self, response, exception, spider):return None@@ -95,7 +95,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):def process_spider_output(self, response, result, spider):1/0- self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware())+ self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())result = self._scrape_response()self.assertIsInstance(result, Failure)diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 0f5646a72..6f8727a15 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -45,8 +45,13 @@ class RecoveryMiddleware:# ================================================================================# (1) exceptions from a spider middleware's process_spider_input method-class ProcessSpiderInputSpider(Spider):- name = 'ProcessSpiderInputSpider'+class FailProcessSpiderInputMiddleware:+ def process_spider_input(self, response, spider):+ spider.logger.info('Middleware: will raise IndexError')+ raise IndexError()++class ProcessSpiderInputSpiderWithoutErrback(Spider):+ name = 'ProcessSpiderInputSpiderWithoutErrback'custom_settings = {'SPIDER_MIDDLEWARES': {# spider@@ -58,23 +63,23 @@ class ProcessSpiderInputSpider(Spider):}def start_requests(self):- yield Request(url=self.mockserver.url('/status?n=200'),- callback=self.parse, errback=self.errback)+ yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse)def parse(self, response):return {'from': 'callback'}++class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):+ name = 'ProcessSpiderInputSpiderWithErrback'++ def start_requests(self):+ yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse, errback=self.errback)+def errback(self, failure):self.logger.info('Got a Failure on the Request errback')return {'from': 'errback'}-class FailProcessSpiderInputMiddleware:- def process_spider_input(self, response, spider):- spider.logger.info('Middleware: will raise IndexError')- raise IndexError()--# ================================================================================# (2) exceptions from a spider callback (generator)class GeneratorCallbackSpider(Spider):@@ -278,12 +283,22 @@ class TestSpiderMiddleware(TestCase):self.assertIn("'item_scraped_count': 3", str(log))@defer.inlineCallbacks- def test_process_spider_input_errback(self):+ def test_process_spider_input_without_errback(self):+ """+ (1.1) An exception from the process_spider_input chain should be caught by the+ process_spider_exception chain from the start if the Request has no errback+ """+ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)+ self.assertIn("Middleware: will raise IndexError", str(log1))+ self.assertIn("Middleware: IndexError exception caught", str(log1))++ @defer.inlineCallbacks+ def test_process_spider_input_with_errback(self):"""- (1) An exception from the process_spider_input chain should not be caught by the- process_spider_exception chain, it should go directly to the Request errback+ (1.2) An exception from the process_spider_input chain should not be caught by the+ process_spider_exception chain if the Request has an errback"""- log1 = yield self.crawl_log(ProcessSpiderInputSpider)+ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback)self.assertNotIn("Middleware: IndexError exception caught", str(log1))self.assertIn("Middleware: will raise IndexError", str(log1))self.assertIn("Got a Failure on the Request errback", str(log1))