Scrapy, a fast high-level web crawling & scraping framework for Python.
Spider middleware: process_spider_exception on generators
Eugenio Lacuesta706ed0e049fe
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rstindex cc02369d4..a3ff72827 100644--- a/docs/topics/exceptions.rst+++ b/docs/topics/exceptions.rst@@ -62,6 +62,17 @@ remain disabled. Those components include:The exception must be raised in the component's ``__init__`` method.+InvalidOutput+-------------++.. exception:: InvalidOutput++This exception can be raised by a downloader or spider middleware to+indicate that some method returned a value not suported by the processing+chain.+See :ref:`topics-spider-middleware` and :ref:`topics-downloader-middleware`+for a list of supported output values.+NotSupported------------diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 8360827e8..fc7669437 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -112,11 +112,12 @@ following methods:.. method:: process_spider_exception(response, exception, spider)- This method is called when a spider or :meth:`process_spider_input`- method (from other spider middleware) raises an exception.+ This method is called when when a spider or :meth:`process_spider_input`/+ :meth:`process_spider_output` method (from other spider middleware)+ raises an exception.:meth:`process_spider_exception` should return either ``None`` or an- iterable of :class:`~scrapy.http.Response`, dict or+ iterable of :class:`~scrapy.http.Request`, dict or:class:`~scrapy.item.Item` objects.If it returns ``None``, Scrapy will continue processing this exception,@@ -125,7 +126,8 @@ following methods:exception reaches the engine (where it's logged and discarded).If it returns an iterable the :meth:`process_spider_output` pipeline- kicks in, and no other :meth:`process_spider_exception` will be called.+ kicks in, starting with the last non-executed method, and no other+ :meth:`process_spider_exception` will be called.:param response: the response being processed when the exception wasraiseddiff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex a206e4b0c..0f03a7b36 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -5,6 +5,7 @@ See documentation in docs/topics/spider-middleware.rst"""import sixfrom twisted.python.failure import Failure+from scrapy.exceptions import InvalidOutputfrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.defer import mustbe_deferredfrom scrapy.utils.conf import build_component_list@@ -40,31 +41,47 @@ class SpiderMiddlewareManager(MiddlewareManager):for method in self.methods['process_spider_input']:try:result = method(response=response, spider=spider)- assert result is None, \- 'Middleware %s must returns None or ' \- 'raise an exception, got %s ' \- % (fname(method), type(result))+ if result is not None:+ raise InvalidOutput('Middleware {} must return None or raise ' \+ 'an exception, got {}'.format(fname(method), type(result)))except:return scrape_func(Failure(), request, spider)return scrape_func(response, request, spider)def process_spider_exception(_failure):exception = _failure.value+ # don't handle InvalidOutput exception+ if isinstance(exception, InvalidOutput):+ return _failurefor method in self.methods['process_spider_exception']:result = method(response=response, exception=exception, spider=spider)- assert result is None or _isiterable(result), \- 'Middleware %s must returns None, or an iterable object, got %s ' % \- (fname(method), type(result))+ if result is not None and not _isiterable(result):+ raise InvalidOutput('Middleware {} must return None or an iterable ' \+ 'object, got {}'.format(fname(method), type(result)))+ # stop exception handling by handing control over to the+ # process_spider_output chain if an iterable has been returnedif result is not None:return resultreturn _failuredef process_spider_output(result):+ def wrapper(result_iterable):+ try:+ for r in result_iterable:+ yield r+ except Exception as ex:+ exception_result = process_spider_exception(Failure(ex))+ if exception_result is None or isinstance(exception_result, Failure):+ raise+ for output in exception_result:+ yield outputfor method in self.methods['process_spider_output']:result = method(response=response, result=result, spider=spider)- assert _isiterable(result), \- 'Middleware %s must returns an iterable object, got %s ' % \- (fname(method), type(result))+ if _isiterable(result):+ result = wrapper(result)+ else:+ raise InvalidOutput('Middleware {} must return an iterable object, ' \+ 'got {}'.format(fname(method), type(result)))return resultdfd = mustbe_deferred(process_spider_input, response)diff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex 4bcecd994..ba7272255 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -11,6 +11,11 @@ class NotConfigured(Exception):"""Indicates a missing configuration situation"""pass+class InvalidOutput(TypeError):+ """Indicates an invalid value has been returned+ by a middleware's processing method"""+ pass+# HTTP and crawlingclass IgnoreRequest(Exception):diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pynew file mode 100644index 000000000..8ec9583d8--- /dev/null+++ b/tests/test_spidermiddleware.py@@ -0,0 +1,340 @@++import logging++from testfixtures import LogCapture+from twisted.trial.unittest import TestCase+from twisted.internet import defer++from scrapy.spiders import Spider+from scrapy.item import Item, Field+from scrapy.http import Request+from scrapy.utils.test import get_crawler+++class TestItem(Item):+ value = Field()+++# ================================================================================+# exceptions from a spider's parse method+class BaseExceptionFromParseMethodSpider(Spider):+ start_urls = ["http://example.com/"]+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540}+ }+++class NotAGeneratorSpider(BaseExceptionFromParseMethodSpider):+ """ return value is NOT a generator """+ name = 'not_a_generator'++ def parse(self, response):+ raise AssertionError+++class GeneratorErrorBeforeItemsSpider(BaseExceptionFromParseMethodSpider):+ """ return value is a generator; the exception is raised+ before the items are yielded: no items should be scraped """+ name = 'generator_error_before_items'++ def parse(self, response):+ raise ValueError+ for i in range(3):+ yield {'value': i}+++class GeneratorErrorAfterItemsSpider(BaseExceptionFromParseMethodSpider):+ """ return value is a generator; the exception is raised+ after the items are yielded: 3 items should be scraped """+ name = 'generator_error_after_items'++ def parse(self, response):+ for i in range(3):+ yield {'value': i}+ raise FloatingPointError+++class CatchExceptionMiddleware(object):+ def process_spider_exception(self, response, exception, spider):+ """ catch an exception and log it """+ logging.warn('{} exception caught'.format(exception.__class__.__name__))+ return None+++# ================================================================================+# exception from a previous middleware's process_spider_input method+# process_spider_input is not expected to return an iterable, so there are no+# separate tests for generator/non-generator implementations+class FromPreviousMiddlewareInputSpider(Spider):+ start_urls = ["http://example.com/"]+ name = 'not_a_generator_from_previous_middleware_input'+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ # engine side+ 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,+ 'tests.test_spidermiddleware.RaiseExceptionOnInputMiddleware': 545,+ # spider side+ }+ }++ def parse(self, response):+ return None+++class RaiseExceptionOnInputMiddleware(object):+ def process_spider_input(self, response, spider):+ raise LookupError+++# ================================================================================+# exception from a previous middleware's process_spider_output method (not a generator)+class NotAGeneratorFromPreviousMiddlewareOutputSpider(Spider):+ start_urls = ["http://example.com/"]+ name = 'not_a_generator_from_previous_middleware_output'+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ # engine side+ 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,+ 'tests.test_spidermiddleware.RaiseExceptionOnOutputNotAGeneratorMiddleware': 545,+ # spider side+ }+ }++ def parse(self, response):+ return [{'value': i} for i in range(3)]+++class RaiseExceptionOnOutputNotAGeneratorMiddleware(object):+ def process_spider_output(self, response, result, spider):+ raise UnicodeError+++# ================================================================================+# exception from a previous middleware's process_spider_output method (generator)+class GeneratorFromPreviousMiddlewareOutputSpider(Spider):+ start_urls = ["http://example.com/"]+ name = 'generator_from_previous_middleware_output'+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ # engine side+ 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,+ 'tests.test_spidermiddleware.RaiseExceptionOnOutputGeneratorMiddleware': 545,+ # spider side+ }+ }++ def parse(self, response):+ return [{'value': i} for i in range(10, 13)]+++class RaiseExceptionOnOutputGeneratorMiddleware(object):+ def process_spider_output(self, response, result, spider):+ for r in result:+ yield r+ raise NameError+++# ================================================================================+# do something useful from the exception handler+class DoSomethingSpider(Spider):+ start_urls = ["http://example.com"]+ name = 'do_something'+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ # engine side+ 'tests.test_spidermiddleware.DoSomethingMiddleware': 540,+ 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545,+ # spider side+ }+ }++ def parse(self, response):+ yield {'value': response.url}+ raise ImportError+++class DoSomethingMiddleware(object):+ def process_spider_exception(self, response, exception, spider):+ return [Request('http://example.org'), {'value': 10}, TestItem(value='asdf')]+++# ================================================================================+# don't catch InvalidOutput from scrapy's spider middleware manager+class InvalidReturnValueFromPreviousMiddlewareInputSpider(Spider):+ start_urls = ["http://example.com/"]+ name = 'invalid_return_value_from_previous_middleware_input'+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ # engine side+ 'tests.test_spidermiddleware.InvalidReturnValueInputMiddleware': 540,+ 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545,+ # spider side+ }+ }++ def parse(self, response):+ return None+++class InvalidReturnValueInputMiddleware(object):+ def process_spider_input(self, response, spider):+ return 1.0 # <type 'float'>, not None+++class InvalidReturnValueFromPreviousMiddlewareOutputSpider(Spider):+ start_urls = ["http://example.com/"]+ name = 'invalid_return_value_from_previous_middleware_output'+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ # engine side+ 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540,+ 'tests.test_spidermiddleware.InvalidReturnValueOutputMiddleware': 545,+ # spider side+ }+ }++ def parse(self, response):+ return None+++class InvalidReturnValueOutputMiddleware(object):+ def process_spider_output(self, response, result, spider):+ return 1 # <type 'int'>, not an iterable+++# ================================================================================+# make sure only non already called process_spider_output methods+# are called if process_spider_exception returns an iterable+class ExecutionChainSpider(Spider):+ start_urls = ["http://example.com"]+ name = 'execution_chain'+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ # engine side+ 'tests.test_spidermiddleware.ThirdMiddleware': 540,+ 'tests.test_spidermiddleware.SecondMiddleware': 541,+ 'tests.test_spidermiddleware.FirstMiddleware': 542+ # spider side+ },+ }++ def parse(self, response):+ return None+++class FirstMiddleware(object):+ def process_spider_output(self, response, result, spider):+ for r in result:+ if isinstance(r, dict):+ r['handled_by_first_middleware'] = True+ yield r++ def process_spider_exception(self, response, exception, spider):+ # log exception, handle control to the next middleware's process_spider_exception+ logging.warn('{} exception caught'.format(exception.__class__.__name__))+ return None+++class SecondMiddleware(object):+ def process_spider_output(self, response, result, spider):+ for r in result:+ if isinstance(r, dict):+ r['handled_by_second_middleware'] = True+ yield r+ raise MemoryError+++class ThirdMiddleware(object):+ def process_spider_output(self, response, result, spider):+ for r in result:+ if isinstance(r, dict):+ r['handled_by_third_middleware'] = True+ yield r++ def process_spider_exception(self, response, exception, spider):+ # handle control to the next middleware's process_spider_output+ return [{'item': i} for i in range(3)]+++class TestSpiderMiddleware(TestCase):++ @defer.inlineCallbacks+ def test_process_spider_exception_from_parse_method(self):+ # non-generator return value+ crawler = get_crawler(NotAGeneratorSpider)+ with LogCapture() as log:+ yield crawler.crawl()+ self.assertIn("AssertionError exception caught", str(log))+ self.assertIn("spider_exceptions/AssertionError", str(log))+ # generator return value, no items before the error+ crawler = get_crawler(GeneratorErrorBeforeItemsSpider)+ with LogCapture() as log:+ yield crawler.crawl()+ self.assertIn("ValueError exception caught", str(log))+ self.assertIn("spider_exceptions/ValueError", str(log))+ # generator return value, 3 items before the error+ crawler = get_crawler(GeneratorErrorAfterItemsSpider)+ with LogCapture() as log:+ yield crawler.crawl()+ self.assertIn("'item_scraped_count': 3", str(log))+ self.assertIn("FloatingPointError exception caught", str(log))+ self.assertIn("spider_exceptions/FloatingPointError", str(log))++ @defer.inlineCallbacks+ def test_process_spider_exception_from_previous_middleware_input(self):+ crawler = get_crawler(FromPreviousMiddlewareInputSpider)+ with LogCapture() as log:+ yield crawler.crawl()+ self.assertIn("LookupError exception caught", str(log))++ @defer.inlineCallbacks+ def test_process_spider_exception_from_previous_middleware_output(self):+ # non-generator output value+ crawler = get_crawler(NotAGeneratorFromPreviousMiddlewareOutputSpider)+ with LogCapture() as log:+ yield crawler.crawl()+ self.assertNotIn("UnicodeError exception caught", str(log))+ # generator output value+ crawler = get_crawler(GeneratorFromPreviousMiddlewareOutputSpider)+ with LogCapture() as log:+ yield crawler.crawl()+ self.assertIn("'item_scraped_count': 3", str(log))+ self.assertIn("NameError exception caught", str(log))++ @defer.inlineCallbacks+ def test_process_spider_exception_do_something(self):+ crawler = get_crawler(DoSomethingSpider)+ with LogCapture() as log:+ yield crawler.crawl()+ self.assertIn("ImportError exception caught", str(log))+ self.assertIn("{'value': 10}", str(log))+ self.assertIn("{'value': 'asdf'}", str(log))+ self.assertIn("{'value': 'http://example.com'}", str(log))+ self.assertIn("{'value': 'http://example.org'}", str(log))++ @defer.inlineCallbacks+ def test_process_spider_exception_invalid_return_value_previous_middleware(self):+ """ don't catch InvalidOutput from middleware """+ # on middleware's input+ crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider)+ with LogCapture() as log1:+ yield crawler1.crawl()+ self.assertNotIn("InvalidOutput exception caught", str(log1))+ self.assertIn("'spider_exceptions/InvalidOutput'", str(log1))+ # on middleware's output+ crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider)+ with LogCapture() as log2:+ yield crawler2.crawl()+ self.assertNotIn("InvalidOutput exception caught", str(log2))+ self.assertIn("'spider_exceptions/InvalidOutput'", str(log2))++ @defer.inlineCallbacks+ def test_process_spider_exception_execution_chain(self):+ # on middleware's input+ crawler1 = get_crawler(ExecutionChainSpider)+ with LogCapture() as log1:+ yield crawler1.crawl()+ self.assertNotIn("handled_by_first_middleware", str(log1))+ self.assertNotIn("handled_by_second_middleware", str(log1))+ self.assertIn("MemoryError exception caught", str(log1))+ self.assertIn("handled_by_third_middleware", str(log1))