Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix bad exception handling, add tests
Eugenio Lacuesta801d3c07b4b7
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 8607ed620..1b67af130 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -66,6 +66,8 @@ class SpiderMiddlewareManager(MiddlewareManager):if result is not None:raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \.format(fname(method), type(result)))+ except _InvalidOutput:+ raiseexcept:return scrape_func(Failure(), request, spider)return scrape_func(response, request, spider)diff --git a/tests/test_spidermiddleware_invalid_values.py b/tests/test_spidermiddleware_invalid_values.pynew file mode 100644index 000000000..0d9af8951--- /dev/null+++ b/tests/test_spidermiddleware_invalid_values.py@@ -0,0 +1,82 @@+from twisted.trial.unittest import TestCase+from twisted.python.failure import Failure++from scrapy.spiders import Spider+from scrapy.http import Request, Response+from scrapy.exceptions import _InvalidOutput+from scrapy.utils.test import get_crawler+from scrapy.core.spidermw import SpiderMiddlewareManager+from tests import mock+++class SpiderMiddlewareTestCase(TestCase):++ def setUp(self):+ self.request = Request('http://example.com/index.html')+ self.response = Response(self.request.url, request=self.request)+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider('foo')+ self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)++ def _scrape_response(self):+ """Execute spider mw manager's scrape_response method and return the result.+ Raise exception in case of failure.+ """+ scrape_func = mock.MagicMock()+ dfd = self.mwman.scrape_response(scrape_func, self.response, self.request, self.spider)+ # catch deferred result and return the value+ results = []+ dfd.addBoth(results.append)+ self._wait(dfd)+ ret = results[0]+ return ret+++class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):+ """Invalid return value for process_spider_input method"""++ def test_invalid_process_spider_input(self):++ class InvalidProcessSpiderInputMiddleware:+ def process_spider_input(self, response, spider):+ return 1++ self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())+ result = self._scrape_response()+ self.assertIsInstance(result, Failure)+ self.assertIsInstance(result.value, _InvalidOutput)+++class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):+ """Invalid return value for process_spider_output method"""++ def test_invalid_process_spider_output(self):++ class InvalidProcessSpiderOutputMiddleware:+ def process_spider_output(self, response, result, spider):+ return 1++ self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())+ result = self._scrape_response()+ self.assertIsInstance(result, Failure)+ self.assertIsInstance(result.value, _InvalidOutput)+++class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):+ """Invalid return value for process_spider_exception method"""++ def test_invalid_process_spider_exception(self):++ class InvalidProcessSpiderOutputExceptionMiddleware:+ def process_spider_exception(self, response, exception, spider):+ return 1++ class RaiseExceptionProcessSpiderOutputMiddleware:+ def process_spider_output(self, response, result, spider):+ raise Exception()++ self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())+ self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())+ result = self._scrape_response()+ self.assertIsInstance(result, Failure)+ self.assertIsInstance(result.value, _InvalidOutput)