Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Simplify check for re-raised exception. Add tests.

Eugenio Lacuesta8c55f5eb159a
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 1b67af130..4268c91d6 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
yield r
except Exception as ex:
exception_result = process_spider_exception(Failure(ex), index+1)
- if exception_result is None or isinstance(exception_result, Failure):
+ if isinstance(exception_result, Failure):
raise
recovered.extend(exception_result)
@@ -115,7 +115,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
result = method(response=response, result=result, spider=spider)
except Exception as ex:
exception_result = process_spider_exception(Failure(ex), method_index+1)
- if exception_result is None or isinstance(exception_result, Failure):
+ if isinstance(exception_result, Failure):
raise
return exception_result
if _isiterable(result):
diff --git a/tests/test_spidermiddleware_invalid_values.py b/tests/test_spidermiddleware.py
similarity index 79%
rename from tests/test_spidermiddleware_invalid_values.py
rename to tests/test_spidermiddleware.py
index 0d9af8951..54756f2ff 100644
--- a/tests/test_spidermiddleware_invalid_values.py
+++ b/tests/test_spidermiddleware.py
@@ -80,3 +80,23 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
result = self._scrape_response()
self.assertIsInstance(result, Failure)
self.assertIsInstance(result.value, _InvalidOutput)
+
+
+class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
+ """Re raise the exception by returning None"""
+
+ def test_process_spider_exception_return_none(self):
+
+ class ProcessSpiderOutputExceptionReturnNoneMiddleware:
+ def process_spider_exception(self, response, exception, spider):
+ return None
+
+ class RaiseExceptionProcessSpiderOutputMiddleware:
+ def process_spider_output(self, response, result, spider):
+ 1/0
+
+ self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware())
+ self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
+ result = self._scrape_response()
+ self.assertIsInstance(result, Failure)
+ self.assertIsInstance(result.value, ZeroDivisionError)