Scrapy, a fast high-level web crawling & scraping framework for Python.
Some cleanup, make sync middlewares fail earlier.
Andrey Rakhmatullin30ed7fa34921
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 230332673..b24ccf6ae 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -20,10 +20,6 @@ def _isiterable(possible_iterator):return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')-def _fname(f):- return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"--class SpiderMiddlewareManager(MiddlewareManager):component_name = 'spider middleware'@@ -48,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager):try:result = method(response=response, spider=spider)if result is not None:- msg = (f"Middleware {_fname(method)} must return None "+ msg = (f"Middleware {method.__qualname__} must return None "f"or raise an exception, got {type(result)}")raise _InvalidOutput(msg)except _InvalidOutput:@@ -88,7 +84,7 @@ class SpiderMiddlewareManager(MiddlewareManager):elif result is None:continueelse:- msg = (f"Middleware {_fname(method)} must return None "+ msg = (f"Middleware {method.__qualname__} must return None "f"or an iterable, got {type(result)}")raise _InvalidOutput(msg)return _failure@@ -96,7 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):def _process_spider_output(self, response, spider, result, start_index=0):# items in this iterable do not need to go through the process_spider_output# chain, they went through it already from the process_spider_exception method- if isinstance(result, collections.abc.AsyncIterator):+ result_async = isinstance(result, collections.abc.AsyncIterator)+ if result_async:recovered = MutableAsyncChain()else:recovered = MutableChain()@@ -116,9 +113,11 @@ class SpiderMiddlewareManager(MiddlewareManager):if _isiterable(result):result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)else:- msg = (f"Middleware {_fname(method)} must return an "+ msg = (f"Middleware {method.__qualname__} must return an "f"iterable, got {type(result)}")raise _InvalidOutput(msg)+ if result_async and isinstance(result, collections.abc.Iterator):+ raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")# check this again as the middlewares could change "result" from sync to asyncif isinstance(result, collections.abc.AsyncIterator):diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 2584dec21..0a6b96c0c 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -122,11 +122,6 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)return result- def assertAsyncGeneratorNotIterable(self, o):- with self.assertRaisesRegex(TypeError,- "'(async_generator|MutableAsyncChain)' object is not iterable"):- list(o)-@defer.inlineCallbacksdef _test_simple_base(self, *mw_classes):result = yield self._get_middleware_result(*mw_classes)@@ -145,9 +140,8 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):@defer.inlineCallbacksdef _test_asyncgen_fail(self, *mw_classes):- result = yield self._get_middleware_result(*mw_classes)- self.assertIsInstance(result, collections.abc.Iterable)- self.assertAsyncGeneratorNotIterable(result)+ with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"):+ yield self._get_middleware_result(*mw_classes)class ProcessSpiderOutputSimpleMiddleware:@@ -238,14 +232,10 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):""" Simple mw; cannot work """return self._test_asyncgen_fail(self.MW_SIMPLE)- @defer.inlineCallbacksdef test_simple_asyncgen(self):""" Simple mw -> asyncgen mw; cannot work """- result = yield self._get_middleware_result(- self.MW_ASYNCGEN,- self.MW_SIMPLE)- self.assertIsInstance(result, collections.abc.AsyncIterable)- self.assertAsyncGeneratorNotIterable(result)+ return self._test_asyncgen_fail(self.MW_ASYNCGEN,+ self.MW_SIMPLE)def test_universal(self):""" Universal mw """