Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Some cleanup, make sync middlewares fail earlier.

Andrey Rakhmatullin30ed7fa34921
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 230332673..b24ccf6ae 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -20,10 +20,6 @@ def _isiterable(possible_iterator):
return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')
-def _fname(f):
- return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
-
-
class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
@@ -48,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
try:
result = method(response=response, spider=spider)
if result is not None:
- msg = (f"Middleware {_fname(method)} must return None "
+ msg = (f"Middleware {method.__qualname__} must return None "
f"or raise an exception, got {type(result)}")
raise _InvalidOutput(msg)
except _InvalidOutput:
@@ -88,7 +84,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None:
continue
else:
- msg = (f"Middleware {_fname(method)} must return None "
+ msg = (f"Middleware {method.__qualname__} must return None "
f"or an iterable, got {type(result)}")
raise _InvalidOutput(msg)
return _failure
@@ -96,7 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_output(self, response, spider, result, start_index=0):
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
- if isinstance(result, collections.abc.AsyncIterator):
+ result_async = isinstance(result, collections.abc.AsyncIterator)
+ if result_async:
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
@@ -116,9 +113,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
else:
- msg = (f"Middleware {_fname(method)} must return an "
+ msg = (f"Middleware {method.__qualname__} must return an "
f"iterable, got {type(result)}")
raise _InvalidOutput(msg)
+ if result_async and isinstance(result, collections.abc.Iterator):
+ raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
# check this again as the middlewares could change "result" from sync to async
if isinstance(result, collections.abc.AsyncIterator):
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2584dec21..0a6b96c0c 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -122,11 +122,6 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
return result
- def assertAsyncGeneratorNotIterable(self, o):
- with self.assertRaisesRegex(TypeError,
- "'(async_generator|MutableAsyncChain)' object is not iterable"):
- list(o)
-
@defer.inlineCallbacks
def _test_simple_base(self, *mw_classes):
result = yield self._get_middleware_result(*mw_classes)
@@ -145,9 +140,8 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
@defer.inlineCallbacks
def _test_asyncgen_fail(self, *mw_classes):
- result = yield self._get_middleware_result(*mw_classes)
- self.assertIsInstance(result, collections.abc.Iterable)
- self.assertAsyncGeneratorNotIterable(result)
+ with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"):
+ yield self._get_middleware_result(*mw_classes)
class ProcessSpiderOutputSimpleMiddleware:
@@ -238,14 +232,10 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
""" Simple mw; cannot work """
return self._test_asyncgen_fail(self.MW_SIMPLE)
- @defer.inlineCallbacks
def test_simple_asyncgen(self):
""" Simple mw -> asyncgen mw; cannot work """
- result = yield self._get_middleware_result(
- self.MW_ASYNCGEN,
- self.MW_SIMPLE)
- self.assertIsInstance(result, collections.abc.AsyncIterable)
- self.assertAsyncGeneratorNotIterable(result)
+ return self._test_asyncgen_fail(self.MW_ASYNCGEN,
+ self.MW_SIMPLE)
def test_universal(self):
""" Universal mw """