Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

More robust sync/async middleware mix checking.

Andrey Rakhmatullin7bd1d888d49d
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b24ccf6ae..d0d292007 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -92,8 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_output(self, response, spider, result, start_index=0):
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
- result_async = isinstance(result, collections.abc.AsyncIterator)
- if result_async:
+ last_result_async = isinstance(result, collections.abc.AsyncIterator)
+ if last_result_async:
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
@@ -116,11 +116,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
msg = (f"Middleware {method.__qualname__} must return an "
f"iterable, got {type(result)}")
raise _InvalidOutput(msg)
- if result_async and isinstance(result, collections.abc.Iterator):
+ if last_result_async and isinstance(result, collections.abc.Iterator):
raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
+ last_result_async = isinstance(result, collections.abc.AsyncIterator)
- # check this again as the middlewares could change "result" from sync to async
- if isinstance(result, collections.abc.AsyncIterator):
+ if last_result_async:
return MutableAsyncChain(result, recovered)
else:
return MutableChain(result, recovered)