Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve diagnostics for sync-only spider middlewares.

Andrey Rakhmatullind8978d405c32
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 4b2520aa1..c7706bb7b 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -27,7 +27,7 @@ from scrapy.utils.defer import (
maybe_deferred_to_future,
mustbe_deferred,
)
-from scrapy.utils.python import MutableAsyncChain, MutableChain
+from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name
if TYPE_CHECKING:
from collections.abc import Generator
@@ -51,10 +51,6 @@ def _isiterable(o: Any) -> bool:
class SpiderMiddlewareManager(MiddlewareManager):
component_name = "spider middleware"
- def __init__(self, *middlewares: Any):
- super().__init__(*middlewares)
- self.downgrade_warning_done = False
-
@classmethod
def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]:
return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))
@@ -227,12 +223,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
# Iterable -> AsyncIterable
result = as_async_generator(result)
elif need_downgrade:
- if not self.downgrade_warning_done:
- logger.warning(
- f"Async iterable passed to {method.__qualname__} "
- f"was downgraded to a non-async one"
- )
- self.downgrade_warning_done = True
+ logger.warning(
+ f"Async iterable passed to {method.__qualname__} was"
+ f" downgraded to a non-async one. This is deprecated and will"
+ f" stop working in a future version of Scrapy. Please see"
+ f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares"
+ f" for more information."
+ )
assert isinstance(result, AsyncIterable)
# AsyncIterable -> Iterable
result = yield deferred_from_coro(collect_asyncgen(result))
@@ -340,10 +337,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
methodname_async = methodname + "_async"
async_method: Callable | None = getattr(mw, methodname_async, None)
if not async_method:
+ if normal_method and not isasyncgenfunction(normal_method):
+ logger.warning(
+ f"Middleware {global_object_name(mw.__class__)} doesn't support"
+ f" asynchronous spider output, this is deprecated and will stop"
+ f" working in a future version of Scrapy. The middleware should"
+ f" be updated to support it. Please see"
+ f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares"
+ f" for more information."
+ )
return normal_method
if not normal_method:
logger.error(
- f"Middleware {mw.__qualname__} has {methodname_async} "
+ f"Middleware {global_object_name(mw.__class__)} has {methodname_async} "
f"without {methodname}, skipping this method."
)
return None
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index ba64ba721..a8507c789 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -152,6 +152,10 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
self.assertEqual(len(result_list), self.RESULT_COUNT)
self.assertIsInstance(result_list[0], self.ITEM_TYPE)
self.assertEqual("downgraded to a non-async" in str(log), downgrade)
+ self.assertEqual(
+ "doesn't support asynchronous spider output" in str(log),
+ ProcessSpiderOutputSimpleMiddleware in mw_classes,
+ )
@defer.inlineCallbacks
def _test_asyncgen_base(
@@ -376,21 +380,21 @@ class UniversalMiddlewareManagerTest(TestCase):
self.mwman = SpiderMiddlewareManager()
def test_simple_mw(self):
- mw = ProcessSpiderOutputSimpleMiddleware
+ mw = ProcessSpiderOutputSimpleMiddleware()
self.mwman._add_middleware(mw)
self.assertEqual(
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
)
def test_async_mw(self):
- mw = ProcessSpiderOutputAsyncGenMiddleware
+ mw = ProcessSpiderOutputAsyncGenMiddleware()
self.mwman._add_middleware(mw)
self.assertEqual(
self.mwman.methods["process_spider_output"][0], mw.process_spider_output
)
def test_universal_mw(self):
- mw = ProcessSpiderOutputUniversalMiddleware
+ mw = ProcessSpiderOutputUniversalMiddleware()
self.mwman._add_middleware(mw)
self.assertEqual(
self.mwman.methods["process_spider_output"][0],
@@ -399,7 +403,7 @@ class UniversalMiddlewareManagerTest(TestCase):
def test_universal_mw_no_sync(self):
with LogCapture() as log:
- self.mwman._add_middleware(UniversalMiddlewareNoSync)
+ self.mwman._add_middleware(UniversalMiddlewareNoSync())
self.assertIn(
"UniversalMiddlewareNoSync has process_spider_output_async"
" without process_spider_output",
@@ -408,7 +412,7 @@ class UniversalMiddlewareManagerTest(TestCase):
self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
def test_universal_mw_both_sync(self):
- mw = UniversalMiddlewareBothSync
+ mw = UniversalMiddlewareBothSync()
with LogCapture() as log:
self.mwman._add_middleware(mw)
self.assertIn(
@@ -422,7 +426,7 @@ class UniversalMiddlewareManagerTest(TestCase):
def test_universal_mw_both_async(self):
with LogCapture() as log:
- self.mwman._add_middleware(UniversalMiddlewareBothAsync)
+ self.mwman._add_middleware(UniversalMiddlewareBothAsync())
self.assertIn(
"UniversalMiddlewareBothAsync.process_spider_output "
"is an async generator function while process_spider_output_async exists",