Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve diagnostics for sync-only spider middlewares.
Andrey Rakhmatullind8978d405c32
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 4b2520aa1..c7706bb7b 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -27,7 +27,7 @@ from scrapy.utils.defer import (maybe_deferred_to_future,mustbe_deferred,)-from scrapy.utils.python import MutableAsyncChain, MutableChain+from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_nameif TYPE_CHECKING:from collections.abc import Generator@@ -51,10 +51,6 @@ def _isiterable(o: Any) -> bool:class SpiderMiddlewareManager(MiddlewareManager):component_name = "spider middleware"- def __init__(self, *middlewares: Any):- super().__init__(*middlewares)- self.downgrade_warning_done = False-@classmethoddef _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]:return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))@@ -227,12 +223,13 @@ class SpiderMiddlewareManager(MiddlewareManager):# Iterable -> AsyncIterableresult = as_async_generator(result)elif need_downgrade:- if not self.downgrade_warning_done:- logger.warning(- f"Async iterable passed to {method.__qualname__} "- f"was downgraded to a non-async one"- )- self.downgrade_warning_done = True+ logger.warning(+ f"Async iterable passed to {method.__qualname__} was"+ f" downgraded to a non-async one. This is deprecated and will"+ f" stop working in a future version of Scrapy. Please see"+ f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares"+ f" for more information."+ )assert isinstance(result, AsyncIterable)# AsyncIterable -> Iterableresult = yield deferred_from_coro(collect_asyncgen(result))@@ -340,10 +337,19 @@ class SpiderMiddlewareManager(MiddlewareManager):methodname_async = methodname + "_async"async_method: Callable | None = getattr(mw, methodname_async, None)if not async_method:+ if normal_method and not isasyncgenfunction(normal_method):+ logger.warning(+ f"Middleware {global_object_name(mw.__class__)} doesn't support"+ f" asynchronous spider output, this is deprecated and will stop"+ f" working in a future version of Scrapy. The middleware should"+ f" be updated to support it. Please see"+ f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares"+ f" for more information."+ )return normal_methodif not normal_method:logger.error(- f"Middleware {mw.__qualname__} has {methodname_async} "+ f"Middleware {global_object_name(mw.__class__)} has {methodname_async} "f"without {methodname}, skipping this method.")return Nonediff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex ba64ba721..a8507c789 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -152,6 +152,10 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):self.assertEqual(len(result_list), self.RESULT_COUNT)self.assertIsInstance(result_list[0], self.ITEM_TYPE)self.assertEqual("downgraded to a non-async" in str(log), downgrade)+ self.assertEqual(+ "doesn't support asynchronous spider output" in str(log),+ ProcessSpiderOutputSimpleMiddleware in mw_classes,+ )@defer.inlineCallbacksdef _test_asyncgen_base(@@ -376,21 +380,21 @@ class UniversalMiddlewareManagerTest(TestCase):self.mwman = SpiderMiddlewareManager()def test_simple_mw(self):- mw = ProcessSpiderOutputSimpleMiddleware+ mw = ProcessSpiderOutputSimpleMiddleware()self.mwman._add_middleware(mw)self.assertEqual(self.mwman.methods["process_spider_output"][0], mw.process_spider_output)def test_async_mw(self):- mw = ProcessSpiderOutputAsyncGenMiddleware+ mw = ProcessSpiderOutputAsyncGenMiddleware()self.mwman._add_middleware(mw)self.assertEqual(self.mwman.methods["process_spider_output"][0], mw.process_spider_output)def test_universal_mw(self):- mw = ProcessSpiderOutputUniversalMiddleware+ mw = ProcessSpiderOutputUniversalMiddleware()self.mwman._add_middleware(mw)self.assertEqual(self.mwman.methods["process_spider_output"][0],@@ -399,7 +403,7 @@ class UniversalMiddlewareManagerTest(TestCase):def test_universal_mw_no_sync(self):with LogCapture() as log:- self.mwman._add_middleware(UniversalMiddlewareNoSync)+ self.mwman._add_middleware(UniversalMiddlewareNoSync())self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async"" without process_spider_output",@@ -408,7 +412,7 @@ class UniversalMiddlewareManagerTest(TestCase):self.assertEqual(self.mwman.methods["process_spider_output"][0], None)def test_universal_mw_both_sync(self):- mw = UniversalMiddlewareBothSync+ mw = UniversalMiddlewareBothSync()with LogCapture() as log:self.mwman._add_middleware(mw)self.assertIn(@@ -422,7 +426,7 @@ class UniversalMiddlewareManagerTest(TestCase):def test_universal_mw_both_async(self):with LogCapture() as log:- self.mwman._add_middleware(UniversalMiddlewareBothAsync)+ self.mwman._add_middleware(UniversalMiddlewareBothAsync())self.assertIn("UniversalMiddlewareBothAsync.process_spider_output ""is an async generator function while process_spider_output_async exists",