Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate passing coros to iterate_spider_output(). (#8394)
Andrey Rakhmatullin171ccbedb734
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 0d6274e6e..723cc46ec 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -2,17 +2,19 @@ from __future__ import annotationsimport inspectimport logging+import warningsfrom collections.abc import AsyncIteratorfrom typing import TYPE_CHECKING, Any, TypeVar, cast, overload+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.spiders import Spiderfrom scrapy.utils.asyncgen import collect_asyncgenfrom scrapy.utils.defer import _is_awaitable, deferred_from_corofrom scrapy.utils.misc import arg_to_iterif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Iterable- from types import CoroutineType, ModuleType+ from collections.abc import AsyncGenerator, Coroutine, Iterable+ from types import ModuleTypefrom twisted.internet.defer import Deferred@@ -31,7 +33,9 @@ def iterate_spider_output(result: AsyncGenerator[_T]) -> AsyncGenerator[_T]: ...@overload-def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...+def iterate_spider_output(+ result: Coroutine[Any, Any, Any],+) -> Deferred[Iterable[Any]]: ...@overload@@ -40,11 +44,19 @@ def iterate_spider_output(result: _T) -> Iterable[Any]: ...def iterate_spider_output(result: Any,-) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]:+) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[Iterable[Any]]:if inspect.isasyncgen(result):return cast("AsyncGenerator[_T]", result)d: Deferred[_T] = deferred_from_coro(result)if inspect.iscoroutine(result):+ warnings.warn(+ "Passing a coroutine to iterate_spider_output() is deprecated."+ " For a coroutine it returns a Deferred instead of an iterable, so"+ " iterating over the result does not work. Await the coroutine and"+ " pass its result instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )return d.addCallback(iterate_spider_output)return arg_to_iter(d)diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex 2da2f716b..96c24f2f9 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -1,11 +1,12 @@from __future__ import annotationsimport logging-from typing import TYPE_CHECKING, cast+from typing import TYPE_CHECKINGimport pytestfrom scrapy import Spider+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Requestfrom scrapy.item import Itemfrom scrapy.settings import Settings@@ -19,8 +20,6 @@ from scrapy.utils.spider import (from tests.utils.decorators import coroutine_testif TYPE_CHECKING:- from twisted.internet.defer import Deferred-from scrapy.spiderloader import SpiderLoaderProtocol@@ -55,7 +54,11 @@ async def test_iterate_spider_output_coroutine() -> None:async def callback() -> dict[str, str]:return {"foo": "bar"}- d = cast("Deferred[list[dict[str, str]]]", iterate_spider_output(callback()))+ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"Passing a coroutine to iterate_spider_output\(\) is deprecated",+ ):+ d = iterate_spider_output(callback())assert await maybe_deferred_to_future(d) == [{"foo": "bar"}]