Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate passing coros to iterate_spider_output(). (#8394)

Andrey Rakhmatullin171ccbedb734
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 0d6274e6e..723cc46ec 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -2,17 +2,19 @@ from __future__ import annotations
import inspect
import logging
+import warnings
from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any, TypeVar, cast, overload
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import _is_awaitable, deferred_from_coro
from scrapy.utils.misc import arg_to_iter
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Iterable
- from types import CoroutineType, ModuleType
+ from collections.abc import AsyncGenerator, Coroutine, Iterable
+ from types import ModuleType
from twisted.internet.defer import Deferred
@@ -31,7 +33,9 @@ def iterate_spider_output(result: AsyncGenerator[_T]) -> AsyncGenerator[_T]: ...
@overload
-def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...
+def iterate_spider_output(
+ result: Coroutine[Any, Any, Any],
+) -> Deferred[Iterable[Any]]: ...
@overload
@@ -40,11 +44,19 @@ def iterate_spider_output(result: _T) -> Iterable[Any]: ...
def iterate_spider_output(
result: Any,
-) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]:
+) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[Iterable[Any]]:
if inspect.isasyncgen(result):
return cast("AsyncGenerator[_T]", result)
d: Deferred[_T] = deferred_from_coro(result)
if inspect.iscoroutine(result):
+ warnings.warn(
+ "Passing a coroutine to iterate_spider_output() is deprecated."
+ " For a coroutine it returns a Deferred instead of an iterable, so"
+ " iterating over the result does not work. Await the coroutine and"
+ " pass its result instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return d.addCallback(iterate_spider_output)
return arg_to_iter(d)
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 2da2f716b..96c24f2f9 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -1,11 +1,12 @@
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
import pytest
from scrapy import Spider
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.item import Item
from scrapy.settings import Settings
@@ -19,8 +20,6 @@ from scrapy.utils.spider import (
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
- from twisted.internet.defer import Deferred
-
from scrapy.spiderloader import SpiderLoaderProtocol
@@ -55,7 +54,11 @@ async def test_iterate_spider_output_coroutine() -> None:
async def callback() -> dict[str, str]:
return {"foo": "bar"}
- d = cast("Deferred[list[dict[str, str]]]", iterate_spider_output(callback()))
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match=r"Passing a coroutine to iterate_spider_output\(\) is deprecated",
+ ):
+ d = iterate_spider_output(callback())
assert await maybe_deferred_to_future(d) == [{"foo": "bar"}]