Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate sync Scraper.handle_spider_output_async() input. (#8288)

Andrey Rakhmatulline8d545a276f7
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index bf51c9a49..a9e35cb12 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -22,6 +22,7 @@ from scrapy.exceptions import (
)
from scrapy.http import Request, Response
from scrapy.pipelines import ItemPipelineManager
+from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available
from scrapy.utils.defer import (
_process_pending_io_before_callback,
@@ -29,9 +30,7 @@ from scrapy.utils.defer import (
aiter_errback,
deferred_from_coro,
ensure_awaitable,
- iter_errback,
maybe_deferred_to_future,
- parallel,
parallel_async,
)
from scrapy.utils.deprecate import method_is_overridden
@@ -439,30 +438,21 @@ class Scraper:
.. versionadded:: 2.13
"""
- it: Iterable[_T] | AsyncIterator[_T]
+ if not isinstance(result, AsyncIterator): # pragma: no cover
+ warnings.warn(
+ "Passing sync iterables to Scraper.handle_spider_output_async() is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ result = as_async_generator(result)
+ it = aiter_errback(result, self.handle_spider_error, request, response)
if is_asyncio_available():
- if isinstance(result, AsyncIterator):
- it = aiter_errback(result, self.handle_spider_error, request, response)
- else:
- it = iter_errback(result, self.handle_spider_error, request, response)
await _parallel_asyncio(
it, self.concurrent_items, self._process_spidermw_output_async, response
)
return
- if isinstance(result, AsyncIterator):
- it = aiter_errback(result, self.handle_spider_error, request, response)
- await maybe_deferred_to_future(
- parallel_async(
- it,
- self.concurrent_items,
- self._process_spidermw_output,
- response,
- )
- )
- return
- it = iter_errback(result, self.handle_spider_error, request, response)
await maybe_deferred_to_future(
- parallel(
+ parallel_async(
it,
self.concurrent_items,
self._process_spidermw_output,
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index c88566c2a..508afec81 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -28,7 +28,7 @@ from scrapy.utils.defer import (
deferred_from_coro,
maybe_deferred_to_future,
)
-from scrapy.utils.python import MutableAsyncChain, global_object_name
+from scrapy.utils.python import _MutableAsyncChain, global_object_name
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
@@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
response: Response | None,
iterable: AsyncIterator[_T],
exception_processor_index: int,
- recover_to: MutableAsyncChain[_T],
+ recover_to: _MutableAsyncChain[_T],
) -> AsyncIterator[_T]:
try:
async for r in iterable:
@@ -111,7 +111,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
except Exception as ex:
if getattr(ex, "_spidermw_unhandled", False):
raise
- exception_result: MutableAsyncChain[_T] = self._process_spider_exception(
+ exception_result: _MutableAsyncChain[_T] = self._process_spider_exception(
response, ex, exception_processor_index
)
recover_to.extend(exception_result)
@@ -121,7 +121,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
response: Response | None,
exception: Exception,
start_index: int = 0,
- ) -> MutableAsyncChain[_T]:
+ ) -> _MutableAsyncChain[_T]:
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
raise exception
@@ -168,10 +168,10 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
response: Response | None,
result: AsyncIterator[_T],
start_index: int = 0,
- ) -> MutableAsyncChain[_T]:
+ ) -> _MutableAsyncChain[_T]:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
- recovered: MutableAsyncChain[_T] = MutableAsyncChain()
+ recovered: _MutableAsyncChain[_T] = _MutableAsyncChain()
method_list = islice(self.methods["process_spider_output"], start_index, None)
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
@@ -183,15 +183,15 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
result = self._evaluate_iterable(
response, result, method_index + 1, recovered
)
- return MutableAsyncChain(result, recovered)
+ return _MutableAsyncChain(result, recovered)
async def _process_callback_output(
self, response: Response | None, result: AsyncIterator[_T]
- ) -> MutableAsyncChain[_T]:
- recovered: MutableAsyncChain[_T] = MutableAsyncChain()
+ ) -> _MutableAsyncChain[_T]:
+ recovered: _MutableAsyncChain[_T] = _MutableAsyncChain()
result = self._evaluate_iterable(response, result, 0, recovered)
result = self._process_spider_output(response, result)
- return MutableAsyncChain(result, recovered)
+ return _MutableAsyncChain(result, recovered)
def scrape_response(
self,
@@ -202,7 +202,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
response: Response,
request: Request,
spider: Spider,
- ) -> Deferred[MutableAsyncChain[_T]]: # pragma: no cover
+ ) -> Deferred[_MutableAsyncChain[_T]]: # pragma: no cover
warn(
"SpiderMiddlewareManager.scrape_response() is deprecated, use scrape_response_async() instead",
ScrapyDeprecationWarning,
@@ -225,7 +225,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
scrape_func: ScrapeFunc[_T],
response: Response,
request: Request,
- ) -> MutableAsyncChain[_T]:
+ ) -> _MutableAsyncChain[_T]:
if not self.crawler:
raise RuntimeError(
"scrape_response_async() called on a SpiderMiddlewareManager"
@@ -246,7 +246,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-me
scrape_func: ScrapeFunc[_T],
failure: Failure,
request: Request,
- ) -> MutableAsyncChain[_T]:
+ ) -> _MutableAsyncChain[_T]:
# There is no response to run the process_spider_input chain on, so the
# errback output enters the process_spider_output chain with None as the
# response. Exceptions from the errback itself are left to the caller,
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index a002442cf..8b1d3e343 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -215,7 +215,7 @@ def parallel(
callable: Callable[Concatenate[_T, _P], _T2], # noqa: A002
*args: _P.args,
**named: _P.kwargs,
-) -> Deferred[None]:
+) -> Deferred[None]: # pragma: no cover
"""Execute a callable over the objects in the given iterable, in parallel,
using no more than ``count`` concurrent calls.
"""
@@ -225,6 +225,12 @@ def parallel(
tasks.start()
yield callable(elem, *args, **named)
+ warnings.warn(
+ "scrapy.utils.defer.parallel() is deprecated.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
tasks = _ParallelTasks(work(), count)
tasks.start()
return tasks.finished
@@ -405,10 +411,15 @@ def iter_errback(
errback: Callable[Concatenate[Failure, _P], Any],
*a: _P.args,
**kw: _P.kwargs,
-) -> Iterable[_T]:
+) -> Iterable[_T]: # pragma: no cover
"""Wrap an iterable calling an errback if an error is caught while
iterating it.
"""
+ warnings.warn(
+ "scrapy.utils.defer.iter_errback() is deprecated.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
it = iter(iterable)
while True:
try:
@@ -426,8 +437,7 @@ async def aiter_errback(
**kw: _P.kwargs,
) -> AsyncIterator[_T]:
"""Wrap an async iterable calling an errback if an error is caught while
- iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`.
- """
+ iterating it."""
it = aiter(aiterable)
while True:
try:
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index e2d627781..a163e1021 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -15,6 +15,8 @@ from functools import partial, wraps
from itertools import chain
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, Self, TypeVar, overload
+from formerly import deprecated_class
+
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncgen import as_async_generator
@@ -345,11 +347,7 @@ async def _async_chain(
yield o
-class MutableAsyncChain(AsyncIterator[_T]):
- """
- Similar to MutableChain but for async iterables
- """
-
+class _MutableAsyncChain(AsyncIterator[_T]):
def __init__(self, *args: Iterable[_T] | AsyncIterator[_T]):
self.data: AsyncIterator[_T] = _async_chain(*args)
@@ -363,6 +361,15 @@ class MutableAsyncChain(AsyncIterator[_T]):
return await anext(self.data)
+MutableAsyncChain = deprecated_class(
+ "MutableAsyncChain",
+ _MutableAsyncChain,
+ category=ScrapyDeprecationWarning,
+ subclass_message="{old} is deprecated.",
+ instance_message="{cls} is deprecated.",
+)
+
+
def _looks_like_import_path(value: str) -> bool:
"""Return True if **value** looks like a valid Python import path or False
otherwise."""
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 140b5be80..4143f3888 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -22,11 +22,9 @@ from scrapy.utils.defer import (
deferred_f_from_coro_f,
deferred_from_coro,
deferred_to_future,
- iter_errback,
maybe_deferred_to_future,
maybeDeferred_coro,
mustbe_deferred,
- parallel,
parallel_async,
)
from tests.utils.decorators import coroutine_test, inline_callbacks_test
@@ -77,30 +75,6 @@ class TestMustbeDeferred:
yield dfd
-class TestIterErrback:
- def test_iter_errback_good(self):
- def itergood() -> Generator[int, None, None]:
- yield from range(10)
-
- errors: list[Failure] = []
- out = list(iter_errback(itergood(), errors.append))
- assert out == list(range(10))
- assert not errors
-
- def test_iter_errback_bad(self):
- def iterbad() -> Generator[int, None, None]:
- for x in range(10):
- if x == 5:
- 1 / 0
- yield x
-
- errors: list[Failure] = []
- out = list(iter_errback(iterbad(), errors.append))
- assert out == [0, 1, 2, 3, 4]
- assert len(errors) == 1
- assert isinstance(errors[0].value, ZeroDivisionError)
-
-
class TestAiterErrback:
@coroutine_test
async def test_aiter_errback_good(self):
@@ -114,7 +88,7 @@ class TestAiterErrback:
assert not errors
@coroutine_test
- async def test_iter_errback_bad(self):
+ async def test_aiter_errback_bad(self):
async def iterbad() -> AsyncGenerator[int, None]:
for x in range(10):
if x == 5:
@@ -139,20 +113,6 @@ class TestAsyncDefTestsuite:
raise RuntimeError("This is expected to be raised")
[email protected]_reactor # parallel() requires a reactor
-class TestParallel:
- @inline_callbacks_test
- def test_count_higher_than_work(self) -> Generator[Deferred[Any], Any, None]:
- results: list[int] = []
- with mock.patch.object(
- Cooperator, "coiterate", autospec=True, side_effect=Cooperator.coiterate
- ) as coiterate:
- yield parallel(range(3), 1_000_000, results.append)
- assert results == [0, 1, 2]
- # One task per item, plus the one that finds no more work.
- assert coiterate.call_count <= 4
-
-
@implementer(IReadDescriptor)
class _ReadTracker:
"""Socket that the event loop finds readable, counting how often it reads it.
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 46b9a69ee..0fedde5f5 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -11,8 +11,8 @@ import pytest
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import aiter_errback
from scrapy.utils.python import (
- MutableAsyncChain,
_looks_like_import_path,
+ _MutableAsyncChain,
binary_is_text,
get_func_args,
get_spec,
@@ -56,7 +56,7 @@ class TestMutableAsyncChain:
@coroutine_test
async def test_mutableasyncchain(self):
- m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))
+ m = _MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))
m.extend(self.g2())
m.extend(self.g3())
@@ -66,7 +66,7 @@ class TestMutableAsyncChain:
@coroutine_test
async def test_mutableasyncchain_exc(self):
- m = MutableAsyncChain(self.g1())
+ m = _MutableAsyncChain(self.g1())
m.extend(self.g4())
m.extend(self.g3())