Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate sync Scraper.handle_spider_output_async() input. (#8288)
Andrey Rakhmatulline8d545a276f7
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex bf51c9a49..a9e35cb12 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -22,6 +22,7 @@ from scrapy.exceptions import ()from scrapy.http import Request, Responsefrom scrapy.pipelines import ItemPipelineManager+from scrapy.utils.asyncgen import as_async_generatorfrom scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_availablefrom scrapy.utils.defer import (_process_pending_io_before_callback,@@ -29,9 +30,7 @@ from scrapy.utils.defer import (aiter_errback,deferred_from_coro,ensure_awaitable,- iter_errback,maybe_deferred_to_future,- parallel,parallel_async,)from scrapy.utils.deprecate import method_is_overridden@@ -439,30 +438,21 @@ class Scraper:.. versionadded:: 2.13"""- it: Iterable[_T] | AsyncIterator[_T]+ if not isinstance(result, AsyncIterator): # pragma: no cover+ warnings.warn(+ "Passing sync iterables to Scraper.handle_spider_output_async() is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ result = as_async_generator(result)+ it = aiter_errback(result, self.handle_spider_error, request, response)if is_asyncio_available():- if isinstance(result, AsyncIterator):- it = aiter_errback(result, self.handle_spider_error, request, response)- else:- it = iter_errback(result, self.handle_spider_error, request, response)await _parallel_asyncio(it, self.concurrent_items, self._process_spidermw_output_async, response)return- if isinstance(result, AsyncIterator):- it = aiter_errback(result, self.handle_spider_error, request, response)- await maybe_deferred_to_future(- parallel_async(- it,- self.concurrent_items,- self._process_spidermw_output,- response,- )- )- return- it = iter_errback(result, self.handle_spider_error, request, response)await maybe_deferred_to_future(- parallel(+ parallel_async(it,self.concurrent_items,self._process_spidermw_output,diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex c88566c2a..508afec81 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -28,7 +28,7 @@ from scrapy.utils.defer import (deferred_from_coro,maybe_deferred_to_future,)-from scrapy.utils.python import MutableAsyncChain, global_object_name+from scrapy.utils.python import _MutableAsyncChain, global_object_nameif TYPE_CHECKING:from twisted.internet.defer import Deferred@@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-meresponse: Response | None,iterable: AsyncIterator[_T],exception_processor_index: int,- recover_to: MutableAsyncChain[_T],+ recover_to: _MutableAsyncChain[_T],) -> AsyncIterator[_T]:try:async for r in iterable:@@ -111,7 +111,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-meexcept Exception as ex:if getattr(ex, "_spidermw_unhandled", False):raise- exception_result: MutableAsyncChain[_T] = self._process_spider_exception(+ exception_result: _MutableAsyncChain[_T] = self._process_spider_exception(response, ex, exception_processor_index)recover_to.extend(exception_result)@@ -121,7 +121,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-meresponse: Response | None,exception: Exception,start_index: int = 0,- ) -> MutableAsyncChain[_T]:+ ) -> _MutableAsyncChain[_T]:# don't handle _InvalidOutput exceptionif isinstance(exception, _InvalidOutput):raise exception@@ -168,10 +168,10 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-meresponse: Response | None,result: AsyncIterator[_T],start_index: int = 0,- ) -> MutableAsyncChain[_T]:+ ) -> _MutableAsyncChain[_T]:# items in this iterable do not need to go through the process_spider_output# chain, they went through it already from the process_spider_exception method- recovered: MutableAsyncChain[_T] = MutableAsyncChain()+ recovered: _MutableAsyncChain[_T] = _MutableAsyncChain()method_list = islice(self.methods["process_spider_output"], start_index, None)for method_index, method in enumerate(method_list, start=start_index):if method is None:@@ -183,15 +183,15 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-meresult = self._evaluate_iterable(response, result, method_index + 1, recovered)- return MutableAsyncChain(result, recovered)+ return _MutableAsyncChain(result, recovered)async def _process_callback_output(self, response: Response | None, result: AsyncIterator[_T]- ) -> MutableAsyncChain[_T]:- recovered: MutableAsyncChain[_T] = MutableAsyncChain()+ ) -> _MutableAsyncChain[_T]:+ recovered: _MutableAsyncChain[_T] = _MutableAsyncChain()result = self._evaluate_iterable(response, result, 0, recovered)result = self._process_spider_output(response, result)- return MutableAsyncChain(result, recovered)+ return _MutableAsyncChain(result, recovered)def scrape_response(self,@@ -202,7 +202,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-meresponse: Response,request: Request,spider: Spider,- ) -> Deferred[MutableAsyncChain[_T]]: # pragma: no cover+ ) -> Deferred[_MutableAsyncChain[_T]]: # pragma: no coverwarn("SpiderMiddlewareManager.scrape_response() is deprecated, use scrape_response_async() instead",ScrapyDeprecationWarning,@@ -225,7 +225,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-mescrape_func: ScrapeFunc[_T],response: Response,request: Request,- ) -> MutableAsyncChain[_T]:+ ) -> _MutableAsyncChain[_T]:if not self.crawler:raise RuntimeError("scrape_response_async() called on a SpiderMiddlewareManager"@@ -246,7 +246,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # pylint: disable=abstract-mescrape_func: ScrapeFunc[_T],failure: Failure,request: Request,- ) -> MutableAsyncChain[_T]:+ ) -> _MutableAsyncChain[_T]:# There is no response to run the process_spider_input chain on, so the# errback output enters the process_spider_output chain with None as the# response. Exceptions from the errback itself are left to the caller,diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex a002442cf..8b1d3e343 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -215,7 +215,7 @@ def parallel(callable: Callable[Concatenate[_T, _P], _T2], # noqa: A002*args: _P.args,**named: _P.kwargs,-) -> Deferred[None]:+) -> Deferred[None]: # pragma: no cover"""Execute a callable over the objects in the given iterable, in parallel,using no more than ``count`` concurrent calls."""@@ -225,6 +225,12 @@ def parallel(tasks.start()yield callable(elem, *args, **named)+ warnings.warn(+ "scrapy.utils.defer.parallel() is deprecated.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+tasks = _ParallelTasks(work(), count)tasks.start()return tasks.finished@@ -405,10 +411,15 @@ def iter_errback(errback: Callable[Concatenate[Failure, _P], Any],*a: _P.args,**kw: _P.kwargs,-) -> Iterable[_T]:+) -> Iterable[_T]: # pragma: no cover"""Wrap an iterable calling an errback if an error is caught whileiterating it."""+ warnings.warn(+ "scrapy.utils.defer.iter_errback() is deprecated.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )it = iter(iterable)while True:try:@@ -426,8 +437,7 @@ async def aiter_errback(**kw: _P.kwargs,) -> AsyncIterator[_T]:"""Wrap an async iterable calling an errback if an error is caught while- iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`.- """+ iterating it."""it = aiter(aiterable)while True:try:diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex e2d627781..a163e1021 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -15,6 +15,8 @@ from functools import partial, wrapsfrom itertools import chainfrom typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, Self, TypeVar, overload+from formerly import deprecated_class+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.asyncgen import as_async_generator@@ -345,11 +347,7 @@ async def _async_chain(yield o-class MutableAsyncChain(AsyncIterator[_T]):- """- Similar to MutableChain but for async iterables- """-+class _MutableAsyncChain(AsyncIterator[_T]):def __init__(self, *args: Iterable[_T] | AsyncIterator[_T]):self.data: AsyncIterator[_T] = _async_chain(*args)@@ -363,6 +361,15 @@ class MutableAsyncChain(AsyncIterator[_T]):return await anext(self.data)+MutableAsyncChain = deprecated_class(+ "MutableAsyncChain",+ _MutableAsyncChain,+ category=ScrapyDeprecationWarning,+ subclass_message="{old} is deprecated.",+ instance_message="{cls} is deprecated.",+)++def _looks_like_import_path(value: str) -> bool:"""Return True if **value** looks like a valid Python import path or Falseotherwise."""diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 140b5be80..4143f3888 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -22,11 +22,9 @@ from scrapy.utils.defer import (deferred_f_from_coro_f,deferred_from_coro,deferred_to_future,- iter_errback,maybe_deferred_to_future,maybeDeferred_coro,mustbe_deferred,- parallel,parallel_async,)from tests.utils.decorators import coroutine_test, inline_callbacks_test@@ -77,30 +75,6 @@ class TestMustbeDeferred:yield dfd-class TestIterErrback:- def test_iter_errback_good(self):- def itergood() -> Generator[int, None, None]:- yield from range(10)-- errors: list[Failure] = []- out = list(iter_errback(itergood(), errors.append))- assert out == list(range(10))- assert not errors-- def test_iter_errback_bad(self):- def iterbad() -> Generator[int, None, None]:- for x in range(10):- if x == 5:- 1 / 0- yield x-- errors: list[Failure] = []- out = list(iter_errback(iterbad(), errors.append))- assert out == [0, 1, 2, 3, 4]- assert len(errors) == 1- assert isinstance(errors[0].value, ZeroDivisionError)--class TestAiterErrback:@coroutine_testasync def test_aiter_errback_good(self):@@ -114,7 +88,7 @@ class TestAiterErrback:assert not errors@coroutine_test- async def test_iter_errback_bad(self):+ async def test_aiter_errback_bad(self):async def iterbad() -> AsyncGenerator[int, None]:for x in range(10):if x == 5:@@ -139,20 +113,6 @@ class TestAsyncDefTestsuite:raise RuntimeError("This is expected to be raised")[email protected]_reactor # parallel() requires a reactor-class TestParallel:- @inline_callbacks_test- def test_count_higher_than_work(self) -> Generator[Deferred[Any], Any, None]:- results: list[int] = []- with mock.patch.object(- Cooperator, "coiterate", autospec=True, side_effect=Cooperator.coiterate- ) as coiterate:- yield parallel(range(3), 1_000_000, results.append)- assert results == [0, 1, 2]- # One task per item, plus the one that finds no more work.- assert coiterate.call_count <= 4--@implementer(IReadDescriptor)class _ReadTracker:"""Socket that the event loop finds readable, counting how often it reads it.diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 46b9a69ee..0fedde5f5 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -11,8 +11,8 @@ import pytestfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.defer import aiter_errbackfrom scrapy.utils.python import (- MutableAsyncChain,_looks_like_import_path,+ _MutableAsyncChain,binary_is_text,get_func_args,get_spec,@@ -56,7 +56,7 @@ class TestMutableAsyncChain:@coroutine_testasync def test_mutableasyncchain(self):- m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))+ m = _MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))m.extend(self.g2())m.extend(self.g3())@@ -66,7 +66,7 @@ class TestMutableAsyncChain:@coroutine_testasync def test_mutableasyncchain_exc(self):- m = MutableAsyncChain(self.g1())+ m = _MutableAsyncChain(self.g1())m.extend(self.g4())m.extend(self.g3())