Scrapy, a fast high-level web crawling & scraping framework for Python.
Full coro support for stat_file() and item_completed() (#8413)
Andrey Rakhmatullin3e4d0faf0eef
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rstindex df07c7dae..683fa7751 100644--- a/docs/topics/coroutines.rst+++ b/docs/topics/coroutines.rst@@ -74,6 +74,37 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):.. versionadded:: VERSION+- The following methods of subclasses of+ :class:`scrapy.pipelines.media.MediaPipeline`:++ - ``media_to_download()``++ .. versionadded:: 2.14++ - ``media_downloaded()``++ .. versionadded:: 2.15++ - ``file_downloaded()``, in subclasses of+ :class:`scrapy.pipelines.files.FilesPipeline`++ .. versionadded:: 2.15++ - ``item_completed()``++ .. versionadded:: VERSION++- The following methods of :ref:`custom media storages+ <media-pipeline-custom-storage>`:++ - ``persist_file()``++ .. versionadded:: 2.15++ - ``stat_file()``++ .. versionadded:: VERSION+Most callables above may also :ref:`await a request <inline-requests>` to sendit and get its response.@@ -129,23 +160,6 @@ return coroutines are listed in :ref:`coroutine-support`):- ``close()``-- Custom feed storages (see :setting:`FEED_STORAGES`):-- - :meth:`~scrapy.extensions.feedexport.FeedStorageProtocol.store`--- Subclasses of :class:`scrapy.pipelines.media.MediaPipeline`:-- - ``media_to_download()``-- - ``item_completed()``--- Custom storages used by subclasses of- :class:`scrapy.pipelines.files.FilesPipeline`:-- - ``persist_file()``-- - ``stat_file()``-In most cases, you can use these APIs in code that otherwise uses coroutines bywrapping a :class:`~twisted.internet.defer.Deferred` object into a:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` fordiff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex dbbe86188..7c77b2471 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -33,7 +33,6 @@ from typing import (from urllib.parse import urlparsefrom itemadapter import ItemAdapter-from twisted.internet.defer import Deferred, maybeDeferredfrom scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Response@@ -54,7 +53,7 @@ from scrapy.utils.boto import (is_botocore_available,)from scrapy.utils.decorators import _warn_spider_arg-from scrapy.utils.defer import deferred_from_coro, ensure_awaitable+from scrapy.utils.defer import _maybeDeferred_coro, deferred_from_coro, ensure_awaitablefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import load_object@@ -65,6 +64,7 @@ if TYPE_CHECKING:from collections.abc import Awaitable, Callablefrom os import PathLike+ from twisted.internet.defer import Deferredfrom twisted.python.failure import Failurefrom scrapy import Spider@@ -790,8 +790,9 @@ class FilesPipeline(MediaPipeline):self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None) -> Deferred[FileInfo | None] | None:path = self.file_path(request, info=info, item=item)- # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type- dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload]+ dfd: Deferred[StatInfo] = _maybeDeferred_coro(+ self.store.stat_file, False, path, info+ )dfd2: Deferred[FileInfo | None] = dfd.addCallback(functools.partial(self._onsuccess, request=request, info=info, path=path))diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 983c150ca..7da4d8d12 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -168,7 +168,7 @@ class MediaPipeline(ABC):(deferred_from_coro(coro) for coro in coros), consumeErrors=True),)- return self.item_completed(results, item, info)+ return await ensure_awaitable(self.item_completed(results, item, info))async def _process_request(self, request: Request, info: SpiderInfo, item: Anydiff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex f04306ad8..9d06b1e56 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -1,3 +1,4 @@+import asyncioimport base64import dataclassesimport hashlib@@ -227,6 +228,30 @@ class TestFilesPipeline:assert result["files"][0]["checksum"] == "abc"assert result["files"][0]["status"] == "uptodate"+ @pytest.mark.only_asyncio+ @coroutine_test+ async def test_file_not_expired_async_stat_file(self):+ """A coroutine stat_file() can await asyncio futures."""++ async def stat_file(path: str, info: Any) -> dict[str, Any]:+ await asyncio.sleep(0.01)+ return {"checksum": "abc", "last_modified": time.time()}++ item_url = "http://example.com/file.pdf"+ item = _create_item_with_files(item_url)+ with (+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),+ mock.patch.object(self.pipeline.store, "stat_file", stat_file),+ mock.patch.object(+ FilesPipeline,+ "get_media_requests",+ return_value=[_prepare_request_object(item_url)],+ ),+ ):+ result = await self.pipeline.process_item(item)+ assert result["files"][0]["checksum"] == "abc"+ assert result["files"][0]["status"] == "uptodate"+@coroutine_testasync def test_file_not_expired_without_checksum(self, caplog: pytest.LogCaptureFixturediff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex b325f981d..8abbb9eb3 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, castfrom unittest.mock import MagicMockimport pytest+from twisted.internet.defer import succeedfrom twisted.python.failure import Failurefrom scrapy import signals@@ -143,6 +144,33 @@ class TestBaseMediaPipeline:assert new_item is item+class DeferredItemCompletedPipeline(UserDefinedPipeline):+ def item_completed(+ self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo+ ) -> Deferred[Any]:+ return succeed(item)+++class AsyncItemCompletedPipeline(UserDefinedPipeline):+ async def item_completed(+ self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo+ ) -> Any:+ return item+++ "pipeline_class", [DeferredItemCompletedPipeline, AsyncItemCompletedPipeline]+)+@coroutine_test+async def test_awaitable_item_completed(pipeline_class: type[MediaPipeline]) -> None:+ crawler = get_crawler(DefaultSpider)+ crawler.spider = crawler._create_spider()+ pipe = build_from_crawler(pipeline_class, crawler)+ pipe.open_spider()+ item = {"name": "name"}+ assert await pipe.process_item(item) is item++class MockedMediaPipeline(UserDefinedPipeline):def __init__(self, *args: Any, crawler: Crawler, **kwargs: Any):super().__init__(*args, crawler=crawler, **kwargs)