Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Full coro support for stat_file() and item_completed() (#8413)

Andrey Rakhmatullin3e4d0faf0eef
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index df07c7dae..683fa7751 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -74,6 +74,37 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
.. versionadded:: VERSION
+- The following methods of subclasses of
+ :class:`scrapy.pipelines.media.MediaPipeline`:
+
+ - ``media_to_download()``
+
+ .. versionadded:: 2.14
+
+ - ``media_downloaded()``
+
+ .. versionadded:: 2.15
+
+ - ``file_downloaded()``, in subclasses of
+ :class:`scrapy.pipelines.files.FilesPipeline`
+
+ .. versionadded:: 2.15
+
+ - ``item_completed()``
+
+ .. versionadded:: VERSION
+
+- The following methods of :ref:`custom media storages
+ <media-pipeline-custom-storage>`:
+
+ - ``persist_file()``
+
+ .. versionadded:: 2.15
+
+ - ``stat_file()``
+
+ .. versionadded:: VERSION
+
Most callables above may also :ref:`await a request <inline-requests>` to send
it and get its response.
@@ -129,23 +160,6 @@ return coroutines are listed in :ref:`coroutine-support`):
- ``close()``
-- Custom feed storages (see :setting:`FEED_STORAGES`):
-
- - :meth:`~scrapy.extensions.feedexport.FeedStorageProtocol.store`
-
-- Subclasses of :class:`scrapy.pipelines.media.MediaPipeline`:
-
- - ``media_to_download()``
-
- - ``item_completed()``
-
-- Custom storages used by subclasses of
- :class:`scrapy.pipelines.files.FilesPipeline`:
-
- - ``persist_file()``
-
- - ``stat_file()``
-
In most cases, you can use these APIs in code that otherwise uses coroutines by
wrapping a :class:`~twisted.internet.defer.Deferred` object into a
:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index dbbe86188..7c77b2471 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -33,7 +33,6 @@ from typing import (
from urllib.parse import urlparse
from itemadapter import ItemAdapter
-from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
@@ -54,7 +53,7 @@ from scrapy.utils.boto import (
is_botocore_available,
)
from scrapy.utils.decorators import _warn_spider_arg
-from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
+from scrapy.utils.defer import _maybeDeferred_coro, deferred_from_coro, ensure_awaitable
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import load_object
@@ -65,6 +64,7 @@ if TYPE_CHECKING:
from collections.abc import Awaitable, Callable
from os import PathLike
+ from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from scrapy import Spider
@@ -790,8 +790,9 @@ class FilesPipeline(MediaPipeline):
self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
) -> Deferred[FileInfo | None] | None:
path = self.file_path(request, info=info, item=item)
- # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
- dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload]
+ dfd: Deferred[StatInfo] = _maybeDeferred_coro(
+ self.store.stat_file, False, path, info
+ )
dfd2: Deferred[FileInfo | None] = dfd.addCallback(
functools.partial(self._onsuccess, request=request, info=info, path=path)
)
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 983c150ca..7da4d8d12 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -168,7 +168,7 @@ class MediaPipeline(ABC):
(deferred_from_coro(coro) for coro in coros), consumeErrors=True
),
)
- return self.item_completed(results, item, info)
+ return await ensure_awaitable(self.item_completed(results, item, info))
async def _process_request(
self, request: Request, info: SpiderInfo, item: Any
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index f04306ad8..9d06b1e56 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,3 +1,4 @@
+import asyncio
import base64
import dataclasses
import hashlib
@@ -227,6 +228,30 @@ class TestFilesPipeline:
assert result["files"][0]["checksum"] == "abc"
assert result["files"][0]["status"] == "uptodate"
+ @pytest.mark.only_asyncio
+ @coroutine_test
+ async def test_file_not_expired_async_stat_file(self):
+ """A coroutine stat_file() can await asyncio futures."""
+
+ async def stat_file(path: str, info: Any) -> dict[str, Any]:
+ await asyncio.sleep(0.01)
+ return {"checksum": "abc", "last_modified": time.time()}
+
+ item_url = "http://example.com/file.pdf"
+ item = _create_item_with_files(item_url)
+ with (
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+ mock.patch.object(self.pipeline.store, "stat_file", stat_file),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
+ ):
+ result = await self.pipeline.process_item(item)
+ assert result["files"][0]["checksum"] == "abc"
+ assert result["files"][0]["status"] == "uptodate"
+
@coroutine_test
async def test_file_not_expired_without_checksum(
self, caplog: pytest.LogCaptureFixture
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index b325f981d..8abbb9eb3 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, cast
from unittest.mock import MagicMock
import pytest
+from twisted.internet.defer import succeed
from twisted.python.failure import Failure
from scrapy import signals
@@ -143,6 +144,33 @@ class TestBaseMediaPipeline:
assert new_item is item
+class DeferredItemCompletedPipeline(UserDefinedPipeline):
+ def item_completed(
+ self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
+ ) -> Deferred[Any]:
+ return succeed(item)
+
+
+class AsyncItemCompletedPipeline(UserDefinedPipeline):
+ async def item_completed(
+ self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
+ ) -> Any:
+ return item
+
+
+ "pipeline_class", [DeferredItemCompletedPipeline, AsyncItemCompletedPipeline]
+)
+@coroutine_test
+async def test_awaitable_item_completed(pipeline_class: type[MediaPipeline]) -> None:
+ crawler = get_crawler(DefaultSpider)
+ crawler.spider = crawler._create_spider()
+ pipe = build_from_crawler(pipeline_class, crawler)
+ pipe.open_spider()
+ item = {"name": "name"}
+ assert await pipe.process_item(item) is item
+
+
class MockedMediaPipeline(UserDefinedPipeline):
def __init__(self, *args: Any, crawler: Crawler, **kwargs: Any):
super().__init__(*args, crawler=crawler, **kwargs)