Scrapy, a fast high-level web crawling & scraping framework for Python.
Refactor MediaPipeline.process_item() to async def (#7177)
Andrey Rakhmatullinc49ae2115afd
diff --git a/docs/news.rst b/docs/news.rstindex 96e5cb2b1..19d990a50 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -74,6 +74,11 @@ Backward-incompatible changes:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` orcalls its methods directly.+- :meth:`scrapy.pipelines.media.MediaPipeline.process_item` now returns a+ coroutine, previously it returned a+ :class:`~twisted.internet.defer.Deferred` object. This+ change only impacts code that calls this method directly.+.. _release-2.13.4:Scrapy 2.13.4 (2025-11-17)diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 25af0aa72..ad4abbbb6 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -12,6 +12,7 @@ import hashlibimport loggingimport mimetypesimport time+import warningsfrom collections import defaultdictfrom contextlib import suppressfrom ftplib import FTP@@ -24,7 +25,7 @@ from itemadapter import ItemAdapterfrom twisted.internet.defer import Deferred, maybeDeferredfrom twisted.internet.threads import deferToThread-from scrapy.exceptions import IgnoreRequest, NotConfigured+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline@@ -36,7 +37,6 @@ from scrapy.utils.python import to_bytesfrom scrapy.utils.request import referer_strif TYPE_CHECKING:- from collections.abc import Callablefrom os import PathLikefrom twisted.python.failure import Failure@@ -44,7 +44,6 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self- from scrapy import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings@@ -443,10 +442,18 @@ class FilesPipeline(MediaPipeline):def __init__(self,store_uri: str | PathLike[str],- download_func: Callable[[Request, Spider], Response] | None = None,+ download_func: None = None,*,crawler: Crawler,):+ if download_func is not None: # pragma: no cover+ warnings.warn(+ "The download_func argument of FilesPipeline.__init__() is ignored"+ " and will be removed in a future Scrapy version.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+if not (store_uri and (store_uri := _to_string(store_uri))):from scrapy.pipelines.images import ImagesPipeline # noqa: PLC0415@@ -476,7 +483,7 @@ class FilesPipeline(MediaPipeline):resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD)- super().__init__(download_func=download_func, crawler=crawler)+ super().__init__(crawler=crawler)@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 687d29fa5..f5d1ad1b9 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -8,20 +8,21 @@ from __future__ import annotationsimport functoolsimport hashlib+import warningsfrom contextlib import suppressfrom io import BytesIOfrom typing import TYPE_CHECKING, Anyfrom itemadapter import ItemAdapter-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FileException, FilesPipeline, _md5sumfrom scrapy.utils.python import to_bytesif TYPE_CHECKING:- from collections.abc import Callable, Iterable+ from collections.abc import Iterablefrom os import PathLikefrom PIL import Image@@ -29,7 +30,6 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self- from scrapy import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.pipelines.media import FileInfoOrError, MediaPipeline@@ -55,10 +55,18 @@ class ImagesPipeline(FilesPipeline):def __init__(self,store_uri: str | PathLike[str],- download_func: Callable[[Request, Spider], Response] | None = None,+ download_func: None = None,*,crawler: Crawler,):+ if download_func is not None: # pragma: no cover+ warnings.warn(+ "The download_func argument of ImagesPipeline.__init__() is ignored"+ " and will be removed in a future Scrapy version.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+try:from PIL import Image, ImageOps # noqa: PLC0415@@ -69,7 +77,7 @@ class ImagesPipeline(FilesPipeline):"ImagesPipeline requires installing Pillow 8.3.2 or later")- super().__init__(store_uri, download_func=download_func, crawler=crawler)+ super().__init__(store_uri, crawler=crawler)settings = crawler.settingsresolve = functools.partial(diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 438e5e7d6..cea6b9c35 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport functoolsimport loggingimport warnings@@ -8,28 +9,27 @@ from collections import defaultdictfrom typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, castfrom twisted import version as twisted_version-from twisted.internet.defer import (- Deferred,- DeferredList,- inlineCallbacks,- maybeDeferred,-)+from twisted.internet.defer import Deferred, DeferredListfrom twisted.python.failure import Failurefrom twisted.python.versions import Versionfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.request import NO_CALLBACK, Request-from scrapy.utils.asyncio import call_later+from scrapy.utils.asyncio import call_later, is_asyncio_availablefrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.decorators import _warn_spider_arg-from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro+from scrapy.utils.defer import (+ _DEFER_DELAY,+ _defer_sleep_async,+ deferred_from_coro,+ ensure_awaitable,+ maybe_deferred_to_future,+)from scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.python import global_object_nameif TYPE_CHECKING:- from collections.abc import Callable, Generator-# typing.Self requires Python 3.11from typing_extensions import Self@@ -68,16 +68,22 @@ class MediaPipeline(ABC):def __init__(self,- download_func: Callable[[Request, Spider], Response] | None = None,+ download_func: None = None,*,crawler: Crawler,):+ if download_func is not None: # pragma: no cover+ warnings.warn(+ "The download_func argument of MediaPipeline.__init__() is ignored"+ " and will be removed in a future Scrapy version.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )self.crawler: Crawler = crawlerassert crawler.request_fingerprinterself._fingerprinter: RequestFingerprinterProtocol = (crawler.request_fingerprinter)- self.download_func = download_funcsettings = crawler.settingsresolve = functools.partial(@@ -119,21 +125,31 @@ class MediaPipeline(ABC):self.spiderinfo = self.SpiderInfo(self.crawler.spider)@_warn_spider_arg- def process_item(- self, item: Any, spider: Spider | None = None- ) -> Deferred[list[FileInfoOrError]]:+ async def process_item(self, item: Any, spider: Spider | None = None) -> Any:info = self.spiderinforequests = arg_to_iter(self.get_media_requests(item, info))- dlist = [self._process_request(r, info, item) for r in requests]- dfd = cast(- "Deferred[list[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True)- )- return dfd.addCallback(self.item_completed, item, info)+ coros = [self._process_request(r, info, item) for r in requests]+ results: list[FileInfoOrError] = []+ if coros:+ if is_asyncio_available():+ results_asyncio = await asyncio.gather(*coros, return_exceptions=True)+ for res in results_asyncio:+ if isinstance(res, BaseException):+ results.append((False, Failure(res)))+ else:+ results.append((True, res))+ else:+ results = await cast(+ "Deferred[list[FileInfoOrError]]",+ DeferredList(+ (deferred_from_coro(coro) for coro in coros), consumeErrors=True+ ),+ )+ return self.item_completed(results, item, info)- @inlineCallbacks- def _process_request(+ async def _process_request(self, request: Request, info: SpiderInfo, item: Any- ) -> Generator[Deferred[Any], Any, FileInfo]:+ ) -> FileInfo:fp = self._fingerprinter.fingerprint(request)eb = request.errback@@ -142,7 +158,7 @@ class MediaPipeline(ABC):# Return cached result if request was already seenif fp in info.downloaded:- yield _defer_sleep()+ await _defer_sleep_async()cached_result = info.downloaded[fp]if isinstance(cached_result, Failure):if eb:@@ -158,27 +174,27 @@ class MediaPipeline(ABC):# Check if request is downloading right now to avoid doing it twiceif fp in info.downloading:- return (yield wad)+ return await maybe_deferred_to_future(wad)# Download request checking media_to_download hook output firstinfo.downloading.add(fp)- yield _defer_sleep()+ await _defer_sleep_async()result: FileInfo | Failuretry:- file_info = yield maybeDeferred(- self.media_to_download, request, info, item=item+ file_info: FileInfo | None = await ensure_awaitable(+ self.media_to_download(request, info, item=item))if file_info:# got a result without downloadingresult = file_infoelse:# download the result- result = yield self._check_media_to_download(request, info, item=item)+ result = await self._check_media_to_download(request, info, item=item)except Exception:result = Failure()logger.exception(result)self._cache_result_and_execute_waiters(result, fp, info)- return (yield wad) # it must return wad at last+ return await maybe_deferred_to_future(wad) # it must return wad at lastdef _modify_media_request(self, request: Request) -> None:if self.handle_httpstatus_list:@@ -186,20 +202,13 @@ class MediaPipeline(ABC):else:request.meta["handle_httpstatus_all"] = True- @inlineCallbacks- def _check_media_to_download( # pylint: disable=inconsistent-return-statements+ async def _check_media_to_download(self, request: Request, info: SpiderInfo, item: Any- ) -> Generator[Deferred[Any], Any, FileInfo]:+ ) -> FileInfo:try:- if self.download_func:- # this ugly code was left only to support tests. TODO: remove- response = yield maybeDeferred(self.download_func, request, info.spider)- else:- self._modify_media_request(request)- assert self.crawler.engine- response = yield deferred_from_coro(- self.crawler.engine.download_async(request)- )+ self._modify_media_request(request)+ assert self.crawler.engine+ response = await self.crawler.engine.download_async(request)return self.media_downloaded(response, request, info, item=item)except Exception:failure = self.media_failed(Failure(), request, info)diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex c27f8bb34..2a09f99bb 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -26,7 +26,7 @@ from twisted.internet.task import Cooperatorfrom twisted.python import failurefrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.asyncio import call_later, is_asyncio_available+from scrapy.utils.asyncio import is_asyncio_availableif TYPE_CHECKING:from collections.abc import AsyncIterator, Callable@@ -84,15 +84,6 @@ def defer_succeed(result: _T) -> Deferred[_T]:return d-def _defer_sleep() -> Deferred[None]:- """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers- before attending pending delayed calls, so do not set delay to zero.- """- d: Deferred[None] = Deferred()- call_later(_DEFER_DELAY, d.callback, None)- return d--async def _defer_sleep_async() -> None:"""Delay by _DEFER_DELAY so reactor has a chance to go through readers and writersbefore attending pending delayed calls, so do not set delay to zero.@@ -100,7 +91,11 @@ async def _defer_sleep_async() -> None:if is_asyncio_available():await asyncio.sleep(_DEFER_DELAY)else:- await _defer_sleep()+ from twisted.internet import reactor++ d: Deferred[None] = Deferred()+ reactor.callLater(_DEFER_DELAY, d.callback, None)+ await ddef defer_result(result: Any) -> Deferred[Any]:diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 966cb320e..6ab5d9a8e 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -13,6 +13,7 @@ from shutil import rmtreefrom tempfile import mkdtempfrom typing import Anyfrom unittest import mock+from unittest.mock import MagicMockfrom urllib.parse import urlparseimport attr@@ -31,6 +32,7 @@ from scrapy.pipelines.files import (S3FilesStore,)from scrapy.settings import Settings+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.ftp import MockFTPServer@@ -83,8 +85,8 @@ class TestFilesPipeline:settings_dict = {"FILES_STORE": self.tempdir}crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)crawler.spider = crawler._create_spider()+ crawler.engine = MagicMock(download_async=_mocked_download_func)self.pipeline = FilesPipeline.from_crawler(crawler)- self.pipeline.download_func = _mocked_download_funcself.pipeline.open_spider()def teardown_method(self):@@ -160,8 +162,8 @@ class TestFilesPipeline:fullpath = Path(self.tempdir, "some", "image", "key.jpg")assert self.pipeline.store._get_filesystem_path(path) == fullpath- @inlineCallbacks- def test_file_not_expired(self):+ @deferred_f_from_coro_f+ async def test_file_not_expired(self):item_url = "http://example.com/file.pdf"item = _create_item_with_files(item_url)patchers = [@@ -180,15 +182,15 @@ class TestFilesPipeline:for p in patchers:p.start()- result = yield self.pipeline.process_item(item)+ result = await self.pipeline.process_item(item)assert result["files"][0]["checksum"] == "abc"assert result["files"][0]["status"] == "uptodate"for p in patchers:p.stop()- @inlineCallbacks- def test_file_expired(self):+ @deferred_f_from_coro_f+ async def test_file_expired(self):item_url = "http://example.com/file2.pdf"item = _create_item_with_files(item_url)patchers = [@@ -211,15 +213,15 @@ class TestFilesPipeline:for p in patchers:p.start()- result = yield self.pipeline.process_item(item)+ result = await self.pipeline.process_item(item)assert result["files"][0]["checksum"] != "abc"assert result["files"][0]["status"] == "downloaded"for p in patchers:p.stop()- @inlineCallbacks- def test_file_cached(self):+ @deferred_f_from_coro_f+ async def test_file_cached(self):item_url = "http://example.com/file3.pdf"item = _create_item_with_files(item_url)patchers = [@@ -242,7 +244,7 @@ class TestFilesPipeline:for p in patchers:p.start()- result = yield self.pipeline.process_item(item)+ result = await self.pipeline.process_item(item)assert result["files"][0]["checksum"] != "abc"assert result["files"][0]["status"] == "cached"diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 9f174cc5f..05ad43d16 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -1,10 +1,10 @@from __future__ import annotationsimport warnings+from unittest.mock import MagicMockimport pytestfrom testfixtures import LogCapture-from twisted.internet.defer import Deferred, inlineCallbacksfrom twisted.python.failure import Failurefrom scrapy import signals@@ -13,17 +13,21 @@ from scrapy.http import Request, Responsefrom scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FileExceptionfrom scrapy.pipelines.media import MediaPipeline-from scrapy.utils.asyncio import call_later+from scrapy.utils.defer import _defer_sleep_async, deferred_f_from_coro_ffrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_allfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler-def _mocked_download_func(request, info):+async def _mocked_download_func(request):assert request.callback is NO_CALLBACKresponse = request.meta.get("response")- return response() if callable(response) else response+ if callable(response):+ response = await response()+ if isinstance(response, Exception):+ raise response+ return responseclass UserDefinedPipeline(MediaPipeline):@@ -50,8 +54,8 @@ class TestBaseMediaPipeline:def setup_method(self):crawler = get_crawler(DefaultSpider, self.settings)crawler.spider = crawler._create_spider()+ crawler.engine = MagicMock(download_async=_mocked_download_func)self.pipe = self.pipeline_class.from_crawler(crawler)- self.pipe.download_func = _mocked_download_funcself.pipe.open_spider()self.info = self.pipe.spiderinfoself.fingerprint = crawler.request_fingerprinter.fingerprint@@ -158,10 +162,10 @@ class TestBaseMediaPipeline:assert new_item is itemassert len(log.records) == 0- @inlineCallbacks- def test_default_process_item(self):+ @deferred_f_from_coro_f+ async def test_default_process_item(self):item = {"name": "name"}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item is item@@ -206,8 +210,8 @@ class TestMediaPipeline(TestBaseMediaPipeline):self.pipe._mockcalled.append("request_errback")return result- @inlineCallbacks- def test_result_succeed(self):+ @deferred_f_from_coro_f+ async def test_result_succeed(self):rsp = Response("http://url1")req = Request("http://url1",@@ -215,7 +219,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):errback=self._errback,)item = {"requests": req}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item["results"] == [(True, {})]assert self.pipe._mockcalled == ["get_media_requests",@@ -224,18 +228,17 @@ class TestMediaPipeline(TestBaseMediaPipeline):"item_completed",]- @inlineCallbacks- def test_result_failure(self):+ @deferred_f_from_coro_f+ async def test_result_failure(self):self.pipe.LOG_FAILED_RESULTS = Falseexc = Exception("foo")- fail = Failure(exc)req = Request("http://url1",- meta={"response": fail},+ meta={"response": exc},errback=self._errback,)item = {"requests": req}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert len(new_item["results"]) == 1assert new_item["results"][0][0] is Falseassert isinstance(new_item["results"][0][1], Failure)@@ -248,16 +251,15 @@ class TestMediaPipeline(TestBaseMediaPipeline):"item_completed",]- @inlineCallbacks- def test_mix_of_success_and_failure(self):+ @deferred_f_from_coro_f+ async def test_mix_of_success_and_failure(self):self.pipe.LOG_FAILED_RESULTS = Falsersp1 = Response("http://url1")req1 = Request("http://url1", meta={"response": rsp1})exc = Exception("foo")- fail = Failure(exc)- req2 = Request("http://url2", meta={"response": fail})+ req2 = Request("http://url2", meta={"response": exc})item = {"requests": [req1, req2]}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert len(new_item["results"]) == 2assert new_item["results"][0] == (True, {})assert new_item["results"][1][0] is False@@ -275,12 +277,12 @@ class TestMediaPipeline(TestBaseMediaPipeline):assert m.count("media_downloaded") == 1assert m.count("media_failed") == 1- @inlineCallbacks- def test_get_media_requests(self):+ @deferred_f_from_coro_f+ async def test_get_media_requests(self):# returns single Request (without callback)req = Request("http://url")item = {"requests": req} # pass a single item- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item is itemassert self.fingerprint(req) in self.info.downloaded@@ -288,17 +290,17 @@ class TestMediaPipeline(TestBaseMediaPipeline):req1 = Request("http://url1")req2 = Request("http://url2")item = {"requests": iter([req1, req2])}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item is itemassert self.fingerprint(req1) in self.info.downloadedassert self.fingerprint(req2) in self.info.downloaded- @inlineCallbacks- def test_results_are_cached_across_multiple_items(self):+ @deferred_f_from_coro_f+ async def test_results_are_cached_across_multiple_items(self):rsp1 = Response("http://url1")req1 = Request("http://url1", meta={"response": rsp1})item = {"requests": req1}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item is itemassert new_item["results"] == [(True, {})]@@ -307,25 +309,25 @@ class TestMediaPipeline(TestBaseMediaPipeline):req1.url, meta={"response": Response("http://donot.download.me")})item = {"requests": req2}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item is itemassert self.fingerprint(req1) == self.fingerprint(req2)assert new_item["results"] == [(True, {})]- @inlineCallbacks- def test_results_are_cached_for_requests_of_single_item(self):+ @deferred_f_from_coro_f+ async def test_results_are_cached_for_requests_of_single_item(self):rsp1 = Response("http://url1")req1 = Request("http://url1", meta={"response": rsp1})req2 = Request(req1.url, meta={"response": Response("http://donot.download.me")})item = {"requests": [req1, req2]}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item is itemassert new_item["results"] == [(True, {}), (True, {})]- @inlineCallbacks- def test_wait_if_request_is_downloading(self):+ @deferred_f_from_coro_f+ async def test_wait_if_request_is_downloading(self):def _check_downloading(response):fp = self.fingerprint(req1)assert fp in self.info.downloading@@ -336,25 +338,24 @@ class TestMediaPipeline(TestBaseMediaPipeline):rsp1 = Response("http://url")- def rsp1_func():- dfd = Deferred().addCallback(_check_downloading)- call_later(0.1, dfd.callback, rsp1)- return dfd+ async def rsp1_func():+ await _defer_sleep_async()+ _check_downloading(rsp1)- def rsp2_func():+ async def rsp2_func():pytest.fail("it must cache rsp1 result and must not try to redownload")req1 = Request("http://url", meta={"response": rsp1_func})req2 = Request(req1.url, meta={"response": rsp2_func})item = {"requests": [req1, req2]}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item["results"] == [(True, {}), (True, {})]- @inlineCallbacks- def test_use_media_to_download_result(self):+ @deferred_f_from_coro_f+ async def test_use_media_to_download_result(self):req = Request("http://url", meta={"result": "ITSME"})item = {"requests": req}- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert new_item["results"] == [(True, "ITSME")]assert self.pipe._mockcalled == ["get_media_requests",@@ -469,45 +470,29 @@ class MediaFailedFailurePipeline(MockedMediaPipeline):return failure # deprecated-class TestMediaFailedFailure:+class TestMediaFailedFailure(TestBaseMediaPipeline):"""Test that media_failed() can return a failure instead of raising."""pipeline_class = MediaFailedFailurePipeline- settings = None-- def setup_method(self):- crawler = get_crawler(DefaultSpider, self.settings)- crawler.spider = crawler._create_spider()- self.pipe = self.pipeline_class.from_crawler(crawler)- self.pipe.download_func = _mocked_download_func- self.pipe.open_spider()- self.info = self.pipe.spiderinfo- self.fingerprint = crawler.request_fingerprinter.fingerprint-- def teardown_method(self):- for name, signal in vars(signals).items():- if not name.startswith("_"):- disconnect_all(signal)def _errback(self, result):self.pipe._mockcalled.append("request_errback")return result- @inlineCallbacks- def test_result_failure(self):+ @deferred_f_from_coro_f+ async def test_result_failure(self):self.pipe.LOG_FAILED_RESULTS = Falseexc = Exception("foo")- fail = Failure(exc)req = Request("http://url1",- meta={"response": fail},+ meta={"response": exc},errback=self._errback,)item = {"requests": req}with pytest.warns(ScrapyDeprecationWarning, match="media_failed returned a Failure instance"):- new_item = yield self.pipe.process_item(item)+ new_item = await self.pipe.process_item(item)assert len(new_item["results"]) == 1assert new_item["results"][0][0] is Falseassert isinstance(new_item["results"][0][1], Failure)