Scrapy, a fast high-level web crawling & scraping framework for Python.
Stop passing media_downloaded() errors to media_failed(). (#8489)
Andrey Rakhmatullin034d9bc9caf9
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex a6264670c..0f0ad8b0d 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -250,12 +250,9 @@ class MediaPipeline(ABC):async def _check_media_to_download(self, request: Request, info: SpiderInfo, item: Any) -> FileInfo:+ self._modify_media_request(request)try:- self._modify_media_request(request)response = await self.crawler.engine.download_async(request)- return await ensure_awaitable(- self.media_downloaded(response, request, info, item=item)- )except Exception:failure = self.media_failed(Failure(), request, info)if isinstance(failure, Failure):@@ -266,6 +263,10 @@ class MediaPipeline(ABC):stacklevel=2,)failure.raiseException()+ else:+ return await ensure_awaitable(+ self.media_downloaded(response, request, info, item=item)+ )def _cache_result_and_execute_waiters(self, result: FileInfo | Failure, fp: bytes, info: SpiderInfodiff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 58cad18f3..00660c524 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -10,6 +10,7 @@ import pytestfrom w3lib.url import add_or_replace_parameterfrom scrapy import Spider, signals+from scrapy.pipelines.media import FileExceptionfrom scrapy.utils.misc import load_objectfrom scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider@@ -17,6 +18,7 @@ from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from scrapy.crawler import Crawler+ from scrapy.http import Responsefrom tests.mockserver.http import MockServer@@ -149,6 +151,7 @@ class TestFileDownloadCrawl:# check that logs do show the failure on the file downloadsfile_dl_failure = f"File (code: {code}): Error downloading file from"assert logs.count(file_dl_failure) == 3+ assert "File (unknown-error)" not in logs# check that no files were written to the media storeassert not list(self.tmpmediastore.iterdir())@@ -179,6 +182,40 @@ class TestFileDownloadCrawl:)self._assert_files_download_failure(crawler, self.items, 404, caplog.text)+ @coroutine_test+ async def test_download_media_failed_response(self, mockserver: MockServer) -> None:+ class ResponseFileException(FileException):+ def __init__(self, response: Response):+ super().__init__(f"download-error ({response.status})")+ self.response = response++ class ResponsePipeline(load_object(self.pipeline_class)): # type: ignore[misc]+ async def media_downloaded(self, response, request, info, *, item=None):+ try:+ return await super().media_downloaded(+ response, request, info, item=item+ )+ except FileException as e:+ raise ResponseFileException(response) from e++ def item_completed(self, results, item, info):+ item["download_errors"] = [+ value.value.response.status+ for ok, value in results+ if not ok and value.check(ResponseFileException)+ ]+ return super().item_completed(results, item, info)++ settings = {**self.settings, "ITEM_PIPELINES": {ResponsePipeline: 1}}+ crawler = self._create_crawler(BrokenLinksMediaDownloadSpider, settings)+ await crawler.crawl_async(+ mockserver.url("/static/files/images/"),+ media_key=self.media_key,+ media_urls_key=self.media_urls_key,+ )+ assert len(self.items) == 1+ assert self.items[0]["download_errors"] == [404, 404, 404]+@coroutine_testasync def test_download_media_redirected_default_failure(self, caplog: pytest.LogCaptureFixture, mockserver: MockServerdiff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex ef0183eab..6a86f4b81 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -570,6 +570,38 @@ class TestBuildFromCrawler:assert pipe._from_crawler_called+class MediaDownloadedErrorPipeline(MockedMediaPipeline):+ def media_downloaded(+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> FileInfo:+ self._mockcalled.append("media_downloaded")+ raise ValueError("foo")+++class TestMediaDownloadedError(TestBaseMediaPipeline):+ pipeline_class = MediaDownloadedErrorPipeline++ @coroutine_test+ async def test_error_skips_media_failed(self):+ url = "https://example.com"+ req = Request(url, meta={"response": Response(url)})+ new_item = await self.pipe.process_item({"requests": req})+ ((ok, failure),) = new_item["results"]+ assert ok is False+ assert isinstance(failure.value, ValueError)+ assert self.mocked_pipe._mockcalled == [+ "get_media_requests",+ "media_to_download",+ "media_downloaded",+ "item_completed",+ ]++class MediaFailedNonePipeline(MockedMediaPipeline):def media_failed( # type: ignore[override]self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo