Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Stop passing media_downloaded() errors to media_failed(). (#8489)

Andrey Rakhmatullin034d9bc9caf9
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index a6264670c..0f0ad8b0d 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -250,12 +250,9 @@ class MediaPipeline(ABC):
async def _check_media_to_download(
self, request: Request, info: SpiderInfo, item: Any
) -> FileInfo:
+ self._modify_media_request(request)
try:
- self._modify_media_request(request)
response = await self.crawler.engine.download_async(request)
- return await ensure_awaitable(
- self.media_downloaded(response, request, info, item=item)
- )
except Exception:
failure = self.media_failed(Failure(), request, info)
if isinstance(failure, Failure):
@@ -266,6 +263,10 @@ class MediaPipeline(ABC):
stacklevel=2,
)
failure.raiseException()
+ else:
+ return await ensure_awaitable(
+ self.media_downloaded(response, request, info, item=item)
+ )
def _cache_result_and_execute_waiters(
self, result: FileInfo | Failure, fp: bytes, info: SpiderInfo
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 58cad18f3..00660c524 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -10,6 +10,7 @@ import pytest
from w3lib.url import add_or_replace_parameter
from scrapy import Spider, signals
+from scrapy.pipelines.media import FileException
from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
@@ -17,6 +18,7 @@ from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.crawler import Crawler
+ from scrapy.http import Response
from tests.mockserver.http import MockServer
@@ -149,6 +151,7 @@ class TestFileDownloadCrawl:
# check that logs do show the failure on the file downloads
file_dl_failure = f"File (code: {code}): Error downloading file from"
assert logs.count(file_dl_failure) == 3
+ assert "File (unknown-error)" not in logs
# check that no files were written to the media store
assert not list(self.tmpmediastore.iterdir())
@@ -179,6 +182,40 @@ class TestFileDownloadCrawl:
)
self._assert_files_download_failure(crawler, self.items, 404, caplog.text)
+ @coroutine_test
+ async def test_download_media_failed_response(self, mockserver: MockServer) -> None:
+ class ResponseFileException(FileException):
+ def __init__(self, response: Response):
+ super().__init__(f"download-error ({response.status})")
+ self.response = response
+
+ class ResponsePipeline(load_object(self.pipeline_class)): # type: ignore[misc]
+ async def media_downloaded(self, response, request, info, *, item=None):
+ try:
+ return await super().media_downloaded(
+ response, request, info, item=item
+ )
+ except FileException as e:
+ raise ResponseFileException(response) from e
+
+ def item_completed(self, results, item, info):
+ item["download_errors"] = [
+ value.value.response.status
+ for ok, value in results
+ if not ok and value.check(ResponseFileException)
+ ]
+ return super().item_completed(results, item, info)
+
+ settings = {**self.settings, "ITEM_PIPELINES": {ResponsePipeline: 1}}
+ crawler = self._create_crawler(BrokenLinksMediaDownloadSpider, settings)
+ await crawler.crawl_async(
+ mockserver.url("/static/files/images/"),
+ media_key=self.media_key,
+ media_urls_key=self.media_urls_key,
+ )
+ assert len(self.items) == 1
+ assert self.items[0]["download_errors"] == [404, 404, 404]
+
@coroutine_test
async def test_download_media_redirected_default_failure(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index ef0183eab..6a86f4b81 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -570,6 +570,38 @@ class TestBuildFromCrawler:
assert pipe._from_crawler_called
+class MediaDownloadedErrorPipeline(MockedMediaPipeline):
+ def media_downloaded(
+ self,
+ response: Response,
+ request: Request,
+ info: MediaPipeline.SpiderInfo,
+ *,
+ item: Any = None,
+ ) -> FileInfo:
+ self._mockcalled.append("media_downloaded")
+ raise ValueError("foo")
+
+
+class TestMediaDownloadedError(TestBaseMediaPipeline):
+ pipeline_class = MediaDownloadedErrorPipeline
+
+ @coroutine_test
+ async def test_error_skips_media_failed(self):
+ url = "https://example.com"
+ req = Request(url, meta={"response": Response(url)})
+ new_item = await self.pipe.process_item({"requests": req})
+ ((ok, failure),) = new_item["results"]
+ assert ok is False
+ assert isinstance(failure.value, ValueError)
+ assert self.mocked_pipe._mockcalled == [
+ "get_media_requests",
+ "media_to_download",
+ "media_downloaded",
+ "item_completed",
+ ]
+
+
class MediaFailedNonePipeline(MockedMediaPipeline):
def media_failed( # type: ignore[override]
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo