Scrapy, a fast high-level web crawling & scraping framework for Python.
MediaPipeline: log media_to_download errors before stripping them (#5068)
Adrián Chaves0e78acb65798
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 75532034a..fc156ab41 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -112,14 +112,14 @@ class MediaPipeline:info.downloading.add(fp)dfd = mustbe_deferred(self.media_to_download, request, info, item=item)dfd.addCallback(self._check_media_to_download, request, info, item=item)+ dfd.addErrback(self._log_exception)dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)- dfd.addErrback(- lambda f: logger.error(- f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider}- )- )return dfd.addBoth(lambda _: wad) # it must return wad at last+ def _log_exception(self, result):+ logger.exception(result)+ return result+def _modify_media_request(self, request):if self.handle_httpstatus_list:request.meta["handle_httpstatus_list"] = self.handle_httpstatus_listdiff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex ed8483483..c41ab483f 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -9,6 +9,7 @@ from w3lib.url import add_or_replace_parameterfrom scrapy import signalsfrom scrapy.crawler import CrawlerRunner+from scrapy.utils.misc import load_objectfrom tests.mockserver import MockServerfrom tests.spiders import SimpleSpider@@ -193,6 +194,29 @@ class FileDownloadCrawlTestCase(TestCase):crawler.stats.get_value("downloader/response_status_count/302"), 3)+ @defer.inlineCallbacks+ def test_download_media_file_path_error(self):+ cls = load_object(self.pipeline_class)++ class ExceptionRaisingMediaPipeline(cls):+ def file_path(self, request, response=None, info=None, *, item=None):+ return 1 / 0++ settings = {+ **self.settings,+ "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},+ }+ runner = CrawlerRunner(settings)+ crawler = self._create_crawler(MediaDownloadSpider, runner=runner)+ with LogCapture() as log:+ yield crawler.crawl(+ self.mockserver.url("/files/images/"),+ media_key=self.media_key,+ media_urls_key=self.media_urls_key,+ mockserver=self.mockserver,+ )+ self.assertIn("ZeroDivisionError", str(log))+skip_pillow: Optional[str]try: