Scrapy, a fast high-level web crawling & scraping framework for Python.
Prefer crawler.settings over settings in media pipelines.
Andrey Rakhmatullin6aa4d2b4ab28
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex f83037e6c..3b730c432 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -453,7 +453,17 @@ class FilesPipeline(MediaPipeline):if not store_uri:raise NotConfigured- if isinstance(settings, dict) or settings is None:+ if crawler is not None:+ if settings is not None:+ warnings.warn(+ f"FilesPipeline.__init__() was called with a crawler instance and a settings instance"+ f" when creating {self.__class__.__qualname__}. The settings instance will be ignored"+ f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ settings = crawler.settings+ elif isinstance(settings, dict) or settings is None:settings = Settings(settings)cls_name = "FilesPipeline"self.store: FilesStoreProtocol = self._get_store(store_uri)@@ -473,7 +483,9 @@ class FilesPipeline(MediaPipeline):)super().__init__(- download_func=download_func, settings=settings, crawler=crawler+ download_func=download_func,+ settings=settings if not crawler else None,+ crawler=crawler,)@classmethod@@ -526,7 +538,7 @@ class FilesPipeline(MediaPipeline):store_uri = settings["FILES_STORE"]if "crawler" in get_func_args(cls.__init__):- o = cls(store_uri, settings=settings, crawler=crawler)+ o = cls(store_uri, crawler=crawler)else:o = cls(store_uri, settings=settings)if crawler:diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 71da6a196..fa26133bb 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -79,10 +79,23 @@ class ImagesPipeline(FilesPipeline):)super().__init__(- store_uri, settings=settings, download_func=download_func, crawler=crawler+ store_uri,+ settings=settings if not crawler else None,+ download_func=download_func,+ crawler=crawler,)- if isinstance(settings, dict) or settings is None:+ if crawler is not None:+ if settings is not None:+ warnings.warn(+ f"ImagesPipeline.__init__() was called with a crawler instance and a settings instance"+ f" when creating {self.__class__.__qualname__}. The settings instance will be ignored"+ f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ settings = crawler.settings+ elif isinstance(settings, dict) or settings is None:settings = Settings(settings)resolve = functools.partial(@@ -140,7 +153,7 @@ class ImagesPipeline(FilesPipeline):store_uri = settings["IMAGES_STORE"]if "crawler" in get_func_args(cls.__init__):- o = cls(store_uri, settings=settings, crawler=crawler)+ o = cls(store_uri, crawler=crawler)else:o = cls(store_uri, settings=settings)if crawler:diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 99abed09e..70c52d090 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -81,7 +81,17 @@ class MediaPipeline(ABC):):self.download_func = download_func- if isinstance(settings, dict) or settings is None:+ if crawler is not None:+ if settings is not None:+ warnings.warn(+ f"MediaPipeline.__init__() was called with a crawler instance and a settings instance"+ f" when creating {self.__class__.__qualname__}. The settings instance will be ignored"+ f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ settings = crawler.settings+ elif isinstance(settings, dict) or settings is None:settings = Settings(settings)resolve = functools.partial(self._key_for_pipe, base_class_name="MediaPipeline", settings=settings@@ -92,7 +102,6 @@ class MediaPipeline(ABC):self._handle_statuses(self.allow_redirects)if crawler:- # TODO use crawler.settingsself._finish_init(crawler)self._modern_init = Trueelse:diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 5e94f9271..9dcb3e4d1 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -755,7 +755,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase):def from_crawler(cls, crawler):settings = crawler.settingsstore_uri = settings["FILES_STORE"]- o = cls(store_uri, settings=settings, crawler=crawler)+ o = cls(store_uri, crawler=crawler)o._from_crawler_called = Truereturn odiff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 3f18c83f7..3ffef4102 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -13,7 +13,6 @@ from twisted.trial import unittestfrom scrapy.http import Request, Responsefrom scrapy.item import Field, Itemfrom scrapy.pipelines.images import ImageException, ImagesPipeline-from scrapy.settings import Settingsfrom scrapy.utils.test import get_crawlerskip_pillow: str | None@@ -392,10 +391,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):have different settings."""custom_settings = self._generate_fake_settings()- default_settings = Settings()- default_sts_pipe = ImagesPipeline(- self.tempdir, settings=default_settings, crawler=get_crawler(None) # TODO- )+ default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None))user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings))for pipe_attr, settings_attr in self.img_cls_attribute_names:expected_default_value = self.default_pipeline_settings.get(pipe_attr)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex a825de92a..58a2d3678 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -378,8 +378,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):def _assert_request_no3xx(self, pipeline_class, settings):- crawler = get_crawler(None, settings)- pipe = pipeline_class(settings=settings, crawler=crawler) # TODO+ pipe = pipeline_class(crawler=get_crawler(None, settings))request = Request("http://url")pipe._modify_media_request(request)