Scrapy, a fast high-level web crawling & scraping framework for Python.
[image_pipeline] style edits
Pawel Miechc22cc1096be5
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex 6cbac913c..f18789ab0 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -191,10 +191,11 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/orIf you need something more complex and want to override the custom pipelinebehaviour, see :ref:`topics-media-pipeline-override`.-If you have multiple image pipelines inheriting from ImagePipeline and you want to have different settings in different pipelines-you can set setting keys preceded with uppercase name of your pipeline class. E.g. if your pipeline is called-MyPipeline and you want to have custom IMAGES_URLS_FIELD you define setting MYPIPELINE_IMAGES_URLS_FIELD and your custom-settings will be used.+If you have multiple image pipelines inheriting from ImagePipeline and you want+to have different settings in different pipelines you can set setting keys+preceded with uppercase name of your pipeline class. E.g. if your pipeline is+called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define+setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.Additional features@@ -219,9 +220,9 @@ specifies the delay in number of days::The default value for both settings is 90 days.-If you have pipeline that subclasses FilesPipeline and you'd like to have different setting-for it you can set setting keys preceded by uppercase class name. E.g. given pipeline class-called MyPipeline you can set setting key:+If you have pipeline that subclasses FilesPipeline and you'd like to have+different setting for it you can set setting keys preceded by uppercase+class name. E.g. given pipeline class called MyPipeline you can set setting key:MYPIPELINE_FILES_EXPIRES = 180diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 73eda5f34..8cdc548f6 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -3,7 +3,7 @@ Files PipelineSee documentation in topics/media-pipeline.rst"""-+import functoolsimport hashlibimport osimport os.path@@ -232,18 +232,20 @@ class FilesPipeline(MediaPipeline):cls_name = "FilesPipeline"self.store = self._get_store(store_uri)+ resolve = functools.partial(self._key_for_pipe,+ base_class_name=cls_name)self.expires = settings.getint(- self._key_for_pipe('FILES_EXPIRES', cls_name), self.EXPIRES+ resolve('FILES_EXPIRES'), self.EXPIRES)if not hasattr(self, "FILES_URLS_FIELD"):self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELDif not hasattr(self, "FILES_RESULT_FIELD"):self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELDself.files_urls_field = settings.get(- self._key_for_pipe('FILES_URLS_FIELD', cls_name), self.FILES_URLS_FIELD+ resolve('FILES_URLS_FIELD'), self.FILES_URLS_FIELD)self.files_result_field = settings.get(- self._key_for_pipe('FILES_RESULT_FIELD', cls_name), self.FILES_RESULT_FIELD+ resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD)super(FilesPipeline, self).__init__(download_func=download_func)diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 73377e2c2..0278ec32f 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -3,7 +3,7 @@ Images PipelineSee documentation in topics/media-pipeline.rst"""-+import functoolsimport hashlibimport six@@ -62,22 +62,24 @@ class ImagesPipeline(FilesPipeline):if not hasattr(self, "IMAGES_URLS_FIELD"):self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD- default_images_urls_field = getattr(self, "IMAGES_URLS_FIELD", "DEFAULT_IMAGES_URLS_FIELD")+ resolve = functools.partial(self._key_for_pipe, base_class_name=cls_name)+self.images_urls_field = settings.get(- self._key_for_pipe('IMAGES_URLS_FIELD', cls_name), default_images_urls_field+ resolve('IMAGES_URLS_FIELD'),+ self.IMAGES_URLS_FIELD)- default_images_result_field = getattr(self, "IMAGES_RESULT_FIELD", "DEFAULT_IMAGES_RESULT_FIELD")self.images_result_field = settings.get(- self._key_for_pipe('IMAGES_RESULT_FIELD', cls_name), default_images_result_field+ resolve('IMAGES_RESULT_FIELD'),+ self.IMAGES_RESULT_FIELD)self.min_width = settings.getint(- self._key_for_pipe('IMAGES_MIN_WIDTH', cls_name), self.MIN_WIDTH+ resolve('IMAGES_MIN_WIDTH'), self.MIN_WIDTH)self.min_height = settings.getint(- self._key_for_pipe('IMAGES_MIN_HEIGHT', cls_name), self.MIN_HEIGHT+ resolve('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT)self.thumbs = settings.get(- self._key_for_pipe('IMAGES_THUMBS', cls_name), self.THUMBS+ resolve('IMAGES_THUMBS'), self.THUMBS)@classmethoddiff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 740312f8f..fe59b9f7c 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -28,18 +28,17 @@ class MediaPipeline(object):self.download_func = download_func- def _key_for_pipe(self, key, base_class_name):+ def _key_for_pipe(self, key, base_class_name=None):"""- Allow setting settings for user defined MediaPipelines that inherit from base.-- User can define setting key:-- MYPIPELINENAME_IMAGE_SETTING_NAME = <some value>-- and it will override default settings and class attributes.+ >>> result = MediaPipeline()._key_for_pipe("IMAGES")+ >>> assert result == "IMAGES"+ >>> class MyPipe(MediaPipeline):+ ... pass+ >>> other_key = MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")+ >>> assert other_key == "MYPIPE_IMAGES""""class_name = self.__class__.__name__- if class_name == base_class_name:+ if class_name == base_class_name or not base_class_name:return keyreturn "{}_{}".format(class_name.upper(), key)