Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[image_pipeline] style edits

Pawel Miechc22cc1096be5
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 6cbac913c..f18789ab0 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -191,10 +191,11 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
If you need something more complex and want to override the custom pipeline
behaviour, see :ref:`topics-media-pipeline-override`.
-If you have multiple image pipelines inheriting from ImagePipeline and you want to have different settings in different pipelines
-you can set setting keys preceded with uppercase name of your pipeline class. E.g. if your pipeline is called
-MyPipeline and you want to have custom IMAGES_URLS_FIELD you define setting MYPIPELINE_IMAGES_URLS_FIELD and your custom
-settings will be used.
+If you have multiple image pipelines inheriting from ImagePipeline and you want
+to have different settings in different pipelines you can set setting keys
+preceded with uppercase name of your pipeline class. E.g. if your pipeline is
+called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define
+setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
Additional features
@@ -219,9 +220,9 @@ specifies the delay in number of days::
The default value for both settings is 90 days.
-If you have pipeline that subclasses FilesPipeline and you'd like to have different setting
-for it you can set setting keys preceded by uppercase class name. E.g. given pipeline class
-called MyPipeline you can set setting key:
+If you have pipeline that subclasses FilesPipeline and you'd like to have
+different setting for it you can set setting keys preceded by uppercase
+class name. E.g. given pipeline class called MyPipeline you can set setting key:
MYPIPELINE_FILES_EXPIRES = 180
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 73eda5f34..8cdc548f6 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -3,7 +3,7 @@ Files Pipeline
See documentation in topics/media-pipeline.rst
"""
-
+import functools
import hashlib
import os
import os.path
@@ -232,18 +232,20 @@ class FilesPipeline(MediaPipeline):
cls_name = "FilesPipeline"
self.store = self._get_store(store_uri)
+ resolve = functools.partial(self._key_for_pipe,
+ base_class_name=cls_name)
self.expires = settings.getint(
- self._key_for_pipe('FILES_EXPIRES', cls_name), self.EXPIRES
+ resolve('FILES_EXPIRES'), self.EXPIRES
)
if not hasattr(self, "FILES_URLS_FIELD"):
self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
if not hasattr(self, "FILES_RESULT_FIELD"):
self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD
self.files_urls_field = settings.get(
- self._key_for_pipe('FILES_URLS_FIELD', cls_name), self.FILES_URLS_FIELD
+ resolve('FILES_URLS_FIELD'), self.FILES_URLS_FIELD
)
self.files_result_field = settings.get(
- self._key_for_pipe('FILES_RESULT_FIELD', cls_name), self.FILES_RESULT_FIELD
+ resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
)
super(FilesPipeline, self).__init__(download_func=download_func)
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 73377e2c2..0278ec32f 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -3,7 +3,7 @@ Images Pipeline
See documentation in topics/media-pipeline.rst
"""
-
+import functools
import hashlib
import six
@@ -62,22 +62,24 @@ class ImagesPipeline(FilesPipeline):
if not hasattr(self, "IMAGES_URLS_FIELD"):
self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD
- default_images_urls_field = getattr(self, "IMAGES_URLS_FIELD", "DEFAULT_IMAGES_URLS_FIELD")
+ resolve = functools.partial(self._key_for_pipe, base_class_name=cls_name)
+
self.images_urls_field = settings.get(
- self._key_for_pipe('IMAGES_URLS_FIELD', cls_name), default_images_urls_field
+ resolve('IMAGES_URLS_FIELD'),
+ self.IMAGES_URLS_FIELD
)
- default_images_result_field = getattr(self, "IMAGES_RESULT_FIELD", "DEFAULT_IMAGES_RESULT_FIELD")
self.images_result_field = settings.get(
- self._key_for_pipe('IMAGES_RESULT_FIELD', cls_name), default_images_result_field
+ resolve('IMAGES_RESULT_FIELD'),
+ self.IMAGES_RESULT_FIELD
)
self.min_width = settings.getint(
- self._key_for_pipe('IMAGES_MIN_WIDTH', cls_name), self.MIN_WIDTH
+ resolve('IMAGES_MIN_WIDTH'), self.MIN_WIDTH
)
self.min_height = settings.getint(
- self._key_for_pipe('IMAGES_MIN_HEIGHT', cls_name), self.MIN_HEIGHT
+ resolve('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT
)
self.thumbs = settings.get(
- self._key_for_pipe('IMAGES_THUMBS', cls_name), self.THUMBS
+ resolve('IMAGES_THUMBS'), self.THUMBS
)
@classmethod
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 740312f8f..fe59b9f7c 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -28,18 +28,17 @@ class MediaPipeline(object):
self.download_func = download_func
- def _key_for_pipe(self, key, base_class_name):
+ def _key_for_pipe(self, key, base_class_name=None):
"""
- Allow setting settings for user defined MediaPipelines that inherit from base.
-
- User can define setting key:
-
- MYPIPELINENAME_IMAGE_SETTING_NAME = <some value>
-
- and it will override default settings and class attributes.
+ >>> result = MediaPipeline()._key_for_pipe("IMAGES")
+ >>> assert result == "IMAGES"
+ >>> class MyPipe(MediaPipeline):
+ ... pass
+ >>> other_key = MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
+ >>> assert other_key == "MYPIPE_IMAGES"
"""
class_name = self.__class__.__name__
- if class_name == base_class_name:
+ if class_name == base_class_name or not base_class_name:
return key
return "{}_{}".format(class_name.upper(), key)