Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[image_pipeline] bring back uppercase pipeline attributes

Pawel Miech4cef1a1d0060
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c94794173..3e6ad554d 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -214,11 +214,14 @@ class FilesPipeline(MediaPipeline):
"""
MEDIA_NAME = "file"
+ EXPIRES = 90
STORE_SCHEMES = {
'': FSFilesStore,
'file': FSFilesStore,
's3': S3FilesStore,
}
+ DEFAULT_FILES_URLS_FIELD = 'file_urls'
+ DEFAULT_FILES_RESULT_FIELD = 'files'
def __init__(self, store_uri, download_func=None, settings=None):
if not store_uri:
@@ -228,9 +231,9 @@ class FilesPipeline(MediaPipeline):
settings = Settings(settings)
self.store = self._get_store(store_uri)
- self.expires = settings.getint('FILES_EXPIRES')
- self.files_urls_field = settings.get('FILES_URLS_FIELD')
- self.files_result_field = settings.get('FILES_RESULT_FIELD')
+ self.expires = settings.getint('FILES_EXPIRES', self.EXPIRES)
+ self.files_urls_field = settings.get('FILES_URLS_FIELD', self.DEFAULT_FILES_URLS_FIELD)
+ self.files_result_field = settings.get('FILES_RESULT_FIELD', self.DEFAULT_FILES_RESULT_FIELD)
super(FilesPipeline, self).__init__(download_func=download_func)
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index c597b6cca..ac78ee6c0 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -37,26 +37,31 @@ class ImagesPipeline(FilesPipeline):
"""
MEDIA_NAME = 'image'
+ MIN_WIDTH = 0
+ MIN_HEIGHT = 0
+ EXPIRES = 0
+ THUMBS = {}
+ IMAGES_URLS_FIELD = 'image_urls'
+ IMAGES_RESULT_FIELD = 'images'
def __init__(self, store_uri, download_func=None, settings=None):
super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)
-
+
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
- self.expires = settings.getint('IMAGES_EXPIRES')
- self.images_urls_field = settings.get('IMAGES_URLS_FIELD')
- self.images_result_field = settings.get('IMAGES_RESULT_FIELD')
- self.min_width = settings.getint('IMAGES_MIN_WIDTH')
- self.min_height = settings.getint('IMAGES_MIN_HEIGHT')
- self.thumbs = settings.get('IMAGES_THUMBS')
+ self.expires = settings.getint('IMAGES_EXPIRES', self.EXPIRES)
+ self.images_urls_field = settings.get('IMAGES_URLS_FIELD', self.IMAGES_URLS_FIELD)
+ self.images_result_field = settings.get('IMAGES_RESULT_FIELD', self.IMAGES_RESULT_FIELD)
+ self.min_width = settings.getint('IMAGES_MIN_WIDTH', self.MIN_WIDTH)
+ self.min_height = settings.getint('IMAGES_MIN_HEIGHT', self.MIN_HEIGHT)
+ self.thumbs = settings.get('IMAGES_THUMBS', self.THUMBS)
@classmethod
def from_settings(cls, settings):
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
-
store_uri = settings['IMAGES_STORE']
return cls(store_uri, settings=settings)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8f064f81e..1e447e9e2 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -179,13 +179,6 @@ HTTPCACHE_GZIP = False
HTTPPROXY_AUTH_ENCODING = 'latin-1'
-IMAGES_MIN_WIDTH = 0
-IMAGES_MIN_HEIGHT = 0
-IMAGES_EXPIRES = 90
-IMAGES_THUMBS = {}
-IMAGES_URLS_FIELD = 'image_urls'
-IMAGES_RESULT_FIELD = 'images'
-
ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
ITEM_PIPELINES = {}
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index f48547b0f..3ce138000 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -224,19 +224,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
def test_images_urls_field(self):
another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
'IMAGES_URLS_FIELD': 'funny_field'}))
- self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD'))
+ default = self.pipeline.IMAGES_URLS_FIELD
+ self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD', default))
self.assertEqual(another_pipeline.images_urls_field, 'funny_field')
def test_images_result_field(self):
another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
'IMAGES_RESULT_FIELD': 'funny_field'}))
- self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD'))
+ default = self.pipeline.IMAGES_RESULT_FIELD
+ self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD', default))
self.assertEqual(another_pipeline.images_result_field, 'funny_field')
def test_min_width(self):
another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
'IMAGES_MIN_WIDTH': 42}))
- self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH'))
+ default = self.pipeline.MIN_WIDTH
+ self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH', default))
self.assertEqual(another_pipeline.min_width, 42)
def test_min_height(self):
@@ -249,7 +252,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
custom_thumbs = {'small': (50, 50), 'big': (270, 270)}
another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,
'IMAGES_THUMBS': custom_thumbs}))
- self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS'))
+ default = self.pipeline.THUMBS
+ self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS', default))
self.assertEqual(another_pipeline.thumbs, custom_thumbs)