Scrapy, a fast high-level web crawling & scraping framework for Python.
[image_pipeline] bring back uppercase pipeline attributes
Pawel Miech4cef1a1d0060
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex c94794173..3e6ad554d 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -214,11 +214,14 @@ class FilesPipeline(MediaPipeline):"""MEDIA_NAME = "file"+ EXPIRES = 90STORE_SCHEMES = {'': FSFilesStore,'file': FSFilesStore,'s3': S3FilesStore,}+ DEFAULT_FILES_URLS_FIELD = 'file_urls'+ DEFAULT_FILES_RESULT_FIELD = 'files'def __init__(self, store_uri, download_func=None, settings=None):if not store_uri:@@ -228,9 +231,9 @@ class FilesPipeline(MediaPipeline):settings = Settings(settings)self.store = self._get_store(store_uri)- self.expires = settings.getint('FILES_EXPIRES')- self.files_urls_field = settings.get('FILES_URLS_FIELD')- self.files_result_field = settings.get('FILES_RESULT_FIELD')+ self.expires = settings.getint('FILES_EXPIRES', self.EXPIRES)+ self.files_urls_field = settings.get('FILES_URLS_FIELD', self.DEFAULT_FILES_URLS_FIELD)+ self.files_result_field = settings.get('FILES_RESULT_FIELD', self.DEFAULT_FILES_RESULT_FIELD)super(FilesPipeline, self).__init__(download_func=download_func)diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex c597b6cca..ac78ee6c0 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -37,26 +37,31 @@ class ImagesPipeline(FilesPipeline):"""MEDIA_NAME = 'image'+ MIN_WIDTH = 0+ MIN_HEIGHT = 0+ EXPIRES = 0+ THUMBS = {}+ IMAGES_URLS_FIELD = 'image_urls'+ IMAGES_RESULT_FIELD = 'images'def __init__(self, store_uri, download_func=None, settings=None):super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)-+if isinstance(settings, dict) or settings is None:settings = Settings(settings)- self.expires = settings.getint('IMAGES_EXPIRES')- self.images_urls_field = settings.get('IMAGES_URLS_FIELD')- self.images_result_field = settings.get('IMAGES_RESULT_FIELD')- self.min_width = settings.getint('IMAGES_MIN_WIDTH')- self.min_height = settings.getint('IMAGES_MIN_HEIGHT')- self.thumbs = settings.get('IMAGES_THUMBS')+ self.expires = settings.getint('IMAGES_EXPIRES', self.EXPIRES)+ self.images_urls_field = settings.get('IMAGES_URLS_FIELD', self.IMAGES_URLS_FIELD)+ self.images_result_field = settings.get('IMAGES_RESULT_FIELD', self.IMAGES_RESULT_FIELD)+ self.min_width = settings.getint('IMAGES_MIN_WIDTH', self.MIN_WIDTH)+ self.min_height = settings.getint('IMAGES_MIN_HEIGHT', self.MIN_HEIGHT)+ self.thumbs = settings.get('IMAGES_THUMBS', self.THUMBS)@classmethoddef from_settings(cls, settings):s3store = cls.STORE_SCHEMES['s3']s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']-store_uri = settings['IMAGES_STORE']return cls(store_uri, settings=settings)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 8f064f81e..1e447e9e2 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -179,13 +179,6 @@ HTTPCACHE_GZIP = FalseHTTPPROXY_AUTH_ENCODING = 'latin-1'-IMAGES_MIN_WIDTH = 0-IMAGES_MIN_HEIGHT = 0-IMAGES_EXPIRES = 90-IMAGES_THUMBS = {}-IMAGES_URLS_FIELD = 'image_urls'-IMAGES_RESULT_FIELD = 'images'-ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'ITEM_PIPELINES = {}diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex f48547b0f..3ce138000 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -224,19 +224,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):def test_images_urls_field(self):another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,'IMAGES_URLS_FIELD': 'funny_field'}))- self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD'))+ default = self.pipeline.IMAGES_URLS_FIELD+ self.assertEqual(self.pipeline.images_urls_field, self.default_settings.get('IMAGES_URLS_FIELD', default))self.assertEqual(another_pipeline.images_urls_field, 'funny_field')def test_images_result_field(self):another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,'IMAGES_RESULT_FIELD': 'funny_field'}))- self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD'))+ default = self.pipeline.IMAGES_RESULT_FIELD+ self.assertEqual(self.pipeline.images_result_field, self.default_settings.get('IMAGES_RESULT_FIELD', default))self.assertEqual(another_pipeline.images_result_field, 'funny_field')def test_min_width(self):another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,'IMAGES_MIN_WIDTH': 42}))- self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH'))+ default = self.pipeline.MIN_WIDTH+ self.assertEqual(self.pipeline.min_width, self.default_settings.getint('IMAGES_MIN_WIDTH', default))self.assertEqual(another_pipeline.min_width, 42)def test_min_height(self):@@ -249,7 +252,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):custom_thumbs = {'small': (50, 50), 'big': (270, 270)}another_pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': self.tempdir,'IMAGES_THUMBS': custom_thumbs}))- self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS'))+ default = self.pipeline.THUMBS+ self.assertEqual(self.pipeline.thumbs, self.default_settings.get('IMAGES_THUMBS', default))self.assertEqual(another_pipeline.thumbs, custom_thumbs)