Scrapy, a fast high-level web crawling & scraping framework for Python.
Change ImagesPipeline class attributes to instance attributes.
djunzu537083524e2a
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 1f6b0c7e7..c597b6cca 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -17,6 +17,7 @@ from PIL import Imagefrom scrapy.utils.misc import md5sumfrom scrapy.utils.python import to_bytesfrom scrapy.http import Request+from scrapy.settings import Settingsfrom scrapy.exceptions import DropItem#TODO: from scrapy.pipelines.media import MediaPipelinefrom scrapy.pipelines.files import FileException, FilesPipeline@@ -37,20 +38,27 @@ class ImagesPipeline(FilesPipeline):MEDIA_NAME = 'image'+ def __init__(self, store_uri, download_func=None, settings=None):+ super(ImagesPipeline, self).__init__(store_uri, settings=settings, download_func=download_func)++ if isinstance(settings, dict) or settings is None:+ settings = Settings(settings)++ self.expires = settings.getint('IMAGES_EXPIRES')+ self.images_urls_field = settings.get('IMAGES_URLS_FIELD')+ self.images_result_field = settings.get('IMAGES_RESULT_FIELD')+ self.min_width = settings.getint('IMAGES_MIN_WIDTH')+ self.min_height = settings.getint('IMAGES_MIN_HEIGHT')+ self.thumbs = settings.get('IMAGES_THUMBS')+@classmethoddef from_settings(cls, settings):- cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH')- cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT')- cls.EXPIRES = settings.getint('IMAGES_EXPIRES')- cls.THUMBS = settings.get('IMAGES_THUMBS')s3store = cls.STORE_SCHEMES['s3']s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']- cls.IMAGES_URLS_FIELD = settings.get('IMAGES_URLS_FIELD')- cls.IMAGES_RESULT_FIELD = settings.get('IMAGES_RESULT_FIELD')store_uri = settings['IMAGES_STORE']- return cls(store_uri)+ return cls(store_uri, settings=settings)def file_downloaded(self, response, request, info):return self.image_downloaded(response, request, info)@@ -73,14 +81,14 @@ class ImagesPipeline(FilesPipeline):orig_image = Image.open(BytesIO(response.body))width, height = orig_image.size- if width < self.MIN_WIDTH or height < self.MIN_HEIGHT:+ if width < self.min_width or height < self.min_height:raise ImageException("Image too small (%dx%d < %dx%d)" %- (width, height, self.MIN_WIDTH, self.MIN_HEIGHT))+ (width, height, self.min_width, self.min_height))image, buf = self.convert_image(orig_image)yield path, image, buf- for thumb_id, size in six.iteritems(self.THUMBS):+ for thumb_id, size in six.iteritems(self.thumbs):thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)thumb_image, thumb_buf = self.convert_image(image, size)yield thumb_path, thumb_image, thumb_buf@@ -102,11 +110,11 @@ class ImagesPipeline(FilesPipeline):return image, bufdef get_media_requests(self, item, info):- return [Request(x) for x in item.get(self.IMAGES_URLS_FIELD, [])]+ return [Request(x) for x in item.get(self.images_urls_field, [])]def item_completed(self, results, item, info):- if isinstance(item, dict) or self.IMAGES_RESULT_FIELD in item.fields:- item[self.IMAGES_RESULT_FIELD] = [x for ok, x in results if ok]+ if isinstance(item, dict) or self.images_result_field in item.fields:+ item[self.images_result_field] = [x for ok, x in results if ok]return itemdef file_path(self, request, response=None, info=None):