Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

makes fix backward compatible

Anubhav Patel2a6bcdb413da
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 3450ee721..ca8ac7b83 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -126,16 +126,42 @@ class ImagesPipeline(FilesPipeline):
if width < self.min_width or height < self.min_height:
raise ImageException("Image too small (%dx%d < %dx%d)" %
(width, height, self.min_width, self.min_height))
+
+ def _is_convert_image_overriden():
+ import inspect
+ if six.PY2:
+ convert_image_signature = inspect.getargspec(self.convert_image)
+ elif six.PY3:
+ convert_image_signature = inspect.getfullargspec(self.convert_image)
+ if 'response_body' not in convert_image_signature.args:
+ return True
+ return False
- image, buf = self.convert_image(orig_image, BytesIO(response.body))
+ def _warn():
+ from scrapy.exceptions import ScrapyDeprecationWarning
+ import warnings
+ warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, '
+ 'overriden method does not accept response_body attribute.',
+ category=ScrapyDeprecationWarning, stacklevel=1)
+
+ convert_image_overriden = _is_convert_image_overriden()
+ if convert_image_overriden:
+ _warn()
+ image, buf = self.convert_image(orig_image)
+ else:
+ image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body))
yield path, image, buf
for thumb_id, size in six.iteritems(self.thumbs):
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
- thumb_image, thumb_buf = self.convert_image(image, buf, size)
+ if convert_image_overriden:
+ _warn()
+ thumb_image, thumb_buf = self.convert_image(image, size)
+ else:
+ thumb_image, thumb_buf = self.convert_image(image, size, buf)
yield thumb_path, thumb_image, thumb_buf
- def convert_image(self, image, response_body, size=None):
+ def convert_image(self, image, size=None, response_body=None):
if image.format == 'PNG' and image.mode == 'RGBA':
background = Image.new('RGBA', image.size, (255, 255, 255))
background.paste(image, image)
@@ -151,9 +177,16 @@ class ImagesPipeline(FilesPipeline):
if size:
image = image.copy()
image.thumbnail(size, Image.ANTIALIAS)
- elif image.format == 'JPEG':
- return image, response_body
-
+ else:
+ if not response_body:
+ from scrapy.exceptions import ScrapyDeprecationWarning
+ import warnings
+ warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, '
+ 'method called without response_body attribute.',
+ category=ScrapyDeprecationWarning, stacklevel=1)
+ elif image.format == 'JPEG':
+ return image, response_body
+
buf = BytesIO()
image.save(buf, 'JPEG')
return image, buf
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 0a2153ddb..3fe71589d 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -80,21 +80,21 @@ class ImagesPipelineTestCase(unittest.TestCase):
# straigh forward case: RGB and JPEG
COLOUR = (0, 127, 255)
im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR)
- converted, converted_buf = self.pipeline.convert_image(im, buf)
+ converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)
self.assertEqual(converted.mode, 'RGB')
self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
# check that we don't convert JPEGs again
self.assertEqual(converted_buf, buf)
# check that thumbnail keep image ratio
- thumbnail, _ = self.pipeline.convert_image(converted, converted_buf, size=(10, 25))
+ thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf)
self.assertEqual(thumbnail.mode, 'RGB')
self.assertEqual(thumbnail.size, (10, 10))
# transparency case: RGBA and PNG
COLOUR = (0, 127, 255, 50)
im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR)
- converted, _ = self.pipeline.convert_image(im, buf)
+ converted, _ = self.pipeline.convert_image(im, response_body=buf)
self.assertEqual(converted.mode, 'RGB')
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
@@ -102,7 +102,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
COLOUR = (0, 127, 255, 50)
im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR)
im = im.convert('P')
- converted, _ = self.pipeline.convert_image(im, buf)
+ converted, _ = self.pipeline.convert_image(im, response_body=buf)
self.assertEqual(converted.mode, 'RGB')
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])