Scrapy, a fast high-level web crawling & scraping framework for Python.
added test case for get_images
drs-116565adc47136
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 47d688c62..09194a0fe 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -18,7 +18,7 @@ from scrapy.pipelines.files import FileException, FilesPipeline# TODO: from scrapy.pipelines.media import MediaPipelinefrom scrapy.settings import Settingsfrom scrapy.utils.misc import md5sum-from scrapy.utils.python import to_bytes, get_func_args+from scrapy.utils.python import get_func_args, to_bytesclass NoimagesDrop(DropItem):@@ -136,7 +136,7 @@ class ImagesPipeline(FilesPipeline):if self._deprecated_convert_image:warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ''overriden method does not accept response_body argument.',- category=ScrapyDeprecationWarning, stacklevel=1)+ category=ScrapyDeprecationWarning)if self._deprecated_convert_image:image, buf = self.convert_image(orig_image)@@ -156,7 +156,7 @@ class ImagesPipeline(FilesPipeline):if response_body is None:warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ''method called without response_body argument.',- category=ScrapyDeprecationWarning, stacklevel=1)+ category=ScrapyDeprecationWarning, stacklevel=2)if image.format == 'PNG' and image.mode == 'RGBA':background = Image.new('RGBA', image.size, (255, 255, 255))diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 6b71b64bc..8e98b8734 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -5,6 +5,7 @@ import warningsfrom shutil import rmtreefrom tempfile import mkdtempfrom unittest import skipIf+from unittest.mock import patchimport attrfrom itemadapter import ItemAdapter@@ -12,7 +13,7 @@ from twisted.trial import unittestfrom scrapy.http import Request, Responsefrom scrapy.item import Field, Item-from scrapy.pipelines.images import ImagesPipeline+from scrapy.pipelines.images import ImageException, ImagesPipelinefrom scrapy.settings import Settingsfrom scrapy.utils.python import to_bytes@@ -93,6 +94,78 @@ class ImagesPipelineTestCase(unittest.TestCase):info=object()),'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')+ def test_get_images_exception(self):+ self.pipeline.min_width = 100+ self.pipeline.min_height = 100++ _, buf1 = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))+ _, buf2 = _create_image('JPEG', 'RGB', (150, 50), (0, 0, 0))+ _, buf3 = _create_image('JPEG', 'RGB', (50, 150), (0, 0, 0))++ resp1 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf1.getvalue())+ resp2 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf2.getvalue())+ resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue())+ req = Request(url="https://dev.mydeco.com/mydeco.gif")++ with self.assertRaises(ImageException):+ next(self.pipeline.get_images(response=resp1, request=req, info=object()))+ with self.assertRaises(ImageException):+ next(self.pipeline.get_images(response=resp2, request=req, info=object()))+ with self.assertRaises(ImageException):+ next(self.pipeline.get_images(response=resp3, request=req, info=object()))++ def test_get_images_new(self):+ self.pipeline.min_width = 0+ self.pipeline.min_height = 0+ self.pipeline.thumbs = {'small': (20, 20)}++ orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))+ orig_thumb, orig_thumb_buf = _create_image('JPEG', 'RGB', (20, 20), (0, 0, 0))+ resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())+ req = Request(url="https://dev.mydeco.com/mydeco.gif")++ get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object())++ path, new_im, new_buf = next(get_images_gen)+ self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')+ self.assertEqual(orig_im, new_im)+ self.assertEqual(buf.getvalue(), new_buf.getvalue())++ thumb_path, thumb_img, thumb_buf = next(get_images_gen)+ self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')+ self.assertEqual(thumb_img, thumb_img)+ self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue())++ def test_get_images_old(self):+ self.pipeline.thumbs = {'small': (20, 20)}+ orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))+ resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())+ req = Request(url="https://dev.mydeco.com/mydeco.gif")++ def overridden_convert_image(image, size=None):+ im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))+ return im, buf++ with patch.object(self.pipeline, 'convert_image', overridden_convert_image):+ with warnings.catch_warnings(record=True) as w:+ warnings.simplefilter('always')+ get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object())+ path, new_im, new_buf = next(get_images_gen)+ self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')+ self.assertEqual(orig_im.mode, new_im.mode)+ self.assertEqual(orig_im.getcolors(), new_im.getcolors())+ self.assertEqual(buf.getvalue(), new_buf.getvalue())++ thumb_path, thumb_img, thumb_buf = next(get_images_gen)+ self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')+ self.assertEqual(orig_im.mode, thumb_img.mode)+ self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())+ self.assertEqual(buf.getvalue(), thumb_buf.getvalue())++ expected_warning_msg = ('ImagesPipeline.convert_image() method overriden in a deprecated way, '+ 'overriden method does not accept response_body argument.')+ self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1)+def test_convert_image_old(self):# tests for old APIwith warnings.catch_warnings(record=True) as w: