Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into 3689-update
Andrey Rahmatullinb03d84e9fbf1
diff --cc scrapy/pipelines/images.pyindex 48e8a7b83,4d59436ab..6a28a3b87--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@@ -146,21 -146,13 +158,21 @@@ class ImagesPipeline(FilesPipeline)yield path, image, buffor thumb_id, size in self.thumbs.items():- thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)+ thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)- thumb_image, thumb_buf = self.convert_image(image, size)+ if self._deprecated_convert_image:+ thumb_image, thumb_buf = self.convert_image(image, size)+ else:+ thumb_image, thumb_buf = self.convert_image(image, size, buf)yield thumb_path, thumb_image, thumb_buf- def convert_image(self, image, size=None):+ def convert_image(self, image, size=None, response_body=None):+ if response_body is None:+ warnings.warn(f'{self.__class__.__name__}.convert_image() method called in a deprecated way, '+ 'method called without response_body argument.',+ category=ScrapyDeprecationWarning, stacklevel=2)+if image.format == 'PNG' and image.mode == 'RGBA':- background = Image.new('RGBA', image.size, (255, 255, 255))+ background = self._Image.new('RGBA', image.size, (255, 255, 255))background.paste(image, image)image = background.convert('RGB')elif image.mode == 'P':@@@ -173,9 -165,14 +185,16 @@@if size:image = image.copy()- image.thumbnail(size, Image.ANTIALIAS)+ try:+ # Image.Resampling.LANCZOS was added in Pillow 9.1.0+ # remove this try except block,+ # when updating the minimum requirements for Pillow.+ resampling_filter = self._Image.Resampling.LANCZOS+ except AttributeError:+ resampling_filter = self._Image.ANTIALIAS+ image.thumbnail(size, resampling_filter)+ elif response_body is not None and image.format == 'JPEG':+ return image, response_bodybuf = BytesIO()image.save(buf, 'JPEG')diff --cc tests/test_pipeline_images.pyindex 0a294db80,c189d08bf..81c9a027a--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@@ -1,19 -2,18 +2,21 @@@ import dataclasseimport hashlibimport ioimport random+import warningsfrom shutil import rmtreefrom tempfile import mkdtemp+from unittest import skipIf+from unittest.mock import patch+ from warnings import catch_warningsimport attrfrom itemadapter import ItemAdapterfrom twisted.trial import unittest+ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.item import Field, Item- from scrapy.pipelines.images import ImageException, ImagesPipeline-from scrapy.pipelines.images import ImagesPipeline, NoimagesDrop++from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDropfrom scrapy.settings import Settingsfrom scrapy.utils.python import to_bytes@@@ -234,6 -133,6 +231,22 @@@ class ImagesPipelineTestCase(unittest.Tself.assertEqual(converted.mode, 'RGB')self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])++ def test_thumbnail_name_from_item(self):++ """++ Custom thumbnail name based on item data, overriding default implementation++ """++++ class CustomImagesPipeline(ImagesPipeline):++ def thumb_path(self, request, thumb_id, response=None, info=None, item=None):++ return f"thumb/{thumb_id}/{item.get('path')}"++++ thumb_path = CustomImagesPipeline.from_settings(Settings(++ {'IMAGES_STORE': self.tempdir}++ )).thumb_path++ item = dict(path='path-to-store-file')++ request = Request("http://example.com")++ self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')++class DeprecatedImagesPipeline(ImagesPipeline):def file_key(self, url):