Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into 3689-update

Andrey Rahmatullinb03d84e9fbf1
diff --cc scrapy/pipelines/images.py
index 48e8a7b83,4d59436ab..6a28a3b87
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@@ -146,21 -146,13 +158,21 @@@ class ImagesPipeline(FilesPipeline)
yield path, image, buf
for thumb_id, size in self.thumbs.items():
- thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
+ thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
- thumb_image, thumb_buf = self.convert_image(image, size)
+ if self._deprecated_convert_image:
+ thumb_image, thumb_buf = self.convert_image(image, size)
+ else:
+ thumb_image, thumb_buf = self.convert_image(image, size, buf)
yield thumb_path, thumb_image, thumb_buf
- def convert_image(self, image, size=None):
+ def convert_image(self, image, size=None, response_body=None):
+ if response_body is None:
+ warnings.warn(f'{self.__class__.__name__}.convert_image() method called in a deprecated way, '
+ 'method called without response_body argument.',
+ category=ScrapyDeprecationWarning, stacklevel=2)
+
if image.format == 'PNG' and image.mode == 'RGBA':
- background = Image.new('RGBA', image.size, (255, 255, 255))
+ background = self._Image.new('RGBA', image.size, (255, 255, 255))
background.paste(image, image)
image = background.convert('RGB')
elif image.mode == 'P':
@@@ -173,9 -165,14 +185,16 @@@
if size:
image = image.copy()
- image.thumbnail(size, Image.ANTIALIAS)
+ try:
+ # Image.Resampling.LANCZOS was added in Pillow 9.1.0
+ # remove this try except block,
+ # when updating the minimum requirements for Pillow.
+ resampling_filter = self._Image.Resampling.LANCZOS
+ except AttributeError:
+ resampling_filter = self._Image.ANTIALIAS
+ image.thumbnail(size, resampling_filter)
+ elif response_body is not None and image.format == 'JPEG':
+ return image, response_body
buf = BytesIO()
image.save(buf, 'JPEG')
diff --cc tests/test_pipeline_images.py
index 0a294db80,c189d08bf..81c9a027a
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@@ -1,19 -2,18 +2,21 @@@ import dataclasse
import hashlib
import io
import random
+import warnings
from shutil import rmtree
from tempfile import mkdtemp
+from unittest import skipIf
+from unittest.mock import patch
+ from warnings import catch_warnings
import attr
from itemadapter import ItemAdapter
from twisted.trial import unittest
+ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.item import Field, Item
- from scrapy.pipelines.images import ImageException, ImagesPipeline
-from scrapy.pipelines.images import ImagesPipeline, NoimagesDrop
++from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop
from scrapy.settings import Settings
from scrapy.utils.python import to_bytes
@@@ -234,6 -133,6 +231,22 @@@ class ImagesPipelineTestCase(unittest.T
self.assertEqual(converted.mode, 'RGB')
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
++ def test_thumbnail_name_from_item(self):
++ """
++ Custom thumbnail name based on item data, overriding default implementation
++ """
++
++ class CustomImagesPipeline(ImagesPipeline):
++ def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
++ return f"thumb/{thumb_id}/{item.get('path')}"
++
++ thumb_path = CustomImagesPipeline.from_settings(Settings(
++ {'IMAGES_STORE': self.tempdir}
++ )).thumb_path
++ item = dict(path='path-to-store-file')
++ request = Request("http://example.com")
++ self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')
++
class DeprecatedImagesPipeline(ImagesPipeline):
def file_key(self, url):