Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make image_format in ImagesPipeline.convert_image() optional. (#8457)

Andrey Rakhmatullin20df334115ca
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e4c73e09a..b0fa77daa 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -21,6 +21,7 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FilesPipeline, _checksum_algorithm
from scrapy.pipelines.media import FileException
from scrapy.utils.defer import ensure_awaitable
+from scrapy.utils.python import get_func_args_dict
if TYPE_CHECKING:
from collections.abc import Iterator
@@ -112,6 +113,18 @@ class ImagesPipeline(FilesPipeline):
self.preserve_format: bool = settings.getbool(
resolve("IMAGES_PRESERVE_FORMAT"), self.PRESERVE_FORMAT
)
+ params = get_func_args_dict(self.convert_image)
+ self._convert_image_takes_format: bool = "image_format" in params or any(
+ param.kind is param.VAR_KEYWORD for param in params.values()
+ )
+ if not self._convert_image_takes_format:
+ warnings.warn(
+ f"{type(self).__name__}.convert_image() does not accept an"
+ f" image_format keyword argument. Overriding convert_image()"
+ f" without it is deprecated.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@@ -185,7 +198,7 @@ class ImagesPipeline(FilesPipeline):
f"{self.min_width}x{self.min_height})"
)
- image, buf = self.convert_image(
+ image, buf = self._convert_image(
transposed_image,
image_format=image_format,
response_body=BytesIO(response.body),
@@ -196,12 +209,12 @@ class ImagesPipeline(FilesPipeline):
thumb_path = self.thumb_path(
request, thumb_id, response=response, info=info, item=item
)
- thumb_image, thumb_buf = self.convert_image(
+ thumb_image, thumb_buf = self._convert_image(
image, size, image_format=image_format, response_body=buf
)
yield thumb_path, thumb_image, thumb_buf
- def convert_image(
+ def _convert_image(
self,
image: Image.Image,
size: tuple[int, int] | None = None,
@@ -209,7 +222,32 @@ class ImagesPipeline(FilesPipeline):
image_format: str,
response_body: BytesIO,
) -> tuple[Image.Image, BytesIO]:
- target_format = image_format if self.preserve_format else "JPEG"
+ if self._convert_image_takes_format:
+ return self.convert_image(
+ image, size, image_format=image_format, response_body=response_body
+ )
+ return self.convert_image(image, size, response_body=response_body)
+
+ def convert_image(
+ self,
+ image: Image.Image,
+ size: tuple[int, int] | None = None,
+ *,
+ image_format: str | None = None,
+ response_body: BytesIO,
+ ) -> tuple[Image.Image, BytesIO]:
+ if image_format is None:
+ warnings.warn(
+ f"Calling {type(self).__name__}.convert_image() without an"
+ f" image_format keyword argument is deprecated.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ image_format = image.format
+ if self.preserve_format and image_format is not None:
+ target_format = image_format
+ else:
+ target_format = "JPEG"
if target_format == "JPEG":
if image_format in {"PNG", "WEBP"} and image.mode == "RGBA":
background = self._Image.new("RGBA", image.size, (255, 255, 255))
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index c23bb7d58..be8aa1f3d 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -16,7 +16,7 @@ import attr
import pytest
from itemadapter import ItemAdapter
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines.files import FilesPipeline, GCSFilesStore, S3FilesStore
@@ -220,6 +220,23 @@ class TestImagesPipeline:
assert thumb_path == "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
assert orig_thumb_buf.getvalue() == thumb_buf.getvalue()
+ def test_get_images_convert_image_without_image_format(self):
+ class OldImagesPipeline(ImagesPipeline):
+ def convert_image(self, image, size=None, *, response_body): # type: ignore[override]
+ return super().convert_image(image, size, response_body=response_body)
+
+ crawler = get_crawler(None, {"IMAGES_STORE": self.tempdir})
+ with pytest.warns(ScrapyDeprecationWarning, match="Overriding convert_image"):
+ pipeline = build_from_crawler(OldImagesPipeline, crawler)
+ _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+ resp = Response(url="https://example.com/image.jpg", body=buf.getvalue())
+ req = Request(url="https://example.com/image.jpg")
+ with pytest.warns(ScrapyDeprecationWarning, match="without an image_format"):
+ _, image, _ = next(
+ pipeline.get_images(response=resp, request=req, info=DUMMY_SPIDER_INFO)
+ )
+ assert image.format == "JPEG"
+
def test_get_transposed_images(self):
orig_im = Image.new("RGB", (2, 2), (0, 0, 0))
orig_im.putpixel((1, 1), (255, 0, 0))
@@ -387,6 +404,14 @@ class TestImagesPipeline:
assert converted.mode == "RGB"
assert converted.getcolors() == [(10000, (128, 128, 128))]
+ def test_convert_image_without_image_format(self):
+ im, buf = _create_image("PNG", "RGBA", (100, 100), (0, 127, 255, 50))
+ with pytest.warns(ScrapyDeprecationWarning, match="without an image_format"):
+ converted, _ = self.pipeline.convert_image(im, response_body=buf)
+ # The format of the image is used, so transparency becomes white.
+ assert converted.mode == "RGB"
+ assert converted.getcolors() == [(10000, (205, 230, 255))]
+
@pytest.mark.parametrize(
"bad_type",
[