Scrapy, a fast high-level web crawling & scraping framework for Python.
Make image_format in ImagesPipeline.convert_image() optional. (#8457)
Andrey Rakhmatullin20df334115ca
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex e4c73e09a..b0fa77daa 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -21,6 +21,7 @@ from scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FilesPipeline, _checksum_algorithmfrom scrapy.pipelines.media import FileExceptionfrom scrapy.utils.defer import ensure_awaitable+from scrapy.utils.python import get_func_args_dictif TYPE_CHECKING:from collections.abc import Iterator@@ -112,6 +113,18 @@ class ImagesPipeline(FilesPipeline):self.preserve_format: bool = settings.getbool(resolve("IMAGES_PRESERVE_FORMAT"), self.PRESERVE_FORMAT)+ params = get_func_args_dict(self.convert_image)+ self._convert_image_takes_format: bool = "image_format" in params or any(+ param.kind is param.VAR_KEYWORD for param in params.values()+ )+ if not self._convert_image_takes_format:+ warnings.warn(+ f"{type(self).__name__}.convert_image() does not accept an"+ f" image_format keyword argument. Overriding convert_image()"+ f" without it is deprecated.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:@@ -185,7 +198,7 @@ class ImagesPipeline(FilesPipeline):f"{self.min_width}x{self.min_height})")- image, buf = self.convert_image(+ image, buf = self._convert_image(transposed_image,image_format=image_format,response_body=BytesIO(response.body),@@ -196,12 +209,12 @@ class ImagesPipeline(FilesPipeline):thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)- thumb_image, thumb_buf = self.convert_image(+ thumb_image, thumb_buf = self._convert_image(image, size, image_format=image_format, response_body=buf)yield thumb_path, thumb_image, thumb_buf- def convert_image(+ def _convert_image(self,image: Image.Image,size: tuple[int, int] | None = None,@@ -209,7 +222,32 @@ class ImagesPipeline(FilesPipeline):image_format: str,response_body: BytesIO,) -> tuple[Image.Image, BytesIO]:- target_format = image_format if self.preserve_format else "JPEG"+ if self._convert_image_takes_format:+ return self.convert_image(+ image, size, image_format=image_format, response_body=response_body+ )+ return self.convert_image(image, size, response_body=response_body)++ def convert_image(+ self,+ image: Image.Image,+ size: tuple[int, int] | None = None,+ *,+ image_format: str | None = None,+ response_body: BytesIO,+ ) -> tuple[Image.Image, BytesIO]:+ if image_format is None:+ warnings.warn(+ f"Calling {type(self).__name__}.convert_image() without an"+ f" image_format keyword argument is deprecated.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ image_format = image.format+ if self.preserve_format and image_format is not None:+ target_format = image_format+ else:+ target_format = "JPEG"if target_format == "JPEG":if image_format in {"PNG", "WEBP"} and image.mode == "RGBA":background = self._Image.new("RGBA", image.size, (255, 255, 255))diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex c23bb7d58..be8aa1f3d 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -16,7 +16,7 @@ import attrimport pytestfrom itemadapter import ItemAdapter-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.item import Field, Itemfrom scrapy.pipelines.files import FilesPipeline, GCSFilesStore, S3FilesStore@@ -220,6 +220,23 @@ class TestImagesPipeline:assert thumb_path == "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"assert orig_thumb_buf.getvalue() == thumb_buf.getvalue()+ def test_get_images_convert_image_without_image_format(self):+ class OldImagesPipeline(ImagesPipeline):+ def convert_image(self, image, size=None, *, response_body): # type: ignore[override]+ return super().convert_image(image, size, response_body=response_body)++ crawler = get_crawler(None, {"IMAGES_STORE": self.tempdir})+ with pytest.warns(ScrapyDeprecationWarning, match="Overriding convert_image"):+ pipeline = build_from_crawler(OldImagesPipeline, crawler)+ _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))+ resp = Response(url="https://example.com/image.jpg", body=buf.getvalue())+ req = Request(url="https://example.com/image.jpg")+ with pytest.warns(ScrapyDeprecationWarning, match="without an image_format"):+ _, image, _ = next(+ pipeline.get_images(response=resp, request=req, info=DUMMY_SPIDER_INFO)+ )+ assert image.format == "JPEG"+def test_get_transposed_images(self):orig_im = Image.new("RGB", (2, 2), (0, 0, 0))orig_im.putpixel((1, 1), (255, 0, 0))@@ -387,6 +404,14 @@ class TestImagesPipeline:assert converted.mode == "RGB"assert converted.getcolors() == [(10000, (128, 128, 128))]+ def test_convert_image_without_image_format(self):+ im, buf = _create_image("PNG", "RGBA", (100, 100), (0, 127, 255, 50))+ with pytest.warns(ScrapyDeprecationWarning, match="without an image_format"):+ converted, _ = self.pipeline.convert_image(im, response_body=buf)+ # The format of the image is used, so transparency becomes white.+ assert converted.mode == "RGB"+ assert converted.getcolors() == [(10000, (205, 230, 255))]+@pytest.mark.parametrize("bad_type",[