Scrapy, a fast high-level web crawling & scraping framework for Python.
Add IMAGES_PRESERVE_FORMAT to skip forced JPEG conversion (#8198)
Adrian4439b82d3e78
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex 362480515..ca2bfae64 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -521,6 +521,26 @@ Example of image files stored using ``small`` and ``big`` thumbnail names::The first one is the full image, as downloaded from the site.+.. _preserving-image-format:++Preserving the original image format+------------------------------------++.. setting:: IMAGES_PRESERVE_FORMAT++By default, the Images Pipeline converts every image to JPEG, flattening+transparency onto a white background. Set :setting:`IMAGES_PRESERVE_FORMAT`+to ``True`` to keep each image, and its thumbnails, in the format it was+downloaded in instead:++.. code-block:: python++ IMAGES_PRESERVE_FORMAT = True++Stored files then get no extension, since the actual format is only known+once the image has been downloaded; see :ref:`file-naming-response` to+derive one from the response instead.+Filtering out small images--------------------------diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 7186fc8de..c97f119ca 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -52,6 +52,7 @@ class ImagesPipeline(FilesPipeline):MIN_HEIGHT: int = 0EXPIRES: int = 90THUMBS: ClassVar[dict[str, tuple[int, int]]] = {}+ PRESERVE_FORMAT: bool = FalseDEFAULT_IMAGES_URLS_FIELD = "image_urls"DEFAULT_IMAGES_RESULT_FIELD = "images"@@ -110,6 +111,9 @@ class ImagesPipeline(FilesPipeline):self.thumbs: dict[str, tuple[int, int]] = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS)+ self.preserve_format: bool = settings.getbool(+ resolve("IMAGES_PRESERVE_FORMAT"), self.PRESERVE_FORMAT+ )@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:@@ -156,13 +160,14 @@ class ImagesPipeline(FilesPipeline):buf.seek(0)checksum = _md5sum(buf)width, height = image.size+ content_type = self._Image.MIME.get(image.format or "JPEG", "image/jpeg")await ensure_awaitable(self.store.persist_file(path,buf,info,meta={"width": width, "height": height},- headers={"Content-Type": "image/jpeg"},+ headers={"Content-Type": content_type},))assert checksum is not None@@ -178,6 +183,8 @@ class ImagesPipeline(FilesPipeline):) -> Iterator[tuple[str, Image.Image, BytesIO]]:path = self.file_path(request, response=response, info=info, item=item)orig_image = self._Image.open(BytesIO(response.body))+ assert orig_image.format is not None+ image_format = orig_image.formattransposed_image = self._ImageOps.exif_transpose(orig_image)width, height = transposed_image.size@@ -189,7 +196,9 @@ class ImagesPipeline(FilesPipeline):)image, buf = self.convert_image(- transposed_image, response_body=BytesIO(response.body)+ transposed_image,+ image_format=image_format,+ response_body=BytesIO(response.body),)yield path, image, buf@@ -197,7 +206,9 @@ class ImagesPipeline(FilesPipeline):thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)- thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf)+ thumb_image, thumb_buf = self.convert_image(+ image, size, image_format=image_format, response_body=buf+ )yield thumb_path, thumb_image, thumb_bufdef convert_image(@@ -205,19 +216,22 @@ class ImagesPipeline(FilesPipeline):image: Image.Image,size: tuple[int, int] | None = None,*,+ image_format: str,response_body: BytesIO,) -> tuple[Image.Image, BytesIO]:- if image.format in {"PNG", "WEBP"} and image.mode == "RGBA":- background = self._Image.new("RGBA", image.size, (255, 255, 255))- background.paste(image, image)- image = background.convert("RGB")- elif image.mode == "P":- image = image.convert("RGBA")- background = self._Image.new("RGBA", image.size, (255, 255, 255))- background.paste(image, image)- image = background.convert("RGB")- elif image.mode != "RGB":- image = image.convert("RGB")+ target_format = image_format if self.preserve_format else "JPEG"+ if target_format == "JPEG":+ if image_format in {"PNG", "WEBP"} and image.mode == "RGBA":+ background = self._Image.new("RGBA", image.size, (255, 255, 255))+ background.paste(image, image)+ image = background.convert("RGB")+ elif image.mode == "P":+ image = image.convert("RGBA")+ background = self._Image.new("RGBA", image.size, (255, 255, 255))+ background.paste(image, image)+ image = background.convert("RGB")+ elif image.mode != "RGB":+ image = image.convert("RGB")if size:image = image.copy()@@ -229,11 +243,13 @@ class ImagesPipeline(FilesPipeline):except AttributeError:resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]image.thumbnail(size, resampling_filter)- elif image.format == "JPEG":+ elif target_format == image_format:+ image.format = target_formatreturn image, response_bodybuf = BytesIO()- image.save(buf, "JPEG")+ image.save(buf, target_format)+ image.format = target_formatreturn image, bufdef get_media_requests(@@ -262,7 +278,8 @@ class ImagesPipeline(FilesPipeline):item: Any = None,) -> str:image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324- return f"full/{image_guid}.jpg"+ ext = "" if self.preserve_format else ".jpg"+ return f"full/{image_guid}{ext}"def thumb_path(self,@@ -274,4 +291,5 @@ class ImagesPipeline(FilesPipeline):item: Any = None,) -> str:thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324- return f"thumbs/{thumb_id}/{thumb_guid}.jpg"+ ext = "" if self.preserve_format else ".jpg"+ return f"thumbs/{thumb_id}/{thumb_guid}{ext}"diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 74b4f3a9b..04c946f57 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -97,6 +97,22 @@ class TestImagesPipeline:== "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg")+ def test_file_path_preserve_format(self):+ pipeline = build_from_crawler(+ ImagesPipeline,+ get_crawler(+ None, {"IMAGES_STORE": self.tempdir, "IMAGES_PRESERVE_FORMAT": True}+ ),+ )+ assert (+ pipeline.file_path(Request("https://dev.mydeco.com/mydeco.gif"))+ == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d"+ )+ assert (+ pipeline.thumb_path(Request("https://dev.mydeco.com/mydeco.gif"), "50")+ == "thumbs/50/3fd165099d8e71b8a48b2683946e64dbfad8b52d"+ )+def test_thumbnail_name(self):thumb_path = self.pipeline.thumb_pathname = "50"@@ -240,12 +256,81 @@ class TestImagesPipeline:assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue()assert Path(self.tempdir, "thumbs", "small", name).exists()+ def test_convert_image_preserve_format(self):+ pipeline = build_from_crawler(+ ImagesPipeline,+ get_crawler(+ None, {"IMAGES_STORE": self.tempdir, "IMAGES_PRESERVE_FORMAT": True}+ ),+ )++ SIZE = (100, 100)+ COLOUR = (0, 127, 255, 50)+ im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)++ converted, converted_buf = pipeline.convert_image(+ im, image_format="PNG", response_body=buf+ )+ assert converted.mode == "RGBA"+ assert converted.format == "PNG"+ # the original bytes are reused, no re-encoding happens+ assert converted_buf == buf++ thumbnail, thumbnail_buf = pipeline.convert_image(+ converted,+ size=(10, 25),+ image_format="PNG",+ response_body=converted_buf,+ )+ assert thumbnail.mode == "RGBA"+ assert thumbnail.format == "PNG"+ assert thumbnail.size == (10, 10)+ assert Image.open(thumbnail_buf).format == "PNG"++ @coroutine_test+ async def test_image_downloaded_preserve_format(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ pipeline = build_from_crawler(+ ImagesPipeline,+ get_crawler(+ None, {"IMAGES_STORE": self.tempdir, "IMAGES_PRESERVE_FORMAT": True}+ ),+ )+ pipeline.thumbs = {"small": (20, 20)}+ _, buf = _create_image("PNG", "RGBA", (50, 50), (0, 127, 255, 50))+ url = "https://dev.mydeco.com/mydeco.gif"+ response = Response(url=url, body=buf.getvalue())++ persisted = []+ monkeypatch.setattr(+ pipeline.store,+ "persist_file",+ lambda path, b, info, meta=None, headers=None: persisted.append(+ (path, b.getvalue(), headers)+ ),+ )++ await pipeline.image_downloaded(response, Request(url=url), DUMMY_SPIDER_INFO)++ assert len(persisted) == 2+ full_path, full_bytes, full_headers = persisted[0]+ assert full_path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d"+ assert full_bytes == buf.getvalue()+ assert full_headers == {"Content-Type": "image/png"}++ _, thumb_bytes, thumb_headers = persisted[1]+ assert Image.open(io.BytesIO(thumb_bytes)).format == "PNG"+ assert thumb_headers == {"Content-Type": "image/png"}+def test_convert_image(self):SIZE = (100, 100)# straight forward case: RGB and JPEGCOLOUR: tuple[int, ...] = (0, 127, 255)im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR)- converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)+ converted, converted_buf = self.pipeline.convert_image(+ im, image_format="JPEG", response_body=buf+ )assert converted.mode == "RGB"assert converted.getcolors() == [(10000, COLOUR)]# check that we don't convert JPEGs again@@ -253,7 +338,7 @@ class TestImagesPipeline:# check that thumbnail keep image ratiothumbnail, _ = self.pipeline.convert_image(- converted, size=(10, 25), response_body=converted_buf+ converted, size=(10, 25), image_format="JPEG", response_body=converted_buf)assert thumbnail.mode == "RGB"assert thumbnail.size == (10, 10)@@ -261,7 +346,9 @@ class TestImagesPipeline:# transparency case: RGBA and PNGCOLOUR = (0, 127, 255, 50)im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)- converted, _ = self.pipeline.convert_image(im, response_body=buf)+ converted, _ = self.pipeline.convert_image(+ im, image_format="PNG", response_body=buf+ )assert converted.mode == "RGB"assert converted.getcolors() == [(10000, (205, 230, 255))]@@ -269,10 +356,20 @@ class TestImagesPipeline:COLOUR = (0, 127, 255, 50)im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)im = im.convert("P")- converted, _ = self.pipeline.convert_image(im, response_body=buf)+ converted, _ = self.pipeline.convert_image(+ im, image_format="PNG", response_body=buf+ )assert converted.mode == "RGB"assert converted.getcolors() == [(10000, (205, 230, 255))]+ # grayscale case: L and PNG+ im, buf = _create_image("PNG", "L", SIZE, 128)+ converted, _ = self.pipeline.convert_image(+ im, image_format="PNG", response_body=buf+ )+ assert converted.mode == "RGB"+ assert converted.getcolors() == [(10000, (128, 128, 128))]+def test_convert_image_legacy_resampling_filter(self, monkeypatch: pytest.MonkeyPatch) -> None:@@ -287,7 +384,9 @@ class TestImagesPipeline:)im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255))- thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf)+ thumbnail, _ = self.pipeline.convert_image(+ im, size=(10, 25), image_format="JPEG", response_body=buf+ )assert thumbnail.size == (10, 10)