Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add IMAGES_PRESERVE_FORMAT to skip forced JPEG conversion (#8198)

Adrian4439b82d3e78
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 362480515..ca2bfae64 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -521,6 +521,26 @@ Example of image files stored using ``small`` and ``big`` thumbnail names::
The first one is the full image, as downloaded from the site.
+.. _preserving-image-format:
+
+Preserving the original image format
+------------------------------------
+
+.. setting:: IMAGES_PRESERVE_FORMAT
+
+By default, the Images Pipeline converts every image to JPEG, flattening
+transparency onto a white background. Set :setting:`IMAGES_PRESERVE_FORMAT`
+to ``True`` to keep each image, and its thumbnails, in the format it was
+downloaded in instead:
+
+.. code-block:: python
+
+ IMAGES_PRESERVE_FORMAT = True
+
+Stored files then get no extension, since the actual format is only known
+once the image has been downloaded; see :ref:`file-naming-response` to
+derive one from the response instead.
+
Filtering out small images
--------------------------
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 7186fc8de..c97f119ca 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -52,6 +52,7 @@ class ImagesPipeline(FilesPipeline):
MIN_HEIGHT: int = 0
EXPIRES: int = 90
THUMBS: ClassVar[dict[str, tuple[int, int]]] = {}
+ PRESERVE_FORMAT: bool = False
DEFAULT_IMAGES_URLS_FIELD = "image_urls"
DEFAULT_IMAGES_RESULT_FIELD = "images"
@@ -110,6 +111,9 @@ class ImagesPipeline(FilesPipeline):
self.thumbs: dict[str, tuple[int, int]] = settings.get(
resolve("IMAGES_THUMBS"), self.THUMBS
)
+ self.preserve_format: bool = settings.getbool(
+ resolve("IMAGES_PRESERVE_FORMAT"), self.PRESERVE_FORMAT
+ )
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@@ -156,13 +160,14 @@ class ImagesPipeline(FilesPipeline):
buf.seek(0)
checksum = _md5sum(buf)
width, height = image.size
+ content_type = self._Image.MIME.get(image.format or "JPEG", "image/jpeg")
await ensure_awaitable(
self.store.persist_file(
path,
buf,
info,
meta={"width": width, "height": height},
- headers={"Content-Type": "image/jpeg"},
+ headers={"Content-Type": content_type},
)
)
assert checksum is not None
@@ -178,6 +183,8 @@ class ImagesPipeline(FilesPipeline):
) -> Iterator[tuple[str, Image.Image, BytesIO]]:
path = self.file_path(request, response=response, info=info, item=item)
orig_image = self._Image.open(BytesIO(response.body))
+ assert orig_image.format is not None
+ image_format = orig_image.format
transposed_image = self._ImageOps.exif_transpose(orig_image)
width, height = transposed_image.size
@@ -189,7 +196,9 @@ class ImagesPipeline(FilesPipeline):
)
image, buf = self.convert_image(
- transposed_image, response_body=BytesIO(response.body)
+ transposed_image,
+ image_format=image_format,
+ response_body=BytesIO(response.body),
)
yield path, image, buf
@@ -197,7 +206,9 @@ class ImagesPipeline(FilesPipeline):
thumb_path = self.thumb_path(
request, thumb_id, response=response, info=info, item=item
)
- thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf)
+ thumb_image, thumb_buf = self.convert_image(
+ image, size, image_format=image_format, response_body=buf
+ )
yield thumb_path, thumb_image, thumb_buf
def convert_image(
@@ -205,19 +216,22 @@ class ImagesPipeline(FilesPipeline):
image: Image.Image,
size: tuple[int, int] | None = None,
*,
+ image_format: str,
response_body: BytesIO,
) -> tuple[Image.Image, BytesIO]:
- if image.format in {"PNG", "WEBP"} and image.mode == "RGBA":
- background = self._Image.new("RGBA", image.size, (255, 255, 255))
- background.paste(image, image)
- image = background.convert("RGB")
- elif image.mode == "P":
- image = image.convert("RGBA")
- background = self._Image.new("RGBA", image.size, (255, 255, 255))
- background.paste(image, image)
- image = background.convert("RGB")
- elif image.mode != "RGB":
- image = image.convert("RGB")
+ target_format = image_format if self.preserve_format else "JPEG"
+ if target_format == "JPEG":
+ if image_format in {"PNG", "WEBP"} and image.mode == "RGBA":
+ background = self._Image.new("RGBA", image.size, (255, 255, 255))
+ background.paste(image, image)
+ image = background.convert("RGB")
+ elif image.mode == "P":
+ image = image.convert("RGBA")
+ background = self._Image.new("RGBA", image.size, (255, 255, 255))
+ background.paste(image, image)
+ image = background.convert("RGB")
+ elif image.mode != "RGB":
+ image = image.convert("RGB")
if size:
image = image.copy()
@@ -229,11 +243,13 @@ class ImagesPipeline(FilesPipeline):
except AttributeError:
resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]
image.thumbnail(size, resampling_filter)
- elif image.format == "JPEG":
+ elif target_format == image_format:
+ image.format = target_format
return image, response_body
buf = BytesIO()
- image.save(buf, "JPEG")
+ image.save(buf, target_format)
+ image.format = target_format
return image, buf
def get_media_requests(
@@ -262,7 +278,8 @@ class ImagesPipeline(FilesPipeline):
item: Any = None,
) -> str:
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324
- return f"full/{image_guid}.jpg"
+ ext = "" if self.preserve_format else ".jpg"
+ return f"full/{image_guid}{ext}"
def thumb_path(
self,
@@ -274,4 +291,5 @@ class ImagesPipeline(FilesPipeline):
item: Any = None,
) -> str:
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324
- return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
+ ext = "" if self.preserve_format else ".jpg"
+ return f"thumbs/{thumb_id}/{thumb_guid}{ext}"
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 74b4f3a9b..04c946f57 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -97,6 +97,22 @@ class TestImagesPipeline:
== "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg"
)
+ def test_file_path_preserve_format(self):
+ pipeline = build_from_crawler(
+ ImagesPipeline,
+ get_crawler(
+ None, {"IMAGES_STORE": self.tempdir, "IMAGES_PRESERVE_FORMAT": True}
+ ),
+ )
+ assert (
+ pipeline.file_path(Request("https://dev.mydeco.com/mydeco.gif"))
+ == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d"
+ )
+ assert (
+ pipeline.thumb_path(Request("https://dev.mydeco.com/mydeco.gif"), "50")
+ == "thumbs/50/3fd165099d8e71b8a48b2683946e64dbfad8b52d"
+ )
+
def test_thumbnail_name(self):
thumb_path = self.pipeline.thumb_path
name = "50"
@@ -240,12 +256,81 @@ class TestImagesPipeline:
assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue()
assert Path(self.tempdir, "thumbs", "small", name).exists()
+ def test_convert_image_preserve_format(self):
+ pipeline = build_from_crawler(
+ ImagesPipeline,
+ get_crawler(
+ None, {"IMAGES_STORE": self.tempdir, "IMAGES_PRESERVE_FORMAT": True}
+ ),
+ )
+
+ SIZE = (100, 100)
+ COLOUR = (0, 127, 255, 50)
+ im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)
+
+ converted, converted_buf = pipeline.convert_image(
+ im, image_format="PNG", response_body=buf
+ )
+ assert converted.mode == "RGBA"
+ assert converted.format == "PNG"
+ # the original bytes are reused, no re-encoding happens
+ assert converted_buf == buf
+
+ thumbnail, thumbnail_buf = pipeline.convert_image(
+ converted,
+ size=(10, 25),
+ image_format="PNG",
+ response_body=converted_buf,
+ )
+ assert thumbnail.mode == "RGBA"
+ assert thumbnail.format == "PNG"
+ assert thumbnail.size == (10, 10)
+ assert Image.open(thumbnail_buf).format == "PNG"
+
+ @coroutine_test
+ async def test_image_downloaded_preserve_format(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ pipeline = build_from_crawler(
+ ImagesPipeline,
+ get_crawler(
+ None, {"IMAGES_STORE": self.tempdir, "IMAGES_PRESERVE_FORMAT": True}
+ ),
+ )
+ pipeline.thumbs = {"small": (20, 20)}
+ _, buf = _create_image("PNG", "RGBA", (50, 50), (0, 127, 255, 50))
+ url = "https://dev.mydeco.com/mydeco.gif"
+ response = Response(url=url, body=buf.getvalue())
+
+ persisted = []
+ monkeypatch.setattr(
+ pipeline.store,
+ "persist_file",
+ lambda path, b, info, meta=None, headers=None: persisted.append(
+ (path, b.getvalue(), headers)
+ ),
+ )
+
+ await pipeline.image_downloaded(response, Request(url=url), DUMMY_SPIDER_INFO)
+
+ assert len(persisted) == 2
+ full_path, full_bytes, full_headers = persisted[0]
+ assert full_path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d"
+ assert full_bytes == buf.getvalue()
+ assert full_headers == {"Content-Type": "image/png"}
+
+ _, thumb_bytes, thumb_headers = persisted[1]
+ assert Image.open(io.BytesIO(thumb_bytes)).format == "PNG"
+ assert thumb_headers == {"Content-Type": "image/png"}
+
def test_convert_image(self):
SIZE = (100, 100)
# straight forward case: RGB and JPEG
COLOUR: tuple[int, ...] = (0, 127, 255)
im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR)
- converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)
+ converted, converted_buf = self.pipeline.convert_image(
+ im, image_format="JPEG", response_body=buf
+ )
assert converted.mode == "RGB"
assert converted.getcolors() == [(10000, COLOUR)]
# check that we don't convert JPEGs again
@@ -253,7 +338,7 @@ class TestImagesPipeline:
# check that thumbnail keep image ratio
thumbnail, _ = self.pipeline.convert_image(
- converted, size=(10, 25), response_body=converted_buf
+ converted, size=(10, 25), image_format="JPEG", response_body=converted_buf
)
assert thumbnail.mode == "RGB"
assert thumbnail.size == (10, 10)
@@ -261,7 +346,9 @@ class TestImagesPipeline:
# transparency case: RGBA and PNG
COLOUR = (0, 127, 255, 50)
im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)
- converted, _ = self.pipeline.convert_image(im, response_body=buf)
+ converted, _ = self.pipeline.convert_image(
+ im, image_format="PNG", response_body=buf
+ )
assert converted.mode == "RGB"
assert converted.getcolors() == [(10000, (205, 230, 255))]
@@ -269,10 +356,20 @@ class TestImagesPipeline:
COLOUR = (0, 127, 255, 50)
im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)
im = im.convert("P")
- converted, _ = self.pipeline.convert_image(im, response_body=buf)
+ converted, _ = self.pipeline.convert_image(
+ im, image_format="PNG", response_body=buf
+ )
assert converted.mode == "RGB"
assert converted.getcolors() == [(10000, (205, 230, 255))]
+ # grayscale case: L and PNG
+ im, buf = _create_image("PNG", "L", SIZE, 128)
+ converted, _ = self.pipeline.convert_image(
+ im, image_format="PNG", response_body=buf
+ )
+ assert converted.mode == "RGB"
+ assert converted.getcolors() == [(10000, (128, 128, 128))]
+
def test_convert_image_legacy_resampling_filter(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -287,7 +384,9 @@ class TestImagesPipeline:
)
im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255))
- thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf)
+ thumbnail, _ = self.pipeline.convert_image(
+ im, size=(10, 25), image_format="JPEG", response_body=buf
+ )
assert thumbnail.size == (10, 10)