Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(image): flatten transparent images onto white before re-encoding as JPEG (#4296)

Rudy Mizrahi Celekli9f5cfcfd2bb9
diff --git a/headroom/image/compressor.py b/headroom/image/compressor.py
index d8e1ffdc8..e39cde434 100644
--- a/headroom/image/compressor.py
+++ b/headroom/image/compressor.py
@@ -334,13 +334,11 @@ class ImageCompressor:
# Resize
resized = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
- # Convert to RGB if needed (for JPEG)
- if resized.mode in ("RGBA", "P"):
- resized = resized.convert("RGB")
-
# Save as JPEG for best compression
+ from .tile_optimizer import _jpeg_ready
+
buf = io.BytesIO()
- resized.save(buf, format="JPEG", quality=quality, optimize=True)
+ _jpeg_ready(resized).save(buf, format="JPEG", quality=quality, optimize=True)
return buf.getvalue(), "image/jpeg"
def _estimate_tokens(self, image_data: bytes, detail: str = "high") -> int:
diff --git a/headroom/image/tile_optimizer.py b/headroom/image/tile_optimizer.py
index 3fa3d2950..44bec31c9 100644
--- a/headroom/image/tile_optimizer.py
+++ b/headroom/image/tile_optimizer.py
@@ -167,6 +167,29 @@ def find_optimal_anthropic_dimensions(width: int, height: int) -> tuple[int, int
# ---------------------------------------------------------------------------
+def _jpeg_ready(img: Any) -> Any:
+ """Return ``img`` in a mode JPEG can store.
+
+ JPEG has no alpha, and ``convert("RGB")`` simply drops it, so transparent
+ pixels show their hidden colour (usually black). Composite them onto
+ white instead: alpha bands, palette transparency and PNG ``tRNS`` on RGB
+ or grayscale images alike. Palette, bilevel and CMYK/YCbCr images convert
+ to RGB. High-bit-depth modes (``I;16``, ``I``, ``F``) have no faithful
+ 8-bit form, so they are returned as-is and the JPEG save fails as before,
+ which makes callers keep the original image.
+ """
+ from PIL import Image
+
+ if img.has_transparency_data:
+ rgba = img.convert("RGBA")
+ flat = Image.new("RGB", rgba.size, (255, 255, 255))
+ flat.paste(rgba, mask=rgba.getchannel("A"))
+ return flat
+ if img.mode in ("1", "P", "CMYK", "YCbCr"):
+ return img.convert("RGB")
+ return img
+
+
def _resize_image_bytes(
image_data: bytes, target_width: int, target_height: int
) -> tuple[bytes, str]:
@@ -182,12 +205,8 @@ def _resize_image_bytes(
resized = img.resize((target_width, target_height), Image.Resampling.LANCZOS)
- # Convert RGBA to RGB for JPEG
- if resized.mode in ("RGBA", "P"):
- resized = resized.convert("RGB")
-
buf = io.BytesIO()
- resized.save(buf, format="JPEG", quality=85, optimize=True)
+ _jpeg_ready(resized).save(buf, format="JPEG", quality=85, optimize=True)
return buf.getvalue(), "image/jpeg"
diff --git a/tests/test_image_compression.py b/tests/test_image_compression.py
index d431c3ebc..7108fc411 100644
--- a/tests/test_image_compression.py
+++ b/tests/test_image_compression.py
@@ -203,6 +203,102 @@ class TestMessageOptimization:
assert len(text_blocks) == 1
assert text_blocks[0]["text"] == "What is this?"
+ @staticmethod
+ def _png_message(img) -> list[dict]:
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+ b64 = base64.b64encode(buf.getvalue()).decode()
+ return [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}
+ ],
+ }
+ ]
+
+ @staticmethod
+ def _decoded(optimized: list[dict]):
+ from PIL import Image
+
+ url = optimized[0]["content"][0]["image_url"]["url"]
+ return Image.open(io.BytesIO(base64.b64decode(url.split(",", 1)[1]))).convert("RGB")
+
+ def test_transparent_png_is_flattened_onto_white(self):
+ """JPEG has no alpha: transparent pixels must not turn black (their hidden RGB)."""
+ from PIL import Image
+
+ img = Image.new("RGBA", (1920, 1080), (0, 0, 0, 0))
+ img.paste((0, 0, 0, 255), (100, 500, 1800, 560)) # black "text" bar
+ optimized, results = optimize_images_in_messages(self._png_message(img), "openai")
+
+ assert len(results) == 1 and results[0].resized
+ out = self._decoded(optimized)
+ assert min(out.getpixel((5, 5))) > 245
+ assert max(out.getpixel((out.width // 2, out.height * 530 // 1080))) < 10
+
+ @pytest.mark.parametrize("mode", ["RGB", "L", "P"])
+ def test_trns_and_palette_transparency_are_flattened_onto_white(self, mode):
+ """PNG tRNS on RGB/L images and a transparent palette entry are transparency too."""
+ from PIL import Image
+
+ img = Image.new("RGBA", (1920, 1080), (0, 0, 0, 0))
+ img.paste((0, 0, 0, 255), (100, 500, 1800, 560))
+ if mode == "P":
+ img = Image.new("P", (1920, 1080), 0)
+ img.putpalette([0, 0, 0, 0, 0, 0]) # index 0 transparent, index 1 opaque black
+ img.paste(1, (100, 500, 1800, 560))
+ img.info["transparency"] = 0
+ else:
+ # tRNS: one colour is transparent; the bar is a different, still dark colour.
+ img = Image.new(mode, (1920, 1080), 0)
+ img.paste(1 if mode == "L" else (1, 1, 1), (100, 500, 1800, 560))
+ img.info["transparency"] = 0 if mode == "L" else (0, 0, 0)
+ optimized, results = optimize_images_in_messages(self._png_message(img), "openai")
+
+ assert len(results) == 1 and results[0].resized
+ out = self._decoded(optimized)
+ assert min(out.getpixel((5, 5))) > 245
+ assert max(out.getpixel((out.width // 2, out.height * 530 // 1080))) < 10
+
+ def test_grayscale_alpha_is_converted_not_raised(self):
+ """LA used to raise "cannot write mode LA as JPEG", skipping the optimization."""
+ from PIL import Image
+
+ img = Image.new("LA", (1920, 1080), (0, 0))
+ img.paste((0, 255), (100, 500, 1800, 560))
+ optimized, results = optimize_images_in_messages(self._png_message(img), "openai")
+
+ assert len(results) == 1 and results[0].resized
+ out = self._decoded(optimized)
+ assert min(out.getpixel((5, 5))) > 245
+
+ def test_high_bit_depth_is_not_squashed_to_8_bit(self):
+ """I;16 has no faithful 8-bit form: leave it unconverted rather than clip to white."""
+ from PIL import Image
+
+ from headroom.image.tile_optimizer import _jpeg_ready
+
+ img = Image.new("I;16", (64, 64), 1000)
+ assert _jpeg_ready(img) is img
+
+ def test_compressor_resize_flattens_transparency_onto_white(self):
+ """ImageCompressor's provider-format resize also writes JPEG."""
+ from PIL import Image
+
+ from headroom.image.compressor import ImageCompressor
+
+ img = Image.new("RGBA", (1600, 1000), (0, 0, 0, 0))
+ img.paste((0, 0, 0, 255), (100, 480, 1500, 520))
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+
+ data, media_type = ImageCompressor()._resize_image(buf.getvalue(), max_dimension=512)
+
+ assert media_type == "image/jpeg"
+ out = Image.open(io.BytesIO(data)).convert("RGB")
+ assert min(out.getpixel((5, 5))) > 245
+
def test_small_image_not_resized(self):
"""Image already at optimal size is not changed."""
msgs = _make_openai_image_message(512, 512)