Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(backends/litellm): stop dropping image blocks from /v1/messages requests (#3753)

Lucas Kim577336d2b66a
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 70e9f4d49..03b94d371 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -588,6 +588,23 @@ def _parse_tool_arguments(arguments: Any) -> Any:
return arguments
+def _anthropic_image_to_openai(block: dict[str, Any]) -> dict[str, Any] | None:
+ """Convert an Anthropic ``image`` block to an OpenAI ``image_url`` part.
+
+ Same mapping as ``AnyLLMBackend._convert_content_blocks``. Returns None for a
+ source type it does not know, so that block is skipped as before.
+ """
+ source = block.get("source") or {}
+ if source.get("type") == "base64":
+ media_type = source.get("media_type", "image/png")
+ url = f"data:{media_type};base64,{source.get('data', '')}"
+ elif source.get("type") == "url":
+ url = source.get("url", "")
+ else:
+ return None
+ return {"type": "image_url", "image_url": {"url": url}}
+
+
def _is_anthropic_family_model(litellm_model: str) -> bool:
"""True when the resolved litellm target speaks the Anthropic Messages
dialect: the Anthropic API, Bedrock-Claude, Vertex-Claude, Azure-Claude.
@@ -884,6 +901,9 @@ class LiteLLMBackend(Backend):
tool_use_blocks = []
tool_result_blocks = []
thinking_blocks: list[dict[str, Any]] = []
+ # Ordered text + image parts; only used when the turn has an image.
+ parts: list[dict[str, Any]] = []
+ has_image = False
for block in content:
if not isinstance(block, dict):
@@ -891,6 +911,12 @@ class LiteLLMBackend(Backend):
block_type = block.get("type", "")
if block_type == "text":
text_parts.append(block.get("text", ""))
+ parts.append({"type": "text", "text": block.get("text", "")})
+ elif block_type == "image":
+ image_part = _anthropic_image_to_openai(block)
+ if image_part:
+ parts.append(image_part)
+ has_image = True
elif block_type == "tool_use":
tool_use_blocks.append(block)
elif block_type == "tool_result":
@@ -966,6 +992,10 @@ class LiteLLMBackend(Backend):
"role": role,
"content": "\n".join(text_parts) if text_parts else "",
}
+ # User turns only: litellm's Bedrock transform raises on an
+ # assistant-turn image, which this code has always dropped.
+ if has_image and role == "user":
+ simple_msg["content"] = parts
if preserve_thinking and thinking_blocks and role == "assistant":
simple_msg["thinking_blocks"] = thinking_blocks
converted.append(simple_msg)
diff --git a/tests/test_backend_bugs.py b/tests/test_backend_bugs.py
index ea1af8db1..e022e8a7b 100644
--- a/tests/test_backend_bugs.py
+++ b/tests/test_backend_bugs.py
@@ -399,6 +399,71 @@ class TestConvertMessagesToolBlocks:
assert converted[0]["content"] == "42"
+class TestConvertMessagesImageBlocks:
+ """Anthropic image blocks must survive _convert_messages_for_litellm."""
+
+ PNG_B64 = "iVBORw0KGgo="
+
+ def _make_backend(self):
+ with patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}):
+ return LiteLLMBackend(provider="bedrock")
+
+ def test_text_and_image_turn_keeps_image(self):
+ """The image reaches the Bedrock Converse body instead of being dropped."""
+ from litellm.litellm_core_utils.prompt_templates.factory import (
+ _bedrock_converse_messages_pt,
+ )
+
+ backend = self._make_backend()
+ image = {
+ "type": "image",
+ "source": {"type": "base64", "media_type": "image/png", "data": self.PNG_B64},
+ }
+ messages = [
+ {"role": "user", "content": [image, {"type": "text", "text": "What color?"}]},
+ ]
+ converted = backend._convert_messages_for_litellm(messages)
+
+ assert converted[0]["content"] == [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{self.PNG_B64}"}},
+ {"type": "text", "text": "What color?"},
+ ]
+ converse = _bedrock_converse_messages_pt(
+ messages=converted, model="us.openai.gpt-6-sol", llm_provider="bedrock_converse"
+ )
+ assert [list(block) for block in converse[0]["content"]] == [["image"], ["text"]]
+
+ def test_image_only_turn_is_not_emptied(self):
+ """An image-only turn must not become "" (litellm then drops the turn)."""
+ backend = self._make_backend()
+ messages = [
+ {"role": "user", "content": [{"type": "text", "text": "Describe the next image."}]},
+ {"role": "assistant", "content": "Send it."},
+ {
+ "role": "user",
+ "content": [{"type": "image", "source": {"type": "url", "url": "https://x/a.png"}}],
+ },
+ ]
+ converted = backend._convert_messages_for_litellm(messages)
+
+ # Text-only block lists still flatten to a string, as before.
+ assert converted[0]["content"] == "Describe the next image."
+ assert converted[2]["content"] == [
+ {"type": "image_url", "image_url": {"url": "https://x/a.png"}},
+ ]
+
+ def test_assistant_turn_image_is_still_dropped(self):
+ """Assistant-turn images keep the old text-only content (litellm's Bedrock
+ transform raises on them)."""
+ backend = self._make_backend()
+ image = {"type": "image", "source": {"type": "url", "url": "https://x/a.png"}}
+ messages = [
+ {"role": "assistant", "content": [image, {"type": "text", "text": "Here is one."}]},
+ ]
+ converted = backend._convert_messages_for_litellm(messages)
+ assert converted[0]["content"] == "Here is one."
+
+
# =============================================================================
# Streaming tool_calls (GitHub Issue — Bug 1)
# =============================================================================