Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(backends/litellm): keep cache_control on text blocks for Bedrock prompt caching (#4565)

Lucas Kim23a5bb1c81dc
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 7e96683cb..617ddac42 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -1037,7 +1037,10 @@ class LiteLLMBackend(Backend):
block_type = block.get("type", "")
if block_type == "text":
text_parts.append(block.get("text", ""))
- parts.append({"type": "text", "text": block.get("text", "")})
+ text_part: dict[str, Any] = {"type": "text", "text": block.get("text", "")}
+ if "cache_control" in block:
+ text_part["cache_control"] = block["cache_control"]
+ parts.append(text_part)
elif block_type == "image":
image_part = _anthropic_image_to_openai(block)
if image_part:
@@ -1131,6 +1134,11 @@ class LiteLLMBackend(Backend):
# assistant-turn image, which this code has always dropped.
if has_image and role == "user":
simple_msg["content"] = parts
+ elif any("cache_control" in p for p in parts):
+ # A joined string can't carry the breakpoint: litellm's Bedrock
+ # Converse transform only emits a cachePoint for a marked content
+ # part, as with the tool_result and system paths.
+ simple_msg["content"] = [p for p in parts if p["type"] == "text"]
if preserve_thinking and thinking_blocks and role == "assistant":
simple_msg["thinking_blocks"] = thinking_blocks
converted.append(simple_msg)
diff --git a/tests/test_bedrock_tool_result_cache_and_streaming_stats.py b/tests/test_bedrock_tool_result_cache_and_streaming_stats.py
index 6475c0d89..e975f6520 100644
--- a/tests/test_bedrock_tool_result_cache_and_streaming_stats.py
+++ b/tests/test_bedrock_tool_result_cache_and_streaming_stats.py
@@ -114,6 +114,40 @@ class TestToolResultCacheControlPreserved:
assert converted[0]["cache_control"] == {"type": "ephemeral"}
+class TestTextBlockCacheControlPreserved:
+ """Plain text blocks must keep cache_control too: a new user prompt puts the
+ breakpoint on a text block, and #1390 (which covered this branch) was closed."""
+
+ def test_marked_text_block_becomes_a_converse_cache_point(self):
+ from litellm.litellm_core_utils.prompt_templates.factory import (
+ _bedrock_converse_messages_pt,
+ )
+
+ backend = _backend()
+ marked = {"type": "text", "text": "Stable prefix.", "cache_control": {"type": "ephemeral"}}
+ messages = [{"role": "user", "content": [marked, {"type": "text", "text": "Question?"}]}]
+ converted = backend._convert_messages_for_litellm(messages)
+
+ converse = _bedrock_converse_messages_pt(
+ messages=converted,
+ model="global.anthropic.claude-sonnet-5",
+ llm_provider="bedrock_converse",
+ )
+ assert converse[0]["content"] == [
+ {"text": "Stable prefix."},
+ {"cachePoint": {"type": "default"}},
+ {"text": "Question?"},
+ ]
+
+ def test_marked_assistant_turn_still_drops_its_image(self):
+ backend = _backend()
+ image = {"type": "image", "source": {"type": "url", "url": "https://x/a.png"}}
+ marked = {"type": "text", "text": "Here is one.", "cache_control": {"type": "ephemeral"}}
+ messages = [{"role": "assistant", "content": [image, marked]}]
+ converted = backend._convert_messages_for_litellm(messages)
+ assert converted[0]["content"] == [marked]
+
+
class TestStreamingCacheStatsCompletion:
"""stream_message must request usage and re-surface real cache stats."""
diff --git a/tests/test_pricing_from_litellm.py b/tests/test_pricing_from_litellm.py
index d9cd7b8a9..bb9117149 100644
--- a/tests/test_pricing_from_litellm.py
+++ b/tests/test_pricing_from_litellm.py
@@ -41,12 +41,6 @@ def test_unwrapped_model_forms_drops_leading_segments() -> None:
("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),
("bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),
("vertex_ai/claude-sonnet-4-5", 3.00, 15.00),
- # Any gateway-prefixed name here must be one litellm still prices:
- # when it prunes a model the unwrap finds nothing and _get_pricing
- # silently returns the $2.50/$10.00 GPT-4o default, which is what
- # this test exists to catch. litellm dropped
- # groq/llama-3.3-70b-versatile on 2026-09-23; see issue #3732.
- ("groq/llama-guard-3-8b", 0.20, 0.20),
# Non-OpenAI models reachable through the OpenAI-compatible passthrough.
("gemini-2.5-flash", 0.30, 2.50),
("deepseek-chat", 0.28, 0.42),
@@ -60,6 +54,25 @@ def test_provider_prices_models_its_table_never_covered(
assert (round(got_in, 2), round(got_out, 2)) == (want_in, want_out)
+def test_gateway_pricing_uses_unwrapped_catalog_entry(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Retiring a vendor model must not break the gateway-resolution regression."""
+ bare_model = "headroom-pricing-fixture"
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ bare_model: {
+ "input_cost_per_token": 0.20 / 1_000_000,
+ "output_cost_per_token": 0.20 / 1_000_000,
+ }
+ },
+ )
+
+ # Only the unwrapped key exists. Returning the unknown-model default instead
+ # of looking through the gateway prefix must still fail this assertion.
+ assert OpenAIProvider()._get_pricing(f"groq/{bare_model}") == pytest.approx((0.20, 0.20))
+
+
def test_explicit_config_outranks_litellm() -> None:
"""A configured price is a decision, not a guess."""
provider = OpenAIProvider()