Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(backends/litellm): keep cache_control on text blocks for Bedrock prompt caching (#4565)
Lucas Kim23a5bb1c81dc
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.pyindex 7e96683cb..617ddac42 100644--- a/headroom/backends/litellm.py+++ b/headroom/backends/litellm.py@@ -1037,7 +1037,10 @@ class LiteLLMBackend(Backend):block_type = block.get("type", "")if block_type == "text":text_parts.append(block.get("text", ""))- parts.append({"type": "text", "text": block.get("text", "")})+ text_part: dict[str, Any] = {"type": "text", "text": block.get("text", "")}+ if "cache_control" in block:+ text_part["cache_control"] = block["cache_control"]+ parts.append(text_part)elif block_type == "image":image_part = _anthropic_image_to_openai(block)if image_part:@@ -1131,6 +1134,11 @@ class LiteLLMBackend(Backend):# assistant-turn image, which this code has always dropped.if has_image and role == "user":simple_msg["content"] = parts+ elif any("cache_control" in p for p in parts):+ # A joined string can't carry the breakpoint: litellm's Bedrock+ # Converse transform only emits a cachePoint for a marked content+ # part, as with the tool_result and system paths.+ simple_msg["content"] = [p for p in parts if p["type"] == "text"]if preserve_thinking and thinking_blocks and role == "assistant":simple_msg["thinking_blocks"] = thinking_blocksconverted.append(simple_msg)diff --git a/tests/test_bedrock_tool_result_cache_and_streaming_stats.py b/tests/test_bedrock_tool_result_cache_and_streaming_stats.pyindex 6475c0d89..e975f6520 100644--- a/tests/test_bedrock_tool_result_cache_and_streaming_stats.py+++ b/tests/test_bedrock_tool_result_cache_and_streaming_stats.py@@ -114,6 +114,40 @@ class TestToolResultCacheControlPreserved:assert converted[0]["cache_control"] == {"type": "ephemeral"}+class TestTextBlockCacheControlPreserved:+ """Plain text blocks must keep cache_control too: a new user prompt puts the+ breakpoint on a text block, and #1390 (which covered this branch) was closed."""++ def test_marked_text_block_becomes_a_converse_cache_point(self):+ from litellm.litellm_core_utils.prompt_templates.factory import (+ _bedrock_converse_messages_pt,+ )++ backend = _backend()+ marked = {"type": "text", "text": "Stable prefix.", "cache_control": {"type": "ephemeral"}}+ messages = [{"role": "user", "content": [marked, {"type": "text", "text": "Question?"}]}]+ converted = backend._convert_messages_for_litellm(messages)++ converse = _bedrock_converse_messages_pt(+ messages=converted,+ model="global.anthropic.claude-sonnet-5",+ llm_provider="bedrock_converse",+ )+ assert converse[0]["content"] == [+ {"text": "Stable prefix."},+ {"cachePoint": {"type": "default"}},+ {"text": "Question?"},+ ]++ def test_marked_assistant_turn_still_drops_its_image(self):+ backend = _backend()+ image = {"type": "image", "source": {"type": "url", "url": "https://x/a.png"}}+ marked = {"type": "text", "text": "Here is one.", "cache_control": {"type": "ephemeral"}}+ messages = [{"role": "assistant", "content": [image, marked]}]+ converted = backend._convert_messages_for_litellm(messages)+ assert converted[0]["content"] == [marked]++class TestStreamingCacheStatsCompletion:"""stream_message must request usage and re-surface real cache stats."""diff --git a/tests/test_pricing_from_litellm.py b/tests/test_pricing_from_litellm.pyindex d9cd7b8a9..bb9117149 100644--- a/tests/test_pricing_from_litellm.py+++ b/tests/test_pricing_from_litellm.py@@ -41,12 +41,6 @@ def test_unwrapped_model_forms_drops_leading_segments() -> None:("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),("bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", 3.00, 15.00),("vertex_ai/claude-sonnet-4-5", 3.00, 15.00),- # Any gateway-prefixed name here must be one litellm still prices:- # when it prunes a model the unwrap finds nothing and _get_pricing- # silently returns the $2.50/$10.00 GPT-4o default, which is what- # this test exists to catch. litellm dropped- # groq/llama-3.3-70b-versatile on 2026-09-23; see issue #3732.- ("groq/llama-guard-3-8b", 0.20, 0.20),# Non-OpenAI models reachable through the OpenAI-compatible passthrough.("gemini-2.5-flash", 0.30, 2.50),("deepseek-chat", 0.28, 0.42),@@ -60,6 +54,25 @@ def test_provider_prices_models_its_table_never_covered(assert (round(got_in, 2), round(got_out, 2)) == (want_in, want_out)+def test_gateway_pricing_uses_unwrapped_catalog_entry(monkeypatch: pytest.MonkeyPatch) -> None:+ """Retiring a vendor model must not break the gateway-resolution regression."""+ bare_model = "headroom-pricing-fixture"+ monkeypatch.setattr(+ litellm,+ "model_cost",+ {+ bare_model: {+ "input_cost_per_token": 0.20 / 1_000_000,+ "output_cost_per_token": 0.20 / 1_000_000,+ }+ },+ )++ # Only the unwrapped key exists. Returning the unknown-model default instead+ # of looking through the gateway prefix must still fail this assertion.+ assert OpenAIProvider()._get_pricing(f"groq/{bare_model}") == pytest.approx((0.20, 0.20))++def test_explicit_config_outranks_litellm() -> None:"""A configured price is a decision, not a guess."""provider = OpenAIProvider()