Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(litellm): opt-in Bedrock prompt caching on the OpenAI-compatible path (#3565)
Ayush Kumar Jha0c19244e38f3
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.pyindex 03b94d371..2bf5d0d79 100644--- a/headroom/backends/litellm.py+++ b/headroom/backends/litellm.py@@ -66,6 +66,7 @@ try:_env_snapshot = set(_os.environ)import litellmfrom litellm import acompletion+ from litellm.utils import supports_prompt_cachingfor _leaked_key in set(_os.environ) - _env_snapshot:del _os.environ[_leaked_key]@@ -76,6 +77,7 @@ except ImportError:LITELLM_AVAILABLE = Falselitellm = None # type: ignoreacompletion = None # type: ignore+ supports_prompt_caching = None # type: ignore# =============================================================================@@ -180,6 +182,49 @@ def _build_openai_extra_body(body: dict[str, Any]) -> dict[str, Any]:}+def _place_system_cache_control(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """Return ``messages`` with an ephemeral cache breakpoint on the first system message.++ litellm turns the marker into a Bedrock Converse ``cachePoint``, which caches+ every tool and system block before it. The first system message is marked+ (not the last) so a client that appends volatile system messages later does+ not turn every turn into a cache write. Returned unchanged when the client+ already placed markers anywhere (it owns breakpoint placement then) or when+ there is no system message with content to mark. Never mutates the input:+ the proxy still reads ``body["messages"]`` after the request is built.+ """+ for message in messages:+ if not isinstance(message, dict):+ continue+ content = message.get("content")+ if "cache_control" in message or (+ isinstance(content, list)+ and any(isinstance(block, dict) and "cache_control" in block for block in content)+ ):+ return messages+ for index, message in enumerate(messages):+ if not isinstance(message, dict) or message.get("role") != "system":+ continue+ content = message.get("content")+ if isinstance(content, str) and content:+ marked = {**message, "cache_control": {"type": "ephemeral"}}+ elif isinstance(content, list):+ # litellm only reads block-level markers off list content.+ blocks = list(content)+ for block_index in range(len(blocks) - 1, -1, -1):+ block = blocks[block_index]+ if isinstance(block, dict) and block.get("type") == "text" and block.get("text"):+ blocks[block_index] = {**block, "cache_control": {"type": "ephemeral"}}+ break+ else:+ continue+ marked = {**message, "content": blocks}+ else:+ continue+ return [*messages[:index], marked, *messages[index + 1 :]]+ return messages++def _fetch_bedrock_inference_profiles(region: str | None, profile_name: str | None = None) -> dict[str, str]:@@ -772,6 +817,15 @@ class LiteLLMBackend(Backend):f"from HEADROOM_BEDROCK_MODEL_MAP: {sorted(self._model_overrides)}")+ # Opt-in (rollout feature): mark the system prompt for Bedrock prompt+ # caching on the OpenAI-format path, where clients such as OpenAI-compat+ # gateways never send `cache_control` themselves.+ from headroom.rollout import resolve_rollout++ self._openai_prompt_caching = provider == "bedrock" and resolve_rollout().is_enabled(+ "bedrock_openai_prompt_caching"+ )+logger.info(f"LiteLLM backend initialized (provider={provider}, region={region})")@property@@ -1666,6 +1720,9 @@ class LiteLLMBackend(Backend):if extra_body:kwargs["extra_body"] = extra_body+ if self._openai_prompt_caching and supports_prompt_caching(model=litellm_model):+ kwargs["messages"] = _place_system_cache_control(kwargs["messages"])+# Provider-specific region configif self.region:if self.provider == "bedrock":@@ -1873,6 +1930,9 @@ class LiteLLMBackend(Backend):if extra_body:kwargs["extra_body"] = extra_body+ if self._openai_prompt_caching and supports_prompt_caching(model=litellm_model):+ kwargs["messages"] = _place_system_cache_control(kwargs["messages"])+# Provider-specific region configif self.region:if self.provider == "bedrock":diff --git a/headroom/rollout.py b/headroom/rollout.pyindex 6c09cabd4..ed71b113c 100644--- a/headroom/rollout.py+++ b/headroom/rollout.py@@ -119,6 +119,17 @@ FEATURES: dict[str, FeatureSpec] = {legacy_env=("HEADROOM_READ_MATURATION",),description="Hold-back Read maturation before provider cache entry.",),+ "bedrock_openai_prompt_caching": FeatureSpec(+ name="bedrock_openai_prompt_caching",+ available_in=RolloutChannel.STABLE,+ # Deliberately NOT default-enabled. A cache breakpoint bills the system+ # prompt at the cache-write rate (1.25x) until it is read back, so a+ # one-shot caller who did not ask for it would pay more, not less.+ description=(+ "Place a system-prompt cache breakpoint on OpenAI-compatible requests "+ "to Bedrock models that support prompt caching (opt-in)."+ ),+ ),}diff --git a/tests/test_backends/test_litellm_bedrock_prompt_caching.py b/tests/test_backends/test_litellm_bedrock_prompt_caching.pynew file mode 100644index 000000000..a3895fbcf--- /dev/null+++ b/tests/test_backends/test_litellm_bedrock_prompt_caching.py@@ -0,0 +1,310 @@+"""Opt-in Bedrock prompt caching on `LiteLLMBackend`'s OpenAI-format path (#3554).++The OpenAI-compatible route forwards `messages` verbatim, so a client that never+sends `cache_control` (OpenAI-compat gateways such as Bifrost) gets no Bedrock+`cachePoint` and therefore never sees `cache_read_input_tokens`. With the+`bedrock_openai_prompt_caching` rollout feature enabled, the backend marks the+first system message when litellm reports the *mapped* model supports prompt+caching. Default behaviour (feature off) is untouched.+"""++from __future__ import annotations++import copy+from types import SimpleNamespace+from typing import Any+from unittest.mock import AsyncMock, patch++import pytest++from tests._dotenv import importorskip_no_env_leak++importorskip_no_env_leak("litellm")++import litellm # noqa: E402++from headroom.backends.litellm import ( # noqa: E402 (must follow importorskip)+ LiteLLMBackend,+ _place_system_cache_control,+)++FEATURE = "bedrock_openai_prompt_caching"+CACHING_MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"+EPHEMERAL = {"type": "ephemeral"}+++class _FakeAsyncStream:+ def __init__(self, items: list[Any]) -> None:+ self._items = list(items)++ def __aiter__(self) -> _FakeAsyncStream:+ self._iter = iter(self._items)+ return self++ async def __anext__(self) -> Any:+ try:+ return next(self._iter)+ except StopIteration as exc:+ raise StopAsyncIteration from exc+++def _make_response() -> SimpleNamespace:+ return SimpleNamespace(+ id="resp_123",+ created=123456,+ choices=[+ SimpleNamespace(+ index=0,+ finish_reason="stop",+ message=SimpleNamespace(role="assistant", content="ok", tool_calls=None),+ )+ ],+ usage=SimpleNamespace(prompt_tokens=2, completion_tokens=3, total_tokens=5),+ )+++def _make_backend(provider: str = "bedrock") -> LiteLLMBackend:+ # Patch the inference-profile fetch so `__init__` doesn't try to talk to AWS.+ with patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}):+ return LiteLLMBackend(provider=provider, region="us-east-1")+++def _request_body(model: str = CACHING_MODEL) -> dict[str, Any]:+ return {+ "model": model,+ "messages": [+ {"role": "system", "content": "You are terse."},+ {"role": "user", "content": "hello"},+ ],+ "max_tokens": 32,+ }+++async def _send(backend: LiteLLMBackend, body: dict[str, Any]) -> dict[str, Any]:+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:+ mock_acomp.return_value = _make_response()+ await backend.send_openai_message(body, {})+ return mock_acomp.await_args.kwargs+++async def _stream(backend: LiteLLMBackend, body: dict[str, Any]) -> dict[str, Any]:+ stream = _FakeAsyncStream(+ [SimpleNamespace(model_dump=lambda **kwargs: {"id": "chunk1", "choices": []})]+ )+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:+ mock_acomp.return_value = stream+ chunks = [chunk async for chunk in backend.stream_openai_message(body, {})]+ assert chunks[-1] == "data: [DONE]\n\n"+ return mock_acomp.await_args.kwargs+++def feature_enabled(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_FEATURES", FEATURE)+ monkeypatch.delenv("HEADROOM_DISABLE_FEATURES", raising=False)+++def feature_default(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("HEADROOM_FEATURES", raising=False)+ monkeypatch.delenv("HEADROOM_DISABLE_FEATURES", raising=False)+++# =============================================================================+# _place_system_cache_control (pure)+# =============================================================================+++def test_marks_first_system_message_string_content() -> None:+ messages = [+ {"role": "system", "content": "a"},+ {"role": "user", "content": "b"},+ {"role": "system", "content": "c"},+ ]+ before = copy.deepcopy(messages)++ out = _place_system_cache_control(messages)++ assert out[0] == {"role": "system", "content": "a", "cache_control": EPHEMERAL}+ assert out[1] is messages[1]+ assert out[2] is messages[2] # only the first system message is marked+ assert messages == before # caller-owned input never mutated+++def test_marks_last_text_block_of_list_content() -> None:+ messages = [+ {+ "role": "system",+ "content": [+ {"type": "text", "text": "a"},+ {"type": "text", "text": "b"},+ {"type": "text", "text": ""},+ ],+ },+ {"role": "user", "content": "hi"},+ ]+ before = copy.deepcopy(messages)++ out = _place_system_cache_control(messages)++ blocks = out[0]["content"]+ assert blocks[0] is messages[0]["content"][0]+ assert blocks[1] == {"type": "text", "text": "b", "cache_control": EPHEMERAL}+ assert blocks[2] is messages[0]["content"][2] # empty text is not a breakpoint+ assert messages == before+++def test_system_message_anywhere_in_the_list() -> None:+ messages = [{"role": "user", "content": "hi"}, {"role": "system", "content": "s"}]++ out = _place_system_cache_control(messages)++ assert out[0] is messages[0]+ assert out[1]["cache_control"] == EPHEMERAL+++ "content",+ ["", [], None, 7, [{"type": "image_url", "image_url": {"url": "x"}}], ["plain string"]],+)+def test_unmarkable_system_content_is_skipped(content: Any) -> None:+ messages = [{"role": "system", "content": content}, {"role": "system", "content": "s"}]++ out = _place_system_cache_control(messages)++ assert out[0] is messages[0]+ assert out[1]["cache_control"] == EPHEMERAL+++ "messages",+ [+ [],+ [{"role": "user", "content": "hi"}],+ [{"role": "system", "content": ""}],+ ["not a dict", {"role": "user", "content": "hi"}],+ # The client already owns breakpoint placement -- anywhere, any shape.+ [+ {"role": "system", "content": "s"},+ {"role": "user", "content": "u", "cache_control": EPHEMERAL},+ ],+ [+ {"role": "system", "content": "s"},+ {+ "role": "user",+ "content": [{"type": "text", "text": "u", "cache_control": EPHEMERAL}],+ },+ ],+ [+ {+ "role": "system",+ "content": [{"type": "text", "text": "s", "cache_control": EPHEMERAL}],+ }+ ],+ ],+)+def test_returns_input_unchanged_when_nothing_to_do(messages: list[Any]) -> None:+ before = copy.deepcopy(messages)++ assert _place_system_cache_control(messages) is messages+ assert messages == before+++# =============================================================================+# Backend gate+# =============================================================================++[email protected]("feature_default")[email protected]("call", [_send, _stream])+async def test_feature_off_by_default_forwards_messages_verbatim(call: Any) -> None:+ body = _request_body()++ kwargs = await call(_make_backend(), body)++ assert kwargs["messages"] is body["messages"]+ assert "cache_control" not in kwargs["messages"][0]++[email protected]("feature_enabled")[email protected]("call", [_send, _stream])+async def test_feature_on_marks_system_prompt_for_caching_model(call: Any) -> None:+ body = _request_body()+ client_messages = copy.deepcopy(body["messages"])++ kwargs = await call(_make_backend(), body)++ assert kwargs["messages"][0]["cache_control"] == EPHEMERAL+ assert kwargs["messages"][1] is body["messages"][1]+ assert body["messages"] == client_messages # proxy re-reads body["messages"] after send++[email protected]("feature_enabled")+async def test_feature_on_skips_models_without_prompt_caching() -> None:+ body = _request_body(model="meta.llama3-1-70b-instruct-v1:0")++ with patch("headroom.backends.litellm.supports_prompt_caching", return_value=False):+ kwargs = await _send(_make_backend(), body)++ assert kwargs["messages"] is body["messages"]++[email protected]("feature_enabled")+async def test_gate_checks_the_mapped_litellm_model() -> None:+ # A bare alias such as `claude-sonnet-4-20250514` is not a Bedrock entry in+ # litellm's registry; the resolved `bedrock/<region>.anthropic...` id is.+ body = _request_body(model="claude-sonnet-4-20250514")+ backend = _make_backend()++ with patch(+ "headroom.backends.litellm.supports_prompt_caching", return_value=True+ ) as mock_supports:+ kwargs = await _send(backend, body)++ mock_supports.assert_called_once_with(model=backend.map_model_id("claude-sonnet-4-20250514"))+ assert kwargs["messages"][0]["cache_control"] == EPHEMERAL++[email protected]("feature_enabled")+async def test_feature_on_is_a_no_op_for_non_bedrock_providers() -> None:+ body = _request_body(model="anthropic/claude-3-5-sonnet-20241022")++ with patch("headroom.backends.litellm.supports_prompt_caching") as mock_supports:+ kwargs = await _send(_make_backend(provider="openrouter"), body)++ mock_supports.assert_not_called()+ assert kwargs["messages"] is body["messages"]++[email protected]("feature_enabled")+async def test_feature_on_respects_client_placed_markers() -> None:+ body = _request_body()+ body["messages"][1]["cache_control"] = EPHEMERAL++ kwargs = await _send(_make_backend(), body)++ assert kwargs["messages"] is body["messages"]+++def test_disable_features_is_the_kill_switch(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_FEATURES", FEATURE)+ monkeypatch.setenv("HEADROOM_DISABLE_FEATURES", FEATURE)++ assert _make_backend()._openai_prompt_caching is False++[email protected]("feature_enabled")+async def test_marker_becomes_a_bedrock_converse_cache_point() -> None:+ """The marker shape must be the one litellm's Converse transform consumes."""+ body = _request_body()++ kwargs = await _send(_make_backend(), body)+ request = litellm.AmazonConverseConfig().transform_request(+ model=CACHING_MODEL,+ messages=kwargs["messages"],+ optional_params={},+ litellm_params={},+ headers={},+ )++ assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}]diff --git a/tests/test_rollout.py b/tests/test_rollout.pyindex f24123c27..50564c3ec 100644--- a/tests/test_rollout.py+++ b/tests/test_rollout.py@@ -396,7 +396,7 @@ def test_cli_json_status_and_strict_error() -> None:assert result.exit_code == 0payload = json.loads(result.output)assert payload["channel"] == "canary"- assert payload["features"][2]["name"] == "tool_result_interceptors"+ assert "tool_result_interceptors" in [feature["name"] for feature in payload["features"]]assert invalid.exit_code != 0assert "unknown rollout feature" in invalid.output