Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(litellm): opt-in Bedrock prompt caching on the OpenAI-compatible path (#3565)

Ayush Kumar Jha0c19244e38f3
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 03b94d371..2bf5d0d79 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -66,6 +66,7 @@ try:
_env_snapshot = set(_os.environ)
import litellm
from litellm import acompletion
+ from litellm.utils import supports_prompt_caching
for _leaked_key in set(_os.environ) - _env_snapshot:
del _os.environ[_leaked_key]
@@ -76,6 +77,7 @@ except ImportError:
LITELLM_AVAILABLE = False
litellm = None # type: ignore
acompletion = None # type: ignore
+ supports_prompt_caching = None # type: ignore
# =============================================================================
@@ -180,6 +182,49 @@ def _build_openai_extra_body(body: dict[str, Any]) -> dict[str, Any]:
}
+def _place_system_cache_control(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Return ``messages`` with an ephemeral cache breakpoint on the first system message.
+
+ litellm turns the marker into a Bedrock Converse ``cachePoint``, which caches
+ every tool and system block before it. The first system message is marked
+ (not the last) so a client that appends volatile system messages later does
+ not turn every turn into a cache write. Returned unchanged when the client
+ already placed markers anywhere (it owns breakpoint placement then) or when
+ there is no system message with content to mark. Never mutates the input:
+ the proxy still reads ``body["messages"]`` after the request is built.
+ """
+ for message in messages:
+ if not isinstance(message, dict):
+ continue
+ content = message.get("content")
+ if "cache_control" in message or (
+ isinstance(content, list)
+ and any(isinstance(block, dict) and "cache_control" in block for block in content)
+ ):
+ return messages
+ for index, message in enumerate(messages):
+ if not isinstance(message, dict) or message.get("role") != "system":
+ continue
+ content = message.get("content")
+ if isinstance(content, str) and content:
+ marked = {**message, "cache_control": {"type": "ephemeral"}}
+ elif isinstance(content, list):
+ # litellm only reads block-level markers off list content.
+ blocks = list(content)
+ for block_index in range(len(blocks) - 1, -1, -1):
+ block = blocks[block_index]
+ if isinstance(block, dict) and block.get("type") == "text" and block.get("text"):
+ blocks[block_index] = {**block, "cache_control": {"type": "ephemeral"}}
+ break
+ else:
+ continue
+ marked = {**message, "content": blocks}
+ else:
+ continue
+ return [*messages[:index], marked, *messages[index + 1 :]]
+ return messages
+
+
def _fetch_bedrock_inference_profiles(
region: str | None, profile_name: str | None = None
) -> dict[str, str]:
@@ -772,6 +817,15 @@ class LiteLLMBackend(Backend):
f"from HEADROOM_BEDROCK_MODEL_MAP: {sorted(self._model_overrides)}"
)
+ # Opt-in (rollout feature): mark the system prompt for Bedrock prompt
+ # caching on the OpenAI-format path, where clients such as OpenAI-compat
+ # gateways never send `cache_control` themselves.
+ from headroom.rollout import resolve_rollout
+
+ self._openai_prompt_caching = provider == "bedrock" and resolve_rollout().is_enabled(
+ "bedrock_openai_prompt_caching"
+ )
+
logger.info(f"LiteLLM backend initialized (provider={provider}, region={region})")
@property
@@ -1666,6 +1720,9 @@ class LiteLLMBackend(Backend):
if extra_body:
kwargs["extra_body"] = extra_body
+ if self._openai_prompt_caching and supports_prompt_caching(model=litellm_model):
+ kwargs["messages"] = _place_system_cache_control(kwargs["messages"])
+
# Provider-specific region config
if self.region:
if self.provider == "bedrock":
@@ -1873,6 +1930,9 @@ class LiteLLMBackend(Backend):
if extra_body:
kwargs["extra_body"] = extra_body
+ if self._openai_prompt_caching and supports_prompt_caching(model=litellm_model):
+ kwargs["messages"] = _place_system_cache_control(kwargs["messages"])
+
# Provider-specific region config
if self.region:
if self.provider == "bedrock":
diff --git a/headroom/rollout.py b/headroom/rollout.py
index 6c09cabd4..ed71b113c 100644
--- a/headroom/rollout.py
+++ b/headroom/rollout.py
@@ -119,6 +119,17 @@ FEATURES: dict[str, FeatureSpec] = {
legacy_env=("HEADROOM_READ_MATURATION",),
description="Hold-back Read maturation before provider cache entry.",
),
+ "bedrock_openai_prompt_caching": FeatureSpec(
+ name="bedrock_openai_prompt_caching",
+ available_in=RolloutChannel.STABLE,
+ # Deliberately NOT default-enabled. A cache breakpoint bills the system
+ # prompt at the cache-write rate (1.25x) until it is read back, so a
+ # one-shot caller who did not ask for it would pay more, not less.
+ description=(
+ "Place a system-prompt cache breakpoint on OpenAI-compatible requests "
+ "to Bedrock models that support prompt caching (opt-in)."
+ ),
+ ),
}
diff --git a/tests/test_backends/test_litellm_bedrock_prompt_caching.py b/tests/test_backends/test_litellm_bedrock_prompt_caching.py
new file mode 100644
index 000000000..a3895fbcf
--- /dev/null
+++ b/tests/test_backends/test_litellm_bedrock_prompt_caching.py
@@ -0,0 +1,310 @@
+"""Opt-in Bedrock prompt caching on `LiteLLMBackend`'s OpenAI-format path (#3554).
+
+The OpenAI-compatible route forwards `messages` verbatim, so a client that never
+sends `cache_control` (OpenAI-compat gateways such as Bifrost) gets no Bedrock
+`cachePoint` and therefore never sees `cache_read_input_tokens`. With the
+`bedrock_openai_prompt_caching` rollout feature enabled, the backend marks the
+first system message when litellm reports the *mapped* model supports prompt
+caching. Default behaviour (feature off) is untouched.
+"""
+
+from __future__ import annotations
+
+import copy
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from tests._dotenv import importorskip_no_env_leak
+
+importorskip_no_env_leak("litellm")
+
+import litellm # noqa: E402
+
+from headroom.backends.litellm import ( # noqa: E402 (must follow importorskip)
+ LiteLLMBackend,
+ _place_system_cache_control,
+)
+
+FEATURE = "bedrock_openai_prompt_caching"
+CACHING_MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"
+EPHEMERAL = {"type": "ephemeral"}
+
+
+class _FakeAsyncStream:
+ def __init__(self, items: list[Any]) -> None:
+ self._items = list(items)
+
+ def __aiter__(self) -> _FakeAsyncStream:
+ self._iter = iter(self._items)
+ return self
+
+ async def __anext__(self) -> Any:
+ try:
+ return next(self._iter)
+ except StopIteration as exc:
+ raise StopAsyncIteration from exc
+
+
+def _make_response() -> SimpleNamespace:
+ return SimpleNamespace(
+ id="resp_123",
+ created=123456,
+ choices=[
+ SimpleNamespace(
+ index=0,
+ finish_reason="stop",
+ message=SimpleNamespace(role="assistant", content="ok", tool_calls=None),
+ )
+ ],
+ usage=SimpleNamespace(prompt_tokens=2, completion_tokens=3, total_tokens=5),
+ )
+
+
+def _make_backend(provider: str = "bedrock") -> LiteLLMBackend:
+ # Patch the inference-profile fetch so `__init__` doesn't try to talk to AWS.
+ with patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}):
+ return LiteLLMBackend(provider=provider, region="us-east-1")
+
+
+def _request_body(model: str = CACHING_MODEL) -> dict[str, Any]:
+ return {
+ "model": model,
+ "messages": [
+ {"role": "system", "content": "You are terse."},
+ {"role": "user", "content": "hello"},
+ ],
+ "max_tokens": 32,
+ }
+
+
+async def _send(backend: LiteLLMBackend, body: dict[str, Any]) -> dict[str, Any]:
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = _make_response()
+ await backend.send_openai_message(body, {})
+ return mock_acomp.await_args.kwargs
+
+
+async def _stream(backend: LiteLLMBackend, body: dict[str, Any]) -> dict[str, Any]:
+ stream = _FakeAsyncStream(
+ [SimpleNamespace(model_dump=lambda **kwargs: {"id": "chunk1", "choices": []})]
+ )
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = stream
+ chunks = [chunk async for chunk in backend.stream_openai_message(body, {})]
+ assert chunks[-1] == "data: [DONE]\n\n"
+ return mock_acomp.await_args.kwargs
+
+
+def feature_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_FEATURES", FEATURE)
+ monkeypatch.delenv("HEADROOM_DISABLE_FEATURES", raising=False)
+
+
+def feature_default(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("HEADROOM_FEATURES", raising=False)
+ monkeypatch.delenv("HEADROOM_DISABLE_FEATURES", raising=False)
+
+
+# =============================================================================
+# _place_system_cache_control (pure)
+# =============================================================================
+
+
+def test_marks_first_system_message_string_content() -> None:
+ messages = [
+ {"role": "system", "content": "a"},
+ {"role": "user", "content": "b"},
+ {"role": "system", "content": "c"},
+ ]
+ before = copy.deepcopy(messages)
+
+ out = _place_system_cache_control(messages)
+
+ assert out[0] == {"role": "system", "content": "a", "cache_control": EPHEMERAL}
+ assert out[1] is messages[1]
+ assert out[2] is messages[2] # only the first system message is marked
+ assert messages == before # caller-owned input never mutated
+
+
+def test_marks_last_text_block_of_list_content() -> None:
+ messages = [
+ {
+ "role": "system",
+ "content": [
+ {"type": "text", "text": "a"},
+ {"type": "text", "text": "b"},
+ {"type": "text", "text": ""},
+ ],
+ },
+ {"role": "user", "content": "hi"},
+ ]
+ before = copy.deepcopy(messages)
+
+ out = _place_system_cache_control(messages)
+
+ blocks = out[0]["content"]
+ assert blocks[0] is messages[0]["content"][0]
+ assert blocks[1] == {"type": "text", "text": "b", "cache_control": EPHEMERAL}
+ assert blocks[2] is messages[0]["content"][2] # empty text is not a breakpoint
+ assert messages == before
+
+
+def test_system_message_anywhere_in_the_list() -> None:
+ messages = [{"role": "user", "content": "hi"}, {"role": "system", "content": "s"}]
+
+ out = _place_system_cache_control(messages)
+
+ assert out[0] is messages[0]
+ assert out[1]["cache_control"] == EPHEMERAL
+
+
+ "content",
+ ["", [], None, 7, [{"type": "image_url", "image_url": {"url": "x"}}], ["plain string"]],
+)
+def test_unmarkable_system_content_is_skipped(content: Any) -> None:
+ messages = [{"role": "system", "content": content}, {"role": "system", "content": "s"}]
+
+ out = _place_system_cache_control(messages)
+
+ assert out[0] is messages[0]
+ assert out[1]["cache_control"] == EPHEMERAL
+
+
+ "messages",
+ [
+ [],
+ [{"role": "user", "content": "hi"}],
+ [{"role": "system", "content": ""}],
+ ["not a dict", {"role": "user", "content": "hi"}],
+ # The client already owns breakpoint placement -- anywhere, any shape.
+ [
+ {"role": "system", "content": "s"},
+ {"role": "user", "content": "u", "cache_control": EPHEMERAL},
+ ],
+ [
+ {"role": "system", "content": "s"},
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "u", "cache_control": EPHEMERAL}],
+ },
+ ],
+ [
+ {
+ "role": "system",
+ "content": [{"type": "text", "text": "s", "cache_control": EPHEMERAL}],
+ }
+ ],
+ ],
+)
+def test_returns_input_unchanged_when_nothing_to_do(messages: list[Any]) -> None:
+ before = copy.deepcopy(messages)
+
+ assert _place_system_cache_control(messages) is messages
+ assert messages == before
+
+
+# =============================================================================
+# Backend gate
+# =============================================================================
+
+
[email protected]("feature_default")
[email protected]("call", [_send, _stream])
+async def test_feature_off_by_default_forwards_messages_verbatim(call: Any) -> None:
+ body = _request_body()
+
+ kwargs = await call(_make_backend(), body)
+
+ assert kwargs["messages"] is body["messages"]
+ assert "cache_control" not in kwargs["messages"][0]
+
+
[email protected]("feature_enabled")
[email protected]("call", [_send, _stream])
+async def test_feature_on_marks_system_prompt_for_caching_model(call: Any) -> None:
+ body = _request_body()
+ client_messages = copy.deepcopy(body["messages"])
+
+ kwargs = await call(_make_backend(), body)
+
+ assert kwargs["messages"][0]["cache_control"] == EPHEMERAL
+ assert kwargs["messages"][1] is body["messages"][1]
+ assert body["messages"] == client_messages # proxy re-reads body["messages"] after send
+
+
[email protected]("feature_enabled")
+async def test_feature_on_skips_models_without_prompt_caching() -> None:
+ body = _request_body(model="meta.llama3-1-70b-instruct-v1:0")
+
+ with patch("headroom.backends.litellm.supports_prompt_caching", return_value=False):
+ kwargs = await _send(_make_backend(), body)
+
+ assert kwargs["messages"] is body["messages"]
+
+
[email protected]("feature_enabled")
+async def test_gate_checks_the_mapped_litellm_model() -> None:
+ # A bare alias such as `claude-sonnet-4-20250514` is not a Bedrock entry in
+ # litellm's registry; the resolved `bedrock/<region>.anthropic...` id is.
+ body = _request_body(model="claude-sonnet-4-20250514")
+ backend = _make_backend()
+
+ with patch(
+ "headroom.backends.litellm.supports_prompt_caching", return_value=True
+ ) as mock_supports:
+ kwargs = await _send(backend, body)
+
+ mock_supports.assert_called_once_with(model=backend.map_model_id("claude-sonnet-4-20250514"))
+ assert kwargs["messages"][0]["cache_control"] == EPHEMERAL
+
+
[email protected]("feature_enabled")
+async def test_feature_on_is_a_no_op_for_non_bedrock_providers() -> None:
+ body = _request_body(model="anthropic/claude-3-5-sonnet-20241022")
+
+ with patch("headroom.backends.litellm.supports_prompt_caching") as mock_supports:
+ kwargs = await _send(_make_backend(provider="openrouter"), body)
+
+ mock_supports.assert_not_called()
+ assert kwargs["messages"] is body["messages"]
+
+
[email protected]("feature_enabled")
+async def test_feature_on_respects_client_placed_markers() -> None:
+ body = _request_body()
+ body["messages"][1]["cache_control"] = EPHEMERAL
+
+ kwargs = await _send(_make_backend(), body)
+
+ assert kwargs["messages"] is body["messages"]
+
+
+def test_disable_features_is_the_kill_switch(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_FEATURES", FEATURE)
+ monkeypatch.setenv("HEADROOM_DISABLE_FEATURES", FEATURE)
+
+ assert _make_backend()._openai_prompt_caching is False
+
+
[email protected]("feature_enabled")
+async def test_marker_becomes_a_bedrock_converse_cache_point() -> None:
+ """The marker shape must be the one litellm's Converse transform consumes."""
+ body = _request_body()
+
+ kwargs = await _send(_make_backend(), body)
+ request = litellm.AmazonConverseConfig().transform_request(
+ model=CACHING_MODEL,
+ messages=kwargs["messages"],
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+
+ assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}]
diff --git a/tests/test_rollout.py b/tests/test_rollout.py
index f24123c27..50564c3ec 100644
--- a/tests/test_rollout.py
+++ b/tests/test_rollout.py
@@ -396,7 +396,7 @@ def test_cli_json_status_and_strict_error() -> None:
assert result.exit_code == 0
payload = json.loads(result.output)
assert payload["channel"] == "canary"
- assert payload["features"][2]["name"] == "tool_result_interceptors"
+ assert "tool_result_interceptors" in [feature["name"] for feature in payload["features"]]
assert invalid.exit_code != 0
assert "unknown rollout feature" in invalid.output