Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): only buffer a stream when a marker is actually redeemable (#3092)
Tejas Choprac502087db702
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex f0946ed2d..2ac5bc7db 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -318,6 +318,44 @@ class AnthropicHandlerMixin:return Truereturn False+ @staticmethod+ def _outgoing_body_has_redeemable_marker(body: Any) -> bool:+ """Does the body about to be sent carry a marker retrieval could expand?++ ``headroom_retrieve`` exists only to expand a ``<<ccr:...>>`` marker, so+ a request carrying none cannot benefit from the buffered path (#3071).++ Ownership is verified rather than shape-matched: the marker shape is not+ unique to Headroom, and adopting another context tool's hash would send+ the model to an endpoint that is guaranteed to miss (#2836). A hash that+ survives ``verify_ownership`` is redeemable right now.++ Errors are swallowed deliberately and answered ``True``. This gates a+ wire-format decision, and the safe direction on an unexpected message+ shape is the long-standing buffered behavior, not a silent change.+ """+ if not isinstance(body, dict):+ return True+ messages = body.get("messages")+ if not isinstance(messages, list) or not messages:+ return False+ try:+ from headroom.ccr.tool_injection import CCRToolInjector++ probe = CCRToolInjector(+ provider="anthropic",+ inject_tool=False,+ inject_system_instructions=False,+ )+ probe.scan_for_markers(messages)+ if not probe.detected_hashes:+ return False+ probe.verify_ownership()+ return bool(probe.detected_hashes)+ except Exception: # pragma: no cover - defensive+ logger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True)+ return True+@staticmethoddef _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:"""Extract observed Anthropic cache-write TTL bucket usage.@@ -3368,13 +3406,40 @@ class AnthropicHandlerMixin:body=body,original_body_bytes=original_body_bytes,)- wants_buffered_stream_ccr = bool(+ # ``headroom_retrieve`` stays resident for the session lifetime so+ # the tools array is byte-stable and the prompt cache survives, so+ # its mere presence is a poor reason to buffer. Once a session went+ # sticky, *every* later streaming turn took the buffered path, and+ # buffering replaces incremental delivery with one write at the+ # end: time-to-last-byte is roughly unchanged, but time-to-first-+ # byte becomes the entire generation. In the traffic reported in+ # #3071 that was 8s on average and up to 100s, on 234 requests in+ # a single day.+ #+ # The tool can only expand a marker that is in the outgoing body+ # and redeemable now, so a turn carrying none cannot benefit from+ # server-side retrieval and should keep streaming. This reads+ # ``body`` rather than the earlier scan of ``optimized_messages``+ # because memory hooks, pre-send extensions and the CCR/tool-search+ # repairs can all replace the message list after that scan; the+ # only list that matters is the one about to go on the wire.+ retrieve_tool_is_offered = (streamand ccr_response_handler_enabledand self._has_headroom_retrieve_tool(tools if tools is not None else body.get("tools")))+ buffered_retrieval_can_help = (+ retrieve_tool_is_offered and self._outgoing_body_has_redeemable_marker(body)+ )+ wants_buffered_stream_ccr = bool(buffered_retrieval_can_help)+ if retrieve_tool_is_offered and not buffered_retrieval_can_help:+ logger.info(+ f"[{request_id}] CCR: headroom_retrieve is resident but this "+ "request carries no redeemable marker, so server-side "+ "retrieval cannot fire; keeping the streaming path (#3071)"+ )buffered_stream_ccr = (wants_buffered_stream_ccr and not outbound_locked_to_client_bytes)diff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.pyindex b90a599d2..586fe6d27 100644--- a/tests/test_ccr_buffered_stream_signed_thinking.py+++ b/tests/test_ccr_buffered_stream_signed_thinking.py@@ -62,8 +62,35 @@ def _config() -> ProxyConfig:)-def _body(*, with_thinking: bool) -> dict:- messages: list[dict] = [{"role": "user", "content": "hi"}]+def ccr_marker() -> str:+ """A marker this proxy actually owns, so retrieval could really fire.++ The buffered path is only taken when the outgoing body carries a redeemable+ marker (#3071) — ``headroom_retrieve`` has nothing to expand otherwise. These+ tests are about what happens *on* that path, so they have to earn it.+ """+ from headroom.cache.backends import InMemoryBackend+ from headroom.cache.compression_store import get_compression_store, reset_compression_store++ reset_compression_store()+ store = get_compression_store(backend=InMemoryBackend())+ hash_key = store.store(+ "the original, uncompressed tool output",+ "<<ccr:placeholder>>",+ original_tokens=100,+ compressed_tokens=5,+ tool_name="Read",+ )+ try:+ yield hash_key+ finally:+ reset_compression_store()+++def _body(*, with_thinking: bool, marker: str | None = None) -> dict:+ first = "hi" if marker is None else f"hi — earlier output is at <<ccr:{marker}>>"+ messages: list[dict] = [{"role": "user", "content": first}]if with_thinking:messages.append(SIGNED_THINKING_TURN)messages.append({"role": "user", "content": "continue"})@@ -85,7 +112,7 @@ def _headers() -> dict[str, str]:[(True, True), (False, False)],)def test_signed_thinking_history_skips_the_buffered_ccr_path(- with_thinking: bool, expect_plain_streaming: bool+ with_thinking: bool, expect_plain_streaming: bool, ccr_marker: str) -> None:"""The buffered path is only chosen when the stream:false flip can land."""calls: dict[str, object] = {}@@ -115,7 +142,9 @@ def test_signed_thinking_history_skips_the_buffered_ccr_path(client.app.state.proxy._stream_response = fake_stream_responseclient.app.state.proxy._retry_request = fake_retryresp = client.post(- "/v1/messages", json=_body(with_thinking=with_thinking), headers=_headers()+ "/v1/messages",+ json=_body(with_thinking=with_thinking, marker=ccr_marker),+ headers=_headers(),)assert resp.status_code == 200, resp.text@@ -133,7 +162,7 @@ def test_signed_thinking_history_skips_the_buffered_ccr_path(@pytest.mark.parametrize("upstream_delay", [0.0, 1.2], ids=["prompt", "past-keepalive"])def test_buffered_ccr_relays_an_unexpected_sse_reply_and_does_not_cache_it(- upstream_delay: float,+ upstream_delay: float, ccr_marker: str) -> None:"""A 200 SSE reply on the buffered path reaches the client as a stream.@@ -153,7 +182,11 @@ def test_buffered_ccr_relays_an_unexpected_sse_reply_and_does_not_cache_it(with TestClient(app) as client:proxy = client.app.state.proxyproxy._retry_request = fake_retry- resp = client.post("/v1/messages", json=_body(with_thinking=False), headers=_headers())+ resp = client.post(+ "/v1/messages",+ json=_body(with_thinking=False, marker=ccr_marker),+ headers=_headers(),+ )assert resp.status_code == 200, resp.textassert resp.headers["content-type"].startswith("text/event-stream")@@ -164,6 +197,76 @@ def test_buffered_ccr_relays_an_unexpected_sse_reply_and_does_not_cache_it(assert len(proxy.cache._cache) == 0, "an unparseable body must never be cached"+ ("marker_kind", "expect_buffered"),+ [+ ("owned", True),+ ("none", False),+ ("foreign", False),+ ],+)+def test_buffering_is_gated_on_a_redeemable_marker(+ marker_kind: str, expect_buffered: bool, ccr_marker: str+) -> None:+ """A resident ``headroom_retrieve`` is not on its own a reason to buffer (#3071).++ The tool is injected once and kept resident so the tools array stays+ byte-stable for the prompt cache. Buffering on its presence alone meant+ every later streaming turn of a sticky session lost incremental delivery —+ time-to-first-byte became the whole generation. Retrieval can only expand a+ marker that is in the outgoing body *and* redeemable now, so that is what+ the wire-format decision keys on.+ """+ marker = {+ "owned": ccr_marker,+ "none": None,+ # Correct shape, not ours: adopting it would send the model to a+ # retrieval that is guaranteed to miss (#2836).+ "foreign": "deadbeefcafe",+ }[marker_kind]++ calls: dict[str, object] = {}++ async def fake_stream_response(url, headers, body, *args, **kwargs): # noqa: ANN001+ calls["stream_body"] = body+ return fastapi.responses.StreamingResponse(iter([SSE_BODY]), media_type="text/event-stream")++ async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001+ calls["buffered_body"] = json.loads(json.dumps(req_body))+ return httpx.Response(+ 200,+ json={+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-20250514",+ "content": [{"type": "text", "text": "ok"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 10, "output_tokens": 5},+ },+ headers={"content-type": "application/json"},+ )++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._stream_response = fake_stream_response+ client.app.state.proxy._retry_request = fake_retry+ resp = client.post(+ "/v1/messages",+ json=_body(with_thinking=False, marker=marker),+ headers=_headers(),+ )++ assert resp.status_code == 200, resp.text+ if expect_buffered:+ assert "buffered_body" in calls, "a redeemable marker must still buffer"+ assert calls["buffered_body"]["stream"] is False+ else:+ assert "stream_body" in calls, "nothing to retrieve — the client must keep streaming"+ assert "buffered_body" not in calls+ assert calls["stream_body"]["stream"] is True++def test_cache_hit_never_replays_a_foreign_content_type() -> None:"""A cache entry cannot hand a caller a wire format it did not ask for."""body = {diff --git a/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py b/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.pyindex 478eaceba..70889b647 100644--- a/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py+++ b/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py@@ -35,6 +35,37 @@ def _make_config() -> ProxyConfig:)[email protected](autouse=True)+def _fresh_compression_store():+ """Each test gets its own store, so seeded markers cannot leak between them."""+ from headroom.cache.backends import InMemoryBackend+ from headroom.cache.compression_store import get_compression_store, reset_compression_store++ reset_compression_store()+ get_compression_store(backend=InMemoryBackend())+ try:+ yield+ finally:+ reset_compression_store()+++def _buffered(text: str) -> str:+ """User content carrying a marker this proxy owns.++ The buffered path engages only when retrieval has something to expand+ (#3071); a resident ``headroom_retrieve`` with no redeemable marker in the+ request keeps streaming. Every test below that is *about* the buffered path+ therefore has to earn it with a real marker rather than the tool alone.+ """+ store = get_compression_store()+ hash_key = store.store(+ original=json.dumps({"earlier": "tool output"}),+ compressed="{}",+ original_item_count=1,+ )+ return f"{text} (earlier output at <<ccr:{hash_key}>>)"++def _message_response(content: list[dict], *, stop_reason: str = "end_turn") -> dict:return {"id": "msg_test",@@ -127,7 +158,7 @@ def test_streaming_headroom_retrieve_is_intercepted_and_returned_as_sse() -> Non"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "retrieve it"}],+ "messages": [{"role": "user", "content": _buffered("retrieve it")}],},)@@ -213,7 +244,7 @@ def test_streaming_with_headroom_retrieve_available_but_unused_returns_sse() ->"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "hello"}],+ "messages": [{"role": "user", "content": _buffered("hello")}],},)@@ -282,7 +313,7 @@ def test_mixed_ccr_and_client_tool_streams_both_blocks_as_sse() -> None:"input_schema": {"type": "object", "properties": {}},},],- "messages": [{"role": "user", "content": "use tools"}],+ "messages": [{"role": "user", "content": _buffered("use tools")}],},)@@ -341,7 +372,7 @@ def test_unresolved_ccr_only_streams_through_as_200() -> None:"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "use tools"}],+ "messages": [{"role": "user", "content": _buffered("use tools")}],},)@@ -368,7 +399,7 @@ async def test_buffered_ccr_withholds_output_until_delayed_upstream_resolves() -"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "wait"}],+ "messages": [{"role": "user", "content": _buffered("wait")}],}request_delivered = False@@ -439,7 +470,7 @@ async def test_buffered_ccr_preserves_early_failure_status_and_headers() -> None"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "fail early"}],+ "messages": [{"role": "user", "content": _buffered("fail early")}],}async def receive():@@ -508,7 +539,7 @@ async def test_buffered_ccr_preserves_late_failure_status_and_headers() -> None:"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "fail late"}],+ "messages": [{"role": "user", "content": _buffered("fail late")}],}async def receive():@@ -588,7 +619,7 @@ def test_buffered_ccr_rejects_malformed_success_as_502() -> None:"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "fail safely"}],+ "messages": [{"role": "user", "content": _buffered("fail safely")}],},)@@ -608,7 +639,7 @@ async def test_buffered_ccr_late_failure_returns_sanitized_json_error() -> None:"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "wait"}],+ "messages": [{"role": "user", "content": _buffered("wait")}],}async def receive():@@ -684,7 +715,7 @@ async def test_buffered_ccr_pre_keepalive_exception_returns_json_error() -> None"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "fail before keepalive"}],+ "messages": [{"role": "user", "content": _buffered("fail before keepalive")}],}async def receive():diff --git a/tests/test_proxy_response_cache_replay.py b/tests/test_proxy_response_cache_replay.pyindex 776d2569d..738b3ad15 100644--- a/tests/test_proxy_response_cache_replay.py+++ b/tests/test_proxy_response_cache_replay.py@@ -31,6 +31,11 @@ httpx = pytest.importorskip("httpx")from fastapi.testclient import TestClient # noqa: E402+from headroom.cache.backends import InMemoryBackend # noqa: E402+from headroom.cache.compression_store import ( # noqa: E402+ get_compression_store,+ reset_compression_store,+)from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402from headroom.proxy.helpers import sanitize_forwarded_response_headers # noqa: E402from headroom.proxy.models import CacheEntry # noqa: E402@@ -245,6 +250,16 @@ def test_buffered_ccr_turn_does_not_write_the_response_cache():},}+ # The buffered conversion needs a marker retrieval could actually expand;+ # a resident `headroom_retrieve` alone keeps the request streaming (#3071).+ reset_compression_store()+ store = get_compression_store(backend=InMemoryBackend())+ marker = store.store(+ original=json.dumps({"earlier": "tool output"}),+ compressed="{}",+ original_item_count=1,+ )+with patch("headroom.proxy.server.AnyLLMBackend"):app = create_app(_ccr_cache_config())with TestClient(app) as client:@@ -273,7 +288,9 @@ def test_buffered_ccr_turn_does_not_write_the_response_cache():"max_tokens": 64,"stream": True,"tools": [create_ccr_tool_definition("anthropic")],- "messages": [{"role": "user", "content": "hello"}],+ "messages": [+ {"role": "user", "content": f"hello (earlier at <<ccr:{marker}>>)"}+ ],},)@@ -282,6 +299,7 @@ def test_buffered_ccr_turn_does_not_write_the_response_cache():assert forwarded_bodies and forwarded_bodies[0]["stream"] is False# ...and nothing was written to the response cache.proxy.cache.set.assert_not_awaited()+ reset_compression_store()# --------------------------------------------------------------------------