Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
"""``headroom_retrieve`` on a streamed turn is served by the proxy (#3587).
A turn whose redeemable markers were all forwarded on earlier turns streams
instead of buffering, so the client gets the first token right away. The model
can still ask for one of those originals. The client never declared
``headroom_retrieve`` (a coding agent answers "No such tool available"), so
the call must not reach it: the stream withholds it, the proxy serves it from
the compression store, and the continuation round streams into the same client
message.
"""
from __future__ import annotations
import json
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
fastapi = pytest.importorskip("fastapi")
from fastapi.testclient import TestClient # noqa: E402
from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402
from headroom.proxy.memory_tool_stream import MemoryToolStreamFilter # noqa: E402
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
RETRIEVE_ONLY = frozenset({"headroom_retrieve"})
ORIGINAL = "line 1 of the original build log\nline 2: the error the model needs"
def _frame(payload: dict[str, Any]) -> bytes:
return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()
def _sse(blocks: list[dict[str, Any]], stop_reason: str) -> bytes:
"""An Anthropic SSE response carrying ``blocks``."""
frames = [
_frame(
{
"type": "message_start",
"message": {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "sonnet-test",
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 10, "output_tokens": 1},
},
}
)
]
for index, block in enumerate(blocks):
if block["type"] == "text":
start = {"type": "text", "text": ""}
delta = {"type": "text_delta", "text": block["text"]}
else:
start = {**block, "input": {}}
delta = {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}
frames.append(
_frame({"type": "content_block_start", "index": index, "content_block": start})
)
frames.append(_frame({"type": "content_block_delta", "index": index, "delta": delta}))
frames.append(_frame({"type": "content_block_stop", "index": index}))
frames.append(
_frame(
{
"type": "message_delta",
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
"usage": {"output_tokens": 5},
}
)
)
frames.append(_frame({"type": "message_stop"}))
return b"".join(frames)
def _events(raw: bytes) -> list[dict[str, Any]]:
return [
json.loads(line[len("data: ") :])
for line in raw.decode().splitlines()
if line.startswith("data: ")
]
def _tool_names(events: list[dict[str, Any]]) -> list[str]:
return [
e["content_block"]["name"]
for e in events
if e["type"] == "content_block_start" and e["content_block"].get("type") == "tool_use"
]
TEXT = {"type": "text", "text": "Let me look at the earlier output."}
BASH = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}
def _retrieve(hash_key: str) -> dict[str, Any]:
return {
"type": "tool_use",
"id": "toolu_ccr",
"name": "headroom_retrieve",
"input": {"hash": hash_key},
}
class TestHoldingFilter:
def test_text_before_the_first_tool_call_streams_live(self) -> None:
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
forwarded = _events(b"".join(flt.feed(_sse([TEXT, BASH], "tool_use"))))
assert [e["type"] for e in forwarded] == [
"message_start",
"content_block_start",
"content_block_delta",
"content_block_stop",
]
assert flt.holds_client_tool_use
def test_without_a_retrieval_the_round_is_byte_identical(self) -> None:
raw = _sse([TEXT, BASH], "tool_use")
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
out = b"".join(flt.feed(raw)) + b"".join(flt.closing_frames())
assert out == raw
def test_dropping_held_client_calls(self) -> None:
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
flt.feed(_sse([BASH, _retrieve("abc")], "tool_use"))
assert flt.hidden_tool_names == ["headroom_retrieve"]
assert flt.release_held(drop_tool_use=True) == []
assert flt.next_index == 0
assert not flt.visible_tool_use
def _sse_response(raw: bytes) -> MagicMock:
response = MagicMock()
response.headers = httpx.Headers({"content-type": "text/event-stream"})
response.status_code = 200
async def aiter_bytes(raw: bytes = raw):
# Split mid-frame so the filter has to reassemble events.
yield raw[:41]
yield raw[41:]
response.aiter_bytes = aiter_bytes
response.aclose = AsyncMock()
return response
@pytest.fixture
def owned_marker():
"""A marker the proxy owns, with its original in the compression store."""
from headroom.cache.backends import InMemoryBackend
from headroom.cache.compression_store import get_compression_store, reset_compression_store
reset_compression_store()
store = get_compression_store(backend=InMemoryBackend())
hash_key = store.store(
ORIGINAL,
"<<ccr:placeholder>>",
original_tokens=100,
compressed_tokens=5,
tool_name="Bash",
)
try:
yield hash_key
finally:
reset_compression_store()
def _two_turn_session(
marker: str, second_turn_rounds: list[bytes]
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:
"""Turn 1 introduces ``marker``; turn 2 only carries it in history.
Returns the client's SSE events for turn 2, the bodies the proxy sent
upstream on turn 2's stream (first round, then continuations), and how
many buffered upstream requests turn 2 made.
"""
first_turn = [
{"role": "user", "content": "build it"},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "tu_1", "name": "Bash", "input": {}}],
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "tu_1",
"content": f"build log compressed at <<ccr:{marker}>>",
}
],
},
]
second_turn = [
*first_turn,
{"role": "assistant", "content": [{"type": "text", "text": "The build failed."}]},
{"role": "user", "content": "what was the exact error?"},
]
buffered_calls: list[int] = []
async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001
buffered_calls.append(1)
return httpx.Response(
200,
json={
"id": "msg_t1",
"type": "message",
"role": "assistant",
"model": "sonnet-test",
"content": [{"type": "text", "text": "The build failed."}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 5},
},
headers={"content-type": "application/json"},
)
config = ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=False,
memory_enabled=False,
)
headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}
tools = [
create_ccr_tool_definition("anthropic"),
{"name": "Bash", "description": "run", "input_schema": {"type": "object"}},
]
def body(messages: list[dict[str, Any]]) -> dict[str, Any]:
return {
"model": "sonnet-test",
"max_tokens": 64,
"stream": True,
"tools": tools,
"messages": messages,
}
app = create_app(config)
with TestClient(app) as client:
proxy = client.app.state.proxy
proxy._retry_request = fake_retry
first = client.post("/v1/messages", json=body(first_turn), headers=headers)
assert first.status_code == 200, first.text
assert len(buffered_calls) == 1, "the turn that introduces the marker buffers"
upstream = MagicMock(spec=httpx.AsyncClient)
upstream.build_request = MagicMock(return_value=MagicMock())
upstream.send = AsyncMock(side_effect=[_sse_response(r) for r in second_turn_rounds])
proxy.http_client = upstream
second = client.post("/v1/messages", json=body(second_turn), headers=headers)
assert second.status_code == 200, second.text
sent = [json.loads(c.kwargs["content"]) for c in upstream.build_request.call_args_list]
return _events(second.content), sent, len(buffered_calls) - 1
def test_later_turn_retrieval_is_served_by_the_proxy(owned_marker: str) -> None:
events, sent, buffered = _two_turn_session(
owned_marker,
[
_sse([TEXT, _retrieve(owned_marker)], "tool_use"),
_sse([{"type": "text", "text": "The error is on line 2."}], "end_turn"),
],
)
assert buffered == 0, "a turn with only already-forwarded markers keeps streaming"
assert "headroom_retrieve" not in _tool_names(events)
texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]
assert texts == ["Let me look at the earlier output.", "The error is on line 2."]
stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]
assert stops == ["end_turn"]
assert [e["type"] for e in events].count("message_start") == 1
assert len(sent) == 2
continuation = sent[1]["messages"]
assert continuation[-2]["content"][-1]["name"] == "headroom_retrieve"
served = continuation[-1]["content"][0]
assert served["type"] == "tool_result" and served["tool_use_id"] == "toolu_ccr"
assert json.loads(served["content"])["original_content"] == ORIGINAL
@pytest.mark.parametrize("client_call_first", [True, False])
def test_retrieval_mixed_with_a_client_tool(owned_marker: str, client_call_first: bool) -> None:
calls = [BASH, _retrieve(owned_marker)]
if not client_call_first:
calls.reverse()
reissued = {**BASH, "id": "toolu_bash_2"}
events, sent, buffered = _two_turn_session(
owned_marker,
[
_sse(calls, "tool_use"),
_sse([{"type": "text", "text": "Now rerun it."}, reissued], "tool_use"),
],
)
assert buffered == 0
# Only the model's decision made with the content in hand reaches the client.
assert _tool_names(events) == ["Bash"]
started = [e["content_block"] for e in events if e["type"] == "content_block_start"]
assert [b.get("id") for b in started if b.get("type") == "tool_use"] == ["toolu_bash_2"]
stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]
assert stops == ["tool_use"]
assert [e["index"] for e in events if e["type"] == "content_block_start"] == [0, 1]
continuation = sent[1]["messages"]
assert [b["name"] for b in continuation[-2]["content"]] == ["headroom_retrieve"]
assert json.loads(continuation[-1]["content"][0]["content"])["original_content"] == ORIGINAL
def test_old_marker_turn_without_retrieval_streams_unchanged(owned_marker: str) -> None:
raw = _sse([{"type": "text", "text": "It failed on line 2."}, BASH], "tool_use")
events, sent, buffered = _two_turn_session(owned_marker, [raw])
assert buffered == 0
assert len(sent) == 1, "no continuation round without a retrieval"
assert events == _events(raw)