Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
"""Per-request backend selection published by an extension.
The property under test throughout is the one that makes this safe to merge:
with nothing published, every path is exactly what it was before.
"""
from __future__ import annotations
import asyncio
import json
from types import SimpleNamespace
import pytest
from headroom.proxy.route_advice import (
BackendResolver,
RouteAdvice,
advice_from,
)
class _Req:
def __init__(self, **state):
self.state = SimpleNamespace(**state)
DEFAULT = object() # stands in for the configured backend
def _sse_contains_error_event(payload: bytes) -> bool:
"""True when any complete SSE event in ``payload`` is a failure event."""
from headroom.proxy.handlers.streaming import _sse_event_outcome
from headroom.proxy.helpers import parse_sse_events_from_byte_buffer
return any(
_sse_event_outcome(event_name, data, "chat")[0]
for event_name, data in parse_sse_events_from_byte_buffer(bytearray(payload))
)
# --- reading what an extension published ------------------------------------
def test_no_extension_means_no_advice():
assert advice_from(_Req()) is None
assert advice_from(object()) is None # no .state at all
assert advice_from(None) is None
def test_advice_is_duck_typed_so_extensions_need_not_import_us():
a = advice_from(
_Req(
headroom_route=SimpleNamespace(
model="moonshot/kimi-k2", provider="moonshot", reason="cheaper"
)
)
)
assert a == RouteAdvice("moonshot/kimi-k2", "moonshot", "cheaper")
def test_an_extension_may_omit_everything_but_the_model():
a = advice_from(_Req(headroom_route=SimpleNamespace(model="gpt-5-mini")))
assert a.model == "gpt-5-mini" and a.provider == ""
def test_malformed_advice_is_ignored_rather_than_raised():
for bad in (
SimpleNamespace(),
SimpleNamespace(model=""),
SimpleNamespace(model=123),
"not an object",
):
assert advice_from(_Req(headroom_route=bad)) is None
def test_advice_needs_a_model():
with pytest.raises(ValueError):
RouteAdvice("")
# --- absent means unchanged, which is the whole safety argument -------------
def test_no_advice_returns_the_configured_backend():
r = BackendResolver(DEFAULT)
assert r.for_request(_Req()) is DEFAULT
def test_no_advice_and_no_configured_backend_stays_none():
"""None is not "no backend", it is the direct-API path. It must survive."""
assert BackendResolver(None).for_request(_Req()) is None
def test_a_native_provider_does_not_switch_backends():
"""Anthropic is the shape the proxy already holds, so a model rewrite is
enough and the extension has already done it."""
r = BackendResolver(DEFAULT)
req = _Req(headroom_route=SimpleNamespace(model="claude-haiku-4-5", provider="anthropic"))
assert r.for_request(req) is DEFAULT
def test_a_bare_anthropic_model_resolves_its_provider_and_stays_put():
r = BackendResolver(DEFAULT)
req = _Req(headroom_route=SimpleNamespace(model="claude-haiku-4-5"))
assert r.for_request(req) is DEFAULT
# --- switching, and refusing to switch --------------------------------------
def test_a_foreign_provider_gets_its_own_backend(monkeypatch):
built = []
class FakeBackend:
def __init__(self, provider):
built.append(provider)
self.provider = provider
monkeypatch.setattr(BackendResolver, "_build", lambda self, p: FakeBackend(p))
r = BackendResolver(DEFAULT)
body = {"model": "claude-opus-5"}
req = _Req(headroom_route=SimpleNamespace(model="moonshot/kimi-k2", provider="moonshot"))
got = r.for_request(req, body=body)
assert isinstance(got, FakeBackend) and got.provider == "moonshot"
# The extension could not safely write a foreign model id; we do it.
assert body["model"] == "moonshot/kimi-k2"
def test_backends_are_built_once_per_provider(monkeypatch):
built = []
monkeypatch.setattr(BackendResolver, "_build", lambda self, p: built.append(p) or object())
r = BackendResolver(DEFAULT)
req = _Req(headroom_route=SimpleNamespace(model="x", provider="moonshot"))
for _ in range(5):
r.for_request(req)
assert built == ["moonshot"], "construction is expensive; cache it"
def test_a_backend_that_will_not_build_falls_back_and_stops_retrying(monkeypatch):
calls = []
monkeypatch.setattr(BackendResolver, "_build", lambda self, p: calls.append(p) or None)
r = BackendResolver(DEFAULT)
req = _Req(headroom_route=SimpleNamespace(model="x", provider="nope"))
for _ in range(5):
assert r.for_request(req) is DEFAULT
assert calls == ["nope"], "a broken provider must not be retried per request"
def test_a_routing_preference_can_never_take_traffic_down(monkeypatch):
"""Missing credentials, missing optional dependency -- whatever the reason,
the request still has to be served."""
def boom(self, provider):
raise RuntimeError("no credentials")
monkeypatch.setattr(BackendResolver, "_build", boom)
r = BackendResolver(DEFAULT)
req = _Req(headroom_route=SimpleNamespace(model="x", provider="moonshot"))
assert r.for_request(req) is DEFAULT
def test_an_unknown_provider_is_rejected_at_resolve_time():
"""LiteLLMBackend accepts ANY provider string -- the registry falls through
to a generic pass-through -- so a typo silently builds a backend that only
fails later, at request time, with an error pointing nowhere near the typo.
Validate the name up front instead."""
r = BackendResolver(DEFAULT)
assert r._build("definitely-not-a-provider-name") is None
req = _Req(headroom_route=SimpleNamespace(model="x", provider="definitely-not-a-provider-name"))
assert r.for_request(req) is DEFAULT
def test_a_real_provider_name_is_accepted():
assert BackendResolver(DEFAULT)._build("moonshot") is not None
# --- streaming, which is the path agents actually take ----------------------
class _Backend:
"""Records that it, and not some other backend, served the request."""
def __init__(self, name):
self.name = name
self.served = False
async def stream_message(self, body, headers):
self.served = True
return
yield # pragma: no cover -- makes this an async generator
async def stream_openai_message(self, body, headers):
self.served = True
return
yield # pragma: no cover
class _FailingBackend(_Backend):
async def stream_message(self, body, headers):
self.served = True
yield SimpleNamespace(
event_type="message_start",
raw_sse="event: message_start\ndata: {}\n\n",
data={},
)
raise RuntimeError("stream finalization failed")
class _ErrorEventBackend(_Backend):
async def stream_message(self, body, headers):
self.served = True
yield SimpleNamespace(
event_type="error",
raw_sse='event: error\ndata: {"error": {"type": "api_error"}}\n\n',
data={"error": {"type": "api_error"}},
)
async def stream_openai_message(self, body, headers):
self.served = True
yield 'data: {"type": "er'
yield 'ror", "message": "provider failed"}\n\n'
yield "data: [DONE]\n\n"
async def _drive_with_outcome(handler, backend):
from headroom.proxy.handlers.streaming import StreamingMixin
response = await StreamingMixin._stream_response_bedrock(
handler,
{"messages": []},
{},
"anthropic",
"m",
"rid",
0,
0,
0,
[],
{},
0.0,
backend=backend,
)
async for _ in response.body_iterator:
pass
async def _drive_openai_with_outcome(handler, backend):
from headroom.proxy.handlers.streaming import StreamingMixin
response = await StreamingMixin._stream_openai_via_backend(
handler,
{"messages": []},
{},
"m",
"rid",
0.0,
0,
0,
0,
[],
{},
0.0,
backend=backend,
)
async for _ in response.body_iterator:
pass
async def _drive(handler, **kw):
from headroom.proxy.handlers.streaming import StreamingMixin
resp = await StreamingMixin._stream_response_bedrock(
handler,
{"messages": []},
{},
"anthropic",
"m",
"rid",
0,
0,
0,
[],
{},
0.0,
**kw,
)
async for _ in resp.body_iterator:
pass
class _Config:
"""Every proxy flag off. Named individually the list would rot; the test
is about which backend served the request, not about config."""
def __getattr__(self, name):
return False
def _handler(default):
return SimpleNamespace(
anthropic_backend=default,
config=_Config(),
_record_request_outcome=lambda outcome: _noop(),
)
async def _noop():
return None
def test_streaming_honours_the_routed_backend():
"""The non-streaming branch was the easy half. body["model"] has already
been rewritten to a foreign id by the time we get here, so streaming to
the configured backend would send e.g. moonshot/kimi-k2 to Anthropic."""
configured, routed = _Backend("anthropic"), _Backend("moonshot")
asyncio.run(_drive(_handler(configured), backend=routed))
assert routed.served and not configured.served
def test_streaming_without_a_route_uses_the_configured_backend():
configured = _Backend("anthropic")
asyncio.run(_drive(_handler(configured)))
assert configured.served
def test_failed_stream_finalization_is_not_counted_as_completed():
backend = _FailingBackend("anthropic")
outcomes = []
async def record_outcome(outcome):
outcomes.append(outcome)
handler = _handler(backend)
handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
handler._record_request_outcome = record_outcome
asyncio.run(_drive_with_outcome(handler, backend))
assert len(outcomes) == 1
assert outcomes[0].status_code == 502
def test_stream_outcome_status_preserves_success_and_reclassifies_failure():
from headroom.proxy.handlers.streaming import _stream_outcome_status
assert _stream_outcome_status(200, True) == 200
assert _stream_outcome_status(200, False) == 502
assert _stream_outcome_status(529, False) == 502
def test_sse_error_detection_handles_native_and_split_events():
assert _sse_contains_error_event(b'event: error\ndata: {"type":"error"}\n\n')
assert _sse_contains_error_event(b'data: {"type":"er' + b'ror"}\n\n')
assert not _sse_contains_error_event(b'data: {"type":"message"}\n\n')
def test_sse_error_text_in_response_content_is_not_an_error():
assert not _sse_contains_error_event(b'data: {"text":"event: error"}\n\n')
def test_responses_failed_envelope_is_an_error():
assert _sse_contains_error_event(
b'data: {"type":"response.failed","response":{"status":"failed",'
b'"error":{"code":"server_error"}}}\n\n'
)
@pytest.mark.parametrize("provider", ["anthropic", "openai"])
@pytest.mark.parametrize("complete", [False, True])
def test_backend_clean_eof_requires_protocol_completion(provider, complete):
from types import MethodType
from headroom.proxy.handlers.streaming import StreamingMixin
class Backend(_Backend):
async def stream_message(self, body, headers):
yield SimpleNamespace(
event_type="message_start", raw_sse="event: message_start\ndata: {}\n\n", data={}
)
if complete:
yield SimpleNamespace(
event_type="message_stop",
raw_sse='event: message_stop\ndata: {"type":"message_stop"}\n\n',
data={"type": "message_stop"},
)
async def stream_openai_message(self, body, headers):
yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'
if complete:
yield "data: [DONE]\n\n"
outcomes = []
async def record(outcome):
outcomes.append(outcome)
handler = _handler(Backend(provider))
handler._record_request_outcome = record
handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
handler._parse_sse_usage_from_buffer = MethodType(
StreamingMixin._parse_sse_usage_from_buffer, handler
)
drive = _drive_with_outcome if provider == "anthropic" else _drive_openai_with_outcome
asyncio.run(drive(handler, handler.anthropic_backend))
assert len(outcomes) == 1
assert outcomes[0].status_code == (200 if complete else 502)
@pytest.mark.parametrize("provider", ["anthropic", "openai"])
def test_backend_client_hangup_after_terminal_event_is_completed(provider):
"""A client may close on the terminal event, before the backend's EOF."""
from types import MethodType
from headroom.proxy.handlers.streaming import StreamingMixin
class Backend(_Backend):
async def stream_message(self, body, headers):
yield SimpleNamespace(
event_type="message_stop",
raw_sse='event: message_stop\ndata: {"type":"message_stop"}\n\n',
data={"type": "message_stop"},
)
await asyncio.Event().wait()
async def stream_openai_message(self, body, headers):
yield "data: [DONE]\n\n"
await asyncio.Event().wait()
outcomes = []
async def record(outcome):
outcomes.append(outcome)
handler = _handler(Backend(provider))
handler._record_request_outcome = record
handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
handler._parse_sse_usage_from_buffer = MethodType(
StreamingMixin._parse_sse_usage_from_buffer, handler
)
async def drive():
arguments = {
"body": {"messages": []},
"headers": {},
"model": "m",
"request_id": "rid",
"original_tokens": 0,
"optimized_tokens": 0,
"tokens_saved": 0,
"transforms_applied": [],
"tags": {},
"optimization_latency": 0.0,
"backend": handler.anthropic_backend,
}
if provider == "anthropic":
response = await StreamingMixin._stream_response_bedrock(
handler, provider=provider, **arguments
)
else:
response = await StreamingMixin._stream_openai_via_backend(
handler, start_time=0.0, **arguments
)
body = response.body_iterator
while b"message_stop" not in (chunk := await body.__anext__()) and b"[DONE]" not in chunk:
pass
await body.aclose()
asyncio.run(drive())
assert len(outcomes) == 1
assert outcomes[0].status_code == 200
@pytest.mark.parametrize("provider", ["anthropic", "openai"])
@pytest.mark.parametrize("complete", [False, True])
def test_backend_prefix_tracker_advances_only_after_completion(provider, complete):
from types import MethodType
from unittest.mock import MagicMock
from headroom.proxy.handlers.streaming import StreamingMixin
class Backend(_Backend):
async def stream_message(self, body, headers):
yield SimpleNamespace(event_type="message_start", raw_sse="", data={})
if complete:
yield SimpleNamespace(event_type="message_stop", raw_sse="", data={})
async def stream_openai_message(self, body, headers):
yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'
if complete:
yield "data: [DONE]\n\n"
backend = Backend(provider)
handler = _handler(backend)
handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
handler._parse_sse_usage_from_buffer = MethodType(
StreamingMixin._parse_sse_usage_from_buffer, handler
)
handler._parse_sse_to_response = lambda *args: None
handler._assistant_message_from_response_json = lambda response: None
tracker = MagicMock()
tracker.classify_cache_miss.return_value = SimpleNamespace(is_miss=False)
async def drive():
arguments = {
"body": {"messages": []},
"headers": {},
"model": "m",
"request_id": "rid",
"original_tokens": 0,
"optimized_tokens": 0,
"tokens_saved": 0,
"transforms_applied": [],
"tags": {},
"optimization_latency": 0.0,
"backend": backend,
"prefix_tracker": tracker,
}
if provider == "anthropic":
response = await StreamingMixin._stream_response_bedrock(
handler, provider=provider, **arguments
)
else:
response = await StreamingMixin._stream_openai_via_backend(
handler, start_time=0.0, **arguments
)
async for _ in response.body_iterator:
pass
asyncio.run(drive())
assert tracker.update_from_response.call_count == int(complete)
@pytest.mark.parametrize(
"event, data, expected_failed, expected_terminal",
[
("response.completed", '{"type":"response.completed"}', False, True),
("response.incomplete", '{"type":"response.incomplete"}', False, True),
("response.failed", '{"type":"response.failed"}', True, False),
(None, "[DONE]", False, False),
],
)
def test_responses_terminal_semantics(event, data, expected_failed, expected_terminal):
from headroom.proxy.handlers.streaming import _sse_event_outcome
assert _sse_event_outcome(event, data, "responses") == (expected_failed, expected_terminal)
@pytest.mark.parametrize("event_type", ['["error"]', '{"unexpected":true}', "7", "true"])
def test_malformed_event_type_is_not_interpreted_as_a_protocol_marker(event_type):
payload = ('data: {"type":' + event_type + "}\n\ndata: [DONE]\n\n").encode()
assert not _sse_contains_error_event(payload)
@pytest.mark.parametrize("wrapped", [False, True])
@pytest.mark.parametrize("finish_reason", ["STOP", "MAX_TOKENS"])
def test_gemini_finish_reason_is_a_terminal_event(wrapped, finish_reason):
import json
from headroom.proxy.handlers.streaming import _sse_event_outcome
payload = {"candidates": [{"index": 0, "finishReason": finish_reason}]}
if wrapped:
payload = {"response": payload}
assert _sse_event_outcome(None, json.dumps(payload), "gemini") == (False, True)
@pytest.mark.parametrize("wrapped", [False, True])
def test_gemini_prompt_block_is_a_failed_outcome(wrapped):
import json
from headroom.proxy.handlers.streaming import _sse_event_outcome
payload = {"promptFeedback": {"blockReason": "SAFETY"}}
if wrapped:
payload = {"response": payload}
failed, _ = _sse_event_outcome(None, json.dumps(payload), "gemini")
assert failed
def test_gemini_completion_requires_all_requested_candidates():
from headroom.proxy.handlers.streaming import StreamingMixin
parser = StreamingMixin()
state = {"sse_buffer": bytearray(), "gemini_candidate_count": 2}
state["sse_buffer"].extend(b'data: {"candidates":[{"index":0,"finishReason":"STOP"}]}\n\n')
parser._parse_sse_usage_from_buffer(state, "gemini")
assert not state.get("stream_terminal", False)
state["sse_buffer"].extend(b'data: {"candidates":[{"index":1,"finishReason":"STOP"}]}\n\n')
parser._parse_sse_usage_from_buffer(state, "gemini")
assert state["stream_terminal"]
def test_outcome_classification_runs_once_for_a_chunked_complete_event(monkeypatch):
from headroom.proxy.handlers import streaming
seen = []
classify = streaming._sse_event_outcome
def record(*args):
seen.append(args)
return classify(*args)
monkeypatch.setattr(streaming, "_sse_event_outcome", record)
state = {"sse_buffer": bytearray()}
parser = streaming.StreamingMixin()
payload = b'data: {"text":"' + b"x" * 10_000 + b'"}\n\n'
for offset in range(0, len(payload), 101):
state["sse_buffer"].extend(payload[offset : offset + 101])
parser._parse_sse_usage_from_buffer(state, "openai")
assert len(seen) == 1
assert not state["sse_buffer"]
def test_backend_error_event_is_not_counted_as_completed():
backend = _ErrorEventBackend("anthropic")
outcomes = []
async def record_outcome(outcome):
outcomes.append(outcome)
handler = _handler(backend)
handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
handler._record_request_outcome = record_outcome
asyncio.run(_drive_with_outcome(handler, backend))
assert backend.served
assert len(outcomes) == 1
assert outcomes[0].status_code == 502
def test_openai_backend_error_event_is_not_counted_as_completed():
backend = _ErrorEventBackend("openai")
outcomes = []
async def record_outcome(outcome):
outcomes.append(outcome)
handler = _handler(backend)
handler._parse_sse_usage_from_buffer = lambda state, provider: None
handler._record_request_outcome = record_outcome
asyncio.run(_drive_openai_with_outcome(handler, backend))
assert backend.served
assert len(outcomes) == 1
assert outcomes[0].status_code == 502
async def _drive_openai(handler, **kw):
from headroom.proxy.handlers.streaming import StreamingMixin
resp = await StreamingMixin._stream_openai_via_backend(
handler,
{"messages": []},
{},
"m",
"rid",
0.0,
0,
0,
0,
[],
{},
0.0,
**kw,
)
async for _ in resp.body_iterator:
pass
def test_openai_streaming_honours_the_routed_backend():
"""opencode and pi can speak either protocol, so the OpenAI chat path
needs the same treatment as the Anthropic one."""
configured, routed = _Backend("openai"), _Backend("moonshot")
asyncio.run(_drive_openai(_handler(configured), backend=routed))
assert routed.served and not configured.served
def test_openai_streaming_without_a_route_uses_the_configured_backend():
configured = _Backend("openai")
asyncio.run(_drive_openai(_handler(configured)))
assert configured.served
def test_the_resolver_follows_a_reassigned_default():
class H:
anthropic_backend = None
h = H()
from headroom.proxy.route_advice import BackendResolver as BR
first = BR(h.anthropic_backend)
assert first.default is None
h.anthropic_backend = DEFAULT
assert BR(h.anthropic_backend).default is DEFAULT
@pytest.mark.parametrize(("written", "inferred"), [(0, False), (20, False), (None, True)])
def test_backend_openai_stream_reads_reported_cache_writes(written, inferred):
"""A reported write, zero included, is the provider's word; only an absent
field falls back to inferring writes from the uncached slice."""
from types import MethodType
from headroom.proxy.handlers.streaming import StreamingMixin
details = {"cached_tokens": 75}
if written is not None:
details["cache_write_tokens"] = written
usage = {"input_tokens": 100, "output_tokens": 5, "input_tokens_details": details}
class Backend(_Backend):
async def stream_openai_message(self, body, headers):
yield f"data: {json.dumps({'type': 'response.completed', 'response': {'usage': usage}})}\n\n"
yield "data: [DONE]\n\n"
outcomes = []
async def record(outcome):
outcomes.append(outcome)
handler = _handler(Backend("openai"))
handler._record_request_outcome = record
handler._parse_sse_usage_from_buffer = MethodType(
StreamingMixin._parse_sse_usage_from_buffer, handler
)
asyncio.run(_drive_openai_with_outcome(handler, handler.anthropic_backend))
o = outcomes[0]
assert o.cache_inferred is inferred
assert o.cache_write_tokens == (25 if written is None else written)