Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): answer headroom_retrieve on the direct chat path instead of forwarding it (#3816)
Tejas Chopra126e144876d2
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 498734780..94c15973c 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -289,6 +289,35 @@ RESPONSES_USAGE_KEYS = {}+class CCRContinuationHTTPError(Exception):+ """A CCR continuation call came back non-2xx.++ ``_retry_request`` deliberately returns rather than raises for the statuses+ it will not retry — any 4xx, an exhausted 429/529, an exhausted 5xx — so a+ continuation failure is indistinguishable from success unless the status is+ checked. Parsing such a body and returning it as the model's answer would+ serialize an upstream *error* under the original call's ``200``.++ Raising instead routes the turn into+ ``CCRResponseHandler.handle_response``'s existing continuation-failure+ path, which forwards the response it already had. That keeps one behaviour+ for every way a continuation can fail: transport error, timeout, and now+ an error status.++ Carries the status and nothing else. An upstream error body is untrusted+ content that routinely contains credential fragments, tenant identifiers+ and excerpts of the request, and the status alone identifies the failure.+ The message matters as much as any explicit log call: ``handle_response``+ logs ``repr(e)`` when a continuation raises+ (``ccr/response_handler.py:541``), so anything placed here reaches the log+ whether or not this module logs it too.+ """++ def __init__(self, status_code: int) -> None:+ super().__init__(f"CCR continuation returned HTTP {status_code}")+ self.status_code = status_code++class TurnHookUsage:"""Upstream calls a turn hook caused that nothing else will account for.@@ -5259,12 +5288,130 @@ class OpenAIHandlerMixin:_thinking = ThinkingTokens()total_latency = (time.time() - start_time) * 1000+ # ── CCR retrieval resolution (direct chat path) ──────────+ # Every other path answers a `headroom_retrieve` call:+ # gateway_turn.py, the Gemini and Anthropic handlers, the custom+ # backend branch above, and Responses. This branch did not, so a+ # tool the proxy itself injects (`ccr_inject_tool` defaults True)+ # reached the client as a `tool_calls` entry for a function it+ # never declared and cannot implement — stalling any agent loop+ # driven off `finish_reason`.+ #+ # The continuation is a real, billed upstream call, so it is+ # recorded into `_hook_usage` exactly like a turn-hook re-drive:+ # `settle(final)` drops the one response the usage block below+ # reads and totals the rest. Counting only the last call would+ # let retrieval hide its own cost, which is the mistake the+ # matching comment on the re-drive block below warns about.+ _ccr_final_json: dict[str, Any] | None = None+ _ccr_cont_error: int | None = None+ if self.ccr_response_handler is not None and response.status_code == 200:+ try:+ _pre_ccr_json = response.json()+ except Exception: # pragma: no cover - non-JSON upstream+ _pre_ccr_json = None+ if _pre_ccr_json and self.ccr_response_handler.has_ccr_tool_calls(+ _pre_ccr_json, "openai"+ ):+ logger.info(+ f"[{request_id}] CCR: retrieval tool call on the direct "+ "chat path, resolving before replying"+ )++ async def _ccr_api_call_fn(+ msgs: list[dict[str, Any]],+ tls: list[dict[str, Any]] | None,+ ) -> dict[str, Any]:+ continuation_body = {**body, "messages": msgs}+ if tls is not None:+ continuation_body["tools"] = tls+ # A continuation is always non-streaming, whatever+ # the client asked for.+ continuation_body.pop("stream", None)+ continuation_headers = {+ k: v+ for k, v in headers.items()+ if k.lower()+ not in (+ "accept",+ "content-type",+ "content-length",+ "content-encoding",+ )+ }+ continuation_headers["content-type"] = "application/json"+ continuation_headers["accept"] = "application/json"+ nonlocal _ccr_cont_error+ cont = await self._retry_request(+ "POST", url, continuation_headers, continuation_body+ )+ # `_retry_request` returns non-2xx verbatim for+ # everything it will not retry, so the status has to+ # be checked before the body is treated as an answer.+ # Without this an upstream 401, or an exhausted 429,+ # reaches the client as HTTP 200 carrying an error+ # object.+ if not 200 <= cont.status_code < 300:+ # Status only. The body is untrusted upstream+ # content and never leaves this branch.+ _ccr_cont_error = cont.status_code+ raise CCRContinuationHTTPError(cont.status_code)+ cont_json: dict[str, Any] = cont.json()+ _hook_usage.record(cont_json, **CHAT_USAGE_KEYS)+ return cont_json++ try:+ _hook_usage.record(_pre_ccr_json, **CHAT_USAGE_KEYS)+ _ccr_final_json = await self.ccr_response_handler.handle_response(+ _pre_ccr_json,+ optimized_messages,+ tools,+ _ccr_api_call_fn,+ provider="openai",+ )+ # `handle_response` swallows a continuation failure+ # and hands back the response it already had, so a+ # failed turn is only detectable here. Identity —+ # not equality — is the signal that nothing was+ # resolved, and it also covers max-rounds+ # exhaustion. Dropping back to `None` makes the+ # return below forward the upstream bytes verbatim+ # instead of re-serializing a dict that came from+ # them, which is what byte-faithful forwarding+ # means on a path that changed nothing.+ if _ccr_cont_error is not None or _ccr_final_json is _pre_ccr_json:+ if _ccr_cont_error is not None:+ logger.warning(+ f"[{request_id}] CCR: continuation returned HTTP "+ f"{_ccr_cont_error}; forwarding the upstream reply "+ "unchanged"+ )+ else:+ logger.warning(+ f"[{request_id}] CCR: retrieval did not resolve; "+ "forwarding the upstream reply unchanged"+ )+ _ccr_final_json = None+ _hook_usage.settle(_pre_ccr_json)+ else:+ _hook_usage.settle(_ccr_final_json)+ except Exception as ccr_err:+ # Fail open to the model's own reply rather than 502 a+ # turn the client could still act on.+ logger.warning(+ f"[{request_id}] CCR: direct-path resolution failed "+ f"({type(ccr_err).__name__}: {ccr_err}); forwarding "+ "the upstream reply unchanged"+ )+ _ccr_final_json = None+ _hook_usage.settle(_pre_ccr_json)+total_input_tokens = optimized_tokens # fallbackoutput_tokens = 0cache_read_tokens = 0resp_json = Nonetry:- resp_json = response.json()+ resp_json = _ccr_final_json if _ccr_final_json is not None else response.json()usage = resp_json.get("usage", {})# Coerce present-but-null counts: the arithmetic below# (`_infer_openai_cache_write_tokens`, `max(...)`) runs@@ -5499,7 +5646,11 @@ class OpenAIHandlerMixin:)return Response(- content=response.content,+ content=(+ json.dumps(_ccr_final_json).encode()+ if _ccr_final_json is not None+ else response.content+ ),status_code=response.status_code,headers=response_headers,)diff --git a/tests/test_direct_chat_ccr_resolution.py b/tests/test_direct_chat_ccr_resolution.pynew file mode 100644index 000000000..b1d002212--- /dev/null+++ b/tests/test_direct_chat_ccr_resolution.py@@ -0,0 +1,420 @@+"""The direct chat-completions path must answer the retrieval tool it injects.++`headroom_retrieve` is injected into a non-streaming chat request by default+(`ccr_inject_tool=True`), but `handlers/openai.py`'s direct HTTP branch had no+CCR tool-call handling at all — the code said so itself. Every other path+resolves the call: `gateway_turn.py:1003`, `handlers/gemini.py:837`,+`handlers/anthropic.py:4286`, `handlers/openai.py` for the custom backend and+for Responses. Only the direct chat branch did not.++So the proxy advertised a tool it would not answer. A model that called it sent+the client a `tool_calls` entry for a tool the client never declared and cannot+implement, which stalls any agent loop driven off `finish_reason`.++The accounting pair is the part worth keeping honest. A resolved turn makes two+billed upstream calls, and the continuation resends the whole conversation, so+counting only the last one hides roughly half the turn. The split matters too:+the forwarded body keeps the provider's own usage untouched (byte-faithful+forwarding), while Headroom's cost tracking folds in the extra call — the same+division the turn-hook re-drive path already uses via `TurnHookUsage`.+"""++from __future__ import annotations++import json+import logging++import pytest++fastapi = pytest.importorskip("fastapi")+httpx = pytest.importorskip("httpx")++from fastapi.testclient import TestClient # noqa: E402++from headroom.cache.compression_store import ( # noqa: E402+ get_compression_store,+ reset_compression_store,+)+from headroom.ccr.tool_injection import CCR_TOOL_NAME # noqa: E402+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++ORIGINAL = "row 1: the original uncompressed rows\nrow 2: more of them"++[email protected](autouse=True)+def reset_store():+ reset_compression_store()+ yield+ reset_compression_store()+++def _stored_hash() -> str:+ return get_compression_store().store(+ original=ORIGINAL,+ compressed="[2 items compressed to 0]",+ original_item_count=2,+ compressed_item_count=0,+ )+++def _config() -> ProxyConfig:+ # No backend configured -> the direct OpenAI HTTP path.+ return ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ ccr_handle_responses=True,+ ccr_inject_tool=True,+ )+++def _retrieve_call_response(hash_key: str) -> dict:+ """An upstream reply in which the model asks to retrieve."""+ return {+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_1",+ "type": "function",+ "function": {+ "name": CCR_TOOL_NAME,+ "arguments": json.dumps({"hash": hash_key}),+ },+ }+ ],+ },+ "finish_reason": "tool_calls",+ }+ ],+ "usage": {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120},+ }+++def _final_response() -> dict:+ """The continuation reply, after the tool result was supplied."""+ return {+ "id": "chatcmpl-2",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "there are 2 rows"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 400, "completion_tokens": 10, "total_tokens": 410},+ }+++class _CollectHandler(logging.Handler):+ """Collect `headroom.*` records regardless of propagation.++ `caplog` attaches to the root logger, but the proxy's own startup+ (`helpers._setup_file_logging`) sets `propagate = False` on `headroom`, so+ records never reach root — and it does that *during* `create_app`, after+ conftest's autouse reset has already run. A handler on the `headroom`+ logger itself still fires, because `propagate` only governs whether records+ are passed further up. Anything asserting on log contents here has to go+ through this, or it silently asserts against an empty string.+ """++ def __init__(self, sink: list[str]) -> None:+ super().__init__(level=logging.DEBUG)+ self._sink = sink+ self.setFormatter(logging.Formatter("%(name)s %(levelname)s %(message)s"))++ def emit(self, record: logging.LogRecord) -> None:+ try:+ self._sink.append(self.format(record))+ except Exception: # pragma: no cover - a test handler must not raise+ pass+++def _run(+ upstream_sequence: list, log_capture: list[str] | None = None+) -> tuple[httpx.Response, list[dict], list]:+ """Post one chat turn, serving `upstream_sequence` to successive calls.++ Each entry is either a payload dict (served as `200`) or an explicit+ `(status, payload)` pair — non-2xx continuations are the whole point of the+ status-handling tests, and `_retry_request` hands those back without+ raising.++ Returns the response, the request bodies that went upstream, and the+ RequestOutcome objects the proxy recorded.+ """+ sent: list[dict] = []+ outcomes: list = []+ remaining = list(upstream_sequence)++ def _next() -> tuple[int, dict]:+ entry = remaining.pop(0) if remaining else upstream_sequence[-1]+ if isinstance(entry, tuple):+ return entry+ return 200, entry++ async def fake_retry(method, url, headers, req_body, *args, **kwargs):+ sent.append(req_body)+ status, payload = _next()+ return httpx.Response(status, json=payload, headers={"content-type": "application/json"})++ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy._retry_request = fake_retry++ # The continuation on this path may go out through the shared client+ # rather than _retry_request; capture both so the test does not depend+ # on which transport the implementation picks.+ class _FakeHTTPClient:+ async def post(self, url, content=None, headers=None, **kwargs): # noqa: ANN001+ import json as _json++ sent.append(_json.loads(content) if content else {})+ status, payload = _next()+ return httpx.Response(+ status, json=payload, headers={"content-type": "application/json"}+ )++ proxy.http_client = _FakeHTTPClient()++ _real_record = proxy._record_request_outcome++ async def _capture(outcome, *args, **kwargs): # noqa: ANN001+ outcomes.append(outcome)+ return await _real_record(outcome, *args, **kwargs)++ proxy._record_request_outcome = _capture++ # After startup, so the proxy's own logging setup cannot undo it.+ _handler: logging.Handler | None = None+ if log_capture is not None:+ _headroom_logger = logging.getLogger("headroom")+ _headroom_logger.disabled = False+ _headroom_logger.setLevel(logging.DEBUG)+ _handler = _CollectHandler(log_capture)+ _headroom_logger.addHandler(_handler)++ try:+ resp = client.post(+ "/v1/chat/completions",+ json={+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "how many rows?"}],+ "stream": False,+ "tools": [+ {+ "type": "function",+ "function": {"name": "Read", "description": "read a file"},+ }+ ],+ },+ headers={"Authorization": "Bearer test-key", "x-api-key": "test-key"},+ )+ finally:+ if _handler is not None:+ logging.getLogger("headroom").removeHandler(_handler)+ return resp, sent, outcomes+++def test_a_retrieval_call_is_resolved_before_the_client_sees_it():+ """The client must never receive a headroom_retrieve tool call."""+ hash_key = _stored_hash()+ resp, sent, _outcomes = _run([_retrieve_call_response(hash_key), _final_response()])++ assert resp.status_code == 200, resp.text+ body = resp.json()+ message = body["choices"][0]["message"]++ tool_calls = message.get("tool_calls") or []+ names = [tc.get("function", {}).get("name") for tc in tool_calls]+ assert CCR_TOOL_NAME not in names, (+ "the proxy injected headroom_retrieve and then handed the model's call "+ "straight to the client, which cannot implement it"+ )+ assert message.get("content") == "there are 2 rows"+ assert len(sent) == 2, f"expected an original call plus a continuation, got {len(sent)}"+++def test_the_continuation_carries_the_retrieved_content():+ """The retrieved original has to reach the model, not just be looked up."""+ hash_key = _stored_hash()+ _resp, sent, _outcomes = _run([_retrieve_call_response(hash_key), _final_response()])++ assert len(sent) == 2+ continuation = sent[1]+ serialized = str(continuation)+ assert ORIGINAL.split("\n")[0] in serialized, (+ "the continuation request did not carry the retrieved original content"+ )+++def test_the_forwarded_body_keeps_the_provider_usage_untouched():+ """Byte-faithful forwarding: the client sees the provider's own final usage.++ Rewriting `usage` in the forwarded body would misreport what the upstream+ call actually returned. Extra calls belong in Headroom's own accounting,+ which the next test covers — this is the same split the turn-hook re-drive+ path already uses.+ """+ hash_key = _stored_hash()+ resp, _sent, _outcomes = _run([_retrieve_call_response(hash_key), _final_response()])+ assert (resp.json().get("usage") or {}).get("prompt_tokens") == 400+++ERROR_BODY = {"error": {"message": "invalid api key", "type": "invalid_request_error"}}++# A value that must never be logged. Upstream error bodies are untrusted and+# routinely carry credential fragments, tenant identifiers and excerpts of the+# request that produced them.+SENTINEL = "sk-live-SENTINEL-must-never-be-logged-8f3a1c"+ERROR_BODY_WITH_SECRET = {+ "error": {+ "message": f"authentication failed for key {SENTINEL} (tenant acme-prod-42)",+ "type": "invalid_request_error",+ }+}++[email protected]("status", [401, 429, 500])+def test_a_failed_continuation_body_never_reaches_the_logs(status):+ """The status is diagnostic enough; the body is untrusted content.++ Two paths would leak it, and both have to stay closed:++ * anything this module logs itself, and+ * the exception's own message — `handle_response` logs `repr(e)` when a+ continuation raises (`ccr/response_handler.py:541`), so a preview placed+ on the exception reaches the log even if this module never logs it.++ The positive assertion at the end is load-bearing: it is what proves the+ capture is actually wired to the `headroom` logger. Without it the negative+ assertions pass against an empty string and the test is worthless.+ """+ hash_key = _stored_hash()+ logs: list[str] = []+ resp, sent, _outcomes = _run(+ [_retrieve_call_response(hash_key), (status, ERROR_BODY_WITH_SECRET)],+ log_capture=logs,+ )+ text = "\n".join(logs)++ assert len(sent) == 2, "the continuation should still have been attempted"+ # Proves the capture works, so the assertions below mean something.+ assert f"HTTP {status}" in text, (+ f"the {status} status was not logged, so either the failure is invisible "+ "or this test is not capturing headroom's logs at all"+ )+ assert SENTINEL not in text, (+ "an upstream error body reached the logs; even a bounded prefix is an "+ "exfiltration path for credentials and tenant identifiers"+ )+ assert "acme-prod-42" not in text, "a tenant identifier reached the logs"+ assert SENTINEL not in resp.text, "the upstream error body reached the client"+++ "status",+ [+ 401, # any 4xx: `_retry_request` returns it without raising+ 429, # retries exhausted, returned verbatim+ 500, # retries exhausted, returned via the HTTPStatusError branch+ ],+)+def test_a_non_2xx_continuation_never_reaches_the_client_as_200(status):+ """An upstream error must not be served under the first call's `200`.++ `_retry_request` deliberately does not raise for anything it will not retry,+ so a continuation failure looks exactly like a success unless the status is+ checked. Parsing that body and returning it as the model's answer produced+ an HTTP 200 whose payload was an error object with no `choices` at all — a+ shape no client expects, and one that hides an auth or quota failure.+ """+ hash_key = _stored_hash()+ resp, sent, _outcomes = _run([_retrieve_call_response(hash_key), (status, ERROR_BODY)])++ assert len(sent) == 2, "the continuation should still have been attempted"+ body = resp.json()+ assert "error" not in body, (+ f"a {status} continuation body was forwarded as the model's reply "+ f"under HTTP {resp.status_code}"+ )+ assert body.get("choices"), "the forwarded body lost its choices"++[email protected]("status", [401, 429, 500])+def test_a_failed_continuation_forwards_the_original_reply_unchanged(status):+ """Fail open, and byte-for-byte.++ `handle_response` swallows a continuation failure and returns the response+ it already had, which is the same contract a transport error or timeout on+ that call already gets. The client therefore sees the model's original+ reply — including the unresolved tool call it cannot run, which is strictly+ better than an error object dressed as a completion, and is the documented+ behaviour for every other way a continuation can fail.++ Byte fidelity matters here too: nothing was resolved, so the upstream bytes+ must be forwarded as they arrived rather than re-serialized from a parsed+ dict.+ """+ hash_key = _stored_hash()+ original = _retrieve_call_response(hash_key)+ resp, _sent, _outcomes = _run([original, (status, ERROR_BODY)])++ assert resp.status_code == 200+ assert resp.json() == original, "the original upstream reply was not forwarded unchanged"+ assert resp.content == httpx.Response(200, json=original).content, (+ "the body was re-serialized instead of forwarded verbatim"+ )++[email protected]("status", [401, 429, 500])+def test_a_failed_continuation_is_not_billed(status):+ """A non-2xx continuation returns no usage, so it must not be counted.++ Only the first call was billed. Recording the error response — or settling+ against the wrong object — would inflate the turn with tokens that were+ never charged, which is the same class of bug in the opposite direction as+ the dropped first call.+ """+ hash_key = _stored_hash()+ _resp, sent, outcomes = _run([_retrieve_call_response(hash_key), (status, ERROR_BODY)])++ assert len(sent) == 2+ assert outcomes, "no RequestOutcome was recorded"+ recorded = outcomes[-1].provider_input_tokens+ assert recorded == 100, (+ f"cost tracking saw {recorded} input tokens; only the first call "+ "(100) was billed, the continuation returned an error with no usage"+ )+++def test_headroom_accounting_counts_both_upstream_calls():+ """A retrieval turn makes two billed calls; cost tracking must see both.++ `handle_response` replaces the response rather than merging usage, so the+ pre-continuation call's tokens are invisible unless they are folded in+ explicitly. The continuation resends the whole conversation, so counting+ only the last call hides roughly half the turn — the same mistake the+ re-drive block warns about ("counting only the last one lets the feature+ hide its own overhead behind the saving it is claiming").+ """+ hash_key = _stored_hash()+ _resp, sent, outcomes = _run([_retrieve_call_response(hash_key), _final_response()])+ assert len(sent) == 2+ assert outcomes, "no RequestOutcome was recorded"++ recorded = outcomes[-1].provider_input_tokens+ assert recorded >= 100 + 400, (+ f"cost tracking saw {recorded} input tokens for a turn that made two "+ "upstream calls costing 100 + 400; the first call was dropped"+ )