Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: skip cross-turn dedup pointers on OpenAI chat streaming (#3191)
Raúl9c30b629624a
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 37bd72fa3..51dfb527d 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -3511,6 +3511,19 @@ class OpenAIHandlerMixin:_compression_failed = Falseoriginal_messages = messages # Preserve for 400-retry fallback+ # Cross-turn dedup rewrites repeated tool-output spans to bare+ # `[↑NL same as msg M]` in-context pointers. Those are recoverable only+ # where the model can resolve the reference; on the streaming chat path+ # the CCR retrieval tool cannot be injected (this path cannot intercept+ # tool calls) and OpenAI-compatible clients never show the model+ # numbered messages, so a folded pointer reads as deleted content and+ # models retry-loop on the "missing" output. Gate the fold on the same+ # recoverability predicate that gates CCR tool injection: the buffered+ # (non-streaming) chat path keeps dedup, the streaming path skips it.+ _dedup_pointers_recoverable = _should_inject_openai_chat_ccr_tool(+ ccr_inject_tool=self.config.ccr_inject_tool,+ stream=stream,+ )_decision = CompressionDecision.decide(headers=request.headers,config=self.config,@@ -3565,6 +3578,7 @@ class OpenAIHandlerMixin:),biases=_hook_biases,compression_policy=compression_policy,+ cross_turn_dedup_recoverable=_dedup_pointers_recoverable,# Thread the savings-profile knobs (e.g.# HEADROOM_SAVINGS_PROFILE=agent-90) onto the live# chat-completions path, matching handlers/@@ -3605,6 +3619,7 @@ class OpenAIHandlerMixin:frozen_message_count=apply_frozen_count,biases=_hook_biases,compression_policy=compression_policy,+ cross_turn_dedup_recoverable=_dedup_pointers_recoverable,# Same savings-profile threading as the token-mode# branch above — the non-token chat path must honor# the configured profile too (#1534).diff --git a/headroom/transforms/cold_prefix.py b/headroom/transforms/cold_prefix.pyindex 3777e1c1d..b58ac4d14 100644--- a/headroom/transforms/cold_prefix.py+++ b/headroom/transforms/cold_prefix.py@@ -179,7 +179,11 @@ def has_plaintext_reasoning(messages: list[dict[str, Any]]) -> bool:def cold_recompact_messages(- messages: list[dict[str, Any]], *, tokenizer: Any, context: str = ""+ messages: list[dict[str, Any]],+ *,+ tokenizer: Any,+ context: str = "",+ cross_turn_dedup_recoverable: bool = True,) -> tuple[list[dict[str, Any]], list[str]]:"""Lossless whole-prefix recompaction for a confirmed-cold turn.@@ -191,6 +195,15 @@ def cold_recompact_messages(preserve nothing. Lossless + prefix-monotonic ⇒ deterministic per content ⇒the recompacted prefix re-caches and stays byte-stable on later warm turns.+ ``cross_turn_dedup_recoverable`` is forwarded to the router's dedup gate:+ pass False on paths where a bare ``[↑NL same as msg M]`` pointer cannot be+ resolved — no CCR retrieval tool can be injected and the client never shows+ the model numbered messages (OpenAI chat-completions streaming, e.g.+ ``wrap copilot``). The fold is then skipped and the bytes stay verbatim;+ the lossless folds still run. The Anthropic cache-mode caller keeps the+ default True (the in-context reference resolves there and the retrieval+ tool is injectable).+Returns (new_messages, transforms_applied). Fail-open: returns the inputunchanged on any error (never breaks the request)."""@@ -200,8 +213,17 @@ def cold_recompact_messages(ContentRouterConfig,)+ # enable_cross_turn_dedup stays on: whether the fold may EMIT pointers+ # is decided per-path by the router's recoverability gate below, not+ # hardcoded here.router = ContentRouter(ContentRouterConfig(lossless=True, enable_cross_turn_dedup=True))- res = router.apply(list(messages), tokenizer, frozen_message_count=0, context=context)+ res = router.apply(+ list(messages),+ tokenizer,+ frozen_message_count=0,+ context=context,+ cross_turn_dedup_recoverable=cross_turn_dedup_recoverable,+ )return res.messages, list(res.transforms_applied)except Exception as e: # never break the requestlog.warning("cold-prefix recompaction failed (%s); leaving prefix unchanged", e)diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex a77446149..6f1fba323 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -1528,6 +1528,11 @@ class ContentRouterConfig:# Runs in both modes: lossless references verbatim/folded content; CCR mode# references the earlier block's kompressed-but-CCR-recoverable form# (deterministic content-hash → stable → still cache-safe, no added loss).+ # Per request the fold is skipped when the caller reports the serving path+ # cannot resolve the in-context `[↑NL same as msg M]` pointer+ # (`apply(cross_turn_dedup_recoverable=False)`, e.g. OpenAI chat-completions+ # streaming, where no CCR retrieval tool can be injected and clients never+ # show the model numbered messages).enable_cross_turn_dedup: bool = False# Lossless-then-lossy. In lossy mode (not `lossless`), after a byte/data# lossless fold (search/log/text) run the aggressive lossy compressor@@ -4757,6 +4762,20 @@ class ContentRouter(Transform):# pass a policy — ``_record_to_toin`` treats that as "no gate"# to preserve pre-F2.2 behaviour for non-proxy callers.self._runtime_compression_policy = kwargs.get("compression_policy")+ # Cross-turn dedup recoverability gate. The fold rewrites a repeated+ # span to a bare in-context pointer (``[↑NL same as msg M]``) that names+ # Headroom's internal message index. That reference is only resolvable+ # where the model can locate the original: on the OpenAI+ # chat-completions streaming path (e.g. ``wrap copilot``) no CCR+ # retrieval tool can be injected (the path cannot intercept tool calls)+ # and the client never shows the model numbered messages, so the+ # pointer reads as deleted content and the model retry-loops on the+ # "missing" output. Same recoverability posture as the lossy+ # ``lossy_unrecoverable_skipped`` guard: when the caller reports the+ # path cannot resolve in-context pointers, skip the fold and keep the+ # bytes verbatim. Default True: every path that does not opt out keeps+ # today's behavior.+ dedup_pointers_recoverable = bool(kwargs.get("cross_turn_dedup_recoverable", True))tokens_before = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)context = kwargs.get("context", "")@@ -5538,7 +5557,7 @@ class ContentRouter(Transform):# later duplicate would carry the same (recoverable) form anyway; dedup# just points to the earlier copy instead of repeating it. Frozen +# cache_control blocks are reference targets only (never rewritten).- if self._cross_turn_dedup_enabled:+ if self._cross_turn_dedup_enabled and dedup_pointers_recoverable:transformed_messages = self._cross_turn_dedup_messages(transformed_messages, frozen_message_count, transforms_applied, route_counts)diff --git a/tests/test_cold_prefix.py b/tests/test_cold_prefix.pynew file mode 100644index 000000000..15d0618e4--- /dev/null+++ b/tests/test_cold_prefix.py@@ -0,0 +1,69 @@+"""Cold-prefix recompaction: the cross-turn dedup fold must be path-aware.++``cold_recompact_messages`` builds its own lossless ContentRouter with+``enable_cross_turn_dedup=True``. The fold rewrites repeated tool-output spans+to bare ``[↑NL same as msg M]`` in-context pointers — unresolvable on paths+where no CCR retrieval tool can be injected and the client never shows the+model numbered messages (OpenAI chat-completions streaming, wrap copilot).+The recompaction therefore takes ``cross_turn_dedup_recoverable`` and forwards+it to the router gate: unrecoverable paths keep the bytes verbatim, the+Anthropic cache-mode caller (default True) keeps folding.+"""++from headroom.transforms.cold_prefix import cold_recompact_messages+++def _mk_tok():+ from headroom.providers import OpenAIProvider+ from headroom.tokenizer import Tokenizer++ return Tokenizer(OpenAIProvider().get_token_counter("gpt-4o"), "gpt-4o")+++def _toolmsg(text, tid):+ return {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": tid, "content": text}],+ }+++def _conversation():+ span = "\n".join(f" result_{i} = compute_overdraft(business_id={i})" for i in range(12))+ return [+ {"role": "user", "content": "fix the overdraft bug"},+ {"role": "assistant", "content": "cat merge.py"},+ _toolmsg(f"$ cat merge.py\n{span}\n# end", "t1"),+ {"role": "assistant", "content": "sed -n range"},+ _toolmsg(f"$ sed -n 1,20p merge.py\n{span}\n# more", "t2"),+ ]+++def test_cold_recompact_folds_by_default():+ # Anthropic cache-mode path (the only caller today): unchanged — the+ # repeated span still folds to an in-context pointer.+ msgs = _conversation()+ out, transforms = cold_recompact_messages(msgs, tokenizer=_mk_tok())+ later = out[-1]["content"][0]["content"]+ assert "[↑" in later+ assert any("cross_turn_dedup" in t for t in transforms)+++def test_cold_recompact_unrecoverable_path_keeps_verbatim_bytes():+ # Unresolvable-pointer path (OpenAI chat streaming shape): the fold is+ # skipped, the repeated span stays byte-verbatim, and no pointer is+ # emitted — while the recompaction itself still runs (message count and+ # order unchanged).+ msgs = _conversation()+ out, transforms = cold_recompact_messages(+ msgs, tokenizer=_mk_tok(), cross_turn_dedup_recoverable=False+ )+ later = out[-1]["content"][0]["content"]+ assert "[↑" not in later+ assert (+ later+ == "$ sed -n 1,20p merge.py\n"+ + "\n".join(f" result_{i} = compute_overdraft(business_id={i})" for i in range(12))+ + "\n# more"+ )+ assert not any("cross_turn_dedup" in t for t in transforms)+ assert len(out) == len(msgs)diff --git a/tests/test_cross_turn_dedup.py b/tests/test_cross_turn_dedup.pyindex 53cd9b13a..391f87eb6 100644--- a/tests/test_cross_turn_dedup.py+++ b/tests/test_cross_turn_dedup.py@@ -399,3 +399,76 @@ def test_dedup_folds_role_function_output():]out = _dedup_only(msgs)assert "[↑" in out[2]["content"]+++# --------------------------------------------------------------------------+# Recoverability gate (unresolvable-pointer paths). The fold rewrites a+# repeated span to a bare `[↑NL same as msg M]` pointer naming Headroom's+# internal message index. On the OpenAI chat-completions streaming path+# (wrap copilot) no CCR retrieval tool can be injected and the client never+# shows the model numbered messages, so the pointer is unresolvable: the+# model reads it as deleted content and retry-loops. `apply()` therefore+# accepts `cross_turn_dedup_recoverable=False` — the same recoverability+# posture as the lossy `lossy_unrecoverable_skipped` guard — and keeps the+# repeated bytes verbatim. Default True preserves every other path.+# --------------------------------------------------------------------------+def _apply_with_recoverable(messages, recoverable):+ import copy++ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig++ r = ContentRouter(ContentRouterConfig(lossless=True, enable_cross_turn_dedup=True))+ return r.apply(+ copy.deepcopy(messages), _mk_tok(), cross_turn_dedup_recoverable=recoverable+ ).messages+++def test_apply_unrecoverable_path_keeps_verbatim_bytes():+ # The OpenAI chat streaming shape (role:tool strings): with dedup ENABLED+ # but the path flagged unrecoverable, the re-read must NOT fold — the+ # request keeps the verbatim bytes, no bare pointer.+ span = _readspan()+ msgs = [+ {"role": "tool", "tool_call_id": "c1", "content": f"$ cat f.py\n{span}"},+ {"role": "assistant", "content": "again"},+ {"role": "tool", "tool_call_id": "c2", "content": f"$ cat f.py\n{span}"},+ ]+ out = _apply_with_recoverable(msgs, recoverable=False)+ assert out[2]["content"] == f"$ cat f.py\n{span}" # verbatim, no pointer+ assert "[↑" not in out[2]["content"]+++def test_apply_unrecoverable_gate_also_covers_tool_result_blocks():+ # Anthropic tool_result block shape, same gate: nothing folds when the+ # caller reports the pointer is unresolvable on this path.+ span = _readspan()+ msgs = [_toolmsg(f"a\n{span}", "t1"), _toolmsg(f"b\n{span}", "t2")]+ out = _apply_with_recoverable(msgs, recoverable=False)+ joined = "".join(b["content"] for m in out for b in m["content"] if isinstance(b, dict))+ assert "[↑" not in joined+ assert out[-1]["content"][0]["content"] == f"b\n{span}" # verbatim bytes kept+++def test_apply_recoverable_default_and_true_still_fold():+ # The recoverable paths (Anthropic, buffered/non-streaming chat — anywhere+ # the reference resolves) keep folding: default kwarg-absent behavior is+ # unchanged, and an explicit True folds too.+ span = _readspan()+ msgs = [+ {"role": "tool", "tool_call_id": "c1", "content": f"$ cat f.py\n{span}"},+ {"role": "assistant", "content": "again"},+ {"role": "tool", "tool_call_id": "c2", "content": f"$ cat f.py\n{span}"},+ ]+ import copy++ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig++ default_out = (+ ContentRouter(ContentRouterConfig(lossless=True, enable_cross_turn_dedup=True))+ .apply(copy.deepcopy(msgs), _mk_tok())+ .messages+ )+ assert "[↑" in default_out[2]["content"] # no kwarg -> still folds++ true_out = _apply_with_recoverable(msgs, recoverable=True)+ assert "[↑" in true_out[2]["content"] # explicit recoverable -> foldsdiff --git a/tests/test_openai_chat_dedup_recoverability.py b/tests/test_openai_chat_dedup_recoverability.pynew file mode 100644index 000000000..9b1f61ee0--- /dev/null+++ b/tests/test_openai_chat_dedup_recoverability.py@@ -0,0 +1,158 @@+"""OpenAI chat-completions: cross-turn dedup pointers are recoverability-gated.++The fold rewrites a repeated tool-output span to a bare ``[↑NL same as msg M]``+pointer naming Headroom's internal message index. On the STREAMING chat path+(``wrap copilot``) the CCR retrieval tool cannot be injected — the path cannot+intercept tool calls — and OpenAI-compatible clients never show the model+numbered messages, so the pointer is unresolvable: models read it as deleted+content and retry-loop. The chat handler therefore threads+``cross_turn_dedup_recoverable=_should_inject_openai_chat_ccr_tool(...)`` into+the router: streaming requests keep the repeated bytes verbatim, while the+buffered (non-streaming) path — where the retrieval tool IS injectable — keeps+folding.++These tests drive the real ``/v1/chat/completions`` handler through a TestClient+with dedup force-enabled (``HEADROOM_DEDUPE=1``) and capture the exact upstream+request body, the same evidence the proxy logs showed when the bug bit.+"""++from __future__ import annotations++import pytest++fastapi = pytest.importorskip("fastapi")+httpx = pytest.importorskip("httpx")++from fastapi.responses import StreamingResponse # noqa: E402+from fastapi.testclient import TestClient # noqa: E402++from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++_SPAN = "\n".join(f" result_{i} = compute_overdraft(business_id={i})" for i in range(12))+++def _messages() -> list[dict]:+ """Two identical multi-line tool outputs — the re-read dedup folds."""+ return [+ {"role": "user", "content": "fix the overdraft bug"},+ {"role": "assistant", "content": "cat merge.py"},+ {"role": "tool", "tool_call_id": "call_1", "content": f"$ cat merge.py\n{_SPAN}\n# end"},+ {"role": "assistant", "content": "sed -n range"},+ {"role": "tool", "tool_call_id": "call_2", "content": f"$ cat merge.py\n{_SPAN}\n# end"},+ ]+++def _config() -> ProxyConfig:+ return ProxyConfig(optimize=True, cache_enabled=False, rate_limit_enabled=False)+++def _post(client: TestClient, *, stream: bool):+ return client.post(+ "/v1/chat/completions",+ json={"model": "gpt-4o", "messages": _messages(), "stream": stream},+ headers={"Authorization": "******"},+ )+++def _sent_text(body: dict) -> str:+ """Concatenate the upstream message contents (parsed, so newlines are real)."""+ return "\n".join(str(m.get("content", "")) for m in body["messages"])+++def test_streaming_chat_keeps_verbatim_bytes_no_dedup_pointer(monkeypatch):+ """The bug: a streaming chat request with a repeated span got a bare+ ``[↑NL same as msg M]`` pointer the model cannot resolve. Now the upstream+ body must carry the repeated bytes verbatim."""+ monkeypatch.setenv("HEADROOM_DEDUPE", "1") # before create_app: router reads env at init+ captured: list[dict] = []++ async def fake_stream(url, headers, body, *args, **kwargs):+ captured.append(body)+ return StreamingResponse(iter([b"data: {}\n\n"]), media_type="text/event-stream")++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._stream_response = fake_stream+ resp = _post(client, stream=True)++ assert resp.status_code == 200, resp.text+ assert captured, "streaming upstream send was not captured"+ sent = _sent_text(captured[0])+ assert "[↑" not in sent # no unresolvable pointer on the streaming path+ assert sent.count(_SPAN) == 2 # both copies forwarded byte-verbatim+++def test_lossless_buffered_chat_also_skips_the_fold(monkeypatch):+ """Coupling lock: --lossless forces ccr_inject_tool=False (server.py), so+ the recoverability predicate is False for buffered chat too and the fold+ is skipped there as well (no retrieval tool exists to redeem anything in+ no-CCR mode). Bytes stay verbatim; the conservative direction is intended."""+ monkeypatch.setenv("HEADROOM_DEDUPE", "1")+ captured: list[dict] = []++ async def fake_retry(method, url, headers, body, *args, **kwargs):+ captured.append(body)+ payload = {+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "done"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 100, "completion_tokens": 5, "total_tokens": 105},+ }+ return httpx.Response(200, json=payload, headers={"content-type": "application/json"})++ config = ProxyConfig(+ optimize=True, lossless=True, cache_enabled=False, rate_limit_enabled=False+ )+ app = create_app(config)+ with TestClient(app) as client:+ client.app.state.proxy._retry_request = fake_retry+ resp = _post(client, stream=False)++ assert resp.status_code == 200, resp.text+ assert captured, "buffered upstream send was not captured"+ sent = _sent_text(captured[0])+ assert "[↑" not in sent # no retrieval tool in lossless mode -> no bare pointer+ assert sent.count(_SPAN) == 2 # both copies forwarded byte-verbatim+++def test_buffered_chat_still_folds_repeated_tool_output(monkeypatch):+ """The recoverable counterpart: non-streaming chat can inject the CCR+ retrieval tool, so the in-context pointer stays resolvable and the+ repeated span still folds (today's behavior, unchanged)."""+ monkeypatch.setenv("HEADROOM_DEDUPE", "1")+ captured: list[dict] = []++ async def fake_retry(method, url, headers, body, *args, **kwargs):+ captured.append(body)+ payload = {+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "done"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 100, "completion_tokens": 5, "total_tokens": 105},+ }+ return httpx.Response(200, json=payload, headers={"content-type": "application/json"})++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._retry_request = fake_retry+ resp = _post(client, stream=False)++ assert resp.status_code == 200, resp.text+ assert captured, "buffered upstream send was not captured"+ sent = _sent_text(captured[0])+ assert "[↑" in sent # fold still fires where the pointer resolves+ assert sent.count(_SPAN) == 1 # earliest copy stays as the in-context original