Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): resolve <<ccr:...>> markers inline when no retrieve-tool path exists (#2512)
Parideboyce8ce8313f8c
diff --git a/headroom/ccr/marker_resolution.py b/headroom/ccr/marker_resolution.pynew file mode 100644index 000000000..aef4a108f--- /dev/null+++ b/headroom/ccr/marker_resolution.py@@ -0,0 +1,83 @@+"""Inline resolution of ``<<ccr:...>>`` markers on the response path.++Normal CCR resolution relies on the ``headroom_retrieve`` tool: a marker is+redeemed when the model calls the tool back. That path assumes there's a+subsequent turn in which the model *can* call it. Callers that never see an+injected tool at all — e.g. Headroom running as a LiteLLM guardrail/proxy hop+with no tool-call turn in between (#2509) — have no way to redeem a marker,+so it leaks through as raw text.++This module provides an explicit, opt-in fallback (``--ccr-inline-resolve``):+scan the outgoing response for markers and substitute the original content+directly, instead of leaving the marker for the model to redeem later.+"""++from __future__ import annotations++import json+import logging+import re+from typing import Any++from ..cache.compression_store import (+ CompressionStore,+ format_retrieval_miss_detail,+ get_compression_store,+)++logger = logging.getLogger(__name__)++# Matches the opaque-blob marker form `<<ccr:HASH,KIND,SIZE>>` (and the+# row-offload form `<<ccr:HASH N_rows_offloaded>>`) emitted by SmartCrusher.+# HASH is 12-24 hex chars; see headroom/ccr/tool_injection.py for the same+# constant used on the injection side.+_MARKER_RE = re.compile(r"<<ccr:([a-f0-9]{12,24})[^>]*>>")+++def resolve_markers_in_text(text: str, *, store: CompressionStore | None = None) -> str:+ """Replace every ``<<ccr:HASH,...>>`` marker in ``text`` with its original content.++ A miss (expired/evicted/unknown hash) can't be reported back to the+ model on this path — there's no tool-call round-trip — so the marker is+ left in place with the miss reason appended rather than raising.+ """+ if "<<ccr:" not in text:+ return text++ resolved_store = store or get_compression_store()++ def _replace(match: re.Match[str]) -> str:+ hash_key = match.group(1)+ entry = resolved_store.retrieve(hash_key)+ if entry is not None:+ original = entry.original_content+ return original if isinstance(original, str) else json.dumps(original)++ get_status = getattr(resolved_store, "get_entry_status", None)+ status = get_status(hash_key, clean_expired=True) if callable(get_status) else None+ detail = format_retrieval_miss_detail(status) if status else "entry not found"+ logger.warning(f"CCR inline-resolve: marker {hash_key} unresolvable ({detail})")+ return f"{match.group(0)} [unresolved: {detail}]"++ return _MARKER_RE.sub(_replace, text)+++def resolve_markers_in_response(response: Any, *, store: CompressionStore | None = None) -> Any:+ """Recursively resolve ``<<ccr:...>>`` markers in every string field of a payload.++ Walks the full response structure rather than picking out+ provider-specific fields (``content`` blocks, ``message.content``,+ Responses-API ``output`` items, ...) so it stays correct regardless of+ where a marker ends up, and doesn't need per-provider maintenance.+ """+ resolved_store = store or get_compression_store()+ if isinstance(response, str):+ return resolve_markers_in_text(response, store=resolved_store)+ if isinstance(response, list):+ return [resolve_markers_in_response(item, store=resolved_store) for item in response]+ if isinstance(response, dict):+ return {+ key: resolve_markers_in_response(value, store=resolved_store)+ for key, value in response.items()+ }+ return responsediff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 30292a4e1..29e964abd 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -313,6 +313,19 @@ def dashboard(port: int, no_open: bool) -> None:"retrieval marker, so no MCP retrieve tool is needed. Env: HEADROOM_LOSSLESS=1."),)+ "--ccr-inline-resolve",+ is_flag=True,+ envvar="HEADROOM_CCR_INLINE_RESOLVE",+ help=(+ "Resolve <<ccr:...>> markers inline on the response path instead of "+ "relying on the model to call headroom_retrieve. For callers with no "+ "tool-call round-trip to redeem a marker (e.g. Headroom running as a "+ "LiteLLM guardrail/proxy hop, see issue #2509). Applies to non-streaming "+ "responses only. Off by default. "+ "Env: HEADROOM_CCR_INLINE_RESOLVE."+ ),+)@click.option("--no-ccr-proactive-expansion",is_flag=True,@@ -934,6 +947,7 @@ def proxy(tpm: int | None,no_ccr: bool,lossless: bool,+ ccr_inline_resolve: bool,no_ccr_proactive_expansion: bool,proxy_extension: tuple[str, ...],compressor: tuple[str, ...],@@ -1206,6 +1220,7 @@ def proxy(# CCR fully on.ccr_inject_tool=not no_ccr,ccr_inject_marker=not no_ccr,+ ccr_resolve_markers_inline=ccr_inline_resolve,lossless=lossless,ccr_proactive_expansion=not no_ccr_proactive_expansion,# Flatten repeat-flag tuple AND any comma-separated values inside it.diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex d19413ac0..5125c0a01 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -25,6 +25,7 @@ import httpxfrom headroom.agent_savings import proxy_pipeline_kwargsfrom headroom.ccr.context_tracker import looks_like_claude_code_compact_summary+from headroom.ccr.marker_resolution import resolve_markers_in_responsefrom headroom.copilot_auth import build_copilot_upstream_urlfrom headroom.pipeline import PipelineStage, summarize_routing_markersfrom headroom.proxy.auth_mode import (@@ -3763,6 +3764,26 @@ class AnthropicHandlerMixin:if _compression_failed:response_headers["x-headroom-compression-failed"] = "true"+ # Inline CCR marker resolution, non-streaming only.+ # Deliberately OUTSIDE the has_ccr_tool_calls gate+ # above: the callers this flag exists for (#2509,+ # Headroom behind a LiteLLM guardrail hop) never get+ # a headroom_retrieve tool-call turn, so gating on+ # one makes the flag a no-op for its own use case.+ # Runs before the security scan so the scanner sees+ # the resolved text, not the marker.+ if (+ getattr(self.config, "ccr_resolve_markers_inline", False)+ and resp_json+ and response.status_code == 200+ ):+ resp_json = resolve_markers_in_response(resp_json)+ response = httpx.Response(+ status_code=200,+ content=json.dumps(resp_json).encode(),+ headers=response_headers,+ )+# Enterprise Security: scan response + de-anonymize.# Gate on a 200 upstream like the sibling CCR/cache/buffered# blocks below: without this, a non-2xx upstream (rate limitdiff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 218ef7ec6..c3e3ff0a3 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -45,6 +45,7 @@ if TYPE_CHECKING:import httpxfrom headroom.agent_savings import proxy_pipeline_kwargs+from headroom.ccr.marker_resolution import resolve_markers_in_responsefrom headroom.config import unwrap_tool_call_namefrom headroom.copilot_auth import (apply_copilot_api_auth,@@ -4201,6 +4202,19 @@ class OpenAIHandlerMixin:# feedback_no_silent_fallbacks.md.raise+ # Inline marker resolution runs OUTSIDE the tool-call+ # gate above: the callers this flag exists for (#2509,+ # Headroom as a LiteLLM guardrail hop) never emit a+ # headroom_retrieve tool call, so gating on one would+ # make the flag a no-op for exactly its use case.+ # Non-streaming responses only.+ if (+ getattr(self.config, "ccr_resolve_markers_inline", False)+ and backend_response.body+ and backend_response.status_code == 200+ ):+ backend_response.body = resolve_markers_in_response(backend_response.body)+# Extract usage from the FINAL backend body (after# any CCR resolution) so the prefix tracker counts# cache stats from the LAST upstream call.@@ -4785,6 +4799,22 @@ class OpenAIHandlerMixin:if _compression_failed:response_headers["x-headroom-compression-failed"] = "true"+ # Inline marker resolution, non-streaming only. The direct+ # path has no CCR tool-call handling at all, which is+ # exactly the #2509 shape: markers would otherwise reach+ # the client as raw text.+ if (+ getattr(self.config, "ccr_resolve_markers_inline", False)+ and resp_json+ and response.status_code == 200+ ):+ resolved_json = resolve_markers_in_response(resp_json)+ return Response(+ content=json.dumps(resolved_json).encode(),+ status_code=response.status_code,+ headers=response_headers,+ )+return Response(content=response.content,status_code=response.status_code,@@ -5957,6 +5987,21 @@ class OpenAIHandlerMixin:headers=sse_headers,)+ # Inline marker resolution, non-streaming only. Runs+ # outside the has_ccr_tool_calls gate above on purpose:+ # the #2509 case has no retrieve tool call at all.+ if (+ getattr(self.config, "ccr_resolve_markers_inline", False)+ and resp_json+ and response.status_code == 200+ ):+ resolved_json = resolve_markers_in_response(resp_json)+ return Response(+ content=json.dumps(resolved_json).encode(),+ status_code=response.status_code,+ headers=response_headers,+ )+return Response(content=response.content,status_code=response.status_code,diff --git a/headroom/proxy/models.py b/headroom/proxy/models.pyindex 069867e2b..e44493bbe 100644--- a/headroom/proxy/models.py+++ b/headroom/proxy/models.py@@ -186,6 +186,13 @@ class ProxyConfig:# markers can be toggled from the CLI (--no-ccr, which also drops the retrieve# tool). Threaded into the router in server.py; default preserves current behavior.ccr_inject_marker: bool = True+ # Explicit opt-in fallback for callers with no path to redeem a marker via+ # the headroom_retrieve tool (e.g. a LiteLLM guardrail/proxy hop with no+ # tool-call turn — issue #2509). Off by default: guessing "this caller+ # can't use tools" is fragile, so operators must opt in with+ # --ccr-inline-resolve / HEADROOM_CCR_INLINE_RESOLVE. Applies to+ # non-streaming responses only; buffered-CCR streaming is untouched.+ ccr_resolve_markers_inline: bool = False# CCR Response Handlingccr_handle_responses: bool = Truediff --git a/headroom/settings_store.py b/headroom/settings_store.pyindex b686d3b86..ad134ebc0 100644--- a/headroom/settings_store.py+++ b/headroom/settings_store.py@@ -136,6 +136,21 @@ SETTINGS: tuple[SettingField, ...] = (help="Disable CCR entirely (no markers, no injected retrieve tool).",tier="basic",),+ SettingField(+ "HEADROOM_CCR_INLINE_RESOLVE",+ "ccr_inline_resolve",+ "Resolve CCR markers inline",+ "Compression",+ "bool",+ default=False,+ help=(+ "Resolve <<ccr:...>> markers on the response path instead of "+ "relying on headroom_retrieve tool calls. For callers with no "+ "tool-call round-trip (e.g. a LiteLLM guardrail/proxy hop). "+ "Non-streaming responses only."+ ),+ tier="advanced",+ ),# --- Limits ---SettingField("HEADROOM_RPM",diff --git a/tests/test_ccr_inline_resolve_handlers.py b/tests/test_ccr_inline_resolve_handlers.pynew file mode 100644index 000000000..51cab54c6--- /dev/null+++ b/tests/test_ccr_inline_resolve_handlers.py@@ -0,0 +1,173 @@+"""Handler-level wiring for ``--ccr-inline-resolve`` (issue #2509).++The pure-module tests in ``test_ccr_marker_resolution.py`` prove the+substitution logic. These tests prove the thing that actually broke: the+resolve call has to run on the response path *when the model never emitted a+``headroom_retrieve`` tool call at all*. That is the whole #2509 shape —+Headroom behind a LiteLLM guardrail hop with no tool-call turn — so any wiring+that sits behind a ``has_ccr_tool_calls`` gate is a no-op for its own use case.++Every response fixture below therefore has zero tool calls.+"""++from __future__ import annotations++import pytest++fastapi = pytest.importorskip("fastapi")+httpx = pytest.importorskip("httpx")++from fastapi.testclient import TestClient # noqa: E402++from headroom.cache.compression_store import ( # noqa: E402+ get_compression_store,+ reset_compression_store,+)+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++ORIGINAL = "the original uncompressed content"++[email protected](autouse=True)+def reset_store():+ reset_compression_store()+ yield+ reset_compression_store()+++def _marker() -> str:+ hash_key = get_compression_store().store(+ original=ORIGINAL,+ compressed="[]",+ original_item_count=1,+ compressed_item_count=0,+ )+ return f"<<ccr:{hash_key},string,23.6KB>>"+++def _config(*, inline_resolve: bool) -> ProxyConfig:+ # No backend -> the "Direct OpenAI API (no backend configured)" path.+ return ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ ccr_resolve_markers_inline=inline_resolve,+ )+++def _chat_response(marker: str) -> dict:+ return {+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": f"here it is: {marker}"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},+ }+++def _responses_response(marker: str) -> dict:+ return {+ "id": "resp_1",+ "object": "response",+ "model": "gpt-4o",+ "status": "completed",+ "output": [+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": f"here it is: {marker}"}],+ }+ ],+ "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},+ }+++def _anthropic_response(marker: str) -> dict:+ return {+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-20250514",+ "content": [{"type": "text", "text": f"here it is: {marker}"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 10, "output_tokens": 5},+ }+++def _run(config: ProxyConfig, path: str, body: dict, upstream: dict) -> httpx.Response:+ async def fake_retry(method, url, headers, req_body, *args, **kwargs):+ return httpx.Response(200, json=upstream, headers={"content-type": "application/json"})++ app = create_app(config)+ with TestClient(app) as client:+ client.app.state.proxy._retry_request = fake_retry+ return client.post(+ path,+ json=body,+ headers={"Authorization": "Bearer test-key", "x-api-key": "test-key"},+ )++[email protected]("inline_resolve", [True, False])+def test_openai_chat_direct_path(inline_resolve):+ marker = _marker()+ resp = _run(+ _config(inline_resolve=inline_resolve),+ "/v1/chat/completions",+ {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}], "stream": False},+ _chat_response(marker),+ )++ assert resp.status_code == 200, resp.text+ text = resp.json()["choices"][0]["message"]["content"]+ if inline_resolve:+ assert text == f"here it is: {ORIGINAL}"+ else:+ assert text == f"here it is: {marker}"++[email protected]("inline_resolve", [True, False])+def test_openai_responses_path(inline_resolve):+ marker = _marker()+ resp = _run(+ _config(inline_resolve=inline_resolve),+ "/v1/responses",+ {"model": "gpt-4o", "input": "hi", "stream": False},+ _responses_response(marker),+ )++ assert resp.status_code == 200, resp.text+ text = resp.json()["output"][0]["content"][0]["text"]+ if inline_resolve:+ assert text == f"here it is: {ORIGINAL}"+ else:+ assert text == f"here it is: {marker}"++[email protected]("inline_resolve", [True, False])+def test_anthropic_messages_path(inline_resolve):+ marker = _marker()+ resp = _run(+ _config(inline_resolve=inline_resolve),+ "/v1/messages",+ {+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 64,+ "messages": [{"role": "user", "content": "hi"}],+ "stream": False,+ },+ _anthropic_response(marker),+ )++ assert resp.status_code == 200, resp.text+ text = resp.json()["content"][0]["text"]+ if inline_resolve:+ assert text == f"here it is: {ORIGINAL}"+ else:+ assert text == f"here it is: {marker}"diff --git a/tests/test_ccr_marker_resolution.py b/tests/test_ccr_marker_resolution.pynew file mode 100644index 000000000..ab2609c3b--- /dev/null+++ b/tests/test_ccr_marker_resolution.py@@ -0,0 +1,99 @@+"""Tests for inline <<ccr:...>> marker resolution (issue #2509)."""++from __future__ import annotations++import json++import pytest++from headroom.cache.compression_store import get_compression_store, reset_compression_store+from headroom.ccr.marker_resolution import (+ resolve_markers_in_response,+ resolve_markers_in_text,+)++[email protected](autouse=True)+def reset_store():+ reset_compression_store()+ yield+ reset_compression_store()+++def _store_entry(original: str) -> str:+ store = get_compression_store()+ return store.store(+ original=original,+ compressed="[]",+ original_item_count=1,+ compressed_item_count=0,+ )+++def test_resolve_markers_in_text_no_marker_is_noop():+ assert resolve_markers_in_text("plain text, no markers here") == "plain text, no markers here"+++def test_resolve_markers_in_text_replaces_hit():+ hash_key = _store_entry("the original uncompressed content")+ text = f"before <<ccr:{hash_key},string,23.6KB>> after"++ resolved = resolve_markers_in_text(text)++ assert resolved == "before the original uncompressed content after"+++def test_resolve_markers_in_text_replaces_multiple_hits():+ hash_a = _store_entry("AAA")+ hash_b = _store_entry("BBB")+ text = f"<<ccr:{hash_a},string,1KB>> and <<ccr:{hash_b},string,1KB>>"++ resolved = resolve_markers_in_text(text)++ assert resolved == "AAA and BBB"+++def test_resolve_markers_in_text_json_array_original_content():+ store = get_compression_store()+ hash_key = store.store(+ original=json.dumps([1, 2, 3]),+ compressed="[]",+ original_item_count=3,+ compressed_item_count=0,+ )+ text = f"<<ccr:{hash_key},array,3>>"++ resolved = resolve_markers_in_text(text)++ assert json.loads(resolved) == [1, 2, 3]+++def test_resolve_markers_in_text_miss_leaves_marker_with_reason():+ text = "<<ccr:deadbeefdeadbeef,string,1KB>>"++ resolved = resolve_markers_in_text(text)++ assert text in resolved+ assert "[unresolved:" in resolved+++def test_resolve_markers_in_response_walks_nested_structure():+ hash_key = _store_entry("full tool output")+ response = {+ "choices": [+ {+ "message": {+ "role": "assistant",+ "content": f"here it is: <<ccr:{hash_key},string,1KB>>",+ }+ }+ ],+ "unrelated": 42,+ "nested": {"list": ["a", f"<<ccr:{hash_key},string,1KB>>", "c"]},+ }++ resolved = resolve_markers_in_response(response)++ assert resolved["choices"][0]["message"]["content"] == "here it is: full tool output"+ assert resolved["nested"]["list"] == ["a", "full tool output", "c"]+ assert resolved["unrelated"] == 42