Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(ccr): resolve <<ccr:...>> markers inline when no retrieve-tool path exists (#2512)

Parideboyce8ce8313f8c
diff --git a/headroom/ccr/marker_resolution.py b/headroom/ccr/marker_resolution.py
new file mode 100644
index 000000000..aef4a108f
--- /dev/null
+++ b/headroom/ccr/marker_resolution.py
@@ -0,0 +1,83 @@
+"""Inline resolution of ``<<ccr:...>>`` markers on the response path.
+
+Normal CCR resolution relies on the ``headroom_retrieve`` tool: a marker is
+redeemed when the model calls the tool back. That path assumes there's a
+subsequent turn in which the model *can* call it. Callers that never see an
+injected tool at all — e.g. Headroom running as a LiteLLM guardrail/proxy hop
+with no tool-call turn in between (#2509) — have no way to redeem a marker,
+so it leaks through as raw text.
+
+This module provides an explicit, opt-in fallback (``--ccr-inline-resolve``):
+scan the outgoing response for markers and substitute the original content
+directly, instead of leaving the marker for the model to redeem later.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+from ..cache.compression_store import (
+ CompressionStore,
+ format_retrieval_miss_detail,
+ get_compression_store,
+)
+
+logger = logging.getLogger(__name__)
+
+# Matches the opaque-blob marker form `<<ccr:HASH,KIND,SIZE>>` (and the
+# row-offload form `<<ccr:HASH N_rows_offloaded>>`) emitted by SmartCrusher.
+# HASH is 12-24 hex chars; see headroom/ccr/tool_injection.py for the same
+# constant used on the injection side.
+_MARKER_RE = re.compile(r"<<ccr:([a-f0-9]{12,24})[^>]*>>")
+
+
+def resolve_markers_in_text(text: str, *, store: CompressionStore | None = None) -> str:
+ """Replace every ``<<ccr:HASH,...>>`` marker in ``text`` with its original content.
+
+ A miss (expired/evicted/unknown hash) can't be reported back to the
+ model on this path — there's no tool-call round-trip — so the marker is
+ left in place with the miss reason appended rather than raising.
+ """
+ if "<<ccr:" not in text:
+ return text
+
+ resolved_store = store or get_compression_store()
+
+ def _replace(match: re.Match[str]) -> str:
+ hash_key = match.group(1)
+ entry = resolved_store.retrieve(hash_key)
+ if entry is not None:
+ original = entry.original_content
+ return original if isinstance(original, str) else json.dumps(original)
+
+ get_status = getattr(resolved_store, "get_entry_status", None)
+ status = get_status(hash_key, clean_expired=True) if callable(get_status) else None
+ detail = format_retrieval_miss_detail(status) if status else "entry not found"
+ logger.warning(f"CCR inline-resolve: marker {hash_key} unresolvable ({detail})")
+ return f"{match.group(0)} [unresolved: {detail}]"
+
+ return _MARKER_RE.sub(_replace, text)
+
+
+def resolve_markers_in_response(response: Any, *, store: CompressionStore | None = None) -> Any:
+ """Recursively resolve ``<<ccr:...>>`` markers in every string field of a payload.
+
+ Walks the full response structure rather than picking out
+ provider-specific fields (``content`` blocks, ``message.content``,
+ Responses-API ``output`` items, ...) so it stays correct regardless of
+ where a marker ends up, and doesn't need per-provider maintenance.
+ """
+ resolved_store = store or get_compression_store()
+ if isinstance(response, str):
+ return resolve_markers_in_text(response, store=resolved_store)
+ if isinstance(response, list):
+ return [resolve_markers_in_response(item, store=resolved_store) for item in response]
+ if isinstance(response, dict):
+ return {
+ key: resolve_markers_in_response(value, store=resolved_store)
+ for key, value in response.items()
+ }
+ return response
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index 30292a4e1..29e964abd 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -313,6 +313,19 @@ def dashboard(port: int, no_open: bool) -> None:
"retrieval marker, so no MCP retrieve tool is needed. Env: HEADROOM_LOSSLESS=1."
),
)
+ "--ccr-inline-resolve",
+ is_flag=True,
+ envvar="HEADROOM_CCR_INLINE_RESOLVE",
+ help=(
+ "Resolve <<ccr:...>> markers inline on the response path instead of "
+ "relying on the model to call headroom_retrieve. For callers with no "
+ "tool-call round-trip to redeem a marker (e.g. Headroom running as a "
+ "LiteLLM guardrail/proxy hop, see issue #2509). Applies to non-streaming "
+ "responses only. Off by default. "
+ "Env: HEADROOM_CCR_INLINE_RESOLVE."
+ ),
+)
@click.option(
"--no-ccr-proactive-expansion",
is_flag=True,
@@ -934,6 +947,7 @@ def proxy(
tpm: int | None,
no_ccr: bool,
lossless: bool,
+ ccr_inline_resolve: bool,
no_ccr_proactive_expansion: bool,
proxy_extension: tuple[str, ...],
compressor: tuple[str, ...],
@@ -1206,6 +1220,7 @@ def proxy(
# CCR fully on.
ccr_inject_tool=not no_ccr,
ccr_inject_marker=not no_ccr,
+ ccr_resolve_markers_inline=ccr_inline_resolve,
lossless=lossless,
ccr_proactive_expansion=not no_ccr_proactive_expansion,
# Flatten repeat-flag tuple AND any comma-separated values inside it.
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index d19413ac0..5125c0a01 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -25,6 +25,7 @@ import httpx
from headroom.agent_savings import proxy_pipeline_kwargs
from headroom.ccr.context_tracker import looks_like_claude_code_compact_summary
+from headroom.ccr.marker_resolution import resolve_markers_in_response
from headroom.copilot_auth import build_copilot_upstream_url
from headroom.pipeline import PipelineStage, summarize_routing_markers
from headroom.proxy.auth_mode import (
@@ -3763,6 +3764,26 @@ class AnthropicHandlerMixin:
if _compression_failed:
response_headers["x-headroom-compression-failed"] = "true"
+ # Inline CCR marker resolution, non-streaming only.
+ # Deliberately OUTSIDE the has_ccr_tool_calls gate
+ # above: the callers this flag exists for (#2509,
+ # Headroom behind a LiteLLM guardrail hop) never get
+ # a headroom_retrieve tool-call turn, so gating on
+ # one makes the flag a no-op for its own use case.
+ # Runs before the security scan so the scanner sees
+ # the resolved text, not the marker.
+ if (
+ getattr(self.config, "ccr_resolve_markers_inline", False)
+ and resp_json
+ and response.status_code == 200
+ ):
+ resp_json = resolve_markers_in_response(resp_json)
+ response = httpx.Response(
+ status_code=200,
+ content=json.dumps(resp_json).encode(),
+ headers=response_headers,
+ )
+
# Enterprise Security: scan response + de-anonymize.
# Gate on a 200 upstream like the sibling CCR/cache/buffered
# blocks below: without this, a non-2xx upstream (rate limit
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 218ef7ec6..c3e3ff0a3 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -45,6 +45,7 @@ if TYPE_CHECKING:
import httpx
from headroom.agent_savings import proxy_pipeline_kwargs
+from headroom.ccr.marker_resolution import resolve_markers_in_response
from headroom.config import unwrap_tool_call_name
from headroom.copilot_auth import (
apply_copilot_api_auth,
@@ -4201,6 +4202,19 @@ class OpenAIHandlerMixin:
# feedback_no_silent_fallbacks.md.
raise
+ # Inline marker resolution runs OUTSIDE the tool-call
+ # gate above: the callers this flag exists for (#2509,
+ # Headroom as a LiteLLM guardrail hop) never emit a
+ # headroom_retrieve tool call, so gating on one would
+ # make the flag a no-op for exactly its use case.
+ # Non-streaming responses only.
+ if (
+ getattr(self.config, "ccr_resolve_markers_inline", False)
+ and backend_response.body
+ and backend_response.status_code == 200
+ ):
+ backend_response.body = resolve_markers_in_response(backend_response.body)
+
# Extract usage from the FINAL backend body (after
# any CCR resolution) so the prefix tracker counts
# cache stats from the LAST upstream call.
@@ -4785,6 +4799,22 @@ class OpenAIHandlerMixin:
if _compression_failed:
response_headers["x-headroom-compression-failed"] = "true"
+ # Inline marker resolution, non-streaming only. The direct
+ # path has no CCR tool-call handling at all, which is
+ # exactly the #2509 shape: markers would otherwise reach
+ # the client as raw text.
+ if (
+ getattr(self.config, "ccr_resolve_markers_inline", False)
+ and resp_json
+ and response.status_code == 200
+ ):
+ resolved_json = resolve_markers_in_response(resp_json)
+ return Response(
+ content=json.dumps(resolved_json).encode(),
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
return Response(
content=response.content,
status_code=response.status_code,
@@ -5957,6 +5987,21 @@ class OpenAIHandlerMixin:
headers=sse_headers,
)
+ # Inline marker resolution, non-streaming only. Runs
+ # outside the has_ccr_tool_calls gate above on purpose:
+ # the #2509 case has no retrieve tool call at all.
+ if (
+ getattr(self.config, "ccr_resolve_markers_inline", False)
+ and resp_json
+ and response.status_code == 200
+ ):
+ resolved_json = resolve_markers_in_response(resp_json)
+ return Response(
+ content=json.dumps(resolved_json).encode(),
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
return Response(
content=response.content,
status_code=response.status_code,
diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py
index 069867e2b..e44493bbe 100644
--- a/headroom/proxy/models.py
+++ b/headroom/proxy/models.py
@@ -186,6 +186,13 @@ class ProxyConfig:
# markers can be toggled from the CLI (--no-ccr, which also drops the retrieve
# tool). Threaded into the router in server.py; default preserves current behavior.
ccr_inject_marker: bool = True
+ # Explicit opt-in fallback for callers with no path to redeem a marker via
+ # the headroom_retrieve tool (e.g. a LiteLLM guardrail/proxy hop with no
+ # tool-call turn — issue #2509). Off by default: guessing "this caller
+ # can't use tools" is fragile, so operators must opt in with
+ # --ccr-inline-resolve / HEADROOM_CCR_INLINE_RESOLVE. Applies to
+ # non-streaming responses only; buffered-CCR streaming is untouched.
+ ccr_resolve_markers_inline: bool = False
# CCR Response Handling
ccr_handle_responses: bool = True
diff --git a/headroom/settings_store.py b/headroom/settings_store.py
index b686d3b86..ad134ebc0 100644
--- a/headroom/settings_store.py
+++ b/headroom/settings_store.py
@@ -136,6 +136,21 @@ SETTINGS: tuple[SettingField, ...] = (
help="Disable CCR entirely (no markers, no injected retrieve tool).",
tier="basic",
),
+ SettingField(
+ "HEADROOM_CCR_INLINE_RESOLVE",
+ "ccr_inline_resolve",
+ "Resolve CCR markers inline",
+ "Compression",
+ "bool",
+ default=False,
+ help=(
+ "Resolve <<ccr:...>> markers on the response path instead of "
+ "relying on headroom_retrieve tool calls. For callers with no "
+ "tool-call round-trip (e.g. a LiteLLM guardrail/proxy hop). "
+ "Non-streaming responses only."
+ ),
+ tier="advanced",
+ ),
# --- Limits ---
SettingField(
"HEADROOM_RPM",
diff --git a/tests/test_ccr_inline_resolve_handlers.py b/tests/test_ccr_inline_resolve_handlers.py
new file mode 100644
index 000000000..51cab54c6
--- /dev/null
+++ b/tests/test_ccr_inline_resolve_handlers.py
@@ -0,0 +1,173 @@
+"""Handler-level wiring for ``--ccr-inline-resolve`` (issue #2509).
+
+The pure-module tests in ``test_ccr_marker_resolution.py`` prove the
+substitution logic. These tests prove the thing that actually broke: the
+resolve call has to run on the response path *when the model never emitted a
+``headroom_retrieve`` tool call at all*. That is the whole #2509 shape —
+Headroom behind a LiteLLM guardrail hop with no tool-call turn — so any wiring
+that sits behind a ``has_ccr_tool_calls`` gate is a no-op for its own use case.
+
+Every response fixture below therefore has zero tool calls.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+httpx = pytest.importorskip("httpx")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.cache.compression_store import ( # noqa: E402
+ get_compression_store,
+ reset_compression_store,
+)
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+ORIGINAL = "the original uncompressed content"
+
+
[email protected](autouse=True)
+def reset_store():
+ reset_compression_store()
+ yield
+ reset_compression_store()
+
+
+def _marker() -> str:
+ hash_key = get_compression_store().store(
+ original=ORIGINAL,
+ compressed="[]",
+ original_item_count=1,
+ compressed_item_count=0,
+ )
+ return f"<<ccr:{hash_key},string,23.6KB>>"
+
+
+def _config(*, inline_resolve: bool) -> ProxyConfig:
+ # No backend -> the "Direct OpenAI API (no backend configured)" path.
+ return ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ ccr_resolve_markers_inline=inline_resolve,
+ )
+
+
+def _chat_response(marker: str) -> dict:
+ return {
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "model": "gpt-4o",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": f"here it is: {marker}"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+ }
+
+
+def _responses_response(marker: str) -> dict:
+ return {
+ "id": "resp_1",
+ "object": "response",
+ "model": "gpt-4o",
+ "status": "completed",
+ "output": [
+ {
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": f"here it is: {marker}"}],
+ }
+ ],
+ "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+ }
+
+
+def _anthropic_response(marker: str) -> dict:
+ return {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-20250514",
+ "content": [{"type": "text", "text": f"here it is: {marker}"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 10, "output_tokens": 5},
+ }
+
+
+def _run(config: ProxyConfig, path: str, body: dict, upstream: dict) -> httpx.Response:
+ async def fake_retry(method, url, headers, req_body, *args, **kwargs):
+ return httpx.Response(200, json=upstream, headers={"content-type": "application/json"})
+
+ app = create_app(config)
+ with TestClient(app) as client:
+ client.app.state.proxy._retry_request = fake_retry
+ return client.post(
+ path,
+ json=body,
+ headers={"Authorization": "Bearer test-key", "x-api-key": "test-key"},
+ )
+
+
[email protected]("inline_resolve", [True, False])
+def test_openai_chat_direct_path(inline_resolve):
+ marker = _marker()
+ resp = _run(
+ _config(inline_resolve=inline_resolve),
+ "/v1/chat/completions",
+ {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}], "stream": False},
+ _chat_response(marker),
+ )
+
+ assert resp.status_code == 200, resp.text
+ text = resp.json()["choices"][0]["message"]["content"]
+ if inline_resolve:
+ assert text == f"here it is: {ORIGINAL}"
+ else:
+ assert text == f"here it is: {marker}"
+
+
[email protected]("inline_resolve", [True, False])
+def test_openai_responses_path(inline_resolve):
+ marker = _marker()
+ resp = _run(
+ _config(inline_resolve=inline_resolve),
+ "/v1/responses",
+ {"model": "gpt-4o", "input": "hi", "stream": False},
+ _responses_response(marker),
+ )
+
+ assert resp.status_code == 200, resp.text
+ text = resp.json()["output"][0]["content"][0]["text"]
+ if inline_resolve:
+ assert text == f"here it is: {ORIGINAL}"
+ else:
+ assert text == f"here it is: {marker}"
+
+
[email protected]("inline_resolve", [True, False])
+def test_anthropic_messages_path(inline_resolve):
+ marker = _marker()
+ resp = _run(
+ _config(inline_resolve=inline_resolve),
+ "/v1/messages",
+ {
+ "model": "claude-sonnet-4-20250514",
+ "max_tokens": 64,
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": False,
+ },
+ _anthropic_response(marker),
+ )
+
+ assert resp.status_code == 200, resp.text
+ text = resp.json()["content"][0]["text"]
+ if inline_resolve:
+ assert text == f"here it is: {ORIGINAL}"
+ else:
+ assert text == f"here it is: {marker}"
diff --git a/tests/test_ccr_marker_resolution.py b/tests/test_ccr_marker_resolution.py
new file mode 100644
index 000000000..ab2609c3b
--- /dev/null
+++ b/tests/test_ccr_marker_resolution.py
@@ -0,0 +1,99 @@
+"""Tests for inline <<ccr:...>> marker resolution (issue #2509)."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from headroom.cache.compression_store import get_compression_store, reset_compression_store
+from headroom.ccr.marker_resolution import (
+ resolve_markers_in_response,
+ resolve_markers_in_text,
+)
+
+
[email protected](autouse=True)
+def reset_store():
+ reset_compression_store()
+ yield
+ reset_compression_store()
+
+
+def _store_entry(original: str) -> str:
+ store = get_compression_store()
+ return store.store(
+ original=original,
+ compressed="[]",
+ original_item_count=1,
+ compressed_item_count=0,
+ )
+
+
+def test_resolve_markers_in_text_no_marker_is_noop():
+ assert resolve_markers_in_text("plain text, no markers here") == "plain text, no markers here"
+
+
+def test_resolve_markers_in_text_replaces_hit():
+ hash_key = _store_entry("the original uncompressed content")
+ text = f"before <<ccr:{hash_key},string,23.6KB>> after"
+
+ resolved = resolve_markers_in_text(text)
+
+ assert resolved == "before the original uncompressed content after"
+
+
+def test_resolve_markers_in_text_replaces_multiple_hits():
+ hash_a = _store_entry("AAA")
+ hash_b = _store_entry("BBB")
+ text = f"<<ccr:{hash_a},string,1KB>> and <<ccr:{hash_b},string,1KB>>"
+
+ resolved = resolve_markers_in_text(text)
+
+ assert resolved == "AAA and BBB"
+
+
+def test_resolve_markers_in_text_json_array_original_content():
+ store = get_compression_store()
+ hash_key = store.store(
+ original=json.dumps([1, 2, 3]),
+ compressed="[]",
+ original_item_count=3,
+ compressed_item_count=0,
+ )
+ text = f"<<ccr:{hash_key},array,3>>"
+
+ resolved = resolve_markers_in_text(text)
+
+ assert json.loads(resolved) == [1, 2, 3]
+
+
+def test_resolve_markers_in_text_miss_leaves_marker_with_reason():
+ text = "<<ccr:deadbeefdeadbeef,string,1KB>>"
+
+ resolved = resolve_markers_in_text(text)
+
+ assert text in resolved
+ assert "[unresolved:" in resolved
+
+
+def test_resolve_markers_in_response_walks_nested_structure():
+ hash_key = _store_entry("full tool output")
+ response = {
+ "choices": [
+ {
+ "message": {
+ "role": "assistant",
+ "content": f"here it is: <<ccr:{hash_key},string,1KB>>",
+ }
+ }
+ ],
+ "unrelated": 42,
+ "nested": {"list": ["a", f"<<ccr:{hash_key},string,1KB>>", "c"]},
+ }
+
+ resolved = resolve_markers_in_response(response)
+
+ assert resolved["choices"][0]["message"]["content"] == "here it is: full tool output"
+ assert resolved["nested"]["list"] == ["a", "full tool output", "c"]
+ assert resolved["unrelated"] == 42