Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache): scope router results to request compression policy (#4590)
Endid5d40fe76471b
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 3359f0c75..d8838af2f 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -1691,13 +1691,30 @@ def _compression_cache_budget_bytes() -> int:return max(1, int(mb * 1024 * 1024))-def content_cache_key(content: str, target_ratio: float | None, gated: bool = False) -> str:- """The compression cache's key for ``content``: a digest, the same in every- process, so the disk tier (and replicas sharing it) can use it. The runtime- target ratio is part of it (a different ratio is a different result), and so- is the reversibility gate tool output passes (#1307)."""+def content_cache_key(+ content: str,+ target_ratio: float | None,+ gated: bool = False,+ *,+ kompress_model: str | None = None,+ force_kompress: bool = False,+ skip_kompress: bool = False,+ bias: float = 1.0,+ context: str = "",+) -> str:+ """A process-stable digest of content and the policy used to compress it.++ Results, skips and frozen verdicts must not cross request-policy boundaries.+ JSON frames the policy fields unambiguously, including arbitrary context and+ model strings, before the content bytes (also used by the shared disk tier).+ """h = hashlib.blake2b(digest_size=16)- h.update(f"{target_ratio!r}|{int(gated)}|".encode())+ h.update(+ json.dumps(+ [target_ratio, gated, kompress_model, force_kompress, skip_kompress, bias, context],+ separators=(",", ":"),+ ).encode("utf-8")+ )h.update(content.encode("utf-8", "surrogatepass"))return h.hexdigest()@@ -6976,14 +6993,14 @@ class ContentRouter(Transform):# Two-tier compression cache.# Tier 1 (skip): known won't-compress → instant skip.# Tier 2 (result): known compresses → reuse compressed text.- # Key on the runtime target_ratio too: the same content compressed at- # a different ratio is a different result, so it must not alias.+ # Key on the request policy and effective compressor inputs too:+ # identical content can produce different results under each.# Tool ground truth is gated against lossy-unrecoverable results below# (#1307). Partition its cache namespace so a gated tool entry is never# served from — or poisons — an ungated entry for byte-identical content.enforce_reversibility = role == "tool"- content_key = content_cache_key(- content, getattr(self, "_runtime_target_ratio", None), gated=enforce_reversibility+ content_key = self._content_cache_key(+ content, gated=enforce_reversibility, bias=msg_bias, context=context)# Tier 1: skip set — instant rejection@@ -7553,6 +7570,21 @@ class ContentRouter(Transform):return Nonereturn folded if len(folded) < len(content) else None+ def _content_cache_key(+ self, content: str, *, gated: bool = False, bias: float = 1.0, context: str = ""+ ) -> str:+ """Scope all message shapes to this request's compression policy."""+ return content_cache_key(+ content,+ self._runtime_target_ratio,+ gated=gated,+ kompress_model=self._runtime_kompress_model,+ force_kompress=self._runtime_force_kompress,+ skip_kompress=self._runtime_skip_kompress,+ bias=bias,+ context=context,+ )+def _get_tool_bias(self, tool_name: str) -> float:"""Look up compression bias for a tool name.@@ -8080,8 +8112,8 @@ class ContentRouter(Transform):# Two-tier compression cache → shared helpercompressed_content, was_compressed = self._compress_block_content(content=tool_text,- content_key=content_cache_key(- tool_text, getattr(self, "_runtime_target_ratio", None)+ content_key=self._content_cache_key(+ tool_text, bias=bias, context=block_context),context=block_context,bias=bias,@@ -8175,8 +8207,8 @@ class ContentRouter(Transform):# Two-tier compression cache → shared helpercompressed_content, _was_compressed = self._compress_block_content(content=text_content,- content_key=content_cache_key(- text_content, getattr(self, "_runtime_target_ratio", None)+ content_key=self._content_cache_key(+ text_content, bias=1.0, context=context),context=context,bias=1.0,diff --git a/tests/test_router_cache_policy.py b/tests/test_router_cache_policy.pynew file mode 100644index 000000000..2db35490a--- /dev/null+++ b/tests/test_router_cache_policy.py@@ -0,0 +1,239 @@+"""Request policy must scope every tier of the router's content cache."""++from __future__ import annotations++import os+import subprocess+import sys+from types import SimpleNamespace++import pytest++from headroom.transforms.content_router import (+ CompressionStrategy,+ ContentRouter,+ ContentRouterConfig,+ content_cache_key,+)++TEXT = "The service reports useful observations about the current deployment. " * 80+++class Tokenizer:+ def count_text(self, text: str) -> int:+ return max(1, len(str(text)) // 4)+++def messages(shape, tool_name="inspect", arguments="{}"):+ call = {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "call_1",+ "type": "function",+ "function": {"name": tool_name, "arguments": arguments},+ }+ ],+ }+ if shape == "tool_result":+ result = {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": TEXT}],+ }+ else:+ content = TEXT if shape == "string" else [{"type": "text", "text": TEXT}]+ result = {"role": "tool", "tool_call_id": "call_1", "content": content}+ return [call, result]+++def output(result, shape):+ content = result.messages[-1]["content"]+ if shape == "string":+ return content+ return content[0]["content" if shape == "tool_result" else "text"]+++def router(monkeypatch):+ # Only the compressor is replaced: apply(), admission, all cache tiers,+ # runtime policy binding and frozen verdict handling remain real.+ router = ContentRouter(ContentRouterConfig(ccr_inject_marker=False))+ monkeypatch.setattr(router, "_kompress_model_ready", lambda: True)+ return router+++def install_compressor(monkeypatch, router, *, skip_first=False, marked=False):+ calls = []++ def compress(content, context="", bias=1.0, **kwargs):+ calls.append((router._runtime_kompress_model, context, bias))+ disabled = router._runtime_kompress_model == "disabled" or router._runtime_skip_kompress+ skip = disabled or (skip_first and len(calls) == 1)+ compressed = content if skip else f"compressed under policy {len(calls)}"+ if marked and not skip:+ compressed += " <<ccr:0123456789abcdef>>"+ return SimpleNamespace(+ original=content,+ compressed=compressed,+ compression_ratio=1.0 if skip else 0.1,+ strategy_used=CompressionStrategy.KOMPRESS,+ strategy_chain=[],+ )++ monkeypatch.setattr(router, "compress", compress)+ return calls++[email protected]("shape", ["string", "tool_result", "text"])[email protected]("freeze", [False, True])+ "changed",+ [+ {"kompress_model": "disabled"},+ {"kompress_model": "another-model"},+ {"skip_kompress": True},+ {"force_kompress": True},+ {"context": "a different question"},+ {"target_ratio": 0.3},+ ],+)+def test_apply_reuses_results_only_under_the_same_policy(+ monkeypatch, router, shape, freeze, changed+):+ monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", str(int(freeze)))+ calls = install_compressor(monkeypatch, router, marked=freeze)+ payload = messages(shape)+ first = output(router.apply(payload, Tokenizer()), shape)+ assert first != TEXT+ assert output(router.apply(payload, Tokenizer()), shape) == first+ assert len(calls) == 1++ second = output(router.apply(payload, Tokenizer(), **changed), shape)+ assert len(calls) == 2, "changed policy must not reuse the previous result"+ assert second != first+ if changed.get("kompress_model") == "disabled" or changed.get("skip_kompress"):+ assert second == TEXT+ assert output(router.apply(payload, Tokenizer(), **changed), shape) == second+ assert len(calls) == 2+ # Returning to the original policy still finds its own cached output.+ assert output(router.apply(payload, Tokenizer()), shape) == first+ assert len(calls) == 2++[email protected]("shape", ["string", "tool_result", "text"])+def test_frozen_verdicts_are_recorded_separately_for_each_policy(monkeypatch, router, shape):+ monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")+ # Lossy output needs a retrieval marker before a verdict is safe to freeze.+ calls = install_compressor(monkeypatch, router, marked=True)+ payload = messages(shape)++ first = output(router.apply(payload, Tokenizer()), shape)+ initial_verdicts = dict(router._frozen_verdicts)+ assert len(initial_verdicts) == 1+ assert list(initial_verdicts.values()) == [True]+ assert output(router.apply(payload, Tokenizer()), shape) == first+ assert router._frozen_verdicts == initial_verdicts+ assert len(calls) == 1++ second = output(router.apply(payload, Tokenizer(), kompress_model="another-model"), shape)+ assert second != first+ assert len(calls) == 2+ assert len(router._frozen_verdicts) == 2+ assert all(verdict is True for verdict in router._frozen_verdicts.values())+ assert initial_verdicts.items() <= router._frozen_verdicts.items()+ assert output(router.apply(payload, Tokenizer()), shape) == first+ assert len(calls) == 2++[email protected]("shape", ["string", "tool_result", "text"])[email protected]("changed", [{"kompress_model": "another-model"}, {"force_kompress": True}])+def test_a_skip_under_one_policy_does_not_suppress_another(monkeypatch, router, shape, changed):+ calls = install_compressor(monkeypatch, router, skip_first=True)+ payload = messages(shape)+ assert output(router.apply(payload, Tokenizer()), shape) == TEXT+ assert output(router.apply(payload, Tokenizer()), shape) == TEXT+ assert len(calls) == 1+ assert output(router.apply(payload, Tokenizer(), **changed), shape) != TEXT+ assert len(calls) == 2+++def test_hook_bias_partitions_string_results(monkeypatch, router):+ calls = install_compressor(monkeypatch, router)+ payload = messages("string")+ first = output(router.apply(payload, Tokenizer()), "string")+ second = output(router.apply(payload, Tokenizer(), biases={1: 2.0}), "string")+ assert second != first+ assert [call[2] for call in calls] == [1.0, 2.0]++[email protected]("shape", ["string", "tool_result"])+def test_effective_tool_bias_partitions_results(monkeypatch, router, shape):+ router.config.tool_profiles = {+ "inspect": SimpleNamespace(bias=1.0),+ "summarize": SimpleNamespace(bias=2.0),+ }+ calls = install_compressor(monkeypatch, router)+ first = output(router.apply(messages(shape), Tokenizer()), shape)+ second = output(router.apply(messages(shape, tool_name="summarize"), Tokenizer()), shape)+ assert second != first+ assert [call[2] for call in calls] == [1.0, 2.0]+++def test_tool_result_relevance_uses_the_enriched_context(monkeypatch, router):+ router.config.relevance_split = True+ calls = install_compressor(monkeypatch, router)+ first = router.apply(messages("tool_result", arguments='{"query":"alpha"}'), Tokenizer())+ second = router.apply(messages("tool_result", arguments='{"query":"beta"}'), Tokenizer())+ assert output(first, "tool_result") != output(second, "tool_result")+ assert len(calls) == 2+ assert calls[0][1] != calls[1][1]+++def test_policy_keys_are_deterministic_across_processes():+ options = {+ "kompress_model": "custom|model",+ "force_kompress": True,+ "skip_kompress": True,+ "bias": 1.5,+ "context": "question|with\x00delimiters",+ }+ expected = content_cache_key(TEXT, 0.5, gated=True, **options)+ code = (+ "from headroom.transforms.content_router import content_cache_key;"+ f"print(content_cache_key({TEXT!r}, 0.5, gated=True, **{options!r}))"+ )+ for seed in ("1", "2"):+ found = subprocess.run(+ [sys.executable, "-c", code],+ capture_output=True,+ text=True,+ check=True,+ env={**os.environ, "PYTHONHASHSEED": seed},+ ).stdout.strip()+ assert found == expected+++def test_context_and_content_have_unambiguous_boundaries():+ assert content_cache_key("b|c", None, context="a") != content_cache_key(+ "c", None, context="a|b"+ )++[email protected]("shape", ["string", "tool_result", "text"])+def test_reopened_disk_results_are_scoped_to_the_request_policy(tmp_path, monkeypatch, shape):+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "policy.db"))+ config = ContentRouterConfig(ccr_inject_marker=False, result_cache_disk=True)+ first = ContentRouter(config)+ calls = install_compressor(monkeypatch, first)+ payload = messages(shape)+ original = output(first.apply(payload, Tokenizer()), shape)+ assert len(calls) == 1++ reopened = ContentRouter(config)+ new_calls = install_compressor(monkeypatch, reopened)+ assert output(reopened.apply(payload, Tokenizer()), shape) == original+ assert new_calls == []+ assert reopened._cache.stats["cache_disk_hits"] == 1+ assert output(reopened.apply(payload, Tokenizer(), kompress_model="disabled"), shape) == TEXT+ assert len(new_calls) == 1