Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache): scope router results to request compression policy (#4590)

Endid5d40fe76471b
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 3359f0c75..d8838af2f 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1691,13 +1691,30 @@ def _compression_cache_budget_bytes() -> int:
return max(1, int(mb * 1024 * 1024))
-def content_cache_key(content: str, target_ratio: float | None, gated: bool = False) -> str:
- """The compression cache's key for ``content``: a digest, the same in every
- process, so the disk tier (and replicas sharing it) can use it. The runtime
- target ratio is part of it (a different ratio is a different result), and so
- is the reversibility gate tool output passes (#1307)."""
+def content_cache_key(
+ content: str,
+ target_ratio: float | None,
+ gated: bool = False,
+ *,
+ kompress_model: str | None = None,
+ force_kompress: bool = False,
+ skip_kompress: bool = False,
+ bias: float = 1.0,
+ context: str = "",
+) -> str:
+ """A process-stable digest of content and the policy used to compress it.
+
+ Results, skips and frozen verdicts must not cross request-policy boundaries.
+ JSON frames the policy fields unambiguously, including arbitrary context and
+ model strings, before the content bytes (also used by the shared disk tier).
+ """
h = hashlib.blake2b(digest_size=16)
- h.update(f"{target_ratio!r}|{int(gated)}|".encode())
+ h.update(
+ json.dumps(
+ [target_ratio, gated, kompress_model, force_kompress, skip_kompress, bias, context],
+ separators=(",", ":"),
+ ).encode("utf-8")
+ )
h.update(content.encode("utf-8", "surrogatepass"))
return h.hexdigest()
@@ -6976,14 +6993,14 @@ class ContentRouter(Transform):
# Two-tier compression cache.
# Tier 1 (skip): known won't-compress → instant skip.
# Tier 2 (result): known compresses → reuse compressed text.
- # Key on the runtime target_ratio too: the same content compressed at
- # a different ratio is a different result, so it must not alias.
+ # Key on the request policy and effective compressor inputs too:
+ # identical content can produce different results under each.
# Tool ground truth is gated against lossy-unrecoverable results below
# (#1307). Partition its cache namespace so a gated tool entry is never
# served from — or poisons — an ungated entry for byte-identical content.
enforce_reversibility = role == "tool"
- content_key = content_cache_key(
- content, getattr(self, "_runtime_target_ratio", None), gated=enforce_reversibility
+ content_key = self._content_cache_key(
+ content, gated=enforce_reversibility, bias=msg_bias, context=context
)
# Tier 1: skip set — instant rejection
@@ -7553,6 +7570,21 @@ class ContentRouter(Transform):
return None
return folded if len(folded) < len(content) else None
+ def _content_cache_key(
+ self, content: str, *, gated: bool = False, bias: float = 1.0, context: str = ""
+ ) -> str:
+ """Scope all message shapes to this request's compression policy."""
+ return content_cache_key(
+ content,
+ self._runtime_target_ratio,
+ gated=gated,
+ kompress_model=self._runtime_kompress_model,
+ force_kompress=self._runtime_force_kompress,
+ skip_kompress=self._runtime_skip_kompress,
+ bias=bias,
+ context=context,
+ )
+
def _get_tool_bias(self, tool_name: str) -> float:
"""Look up compression bias for a tool name.
@@ -8080,8 +8112,8 @@ class ContentRouter(Transform):
# Two-tier compression cache → shared helper
compressed_content, was_compressed = self._compress_block_content(
content=tool_text,
- content_key=content_cache_key(
- tool_text, getattr(self, "_runtime_target_ratio", None)
+ content_key=self._content_cache_key(
+ tool_text, bias=bias, context=block_context
),
context=block_context,
bias=bias,
@@ -8175,8 +8207,8 @@ class ContentRouter(Transform):
# Two-tier compression cache → shared helper
compressed_content, _was_compressed = self._compress_block_content(
content=text_content,
- content_key=content_cache_key(
- text_content, getattr(self, "_runtime_target_ratio", None)
+ content_key=self._content_cache_key(
+ text_content, bias=1.0, context=context
),
context=context,
bias=1.0,
diff --git a/tests/test_router_cache_policy.py b/tests/test_router_cache_policy.py
new file mode 100644
index 000000000..2db35490a
--- /dev/null
+++ b/tests/test_router_cache_policy.py
@@ -0,0 +1,239 @@
+"""Request policy must scope every tier of the router's content cache."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+from headroom.transforms.content_router import (
+ CompressionStrategy,
+ ContentRouter,
+ ContentRouterConfig,
+ content_cache_key,
+)
+
+TEXT = "The service reports useful observations about the current deployment. " * 80
+
+
+class Tokenizer:
+ def count_text(self, text: str) -> int:
+ return max(1, len(str(text)) // 4)
+
+
+def messages(shape, tool_name="inspect", arguments="{}"):
+ call = {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": tool_name, "arguments": arguments},
+ }
+ ],
+ }
+ if shape == "tool_result":
+ result = {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": TEXT}],
+ }
+ else:
+ content = TEXT if shape == "string" else [{"type": "text", "text": TEXT}]
+ result = {"role": "tool", "tool_call_id": "call_1", "content": content}
+ return [call, result]
+
+
+def output(result, shape):
+ content = result.messages[-1]["content"]
+ if shape == "string":
+ return content
+ return content[0]["content" if shape == "tool_result" else "text"]
+
+
+def router(monkeypatch):
+ # Only the compressor is replaced: apply(), admission, all cache tiers,
+ # runtime policy binding and frozen verdict handling remain real.
+ router = ContentRouter(ContentRouterConfig(ccr_inject_marker=False))
+ monkeypatch.setattr(router, "_kompress_model_ready", lambda: True)
+ return router
+
+
+def install_compressor(monkeypatch, router, *, skip_first=False, marked=False):
+ calls = []
+
+ def compress(content, context="", bias=1.0, **kwargs):
+ calls.append((router._runtime_kompress_model, context, bias))
+ disabled = router._runtime_kompress_model == "disabled" or router._runtime_skip_kompress
+ skip = disabled or (skip_first and len(calls) == 1)
+ compressed = content if skip else f"compressed under policy {len(calls)}"
+ if marked and not skip:
+ compressed += " <<ccr:0123456789abcdef>>"
+ return SimpleNamespace(
+ original=content,
+ compressed=compressed,
+ compression_ratio=1.0 if skip else 0.1,
+ strategy_used=CompressionStrategy.KOMPRESS,
+ strategy_chain=[],
+ )
+
+ monkeypatch.setattr(router, "compress", compress)
+ return calls
+
+
[email protected]("shape", ["string", "tool_result", "text"])
[email protected]("freeze", [False, True])
+ "changed",
+ [
+ {"kompress_model": "disabled"},
+ {"kompress_model": "another-model"},
+ {"skip_kompress": True},
+ {"force_kompress": True},
+ {"context": "a different question"},
+ {"target_ratio": 0.3},
+ ],
+)
+def test_apply_reuses_results_only_under_the_same_policy(
+ monkeypatch, router, shape, freeze, changed
+):
+ monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", str(int(freeze)))
+ calls = install_compressor(monkeypatch, router, marked=freeze)
+ payload = messages(shape)
+ first = output(router.apply(payload, Tokenizer()), shape)
+ assert first != TEXT
+ assert output(router.apply(payload, Tokenizer()), shape) == first
+ assert len(calls) == 1
+
+ second = output(router.apply(payload, Tokenizer(), **changed), shape)
+ assert len(calls) == 2, "changed policy must not reuse the previous result"
+ assert second != first
+ if changed.get("kompress_model") == "disabled" or changed.get("skip_kompress"):
+ assert second == TEXT
+ assert output(router.apply(payload, Tokenizer(), **changed), shape) == second
+ assert len(calls) == 2
+ # Returning to the original policy still finds its own cached output.
+ assert output(router.apply(payload, Tokenizer()), shape) == first
+ assert len(calls) == 2
+
+
[email protected]("shape", ["string", "tool_result", "text"])
+def test_frozen_verdicts_are_recorded_separately_for_each_policy(monkeypatch, router, shape):
+ monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
+ # Lossy output needs a retrieval marker before a verdict is safe to freeze.
+ calls = install_compressor(monkeypatch, router, marked=True)
+ payload = messages(shape)
+
+ first = output(router.apply(payload, Tokenizer()), shape)
+ initial_verdicts = dict(router._frozen_verdicts)
+ assert len(initial_verdicts) == 1
+ assert list(initial_verdicts.values()) == [True]
+ assert output(router.apply(payload, Tokenizer()), shape) == first
+ assert router._frozen_verdicts == initial_verdicts
+ assert len(calls) == 1
+
+ second = output(router.apply(payload, Tokenizer(), kompress_model="another-model"), shape)
+ assert second != first
+ assert len(calls) == 2
+ assert len(router._frozen_verdicts) == 2
+ assert all(verdict is True for verdict in router._frozen_verdicts.values())
+ assert initial_verdicts.items() <= router._frozen_verdicts.items()
+ assert output(router.apply(payload, Tokenizer()), shape) == first
+ assert len(calls) == 2
+
+
[email protected]("shape", ["string", "tool_result", "text"])
[email protected]("changed", [{"kompress_model": "another-model"}, {"force_kompress": True}])
+def test_a_skip_under_one_policy_does_not_suppress_another(monkeypatch, router, shape, changed):
+ calls = install_compressor(monkeypatch, router, skip_first=True)
+ payload = messages(shape)
+ assert output(router.apply(payload, Tokenizer()), shape) == TEXT
+ assert output(router.apply(payload, Tokenizer()), shape) == TEXT
+ assert len(calls) == 1
+ assert output(router.apply(payload, Tokenizer(), **changed), shape) != TEXT
+ assert len(calls) == 2
+
+
+def test_hook_bias_partitions_string_results(monkeypatch, router):
+ calls = install_compressor(monkeypatch, router)
+ payload = messages("string")
+ first = output(router.apply(payload, Tokenizer()), "string")
+ second = output(router.apply(payload, Tokenizer(), biases={1: 2.0}), "string")
+ assert second != first
+ assert [call[2] for call in calls] == [1.0, 2.0]
+
+
[email protected]("shape", ["string", "tool_result"])
+def test_effective_tool_bias_partitions_results(monkeypatch, router, shape):
+ router.config.tool_profiles = {
+ "inspect": SimpleNamespace(bias=1.0),
+ "summarize": SimpleNamespace(bias=2.0),
+ }
+ calls = install_compressor(monkeypatch, router)
+ first = output(router.apply(messages(shape), Tokenizer()), shape)
+ second = output(router.apply(messages(shape, tool_name="summarize"), Tokenizer()), shape)
+ assert second != first
+ assert [call[2] for call in calls] == [1.0, 2.0]
+
+
+def test_tool_result_relevance_uses_the_enriched_context(monkeypatch, router):
+ router.config.relevance_split = True
+ calls = install_compressor(monkeypatch, router)
+ first = router.apply(messages("tool_result", arguments='{"query":"alpha"}'), Tokenizer())
+ second = router.apply(messages("tool_result", arguments='{"query":"beta"}'), Tokenizer())
+ assert output(first, "tool_result") != output(second, "tool_result")
+ assert len(calls) == 2
+ assert calls[0][1] != calls[1][1]
+
+
+def test_policy_keys_are_deterministic_across_processes():
+ options = {
+ "kompress_model": "custom|model",
+ "force_kompress": True,
+ "skip_kompress": True,
+ "bias": 1.5,
+ "context": "question|with\x00delimiters",
+ }
+ expected = content_cache_key(TEXT, 0.5, gated=True, **options)
+ code = (
+ "from headroom.transforms.content_router import content_cache_key;"
+ f"print(content_cache_key({TEXT!r}, 0.5, gated=True, **{options!r}))"
+ )
+ for seed in ("1", "2"):
+ found = subprocess.run(
+ [sys.executable, "-c", code],
+ capture_output=True,
+ text=True,
+ check=True,
+ env={**os.environ, "PYTHONHASHSEED": seed},
+ ).stdout.strip()
+ assert found == expected
+
+
+def test_context_and_content_have_unambiguous_boundaries():
+ assert content_cache_key("b|c", None, context="a") != content_cache_key(
+ "c", None, context="a|b"
+ )
+
+
[email protected]("shape", ["string", "tool_result", "text"])
+def test_reopened_disk_results_are_scoped_to_the_request_policy(tmp_path, monkeypatch, shape):
+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "policy.db"))
+ config = ContentRouterConfig(ccr_inject_marker=False, result_cache_disk=True)
+ first = ContentRouter(config)
+ calls = install_compressor(monkeypatch, first)
+ payload = messages(shape)
+ original = output(first.apply(payload, Tokenizer()), shape)
+ assert len(calls) == 1
+
+ reopened = ContentRouter(config)
+ new_calls = install_compressor(monkeypatch, reopened)
+ assert output(reopened.apply(payload, Tokenizer()), shape) == original
+ assert new_calls == []
+ assert reopened._cache.stats["cache_disk_hits"] == 1
+ assert output(reopened.apply(payload, Tokenizer(), kompress_model="disabled"), shape) == TEXT
+ assert len(new_calls) == 1