Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(kompress): append the CCR marker whenever the saving pays for it (#3484)
gglucass7bd4dbaf8f40
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 770b32476..29d0486ef 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -1294,6 +1294,41 @@ class KompressResult:return (self.tokens_saved / self.original_tokens) * 100+_payload_encoder: Any = None+++def payload_tokens(text: str) -> int:+ """Token count of a complete payload, in one consistent unit.++ The unit is cl100k_base (tiktoken, a hard dependency), used as a fixed+ estimate. Actual model tokenizers, including those of other OpenAI+ models, can differ. Without an encoder, the fallback compares character+ counts; that heuristic is not a bound on provider token counts.++ The CCR gate measures the whole original and the whole candidate-plus-+ marker with this, never a marker-only cost against a word count: the+ marker is 36-45 tokens for 12 words, the words Kompress drops can be one+ token each, and the word left at the head of the candidate can tokenize+ differently from its space-prefixed form in the source. Only a comparison+ of the two complete texts in one unit establishes that the shipped+ payload is smaller.+ """+ global _payload_encoder+ if _payload_encoder is None:+ try:+ import tiktoken++ _payload_encoder = tiktoken.get_encoding("cl100k_base")+ except Exception:+ _payload_encoder = False+ if _payload_encoder:+ try:+ return len(_payload_encoder.encode(text, disallowed_special=()))+ except Exception:+ pass+ return len(text)++def ccr_retrieval_marker(n_words: int, compressed_count: int, ccr_source: str, cache_key: str) -> str:@@ -1725,30 +1760,43 @@ class KompressCompressor(Transform):compressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]compressed = " ".join(compressed_words)compressed_count = len(compressed_words)- ratio = compressed_count / n_words if n_words else 1.0-- result = KompressResult(- compressed=compressed,- original=content,- original_tokens=n_words,- compressed_tokens=compressed_count,- compression_ratio=ratio,- model_used=self.config.model_id,- )-- # CCR marker- if self.config.enable_ccr and ratio < 0.8:+ cache_key: str | None = None+ original_tokens = n_words+ compressed_tokens = compressed_count++ # CCR marker: anything the lossy pass shrank must stay retrievable,+ # and the complete marked payload must be smaller than the+ # original. Both are measured whole, in one unit (payload_tokens);+ # a candidate that is not is passed through. The accounting then+ # reports that same measurement, so ``tokens_saved`` describes+ # the shipped payload rather than a word count.+ if self.config.enable_ccr:ccr_source = ccr_original if ccr_original is not None else contentccr_source_tokens = len(ccr_source.split())cache_key = self._store_in_ccr(ccr_source, compressed, ccr_source_tokens)if cache_key:- result.cache_key = cache_key# Report the source line span so a reader can tell content was# compressed away rather than absent — "items" counts words, which# does not map to lines and reads as evidence of absence (#2586).- result.compressed += ccr_retrieval_marker(+ marked = compressed + ccr_retrieval_marker(n_words, compressed_count, ccr_source, cache_key)+ original_tokens = payload_tokens(content)+ compressed_tokens = payload_tokens(marked)+ if compressed_tokens >= original_tokens:+ return self._passthrough(content, n_words)+ compressed = marked++ ratio = compressed_tokens / original_tokens if original_tokens else 1.0+ result = KompressResult(+ compressed=compressed,+ original=content,+ original_tokens=original_tokens,+ compressed_tokens=compressed_tokens,+ compression_ratio=ratio,+ cache_key=cache_key,+ model_used=self.config.model_id,+ )if inference_ms >= 1000.0:logger.info(@@ -2119,33 +2167,40 @@ class KompressCompressor(Transform):compressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]compressed = " ".join(compressed_words)compressed_count = len(compressed_words)- comp_ratio = compressed_count / n_words if n_words else 1.0+ cache_key: str | None = None+ original_tokens = n_words+ compressed_tokens = compressed_count- result = KompressResult(- compressed=compressed,- original=content,- original_tokens=n_words,- compressed_tokens=compressed_count,- compression_ratio=comp_ratio,- model_used=self.config.model_id,- )-- if self.config.enable_ccr and comp_ratio < 0.8:+ # Same gate and accounting as the single path.+ if self.config.enable_ccr:ccr_source = ccr_sources[text_idx]if ccr_source is None:ccr_source = contentccr_source_tokens = len(ccr_source.split())cache_key = self._store_in_ccr(ccr_source, compressed, ccr_source_tokens)if cache_key:- result.cache_key = cache_key# Report the source line span so a reader can tell content was# compressed away rather than absent — "items" counts words, which# does not map to lines and reads as evidence of absence (#2586).- result.compressed += ccr_retrieval_marker(+ marked = compressed + ccr_retrieval_marker(n_words, compressed_count, ccr_source, cache_key)+ original_tokens = payload_tokens(content)+ compressed_tokens = payload_tokens(marked)+ if compressed_tokens >= original_tokens:+ results[text_idx] = self._passthrough(content, n_words)+ continue+ compressed = marked- results[text_idx] = result+ results[text_idx] = KompressResult(+ compressed=compressed,+ original=content,+ original_tokens=original_tokens,+ compressed_tokens=compressed_tokens,+ compression_ratio=compressed_tokens / original_tokens if original_tokens else 1.0,+ cache_key=cache_key,+ model_used=self.config.model_id,+ )# Safety: every slot must be populated.final: list[KompressResult] = []diff --git a/headroom/transforms/kompress_remote.py b/headroom/transforms/kompress_remote.pyindex 984d9db9f..0d1be810d 100644--- a/headroom/transforms/kompress_remote.py+++ b/headroom/transforms/kompress_remote.py@@ -57,7 +57,13 @@ import loggingimport httpx-from .kompress_compressor import KompressConfig, KompressResult, store_kompress_in_ccr+from .kompress_compressor import (+ KompressConfig,+ KompressResult,+ ccr_retrieval_marker,+ payload_tokens,+ store_kompress_in_ccr,+)logger = logging.getLogger(__name__)@@ -94,7 +100,6 @@ _MIN_WORDS = 10# Accept-any-shrink CCR gate, identical to KompressCompressor.compress: only# store + mark when the shrink is worth the retrieval marker's own cost.-_CCR_RATIO_GATE = 0.8class RemoteKompressCompressor:@@ -238,7 +243,9 @@ class RemoteKompressCompressor:# CCR stays PROXY-LOCAL: endpoint is stateless (enable_ccr=False), so we# store the mapping + append the retrieval marker here — same policy and# marker format as KompressCompressor.compress.- if self.config.enable_ccr and result.compression_ratio < _CCR_RATIO_GATE:+ if self.config.enable_ccr and compressed != content:+ # Same gate as KompressCompressor: the marked payload must save+ # tokens, and a result that cannot pay for the marker passes through.# Store the PRE-protection text when the caller supplied it. ``content``# may be the tag-protected placeholder intermediate, and storing that# makes a later full retrieval hand back {{HEADROOM_TAG_N}} instead of@@ -253,15 +260,23 @@ class RemoteKompressCompressor:)cache_key = store_kompress_in_ccr(ccr_source, compressed, ccr_source_tokens)if cache_key:- result.cache_key = cache_key# Report the source line span so a reader can tell content was# compressed away rather than absent (#2586).- source_lines = ccr_source.count("\n") + 1- line_word = "line" if source_lines == 1 else "lines"- result.compressed += (- f"\n[{result.original_tokens} words compressed to "- f"{result.compressed_tokens} (from {source_lines} source {line_word})."- f" Retrieve more: hash={cache_key}]"+ marked = compressed + ccr_retrieval_marker(+ result.original_tokens, result.compressed_tokens, ccr_source, cache_key+ )+ # Whole original against whole marked candidate, one unit,+ # and the accounting reports that measurement.+ original_tokens = payload_tokens(content)+ compressed_tokens = payload_tokens(marked)+ if compressed_tokens >= original_tokens:+ return self._passthrough(content, n_words)+ result.cache_key = cache_key+ result.compressed = marked+ result.original_tokens = original_tokens+ result.compressed_tokens = compressed_tokens+ result.compression_ratio = (+ compressed_tokens / original_tokens if original_tokens else 1.0)return resultdiff --git a/tests/test_ccr_tag_placeholder_regression.py b/tests/test_ccr_tag_placeholder_regression.pyindex 2a3068bb4..94ad18a0e 100644--- a/tests/test_ccr_tag_placeholder_regression.py+++ b/tests/test_ccr_tag_placeholder_regression.py@@ -142,8 +142,9 @@ def test_compress_batch_validates_ccr_originals_length():# it keeps the first two words of each chunk so the compression ratio clears the# ``< 0.8`` CCR-store threshold.-_RAW = "<system-reminder>CRITICAL: invoke the skill</system-reminder> " + " ".join(["filler"] * 40)-_PLACEHOLDER = "{{HEADROOM_TAG_0}} " + " ".join(["filler"] * 40)+# 100 filler words: keeping two saves ~99, well past the marker cost gate.+_RAW = "<system-reminder>CRITICAL: invoke the skill</system-reminder> " + " ".join(["filler"] * 100)+_PLACEHOLDER = "{{HEADROOM_TAG_0}} " + " ".join(["filler"] * 100)class _FakeEncoding:diff --git a/tests/test_kompress_marker_gate.py b/tests/test_kompress_marker_gate.pynew file mode 100644index 000000000..8cea9356b--- /dev/null+++ b/tests/test_kompress_marker_gate.py@@ -0,0 +1,154 @@+"""The CCR retrieval marker follows the saving, and the saving must be real.++A lossy Kompress result that shipped without a marker was discarded by the+router's lossy-unrecoverable guard (#1307), so every pass that shrank a block+by less than 20 percent ran the model and saved nothing. The gate now compares+the whole original with the whole candidate-plus-marker in one token unit+(cl100k_base, an estimate for non-OpenAI providers; one token per character+without an encoder) and passes the candidate through unless it is smaller. A+marked result reports that same measurement. Word counts cannot do this: the+marker is 36-45 tokens for 12 words, dropped words can be one token each, and+the word left at the head of the candidate can tokenize differently from its+space-prefixed form in the source.+"""++from __future__ import annotations++import hashlib++import pytest++from headroom.transforms import kompress_compressor as kc+from headroom.transforms.kompress_compressor import (+ KompressCompressor,+ KompressConfig,+ ccr_retrieval_marker,+ payload_tokens,+)+++class _Enc(dict):+ def word_ids(self, batch_index=0):+ return self["_word_ids"][batch_index]+++class _Tok:+ def __call__(self, chunk_words, **kw):+ batch_words = (+ chunk_words if chunk_words and isinstance(chunk_words[0], list) else [chunk_words]+ )+ return _Enc(+ input_ids=[[0] * len(words) for words in batch_words],+ attention_mask=[[1] * len(words) for words in batch_words],+ _word_ids=[list(range(len(words))) for words in batch_words],+ )+++class _DropFirst:+ """Keep every word except the first ``drop`` of each row."""++ def __init__(self, drop: int) -> None:+ self.drop = drop++ def get_keep_mask(self, input_ids, attention_mask):+ return [[idx >= self.drop for idx, _ in enumerate(row)] for row in input_ids]++ def get_scores(self, input_ids, attention_mask):+ return [[0.0 if idx < self.drop else 1.0 for idx, _ in enumerate(row)] for row in input_ids]+++def _install(monkeypatch, drop: int) -> None:+ monkeypatch.setattr(kc, "_load_kompress", lambda *a, **k: (_DropFirst(drop), _Tok(), "onnx"))+ monkeypatch.setattr(kc, "_model_device_type", lambda *a, **k: "cpu")+++def _prose(n: int) -> str:+ # Plain lowercase words: nothing the must-keep override would pin.+ return " ".join(f"w{chr(97 + i % 26)}{chr(97 + (i // 26) % 26)}" for i in range(n))+++def _real_key(source: str) -> str:+ """The CCR store's own key: the SHA-256 prefix of the stored source."""+ return hashlib.sha256(source.encode()).hexdigest()[:24]+++def _compressor(monkeypatch, **config) -> KompressCompressor:+ compressor = KompressCompressor(KompressConfig(min_input_words=10, **config))+ monkeypatch.setattr(compressor, "_should_batch_single_content", lambda *a, **k: False)+ monkeypatch.setattr(compressor, "_should_use_sequential_fallback", lambda: False)+ monkeypatch.setattr(compressor, "_store_in_ccr", lambda source, *a, **k: _real_key(source))+ return compressor+++# 100 single-token words: the tightest source there is. Dropping 41 saves 41+# tokens, and the marker for it (with its real source-derived hash) costs 43.+SINGLE_TOKEN_SOURCE = " ".join(["alpha"] * 99 + ["nfs"])+# Dropping 36 leaves "bureaucratic" at the head of the candidate, where it+# tokenizes differently from " bureaucratic" in the source: 100 -> 103 with+# the marker, though a marker-only cost against saved words called it 99.+HEAD_WORD_SOURCE = " ".join(["alpha"] * 36 + ["bureaucratic"] + ["alpha"] * 63)+++def _tok(text: str) -> int:+ enc = pytest.importorskip("tiktoken").get_encoding("cl100k_base")+ return len(enc.encode(text))+++def test_payload_tokens_is_cl100k_or_one_per_character(monkeypatch):+ marker = ccr_retrieval_marker(100, 59, SINGLE_TOKEN_SOURCE, _real_key(SINGLE_TOKEN_SOURCE))+ assert _real_key(SINGLE_TOKEN_SOURCE) == "7dbb8f8de9f3e1d7c6f3a6e1"+ assert payload_tokens(marker) == _tok(marker) == 43+ assert payload_tokens(SINGLE_TOKEN_SOURCE) == _tok(SINGLE_TOKEN_SOURCE) == 100+ monkeypatch.setattr(kc, "_payload_encoder", False)+ assert payload_tokens(marker) == len(marker)+++ ("source", "drop"),+ [(SINGLE_TOKEN_SOURCE, 41), (HEAD_WORD_SOURCE, 36)],+ ids=["single-token-words", "retokenized-head-word"],+)+def test_candidates_that_do_not_shrink_the_whole_payload_pass_through(monkeypatch, source, drop):+ kept = " ".join(source.split()[drop:])+ marked = kept + ccr_retrieval_marker(100, 100 - drop, source, _real_key(source))+ assert _tok(marked) > _tok(source) # the reviewer's measurement+ _install(monkeypatch, drop=drop)+ result = _compressor(monkeypatch).compress(source)+ assert result.compressed == source+ assert result.cache_key is None+ assert result.compression_ratio == 1.0+ [batched] = _compressor(monkeypatch).compress_batch([source], batch_size=8)+ assert batched.compressed == source and batched.compression_ratio == 1.0+++def test_marked_result_reports_the_whole_payload_measurement(monkeypatch):+ # 300 single-token words, drop 60: 240 kept plus a ~43-token marker is+ # smaller than 300, and the accounting is that measurement, not words.+ source = " ".join(["alpha"] * 299 + ["nfs"])+ _install(monkeypatch, drop=60)+ result = _compressor(monkeypatch).compress(source)+ marked = " ".join(source.split()[60:]) + ccr_retrieval_marker(+ 300, 240, source, _real_key(source)+ )+ assert result.compressed == marked+ assert result.cache_key == _real_key(source)+ assert (result.original_tokens, result.compressed_tokens) == (_tok(source), _tok(marked))+ assert result.compression_ratio == _tok(marked) / _tok(source)+ [batched] = _compressor(monkeypatch).compress_batch([source], batch_size=8)+ assert batched.compressed == result.compressed+ assert (batched.original_tokens, batched.compressed_tokens) == (+ result.original_tokens,+ result.compressed_tokens,+ )+++def test_no_ccr_mode_ships_unmarked_lossy_unchanged(monkeypatch):+ # Without CCR there is no marker and no whole-payload gate: the deliberate+ # output is the bare lossy result in word counts, on both paths, as before.+ _install(monkeypatch, drop=41)+ compressor = _compressor(monkeypatch, enable_ccr=False)+ result = compressor.compress(SINGLE_TOKEN_SOURCE)+ assert (result.original_tokens, result.compressed_tokens) == (100, 59)+ assert "Retrieve more" not in result.compressed+ [batched] = compressor.compress_batch([SINGLE_TOKEN_SOURCE], batch_size=8)+ assert batched.compressed_tokens == 59diff --git a/tests/test_kompress_request_nonblocking.py b/tests/test_kompress_request_nonblocking.pyindex c01a8405f..5c92a146d 100644--- a/tests/test_kompress_request_nonblocking.py+++ b/tests/test_kompress_request_nonblocking.py@@ -235,7 +235,9 @@ def test_capacity_available_still_compresses(monkeypatch):lambda *args, **kwargs: (_FakeModel(), _FakeTokenizer(), "onnx"),)- result = KompressCompressor(KompressConfig(min_input_words=10)).compress(+ # No CCR: this test is about capacity, and 10 saved words would not pay+ # for a retrieval marker (CCR_MARKER_COST_WORDS), which is a passthrough.+ result = KompressCompressor(KompressConfig(min_input_words=10, enable_ccr=False)).compress(" ".join(["word"] * 20), allow_download=False)assert 0 < result.compression_ratio < 1.0diff --git a/tests/test_remote_kompress_dropin.py b/tests/test_remote_kompress_dropin.pyindex d423bfa3c..6b295e961 100644--- a/tests/test_remote_kompress_dropin.py+++ b/tests/test_remote_kompress_dropin.py@@ -117,8 +117,9 @@ def _compressor(monkeypatch, *, enable_ccr: bool, payload: dict):return c-ORIGINAL = "real secret block " * 20-PLACEHOLDER = "{{HEADROOM_TAG_0}} " * 20+# 60 words: "short" saves 59, past the marker cost gate (CCR_MARKER_COST_WORDS).+ORIGINAL = "real secret block " * 60+PLACEHOLDER = "{{HEADROOM_TAG_0}} " * 60def test_passing_ccr_original_no_longer_raises(monkeypatch) -> None:@@ -190,3 +191,39 @@ def test_the_common_path_without_an_override_is_unchanged(monkeypatch) -> None:assert stored["original"] == ORIGINAL# Still the endpoint's own count when no override was supplied.assert stored["tokens"] == 999+++def test_remote_marker_gate_measures_the_whole_payload_like_local(monkeypatch) -> None:+ """Parity with KompressCompressor: the whole original against the whole+ marked candidate in one token unit. Both boundary sources pass through+ (41 single-token words saved against a 43-token marker; a head word that+ retokenizes: 100 -> 103), and a real saving reports that measurement."""+ import hashlib++ from headroom.transforms.kompress_compressor import ccr_retrieval_marker, payload_tokens++ def _store(original, compressed, original_tokens): # noqa: ANN001+ return hashlib.sha256(original.encode()).hexdigest()[:24]++ monkeypatch.setattr("headroom.transforms.kompress_remote.store_kompress_in_ccr", _store)+ for source, drop in (+ (" ".join(["alpha"] * 99 + ["nfs"]), 41),+ (" ".join(["alpha"] * 36 + ["bureaucratic"] + ["alpha"] * 63), 36),+ ):+ kept = " ".join(source.split()[drop:])+ c = _compressor(monkeypatch, enable_ccr=True, payload={"compressed": kept})+ result = c.compress(source)+ assert result.compressed == source+ assert result.cache_key is None+ assert result.compression_ratio == 1.0++ big = " ".join(["alpha"] * 299 + ["nfs"])+ kept = " ".join(big.split()[60:])+ c = _compressor(monkeypatch, enable_ccr=True, payload={"compressed": kept})+ result = c.compress(big)+ marked = kept + ccr_retrieval_marker(300, 240, big, _store(big, kept, 300))+ assert result.compressed == marked+ assert (result.original_tokens, result.compressed_tokens) == (+ payload_tokens(big),+ payload_tokens(marked),+ )diff --git a/tests/test_transforms/test_kompress_deadline.py b/tests/test_transforms/test_kompress_deadline.pyindex b420fe362..2ae94f458 100644--- a/tests/test_transforms/test_kompress_deadline.py+++ b/tests/test_transforms/test_kompress_deadline.py@@ -10,6 +10,10 @@ one that leaks.from __future__ import annotations+import hashlib++import pytest+from headroom.transforms import kompress_compressor as kc@@ -32,14 +36,26 @@ def test_compress_bails_at_deadline_keeping_tail_verbatim(monkeypatch):assert result.compressed.split() == content.split()-def test_compress_partial_run_keeps_processed_head_plus_verbatim_tail(monkeypatch):- # The real partial case: chunk 0 processes (gets compressed), chunk 1 trips- # the deadline (kept verbatim). Output must be compressed-head + verbatim-tail.- # Clock: call1=t_deadline(0); calls 2-4 are chunk-0's check+inference reads- # (under budget); call 5+ is chunk-1's check -> trips.- # Robust clock: jump past the deadline only AFTER chunk 0 is processed- # (tracked via the model mock), so adding perf_counter calls inside the chunk- # body -- e.g. sub-stage timing -- can't shift when the deadline trips.+ ("n_words", "net_saving"),+ [(200, True), (20, False)],+ ids=["net-saving", "no-net-saving"],+)+def test_compress_partial_run_keeps_processed_head_plus_verbatim_tail(+ monkeypatch, n_words, net_saving+):+ # real partial case: chunk 0 processes (gets compressed), chunk 1 trips the+ # deadline (kept verbatim). Output must be compressed-head + verbatim-tail.+ # Clock: call 1 = t_deadline (0); calls 2-4 chunk-0's check + inference+ # reads (under budget); call 5+ chunk-1's check -> trips.+ # Robust clock: jump past the deadline only AFTER chunk 0 processed+ # (tracked via the model mock), so adding perf_counter calls inside the+ # chunk body -- e.g. sub-stage timing -- can't shift when the deadline trips.+ #+ # Two sizes: at 200 words the marked partial result is smaller than the+ # original and ships; at 20 words (150 -> 300-odd tokens either way, plus a+ # ~43-token marker) the CCR gate finds no net saving and passes the whole+ # payload through, which is the other half of the contract.state = {"chunks_done": 0}def fake_clock():@@ -68,15 +84,27 @@ def test_compress_partial_run_keeps_processed_head_plus_verbatim_tail(monkeypatcmonkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")comp = kc.KompressCompressor(kc.KompressConfig(min_input_words=10))- comp.config.chunk_words = 10 # 20 words -> 2 chunks+ comp.config.chunk_words = n_words // 2 # two chunksmonkeypatch.setattr(comp, "_should_batch_single_content", lambda *a, **k: False)-- words = [f"w{i}" for i in range(20)]- out = comp.compress(" ".join(words)).compressed.split()-- # chunk 0 processed: first half kept (w0..w4), second half dropped (w5..w9)- assert "w0" in out and "w4" in out- assert "w5" not in out and "w9" not in out- # chunk 1 tripped the deadline -> its words kept verbatim (w10..w19 all present)- for i in range(10, 20):+ monkeypatch.setattr(+ comp,+ "_store_in_ccr",+ lambda source, *a, **k: hashlib.sha256(source.encode()).hexdigest()[:24],+ )++ words = [f"w{i}" for i in range(n_words)]+ result = comp.compress(" ".join(words))+ if not net_saving:+ assert result.compressed == " ".join(words)+ assert result.compression_ratio == 1.0+ return++ out = result.compressed.split()+ half = n_words // 2+ assert result.cache_key is not None and "Retrieve more" in result.compressed+ # chunk 0 processed: its first half kept, its second half dropped+ assert "w0" in out and f"w{half // 2 - 1}" in out+ assert f"w{half // 2}" not in out and f"w{half - 1}" not in out+ # chunk 1 tripped the deadline -> its words kept verbatim (all present)+ for i in range(half, n_words):assert f"w{i}" in out