Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: clarify CCR marker content preservation (#3175)
Himanshu Patro005a4e146f94
diff --git a/headroom/ccr/tool_injection.py b/headroom/ccr/tool_injection.pyindex b4933610b..68f4bbfca 100644--- a/headroom/ccr/tool_injection.py+++ b/headroom/ccr/tool_injection.py@@ -235,7 +235,9 @@ class CCRToolInjector:## Standard format: [N <type> compressed to M. Retrieve more: hash=xxx]# Matches items, lines, matches, or any other type- re.compile(r"\[(\d+) \w+ compressed to (\d+)\. Retrieve more: hash=([a-f0-9]{24})\]"),+ re.compile(+ r"\[(\d+) \w+ compressed to (\d+)(?: \([^)]+\))?\.(?: Original content preserved\.)? Retrieve more: hash=([a-f0-9]{24})\]"+ ),# Legacy format without "to M" or "Retrieve more:" (old TextCompressor)re.compile(r"\[(\d+) \w+ compressed\. hash=([a-f0-9]{24})\]"),# Generic fallback: any bracket compression marker with hash (exactly 24 chars)diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 9bea434f5..49588eac4 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -1370,6 +1370,7 @@ def ccr_retrieval_marker(return (f"\n[{n_words} words compressed to {compressed_count}"f" (from {source_lines} source {line_word})."+ " Original content preserved."f" Retrieve more: hash={cache_key}]")diff --git a/tests/test_ccr_marker_policy.py b/tests/test_ccr_marker_policy.pyindex 722a59dc3..03764c9c8 100644--- a/tests/test_ccr_marker_policy.py+++ b/tests/test_ccr_marker_policy.py@@ -34,6 +34,125 @@ def test_source_line_span_marker_is_still_detected() -> None:assert _hashes(marker) == ["c00eb437e5e5c00eb437e5e5"]+def test_new_marker_with_integrity_clause_is_detected() -> None:+ # Issue #3098: marker includes "Original content preserved."+ marker = (+ "[122 items compressed to 27 (from 5 source lines). "+ "Original content preserved. Retrieve more: hash=c00eb437e5e5c00eb437e5e5]"+ )+ assert _hashes(marker) == ["c00eb437e5e5c00eb437e5e5"]+++class _FakeEncoding:+ def __init__(self, word_lists: list[list[str]]):+ self._word_lists = word_lists+ self._ids = [[0] * len(w) for w in word_lists]++ def __getitem__(self, key: str):+ return {"input_ids": self._ids, "attention_mask": self._ids}[key]++ def word_ids(self, batch_index: int = 0) -> list[int]:+ return list(range(len(self._word_lists[batch_index])))+++class _FakeTokenizer:+ def __call__(self, words: list[str] | list[list[str]], **_kwargs) -> _FakeEncoding:+ batch = words if (words and isinstance(words[0], list)) else [words] # type: ignore[list-item]+ return _FakeEncoding(batch) # type: ignore[arg-type]+++class _FakeModel:+ """Keep the first two words of each chunk -> ratio < 0.8 (CCR fires)."""++ def get_keep_mask(+ self, input_ids: list[list[int]], _attention_mask: object+ ) -> list[list[bool]]:+ return [[i < 2 for i in range(len(input_ids[0]))]]++ def get_scores(self, input_ids: list[list[int]], _attention_mask: object) -> list[list[float]]:+ return [[1.0 if i < 2 else 0.0 for i in range(len(row))] for row in input_ids]+++def test_remote_kompress_compressor_emits_original_content_preserved(monkeypatch) -> None:+ from headroom.transforms.kompress_remote import RemoteKompressCompressor++ class _FakeResponse:+ status_code = 200++ def raise_for_status(self) -> None:+ return None++ def json(self) -> dict:+ return {"compressed": "short text", "compression_ratio": 0.2, "original_tokens": 100}++ class _FakeClient:+ def post(self, url, headers=None, json=None): # noqa: A002, ANN001+ return _FakeResponse()++ def close(self) -> None:+ return None++ monkeypatch.setattr(+ "headroom.transforms.kompress_remote.store_kompress_in_ccr",+ lambda original, compressed, tokens: "cafebabecafebabecafebabe",+ )+ c = RemoteKompressCompressor("https://ml.example.invalid")+ c._client = _FakeClient()+ c.config.enable_ccr = True++ result = c.compress("line one\nline two\nline three " * 20)+ assert (+ "Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"+ in result.compressed+ )+++def test_kompress_compressor_emits_original_content_preserved(monkeypatch) -> None:+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ compressor.config.enable_ccr = True+ monkeypatch.setattr(+ compressor,+ "_store_in_ccr",+ lambda original, compressed, tokens: "cafebabecafebabecafebabe",+ )+ monkeypatch.setattr(+ "headroom.transforms.kompress_compressor._load_kompress",+ lambda *a, **k: (_FakeModel(), _FakeTokenizer(), "onnx"),+ )++ result = compressor.compress("line one\nline two\nline three " * 20)+ assert (+ "Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"+ in result.compressed+ )+++def test_kompress_compressor_batch_emits_original_content_preserved(monkeypatch) -> None:+ from headroom.transforms.kompress_compressor import KompressCompressor++ compressor = KompressCompressor()+ compressor.config.enable_ccr = True+ monkeypatch.setattr(compressor, "_should_use_sequential_fallback", lambda: False)+ monkeypatch.setattr(+ compressor,+ "_store_in_ccr",+ lambda original, compressed, tokens: "cafebabecafebabecafebabe",+ )+ monkeypatch.setattr(+ "headroom.transforms.kompress_compressor._load_kompress",+ lambda *a, **k: (_FakeModel(), _FakeTokenizer(), "onnx"),+ )++ results = compressor.compress_batch(["line one\nline two\nline three " * 20])+ assert len(results) == 1+ assert (+ "Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"+ in results[0].compressed+ )++def test_has_new_ccr_markers_detects_hash_not_seen_in_previous_forward() -> None:old = "[100 items compressed to 10. Retrieve more: hash=abc123def456abc123def456]"new = "[50 items compressed to 5. Retrieve more: hash=deadbeefdeadbeefdeadbeef]"diff --git a/tests/test_kompress_marker_gate.py b/tests/test_kompress_marker_gate.pyindex 8cea9356b..f7ee42a63 100644--- a/tests/test_kompress_marker_gate.py+++ b/tests/test_kompress_marker_gate.py@@ -97,7 +97,7 @@ def _tok(text: str) -> int:def test_payload_tokens_is_cl100k_or_one_per_character(monkeypatch):marker = ccr_retrieval_marker(100, 59, SINGLE_TOKEN_SOURCE, _real_key(SINGLE_TOKEN_SOURCE))assert _real_key(SINGLE_TOKEN_SOURCE) == "7dbb8f8de9f3e1d7c6f3a6e1"- assert payload_tokens(marker) == _tok(marker) == 43+ assert payload_tokens(marker) == _tok(marker) == 47assert payload_tokens(SINGLE_TOKEN_SOURCE) == _tok(SINGLE_TOKEN_SOURCE) == 100monkeypatch.setattr(kc, "_payload_encoder", False)assert payload_tokens(marker) == len(marker)