Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix: clarify CCR marker content preservation (#3175)

Himanshu Patro005a4e146f94
diff --git a/headroom/ccr/tool_injection.py b/headroom/ccr/tool_injection.py
index b4933610b..68f4bbfca 100644
--- a/headroom/ccr/tool_injection.py
+++ b/headroom/ccr/tool_injection.py
@@ -235,7 +235,9 @@ class CCRToolInjector:
#
# Standard format: [N <type> compressed to M. Retrieve more: hash=xxx]
# Matches items, lines, matches, or any other type
- re.compile(r"\[(\d+) \w+ compressed to (\d+)\. Retrieve more: hash=([a-f0-9]{24})\]"),
+ re.compile(
+ r"\[(\d+) \w+ compressed to (\d+)(?: \([^)]+\))?\.(?: Original content preserved\.)? Retrieve more: hash=([a-f0-9]{24})\]"
+ ),
# Legacy format without "to M" or "Retrieve more:" (old TextCompressor)
re.compile(r"\[(\d+) \w+ compressed\. hash=([a-f0-9]{24})\]"),
# Generic fallback: any bracket compression marker with hash (exactly 24 chars)
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 9bea434f5..49588eac4 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -1370,6 +1370,7 @@ def ccr_retrieval_marker(
return (
f"\n[{n_words} words compressed to {compressed_count}"
f" (from {source_lines} source {line_word})."
+ " Original content preserved."
f" Retrieve more: hash={cache_key}]"
)
diff --git a/tests/test_ccr_marker_policy.py b/tests/test_ccr_marker_policy.py
index 722a59dc3..03764c9c8 100644
--- a/tests/test_ccr_marker_policy.py
+++ b/tests/test_ccr_marker_policy.py
@@ -34,6 +34,125 @@ def test_source_line_span_marker_is_still_detected() -> None:
assert _hashes(marker) == ["c00eb437e5e5c00eb437e5e5"]
+def test_new_marker_with_integrity_clause_is_detected() -> None:
+ # Issue #3098: marker includes "Original content preserved."
+ marker = (
+ "[122 items compressed to 27 (from 5 source lines). "
+ "Original content preserved. Retrieve more: hash=c00eb437e5e5c00eb437e5e5]"
+ )
+ assert _hashes(marker) == ["c00eb437e5e5c00eb437e5e5"]
+
+
+class _FakeEncoding:
+ def __init__(self, word_lists: list[list[str]]):
+ self._word_lists = word_lists
+ self._ids = [[0] * len(w) for w in word_lists]
+
+ def __getitem__(self, key: str):
+ return {"input_ids": self._ids, "attention_mask": self._ids}[key]
+
+ def word_ids(self, batch_index: int = 0) -> list[int]:
+ return list(range(len(self._word_lists[batch_index])))
+
+
+class _FakeTokenizer:
+ def __call__(self, words: list[str] | list[list[str]], **_kwargs) -> _FakeEncoding:
+ batch = words if (words and isinstance(words[0], list)) else [words] # type: ignore[list-item]
+ return _FakeEncoding(batch) # type: ignore[arg-type]
+
+
+class _FakeModel:
+ """Keep the first two words of each chunk -> ratio < 0.8 (CCR fires)."""
+
+ def get_keep_mask(
+ self, input_ids: list[list[int]], _attention_mask: object
+ ) -> list[list[bool]]:
+ return [[i < 2 for i in range(len(input_ids[0]))]]
+
+ def get_scores(self, input_ids: list[list[int]], _attention_mask: object) -> list[list[float]]:
+ return [[1.0 if i < 2 else 0.0 for i in range(len(row))] for row in input_ids]
+
+
+def test_remote_kompress_compressor_emits_original_content_preserved(monkeypatch) -> None:
+ from headroom.transforms.kompress_remote import RemoteKompressCompressor
+
+ class _FakeResponse:
+ status_code = 200
+
+ def raise_for_status(self) -> None:
+ return None
+
+ def json(self) -> dict:
+ return {"compressed": "short text", "compression_ratio": 0.2, "original_tokens": 100}
+
+ class _FakeClient:
+ def post(self, url, headers=None, json=None): # noqa: A002, ANN001
+ return _FakeResponse()
+
+ def close(self) -> None:
+ return None
+
+ monkeypatch.setattr(
+ "headroom.transforms.kompress_remote.store_kompress_in_ccr",
+ lambda original, compressed, tokens: "cafebabecafebabecafebabe",
+ )
+ c = RemoteKompressCompressor("https://ml.example.invalid")
+ c._client = _FakeClient()
+ c.config.enable_ccr = True
+
+ result = c.compress("line one\nline two\nline three " * 20)
+ assert (
+ "Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"
+ in result.compressed
+ )
+
+
+def test_kompress_compressor_emits_original_content_preserved(monkeypatch) -> None:
+ from headroom.transforms.kompress_compressor import KompressCompressor
+
+ compressor = KompressCompressor()
+ compressor.config.enable_ccr = True
+ monkeypatch.setattr(
+ compressor,
+ "_store_in_ccr",
+ lambda original, compressed, tokens: "cafebabecafebabecafebabe",
+ )
+ monkeypatch.setattr(
+ "headroom.transforms.kompress_compressor._load_kompress",
+ lambda *a, **k: (_FakeModel(), _FakeTokenizer(), "onnx"),
+ )
+
+ result = compressor.compress("line one\nline two\nline three " * 20)
+ assert (
+ "Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"
+ in result.compressed
+ )
+
+
+def test_kompress_compressor_batch_emits_original_content_preserved(monkeypatch) -> None:
+ from headroom.transforms.kompress_compressor import KompressCompressor
+
+ compressor = KompressCompressor()
+ compressor.config.enable_ccr = True
+ monkeypatch.setattr(compressor, "_should_use_sequential_fallback", lambda: False)
+ monkeypatch.setattr(
+ compressor,
+ "_store_in_ccr",
+ lambda original, compressed, tokens: "cafebabecafebabecafebabe",
+ )
+ monkeypatch.setattr(
+ "headroom.transforms.kompress_compressor._load_kompress",
+ lambda *a, **k: (_FakeModel(), _FakeTokenizer(), "onnx"),
+ )
+
+ results = compressor.compress_batch(["line one\nline two\nline three " * 20])
+ assert len(results) == 1
+ assert (
+ "Original content preserved. Retrieve more: hash=cafebabecafebabecafebabe"
+ in results[0].compressed
+ )
+
+
def test_has_new_ccr_markers_detects_hash_not_seen_in_previous_forward() -> None:
old = "[100 items compressed to 10. Retrieve more: hash=abc123def456abc123def456]"
new = "[50 items compressed to 5. Retrieve more: hash=deadbeefdeadbeefdeadbeef]"
diff --git a/tests/test_kompress_marker_gate.py b/tests/test_kompress_marker_gate.py
index 8cea9356b..f7ee42a63 100644
--- a/tests/test_kompress_marker_gate.py
+++ b/tests/test_kompress_marker_gate.py
@@ -97,7 +97,7 @@ def _tok(text: str) -> int:
def test_payload_tokens_is_cl100k_or_one_per_character(monkeypatch):
marker = ccr_retrieval_marker(100, 59, SINGLE_TOKEN_SOURCE, _real_key(SINGLE_TOKEN_SOURCE))
assert _real_key(SINGLE_TOKEN_SOURCE) == "7dbb8f8de9f3e1d7c6f3a6e1"
- assert payload_tokens(marker) == _tok(marker) == 43
+ assert payload_tokens(marker) == _tok(marker) == 47
assert payload_tokens(SINGLE_TOKEN_SOURCE) == _tok(SINGLE_TOKEN_SOURCE) == 100
monkeypatch.setattr(kc, "_payload_encoder", False)
assert payload_tokens(marker) == len(marker)