Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
refactor(transforms): share one CCR persist helper for the Rust-backed compressors (#4094)
Tejas Chopraea947ca421ee
diff --git a/headroom/transforms/base.py b/headroom/transforms/base.pyindex cffad025a..d541d1c7d 100644--- a/headroom/transforms/base.py+++ b/headroom/transforms/base.py@@ -2,12 +2,55 @@from __future__ import annotations+import loggingfrom abc import ABC, abstractmethodfrom typing import Anyfrom ..config import TransformResult+from ..log_safety import describe_exception, safe_idfrom ..tokenizer import Tokenizer+logger = logging.getLogger(__name__)+++def persist_rust_ccr_entry(original: str, compressed: str, cache_key: str, *, source: str) -> None:+ """Store a Rust-emitted CCR entry in the production ``CompressionStore``.++ The Rust search/diff/log compressors emit a retrieval marker but keep the+ original only in their in-memory test store. This writes it through to the+ long-lived Python store so the marker resolves. Failures are logged at+ warning level with ``source`` (the compressor name): a store hiccup must not+ break the response, just degrade retrieval.+ """+ try:+ from ..cache.compression_store import get_compression_store+ except ImportError as e:+ logger.warning(+ "CCR store import failed (%s); cache_key %s won't persist: %s",+ source,+ cache_key,+ safe_id(e),+ )+ logger.debug("CCR store import failure detail: %s", describe_exception(e))+ return+ try:+ store: Any = get_compression_store()+ # The Rust-emitted marker embeds MD5(original)[:24], but store() has+ # defaulted to SHA-256(original)[:24] since PR #395. Pass the marker's+ # key explicitly so retrieving the marker hash finds the entry (#816).+ store.store(original, compressed, explicit_hash=cache_key)+ except Exception as e:+ # Store backends can echo the payload in errors. The WARNING names only+ # the exception type; the DEBUG record has types and code locations, or+ # the full traceback (payload included) once HEADROOM_DEBUG_DUMP=full.+ logger.warning(+ "CCR store write failed (%s, %s); cache_key %s remains in-marker only",+ source,+ safe_id(e),+ cache_key,+ )+ logger.debug("CCR store write failure detail: %s", describe_exception(e))+class Transform(ABC):"""Abstract base class for message transforms."""diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.pyindex 15b906fc4..9bfacc9b8 100644--- a/headroom/transforms/diff_compressor.py+++ b/headroom/transforms/diff_compressor.py@@ -20,12 +20,11 @@ fallback. Build it locally with `scripts/build_rust_extension.sh`from __future__ import annotations-import loggingimport warningsfrom dataclasses import dataclassfrom typing import Any-logger = logging.getLogger(__name__)+from .base import persist_rust_ccr_entry@dataclass@@ -114,7 +113,7 @@ class DiffCompressor:# call. Without this, every diff CCR marker emitted in# production is dangling — the regression fixed in the# audit-cleanup PR.- self._persist_to_python_ccr(content, r.compressed, cache_key)+ persist_rust_ccr_entry(content, r.compressed, cache_key, source="diff")return DiffCompressionResult(compressed=r.compressed,original_line_count=r.original_line_count,@@ -127,31 +126,6 @@ class DiffCompressor:cache_key=cache_key,)- def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:- """Promote a Rust-emitted cache_key into the production Python- CompressionStore. Failures are logged at warning level — a- store hiccup must not break the response, just degrade- retrieval. Mirrors the same helper on log_compressor.py and- search_compressor.py."""- try:- from ..cache.compression_store import get_compression_store- except ImportError as e:- logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)- return- try:- store: Any = get_compression_store()- # The Rust-emitted marker embeds MD5(original)[:24], but- # store() has defaulted to SHA-256(original)[:24] since- # PR #395. Pass the marker's key explicitly so retrieving- # the marker hash actually finds the entry (issue #816).- store.store(original, compressed, explicit_hash=cache_key)- except Exception as e:- logger.warning(- "CCR store write failed; cache_key %s remains in-marker only: %s",- cache_key,- e,- )-def compress_with_stats(self, content: str, context: str = "") -> tuple[DiffCompressionResult, Any]:diff --git a/headroom/transforms/log_compressor.py b/headroom/transforms/log_compressor.pyindex 33cf49420..c5cb05250 100644--- a/headroom/transforms/log_compressor.py+++ b/headroom/transforms/log_compressor.py@@ -37,12 +37,11 @@ exists only for unit testing.from __future__ import annotations-import loggingfrom dataclasses import dataclass, fieldfrom enum import Enum-from typing import Any, cast+from typing import cast-logger = logging.getLogger(__name__)+from .base import persist_rust_ccr_entryclass LogFormat(Enum):@@ -201,7 +200,7 @@ class LogCompressor:rust_result = self._rust.compress(content, bias)cache_key: str | None = rust_result.cache_keyif cache_key is not None:- self._persist_to_python_ccr(content, rust_result.compressed, cache_key)+ persist_rust_ccr_entry(content, rust_result.compressed, cache_key, source="log")stats_dict = {k: int(v) for k, v in cast("dict[str, int]", rust_result.stats).items()}return LogCompressionResult(@@ -215,28 +214,6 @@ class LogCompressor:stats=stats_dict,)- def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:- """Promote a Rust-emitted cache_key into the production Python- CompressionStore. Failures are logged at warning level."""- try:- from ..cache.compression_store import get_compression_store- except ImportError as e:- logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)- return- try:- store: Any = get_compression_store()- # The Rust-emitted marker embeds MD5(original)[:24], but- # store() has defaulted to SHA-256(original)[:24] since- # PR #395. Pass the marker's key explicitly so retrieving- # the marker hash actually finds the entry (issue #816).- store.store(original, compressed, explicit_hash=cache_key)- except Exception as e:- logger.warning(- "CCR store write failed; cache_key %s remains in-marker only: %s",- cache_key,- e,- )-__all__ = ["LogCompressor",diff --git a/headroom/transforms/search_compressor.py b/headroom/transforms/search_compressor.pyindex b8f6214ed..cc98022d5 100644--- a/headroom/transforms/search_compressor.py+++ b/headroom/transforms/search_compressor.py@@ -39,12 +39,10 @@ exactly what the previous Python implementation provided.from __future__ import annotations-import loggingfrom dataclasses import dataclass, field-from typing import Any, cast--logger = logging.getLogger(__name__)+from typing import cast+from .base import persist_rust_ccr_entry# ─── Public dataclasses (preserve existing import surface) ──────────────────@@ -178,7 +176,7 @@ class SearchCompressor:# store. The Rust crate already wrote to its in-memory test# store; promote that to the long-lived Python store so the# marker remains retrievable beyond the request lifecycle.- self._persist_to_python_ccr(content, rust_result.compressed, cache_key)+ persist_rust_ccr_entry(content, rust_result.compressed, cache_key, source="search")summaries = dict(cast("dict[str, str]", rust_result.summaries))return SearchCompressionResult(@@ -192,36 +190,6 @@ class SearchCompressor:summaries=summaries,)- # ─── Internal CCR persistence ───────────────────────────────────────-- def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:- """Promote the Rust-emitted cache_key into the production Python- `CompressionStore`. Failures are surfaced via logging instead of- being silently swallowed (see no-silent-fallbacks rule).-- Note: the Rust path computes the hash and embeds it in the- emitted marker text — the Rust hash IS the canonical one- (MD5(original)[:24]). The store must be keyed by that exact- hash or the marker dangles.- """- try:- from ..cache.compression_store import get_compression_store- except ImportError as e:- logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)- return-- try:- store: Any = get_compression_store()- # The Rust-emitted marker embeds MD5(original)[:24], but- # store() has defaulted to SHA-256(original)[:24] since- # PR #395. Pass the marker's key explicitly so retrieving- # the marker hash actually finds the entry (issue #816).- store.store(original, compressed, explicit_hash=cache_key)- except Exception as e:- logger.warning(- "CCR store write failed; cache_key %s remains in-marker only: %s", cache_key, e- )-__all__ = ["SearchCompressor",diff --git a/tests/test_ccr_rust_marker_hash_bridge.py b/tests/test_ccr_rust_marker_hash_bridge.pyindex 2b98e2e20..4ebd6bf11 100644--- a/tests/test_ccr_rust_marker_hash_bridge.py+++ b/tests/test_ccr_rust_marker_hash_bridge.py@@ -4,18 +4,20 @@ The Rust side embeds ``MD5(original)[:24]`` in the emitted``Retrieve more: hash=...`` marker, but since PR #395``CompressionStore.store()`` defaults to ``SHA-256(original)[:24]``.PR #395 fixed the SmartCrusher path by passing ``explicit_hash``-(see ``test_ccr_row_drop_store_bridge.py``); the three-``_persist_to_python_ccr`` shims on the Rust-accelerated transforms-were never migrated, so every marker they emitted dangled —-retrieval returned "Entry not found or expired" inside any TTL.--These tests pin the cross-language contract at the shim layer: the-store entry must be keyed by the exact hash the marker embeds.+(see ``test_ccr_row_drop_store_bridge.py``); the CCR write-through+used by the Rust-accelerated transforms was never migrated, so every+marker they emitted dangled — retrieval returned "Entry not found or+expired" inside any TTL.++These tests pin the cross-language contract at the shared+``persist_rust_ccr_entry`` helper: the store entry must be keyed by the+exact hash the marker embeds."""from __future__ import annotationsimport hashlib+import loggingimport pytest@@ -23,11 +25,7 @@ from headroom.cache.compression_store import (get_compression_store,reset_compression_store,)-from headroom.transforms.diff_compressor import DiffCompressor-from headroom.transforms.log_compressor import LogCompressor-from headroom.transforms.search_compressor import SearchCompressor--pytest.importorskip("headroom._core", reason="Rust extension required")+from headroom.transforms.base import persist_rust_ccr_entry@pytest.fixture(autouse=True)@@ -37,6 +35,30 @@ def _fresh_store():reset_compression_store()[email protected]("level", [logging.INFO, logging.DEBUG])+def test_persist_warning_rejects_malformed_exception_names(monkeypatch, caplog, level) -> None:+ from headroom.cache import compression_store+ from headroom.transforms import base++ canary = "PRIVATE_CREDENTIAL_CANARY\nFORGED_LOG_ENTRY"+ error_type = type(canary, (RuntimeError,), {})++ class BrokenStore:+ def store(self, original, compressed, **kwargs):+ raise error_type("untrusted backend text")++ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ monkeypatch.setattr(compression_store, "get_compression_store", lambda: BrokenStore())+ with caplog.at_level(level, logger=base.logger.name):+ assert persist_rust_ccr_entry("original", "compressed", "a" * 24, source="diff") is None+ assert len([record for record in caplog.records if record.levelno == logging.WARNING]) == 1+ for record in caplog.records:+ rendered = logging.Formatter().format(record)+ assert "PRIVATE_CREDENTIAL_CANARY" not in rendered+ assert "FORGED_LOG_ENTRY" not in rendered+ assert "\n" not in rendered++def _rust_marker_key(original: str) -> str:"""The hash the Rust side embeds in emitted markers."""return hashlib.md5(original.encode()).hexdigest()[:24]@@ -61,25 +83,105 @@ def _assert_round_trip(original: str) -> None:)-def test_search_compressor_shim_stores_under_marker_key() -> None:- original = "src/app.py:12: def handle_request(payload):\n" * 40- SearchCompressor()._persist_to_python_ccr(- original, "compressed search output", _rust_marker_key(original)- )- _assert_round_trip(original)---def test_diff_compressor_shim_stores_under_marker_key() -> None:- original = "+added line of code\n-removed line of code\n" * 40- DiffCompressor()._persist_to_python_ccr(- original, "compressed diff output", _rust_marker_key(original)+ ("source", "original"),+ [+ ("search", "src/app.py:12: def handle_request(payload):\n" * 40),+ ("diff", "+added line of code\n-removed line of code\n" * 40),+ ("log", "2026-06-11T09:00:00Z INFO worker heartbeat ok seq=1\n" * 40),+ ],+)+def test_persist_stores_under_marker_key(source: str, original: str) -> None:+ persist_rust_ccr_entry(+ original, f"compressed {source} output", _rust_marker_key(original), source=source)_assert_round_trip(original)-def test_log_compressor_shim_stores_under_marker_key() -> None:- original = "2026-06-11T09:00:00Z INFO worker heartbeat ok seq=1\n" * 40- LogCompressor()._persist_to_python_ccr(- original, "compressed log output", _rust_marker_key(original)- )- _assert_round_trip(original)+def test_persist_failure_logs_no_exception_text_at_any_level(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ import logging++ import headroom.cache.compression_store as compression_store+ import headroom.transforms.base as base++ content = "def private_source_canary_4c1e(): pass"+ credential = "sk-test-credential-canary-91bd"++ class _Store:+ def store(self, original: str, *_args: object, **_kwargs: object) -> str:+ try:+ raise ValueError(f"backend rejected key {credential}")+ except ValueError as cause:+ raise RuntimeError(f"cannot store {original}") from cause++ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ monkeypatch.setattr(compression_store, "get_compression_store", lambda: _Store())+ records: list[logging.LogRecord] = []++ class _Capture(logging.Handler):+ def emit(self, record: logging.LogRecord) -> None:+ records.append(record)++ handler = _Capture(level=logging.DEBUG)+ base.logger.addHandler(handler)+ monkeypatch.setattr(base.logger, "level", logging.DEBUG)+ try:+ persist_rust_ccr_entry(content, "comp", "a" * 24, source="diff")+ finally:+ base.logger.removeHandler(handler)++ formatter = logging.Formatter()+ for record in records:+ rendered = formatter.format(record)+ assert content not in rendered+ assert credential not in rendered++ warnings = [r for r in records if r.levelno == logging.WARNING]+ assert len(warnings) == 1+ assert "(diff, <RuntimeError>)" in warnings[0].getMessage()+ assert "a" * 24 in warnings[0].getMessage()+ details = [r for r in records if r.levelno == logging.DEBUG]+ assert len(details) == 1+ assert "RuntimeError" in details[0].getMessage()+ assert "caused by ValueError" in details[0].getMessage()+++def test_persist_warning_stays_one_line_when_full_dump_is_on(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """HEADROOM_DEBUG_DUMP=full puts the traceback in DEBUG only, never the WARNING."""+ import headroom.cache.compression_store as compression_store+ import headroom.transforms.base as base++ class _Store:+ def store(self, *_args: object, **_kwargs: object) -> str:+ raise RuntimeError("cannot store original")++ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ monkeypatch.setattr(compression_store, "get_compression_store", lambda: _Store())+ records: list[logging.LogRecord] = []++ class _Capture(logging.Handler):+ def emit(self, record: logging.LogRecord) -> None:+ records.append(record)++ handler = _Capture(level=logging.DEBUG)+ base.logger.addHandler(handler)+ monkeypatch.setattr(base.logger, "level", logging.DEBUG)+ try:+ persist_rust_ccr_entry("original", "comp", "a" * 24, source="log")+ finally:+ base.logger.removeHandler(handler)++ warnings = [r.getMessage() for r in records if r.levelno == logging.WARNING]+ assert len(warnings) == 1+ assert "(log, <RuntimeError>)" in warnings[0]+ assert "\n" not in warnings[0]+ assert "Traceback" not in warnings[0]+ assert "cannot store original" not in warnings[0]+ details = [r.getMessage() for r in records if r.levelno == logging.DEBUG]+ assert len(details) == 1+ assert "Traceback" in details[0]+ assert "cannot store original" in details[0]diff --git a/tests/test_transforms_search_compressor.py b/tests/test_transforms_search_compressor.pyindex c648188d3..207260341 100644--- a/tests/test_transforms_search_compressor.py+++ b/tests/test_transforms_search_compressor.py@@ -4,6 +4,7 @@ from types import SimpleNamespaceimport pytest+from headroom.transforms.base import persist_rust_ccr_entryfrom headroom.transforms.search_compressor import (SearchCompressionResult,SearchCompressor,@@ -73,12 +74,9 @@ def test_search_compressor_compress_paths_and_ccr() -> None:assert len(result.summaries) >= 1-def test_search_compressor_persist_to_python_ccr(monkeypatch: pytest.MonkeyPatch) -> None:- """Phase 3e.2: CCR persistence is now in `_persist_to_python_ccr`,- which delegates to the production `CompressionStore`. Failures are- logged (not silently swallowed) — this pins both paths."""- compressor = SearchCompressor()-+def test_persist_rust_ccr_entry(monkeypatch: pytest.MonkeyPatch) -> None:+ """CCR write-through delegates to the production `CompressionStore`.+ Failures are logged (not silently swallowed) — this pins both paths."""seen: dict[str, tuple[str, str, str | None]] = {}monkeypatch.setitem(__import__("sys").modules,@@ -91,7 +89,7 @@ def test_search_compressor_persist_to_python_ccr(monkeypatch: pytest.MonkeyPatch)),)- compressor._persist_to_python_ccr("orig", "comp", "abc123")+ persist_rust_ccr_entry("orig", "comp", "abc123", source="search")# explicit_hash carries the Rust marker key so retrieval of the# marker hash finds the entry (issue #816).assert seen["call"] == ("orig", "comp", "abc123")@@ -106,7 +104,7 @@ def test_search_compressor_persist_to_python_ccr(monkeypatch: pytest.MonkeyPatch"headroom.cache.compression_store",SimpleNamespace(get_compression_store=broken_store),)- compressor._persist_to_python_ccr("orig", "comp", "abc123") # must not raise+ persist_rust_ccr_entry("orig", "comp", "abc123", source="search") # must not raisedef test_search_compression_result_properties() -> None: