Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor(transforms): share one CCR persist helper for the Rust-backed compressors (#4094)

Tejas Chopraea947ca421ee
diff --git a/headroom/transforms/base.py b/headroom/transforms/base.py
index cffad025a..d541d1c7d 100644
--- a/headroom/transforms/base.py
+++ b/headroom/transforms/base.py
@@ -2,12 +2,55 @@
from __future__ import annotations
+import logging
from abc import ABC, abstractmethod
from typing import Any
from ..config import TransformResult
+from ..log_safety import describe_exception, safe_id
from ..tokenizer import Tokenizer
+logger = logging.getLogger(__name__)
+
+
+def persist_rust_ccr_entry(original: str, compressed: str, cache_key: str, *, source: str) -> None:
+ """Store a Rust-emitted CCR entry in the production ``CompressionStore``.
+
+ The Rust search/diff/log compressors emit a retrieval marker but keep the
+ original only in their in-memory test store. This writes it through to the
+ long-lived Python store so the marker resolves. Failures are logged at
+ warning level with ``source`` (the compressor name): a store hiccup must not
+ break the response, just degrade retrieval.
+ """
+ try:
+ from ..cache.compression_store import get_compression_store
+ except ImportError as e:
+ logger.warning(
+ "CCR store import failed (%s); cache_key %s won't persist: %s",
+ source,
+ cache_key,
+ safe_id(e),
+ )
+ logger.debug("CCR store import failure detail: %s", describe_exception(e))
+ return
+ try:
+ store: Any = get_compression_store()
+ # The Rust-emitted marker embeds MD5(original)[:24], but store() has
+ # defaulted to SHA-256(original)[:24] since PR #395. Pass the marker's
+ # key explicitly so retrieving the marker hash finds the entry (#816).
+ store.store(original, compressed, explicit_hash=cache_key)
+ except Exception as e:
+ # Store backends can echo the payload in errors. The WARNING names only
+ # the exception type; the DEBUG record has types and code locations, or
+ # the full traceback (payload included) once HEADROOM_DEBUG_DUMP=full.
+ logger.warning(
+ "CCR store write failed (%s, %s); cache_key %s remains in-marker only",
+ source,
+ safe_id(e),
+ cache_key,
+ )
+ logger.debug("CCR store write failure detail: %s", describe_exception(e))
+
class Transform(ABC):
"""Abstract base class for message transforms."""
diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.py
index 15b906fc4..9bfacc9b8 100644
--- a/headroom/transforms/diff_compressor.py
+++ b/headroom/transforms/diff_compressor.py
@@ -20,12 +20,11 @@ fallback. Build it locally with `scripts/build_rust_extension.sh`
from __future__ import annotations
-import logging
import warnings
from dataclasses import dataclass
from typing import Any
-logger = logging.getLogger(__name__)
+from .base import persist_rust_ccr_entry
@dataclass
@@ -114,7 +113,7 @@ class DiffCompressor:
# call. Without this, every diff CCR marker emitted in
# production is dangling — the regression fixed in the
# audit-cleanup PR.
- self._persist_to_python_ccr(content, r.compressed, cache_key)
+ persist_rust_ccr_entry(content, r.compressed, cache_key, source="diff")
return DiffCompressionResult(
compressed=r.compressed,
original_line_count=r.original_line_count,
@@ -127,31 +126,6 @@ class DiffCompressor:
cache_key=cache_key,
)
- def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:
- """Promote a Rust-emitted cache_key into the production Python
- CompressionStore. Failures are logged at warning level — a
- store hiccup must not break the response, just degrade
- retrieval. Mirrors the same helper on log_compressor.py and
- search_compressor.py."""
- try:
- from ..cache.compression_store import get_compression_store
- except ImportError as e:
- logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)
- return
- try:
- store: Any = get_compression_store()
- # The Rust-emitted marker embeds MD5(original)[:24], but
- # store() has defaulted to SHA-256(original)[:24] since
- # PR #395. Pass the marker's key explicitly so retrieving
- # the marker hash actually finds the entry (issue #816).
- store.store(original, compressed, explicit_hash=cache_key)
- except Exception as e:
- logger.warning(
- "CCR store write failed; cache_key %s remains in-marker only: %s",
- cache_key,
- e,
- )
-
def compress_with_stats(
self, content: str, context: str = ""
) -> tuple[DiffCompressionResult, Any]:
diff --git a/headroom/transforms/log_compressor.py b/headroom/transforms/log_compressor.py
index 33cf49420..c5cb05250 100644
--- a/headroom/transforms/log_compressor.py
+++ b/headroom/transforms/log_compressor.py
@@ -37,12 +37,11 @@ exists only for unit testing.
from __future__ import annotations
-import logging
from dataclasses import dataclass, field
from enum import Enum
-from typing import Any, cast
+from typing import cast
-logger = logging.getLogger(__name__)
+from .base import persist_rust_ccr_entry
class LogFormat(Enum):
@@ -201,7 +200,7 @@ class LogCompressor:
rust_result = self._rust.compress(content, bias)
cache_key: str | None = rust_result.cache_key
if cache_key is not None:
- self._persist_to_python_ccr(content, rust_result.compressed, cache_key)
+ persist_rust_ccr_entry(content, rust_result.compressed, cache_key, source="log")
stats_dict = {k: int(v) for k, v in cast("dict[str, int]", rust_result.stats).items()}
return LogCompressionResult(
@@ -215,28 +214,6 @@ class LogCompressor:
stats=stats_dict,
)
- def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:
- """Promote a Rust-emitted cache_key into the production Python
- CompressionStore. Failures are logged at warning level."""
- try:
- from ..cache.compression_store import get_compression_store
- except ImportError as e:
- logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)
- return
- try:
- store: Any = get_compression_store()
- # The Rust-emitted marker embeds MD5(original)[:24], but
- # store() has defaulted to SHA-256(original)[:24] since
- # PR #395. Pass the marker's key explicitly so retrieving
- # the marker hash actually finds the entry (issue #816).
- store.store(original, compressed, explicit_hash=cache_key)
- except Exception as e:
- logger.warning(
- "CCR store write failed; cache_key %s remains in-marker only: %s",
- cache_key,
- e,
- )
-
__all__ = [
"LogCompressor",
diff --git a/headroom/transforms/search_compressor.py b/headroom/transforms/search_compressor.py
index b8f6214ed..cc98022d5 100644
--- a/headroom/transforms/search_compressor.py
+++ b/headroom/transforms/search_compressor.py
@@ -39,12 +39,10 @@ exactly what the previous Python implementation provided.
from __future__ import annotations
-import logging
from dataclasses import dataclass, field
-from typing import Any, cast
-
-logger = logging.getLogger(__name__)
+from typing import cast
+from .base import persist_rust_ccr_entry
# ─── Public dataclasses (preserve existing import surface) ──────────────────
@@ -178,7 +176,7 @@ class SearchCompressor:
# store. The Rust crate already wrote to its in-memory test
# store; promote that to the long-lived Python store so the
# marker remains retrievable beyond the request lifecycle.
- self._persist_to_python_ccr(content, rust_result.compressed, cache_key)
+ persist_rust_ccr_entry(content, rust_result.compressed, cache_key, source="search")
summaries = dict(cast("dict[str, str]", rust_result.summaries))
return SearchCompressionResult(
@@ -192,36 +190,6 @@ class SearchCompressor:
summaries=summaries,
)
- # ─── Internal CCR persistence ───────────────────────────────────────
-
- def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:
- """Promote the Rust-emitted cache_key into the production Python
- `CompressionStore`. Failures are surfaced via logging instead of
- being silently swallowed (see no-silent-fallbacks rule).
-
- Note: the Rust path computes the hash and embeds it in the
- emitted marker text — the Rust hash IS the canonical one
- (MD5(original)[:24]). The store must be keyed by that exact
- hash or the marker dangles.
- """
- try:
- from ..cache.compression_store import get_compression_store
- except ImportError as e:
- logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)
- return
-
- try:
- store: Any = get_compression_store()
- # The Rust-emitted marker embeds MD5(original)[:24], but
- # store() has defaulted to SHA-256(original)[:24] since
- # PR #395. Pass the marker's key explicitly so retrieving
- # the marker hash actually finds the entry (issue #816).
- store.store(original, compressed, explicit_hash=cache_key)
- except Exception as e:
- logger.warning(
- "CCR store write failed; cache_key %s remains in-marker only: %s", cache_key, e
- )
-
__all__ = [
"SearchCompressor",
diff --git a/tests/test_ccr_rust_marker_hash_bridge.py b/tests/test_ccr_rust_marker_hash_bridge.py
index 2b98e2e20..4ebd6bf11 100644
--- a/tests/test_ccr_rust_marker_hash_bridge.py
+++ b/tests/test_ccr_rust_marker_hash_bridge.py
@@ -4,18 +4,20 @@ The Rust side embeds ``MD5(original)[:24]`` in the emitted
``Retrieve more: hash=...`` marker, but since PR #395
``CompressionStore.store()`` defaults to ``SHA-256(original)[:24]``.
PR #395 fixed the SmartCrusher path by passing ``explicit_hash``
-(see ``test_ccr_row_drop_store_bridge.py``); the three
-``_persist_to_python_ccr`` shims on the Rust-accelerated transforms
-were never migrated, so every marker they emitted dangled —
-retrieval returned "Entry not found or expired" inside any TTL.
-
-These tests pin the cross-language contract at the shim layer: the
-store entry must be keyed by the exact hash the marker embeds.
+(see ``test_ccr_row_drop_store_bridge.py``); the CCR write-through
+used by the Rust-accelerated transforms was never migrated, so every
+marker they emitted dangled — retrieval returned "Entry not found or
+expired" inside any TTL.
+
+These tests pin the cross-language contract at the shared
+``persist_rust_ccr_entry`` helper: the store entry must be keyed by the
+exact hash the marker embeds.
"""
from __future__ import annotations
import hashlib
+import logging
import pytest
@@ -23,11 +25,7 @@ from headroom.cache.compression_store import (
get_compression_store,
reset_compression_store,
)
-from headroom.transforms.diff_compressor import DiffCompressor
-from headroom.transforms.log_compressor import LogCompressor
-from headroom.transforms.search_compressor import SearchCompressor
-
-pytest.importorskip("headroom._core", reason="Rust extension required")
+from headroom.transforms.base import persist_rust_ccr_entry
@pytest.fixture(autouse=True)
@@ -37,6 +35,30 @@ def _fresh_store():
reset_compression_store()
[email protected]("level", [logging.INFO, logging.DEBUG])
+def test_persist_warning_rejects_malformed_exception_names(monkeypatch, caplog, level) -> None:
+ from headroom.cache import compression_store
+ from headroom.transforms import base
+
+ canary = "PRIVATE_CREDENTIAL_CANARY\nFORGED_LOG_ENTRY"
+ error_type = type(canary, (RuntimeError,), {})
+
+ class BrokenStore:
+ def store(self, original, compressed, **kwargs):
+ raise error_type("untrusted backend text")
+
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ monkeypatch.setattr(compression_store, "get_compression_store", lambda: BrokenStore())
+ with caplog.at_level(level, logger=base.logger.name):
+ assert persist_rust_ccr_entry("original", "compressed", "a" * 24, source="diff") is None
+ assert len([record for record in caplog.records if record.levelno == logging.WARNING]) == 1
+ for record in caplog.records:
+ rendered = logging.Formatter().format(record)
+ assert "PRIVATE_CREDENTIAL_CANARY" not in rendered
+ assert "FORGED_LOG_ENTRY" not in rendered
+ assert "\n" not in rendered
+
+
def _rust_marker_key(original: str) -> str:
"""The hash the Rust side embeds in emitted markers."""
return hashlib.md5(original.encode()).hexdigest()[:24]
@@ -61,25 +83,105 @@ def _assert_round_trip(original: str) -> None:
)
-def test_search_compressor_shim_stores_under_marker_key() -> None:
- original = "src/app.py:12: def handle_request(payload):\n" * 40
- SearchCompressor()._persist_to_python_ccr(
- original, "compressed search output", _rust_marker_key(original)
- )
- _assert_round_trip(original)
-
-
-def test_diff_compressor_shim_stores_under_marker_key() -> None:
- original = "+added line of code\n-removed line of code\n" * 40
- DiffCompressor()._persist_to_python_ccr(
- original, "compressed diff output", _rust_marker_key(original)
+ ("source", "original"),
+ [
+ ("search", "src/app.py:12: def handle_request(payload):\n" * 40),
+ ("diff", "+added line of code\n-removed line of code\n" * 40),
+ ("log", "2026-06-11T09:00:00Z INFO worker heartbeat ok seq=1\n" * 40),
+ ],
+)
+def test_persist_stores_under_marker_key(source: str, original: str) -> None:
+ persist_rust_ccr_entry(
+ original, f"compressed {source} output", _rust_marker_key(original), source=source
)
_assert_round_trip(original)
-def test_log_compressor_shim_stores_under_marker_key() -> None:
- original = "2026-06-11T09:00:00Z INFO worker heartbeat ok seq=1\n" * 40
- LogCompressor()._persist_to_python_ccr(
- original, "compressed log output", _rust_marker_key(original)
- )
- _assert_round_trip(original)
+def test_persist_failure_logs_no_exception_text_at_any_level(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import logging
+
+ import headroom.cache.compression_store as compression_store
+ import headroom.transforms.base as base
+
+ content = "def private_source_canary_4c1e(): pass"
+ credential = "sk-test-credential-canary-91bd"
+
+ class _Store:
+ def store(self, original: str, *_args: object, **_kwargs: object) -> str:
+ try:
+ raise ValueError(f"backend rejected key {credential}")
+ except ValueError as cause:
+ raise RuntimeError(f"cannot store {original}") from cause
+
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ monkeypatch.setattr(compression_store, "get_compression_store", lambda: _Store())
+ records: list[logging.LogRecord] = []
+
+ class _Capture(logging.Handler):
+ def emit(self, record: logging.LogRecord) -> None:
+ records.append(record)
+
+ handler = _Capture(level=logging.DEBUG)
+ base.logger.addHandler(handler)
+ monkeypatch.setattr(base.logger, "level", logging.DEBUG)
+ try:
+ persist_rust_ccr_entry(content, "comp", "a" * 24, source="diff")
+ finally:
+ base.logger.removeHandler(handler)
+
+ formatter = logging.Formatter()
+ for record in records:
+ rendered = formatter.format(record)
+ assert content not in rendered
+ assert credential not in rendered
+
+ warnings = [r for r in records if r.levelno == logging.WARNING]
+ assert len(warnings) == 1
+ assert "(diff, <RuntimeError>)" in warnings[0].getMessage()
+ assert "a" * 24 in warnings[0].getMessage()
+ details = [r for r in records if r.levelno == logging.DEBUG]
+ assert len(details) == 1
+ assert "RuntimeError" in details[0].getMessage()
+ assert "caused by ValueError" in details[0].getMessage()
+
+
+def test_persist_warning_stays_one_line_when_full_dump_is_on(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """HEADROOM_DEBUG_DUMP=full puts the traceback in DEBUG only, never the WARNING."""
+ import headroom.cache.compression_store as compression_store
+ import headroom.transforms.base as base
+
+ class _Store:
+ def store(self, *_args: object, **_kwargs: object) -> str:
+ raise RuntimeError("cannot store original")
+
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ monkeypatch.setattr(compression_store, "get_compression_store", lambda: _Store())
+ records: list[logging.LogRecord] = []
+
+ class _Capture(logging.Handler):
+ def emit(self, record: logging.LogRecord) -> None:
+ records.append(record)
+
+ handler = _Capture(level=logging.DEBUG)
+ base.logger.addHandler(handler)
+ monkeypatch.setattr(base.logger, "level", logging.DEBUG)
+ try:
+ persist_rust_ccr_entry("original", "comp", "a" * 24, source="log")
+ finally:
+ base.logger.removeHandler(handler)
+
+ warnings = [r.getMessage() for r in records if r.levelno == logging.WARNING]
+ assert len(warnings) == 1
+ assert "(log, <RuntimeError>)" in warnings[0]
+ assert "\n" not in warnings[0]
+ assert "Traceback" not in warnings[0]
+ assert "cannot store original" not in warnings[0]
+ details = [r.getMessage() for r in records if r.levelno == logging.DEBUG]
+ assert len(details) == 1
+ assert "Traceback" in details[0]
+ assert "cannot store original" in details[0]
diff --git a/tests/test_transforms_search_compressor.py b/tests/test_transforms_search_compressor.py
index c648188d3..207260341 100644
--- a/tests/test_transforms_search_compressor.py
+++ b/tests/test_transforms_search_compressor.py
@@ -4,6 +4,7 @@ from types import SimpleNamespace
import pytest
+from headroom.transforms.base import persist_rust_ccr_entry
from headroom.transforms.search_compressor import (
SearchCompressionResult,
SearchCompressor,
@@ -73,12 +74,9 @@ def test_search_compressor_compress_paths_and_ccr() -> None:
assert len(result.summaries) >= 1
-def test_search_compressor_persist_to_python_ccr(monkeypatch: pytest.MonkeyPatch) -> None:
- """Phase 3e.2: CCR persistence is now in `_persist_to_python_ccr`,
- which delegates to the production `CompressionStore`. Failures are
- logged (not silently swallowed) — this pins both paths."""
- compressor = SearchCompressor()
-
+def test_persist_rust_ccr_entry(monkeypatch: pytest.MonkeyPatch) -> None:
+ """CCR write-through delegates to the production `CompressionStore`.
+ Failures are logged (not silently swallowed) — this pins both paths."""
seen: dict[str, tuple[str, str, str | None]] = {}
monkeypatch.setitem(
__import__("sys").modules,
@@ -91,7 +89,7 @@ def test_search_compressor_persist_to_python_ccr(monkeypatch: pytest.MonkeyPatch
)
),
)
- compressor._persist_to_python_ccr("orig", "comp", "abc123")
+ persist_rust_ccr_entry("orig", "comp", "abc123", source="search")
# explicit_hash carries the Rust marker key so retrieval of the
# marker hash finds the entry (issue #816).
assert seen["call"] == ("orig", "comp", "abc123")
@@ -106,7 +104,7 @@ def test_search_compressor_persist_to_python_ccr(monkeypatch: pytest.MonkeyPatch
"headroom.cache.compression_store",
SimpleNamespace(get_compression_store=broken_store),
)
- compressor._persist_to_python_ccr("orig", "comp", "abc123") # must not raise
+ persist_rust_ccr_entry("orig", "comp", "abc123", source="search") # must not raise
def test_search_compression_result_properties() -> None: