Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): pyo3 bridge for SmartCrusher

chopratejas5328d87b1eb7
diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs
index 0e51e1ba9..93d43983d 100644
--- a/crates/headroom-py/src/lib.rs
+++ b/crates/headroom-py/src/lib.rs
@@ -15,6 +15,10 @@
use std::collections::BTreeMap;
+use headroom_core::transforms::smart_crusher::{
+ CrushResult as RustCrushResult, SmartCrusher as RustSmartCrusher,
+ SmartCrusherConfig as RustSmartCrusherConfig,
+};
use headroom_core::transforms::{
DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,
};
@@ -367,6 +371,248 @@ impl PyDiffCompressor {
}
}
+// ─── SmartCrusherConfig ────────────────────────────────────────────────────
+
+/// Mirror of `headroom.transforms.smart_crusher.SmartCrusherConfig`.
+/// Defaults match Python's dataclass byte-for-byte. The constructor
+/// accepts every field as a kwarg with the same name and type so the
+/// Python shim can pass `SmartCrusherConfig(**asdict(py_cfg))`.
+#[pyclass(name = "SmartCrusherConfig", module = "headroom._core")]
+#[derive(Clone)]
+struct PySmartCrusherConfig {
+ inner: RustSmartCrusherConfig,
+}
+
+#[pymethods]
+impl PySmartCrusherConfig {
+ #[new]
+ #[pyo3(signature = (
+ enabled = true,
+ min_items_to_analyze = 5,
+ min_tokens_to_crush = 200,
+ variance_threshold = 2.0,
+ uniqueness_threshold = 0.1,
+ similarity_threshold = 0.8,
+ max_items_after_crush = 15,
+ preserve_change_points = true,
+ factor_out_constants = false,
+ include_summaries = false,
+ use_feedback_hints = true,
+ toin_confidence_threshold = 0.5,
+ dedup_identical_items = true,
+ first_fraction = 0.3,
+ last_fraction = 0.15,
+ relevance_threshold = 0.3,
+ ))]
+ #[allow(clippy::too_many_arguments)]
+ fn new(
+ enabled: bool,
+ min_items_to_analyze: usize,
+ min_tokens_to_crush: usize,
+ variance_threshold: f64,
+ uniqueness_threshold: f64,
+ similarity_threshold: f64,
+ max_items_after_crush: usize,
+ preserve_change_points: bool,
+ factor_out_constants: bool,
+ include_summaries: bool,
+ use_feedback_hints: bool,
+ toin_confidence_threshold: f64,
+ dedup_identical_items: bool,
+ first_fraction: f64,
+ last_fraction: f64,
+ relevance_threshold: f64,
+ ) -> Self {
+ Self {
+ inner: RustSmartCrusherConfig {
+ enabled,
+ min_items_to_analyze,
+ min_tokens_to_crush,
+ variance_threshold,
+ uniqueness_threshold,
+ similarity_threshold,
+ max_items_after_crush,
+ preserve_change_points,
+ factor_out_constants,
+ include_summaries,
+ use_feedback_hints,
+ toin_confidence_threshold,
+ dedup_identical_items,
+ first_fraction,
+ last_fraction,
+ relevance_threshold,
+ },
+ }
+ }
+
+ #[getter]
+ fn enabled(&self) -> bool {
+ self.inner.enabled
+ }
+ #[getter]
+ fn min_items_to_analyze(&self) -> usize {
+ self.inner.min_items_to_analyze
+ }
+ #[getter]
+ fn min_tokens_to_crush(&self) -> usize {
+ self.inner.min_tokens_to_crush
+ }
+ #[getter]
+ fn variance_threshold(&self) -> f64 {
+ self.inner.variance_threshold
+ }
+ #[getter]
+ fn uniqueness_threshold(&self) -> f64 {
+ self.inner.uniqueness_threshold
+ }
+ #[getter]
+ fn similarity_threshold(&self) -> f64 {
+ self.inner.similarity_threshold
+ }
+ #[getter]
+ fn max_items_after_crush(&self) -> usize {
+ self.inner.max_items_after_crush
+ }
+ #[getter]
+ fn preserve_change_points(&self) -> bool {
+ self.inner.preserve_change_points
+ }
+ #[getter]
+ fn factor_out_constants(&self) -> bool {
+ self.inner.factor_out_constants
+ }
+ #[getter]
+ fn include_summaries(&self) -> bool {
+ self.inner.include_summaries
+ }
+ #[getter]
+ fn use_feedback_hints(&self) -> bool {
+ self.inner.use_feedback_hints
+ }
+ #[getter]
+ fn toin_confidence_threshold(&self) -> f64 {
+ self.inner.toin_confidence_threshold
+ }
+ #[getter]
+ fn dedup_identical_items(&self) -> bool {
+ self.inner.dedup_identical_items
+ }
+ #[getter]
+ fn first_fraction(&self) -> f64 {
+ self.inner.first_fraction
+ }
+ #[getter]
+ fn last_fraction(&self) -> f64 {
+ self.inner.last_fraction
+ }
+ #[getter]
+ fn relevance_threshold(&self) -> f64 {
+ self.inner.relevance_threshold
+ }
+
+ fn __repr__(&self) -> String {
+ format!(
+ "SmartCrusherConfig(enabled={}, min_items_to_analyze={}, \
+ min_tokens_to_crush={}, max_items_after_crush={}, \
+ relevance_threshold={})",
+ self.inner.enabled,
+ self.inner.min_items_to_analyze,
+ self.inner.min_tokens_to_crush,
+ self.inner.max_items_after_crush,
+ self.inner.relevance_threshold,
+ )
+ }
+}
+
+// ─── CrushResult ───────────────────────────────────────────────────────────
+
+/// Mirror of `headroom.transforms.smart_crusher.CrushResult`. Read-only;
+/// the Python shim builds its own dataclass instance from these
+/// attributes so callers that destructure with `asdict()` keep working.
+#[pyclass(name = "CrushResult", module = "headroom._core")]
+struct PyCrushResult {
+ inner: RustCrushResult,
+}
+
+#[pymethods]
+impl PyCrushResult {
+ #[getter]
+ fn compressed(&self) -> &str {
+ &self.inner.compressed
+ }
+ #[getter]
+ fn original(&self) -> &str {
+ &self.inner.original
+ }
+ #[getter]
+ fn was_modified(&self) -> bool {
+ self.inner.was_modified
+ }
+ #[getter]
+ fn strategy(&self) -> &str {
+ &self.inner.strategy
+ }
+
+ fn __repr__(&self) -> String {
+ format!(
+ "CrushResult(compressed=<{} chars>, was_modified={}, strategy={:?})",
+ self.inner.compressed.len(),
+ self.inner.was_modified,
+ self.inner.strategy,
+ )
+ }
+}
+
+// ─── SmartCrusher ──────────────────────────────────────────────────────────
+
+/// Mirror of `headroom.transforms.smart_crusher.SmartCrusher`.
+///
+/// Constructor accepts only `config` — Python's `relevance_config`,
+/// `scorer`, and `ccr_config` parameters are handled in the Python
+/// shim (Stage 3c.1 keeps the optional subsystems disabled in Rust;
+/// the shim drops those args to preserve call-site compatibility).
+#[pyclass(name = "SmartCrusher", module = "headroom._core")]
+struct PySmartCrusher {
+ inner: RustSmartCrusher,
+}
+
+#[pymethods]
+impl PySmartCrusher {
+ #[new]
+ #[pyo3(signature = (config = None))]
+ fn new(config: Option<&PySmartCrusherConfig>) -> Self {
+ let cfg = config
+ .map(|c| c.inner.clone())
+ .unwrap_or_default();
+ Self {
+ inner: RustSmartCrusher::new(cfg),
+ }
+ }
+
+ /// `crush(content, query="", bias=1.0) -> CrushResult`. Argument
+ /// order and keyword names mirror the Python implementation.
+ #[pyo3(signature = (content, query = "", bias = 1.0))]
+ fn crush(&self, content: &str, query: &str, bias: f64) -> PyCrushResult {
+ PyCrushResult {
+ inner: self.inner.crush(content, query, bias),
+ }
+ }
+
+ /// `smart_crush_content(content, query="", bias=1.0) -> (str, bool, str)`.
+ /// Mirrors Python's `_smart_crush_content` — used by
+ /// `smart_crush_tool_output` convenience function and direct
+ /// callers that want the tuple form.
+ #[pyo3(signature = (content, query = "", bias = 1.0))]
+ fn smart_crush_content(
+ &self,
+ content: &str,
+ query: &str,
+ bias: f64,
+ ) -> (String, bool, String) {
+ self.inner.smart_crush_content(content, query, bias)
+ }
+}
+
// ─── Module init ───────────────────────────────────────────────────────────
#[pymodule]
@@ -376,5 +622,8 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyDiffCompressionResult>()?;
m.add_class::<PyDiffCompressorStats>()?;
m.add_class::<PyDiffCompressor>()?;
+ m.add_class::<PySmartCrusherConfig>()?;
+ m.add_class::<PyCrushResult>()?;
+ m.add_class::<PySmartCrusher>()?;
Ok(())
}
diff --git a/tests/test_transforms/test_smart_crusher_rust_parity.py b/tests/test_transforms/test_smart_crusher_rust_parity.py
new file mode 100644
index 000000000..8e5a71276
--- /dev/null
+++ b/tests/test_transforms/test_smart_crusher_rust_parity.py
@@ -0,0 +1,87 @@
+"""Parity test: PyO3-backed `SmartCrusher` vs recorded fixtures.
+
+Stage 3c.1b verification — guards the PyO3 bridge against regressions
+by replaying every recorded fixture in
+`tests/parity/fixtures/smart_crusher/` through `headroom._core.SmartCrusher`
+and asserting the output matches the recording byte-for-byte.
+
+Twin of `test_diff_compressor_rust_parity.py`. The Rust side runs the
+same fixtures via `cargo run -p headroom-parity --bin parity-run --
+run --only smart_crusher`; this Python test specifically catches PyO3
+bridge regressions (input/output mistranslation) that the Rust-only
+binary cannot.
+
+Skipped automatically when the `headroom._core` wheel isn't installed
+(e.g. CI lane without the maturin step).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+
+def _has_core() -> bool:
+ try:
+ from headroom._core import SmartCrusher # noqa: F401
+
+ return True
+ except ImportError:
+ return False
+
+
+pytestmark = pytest.mark.skipif(
+ not _has_core(),
+ reason="headroom._core wheel not installed (run `scripts/build_rust_extension.sh`)",
+)
+
+
+_FIXTURES_DIR = Path(__file__).parent.parent / "parity" / "fixtures" / "smart_crusher"
+
+
+def _all_fixtures() -> list[Path]:
+ return sorted(_FIXTURES_DIR.glob("*.json"))
+
+
+def test_at_least_17_fixtures_present():
+ """Sanity check: the recorded fixture suite landed."""
+ fixtures = _all_fixtures()
+ assert len(fixtures) >= 17, (
+ f"expected >= 17 fixtures, found {len(fixtures)}. "
+ "If you re-recorded and got fewer, something deleted them."
+ )
+
+
[email protected]("fixture_path", _all_fixtures(), ids=lambda p: p.name)
+def test_rust_backend_matches_recorded_output(fixture_path: Path):
+ """Replay each recorded input through the PyO3 bridge; every output
+ field must match the recording. Any mismatch is a bridge bug or a
+ Rust regression — cross-check with `cargo run -p headroom-parity`.
+ """
+ from headroom._core import SmartCrusher, SmartCrusherConfig
+
+ fixture = json.loads(fixture_path.read_text())
+ inp = fixture["input"]
+ cfg_dict = fixture["config"]
+ expected = fixture["output"]
+
+ cfg = SmartCrusherConfig(**cfg_dict)
+ crusher = SmartCrusher(cfg)
+ actual = crusher.crush(inp["content"], inp["query"], inp["bias"])
+
+ assert actual.compressed == expected["compressed"], (
+ f"compressed bytes differ for {fixture_path.name}\n"
+ f" expected: {expected['compressed'][:120]!r}\n"
+ f" actual : {actual.compressed[:120]!r}"
+ )
+ assert actual.original == expected["original"], f"original bytes differ for {fixture_path.name}"
+ assert actual.was_modified == expected["was_modified"], (
+ f"was_modified differs for {fixture_path.name}: "
+ f"expected={expected['was_modified']} actual={actual.was_modified}"
+ )
+ assert actual.strategy == expected["strategy"], (
+ f"strategy differs for {fixture_path.name}: "
+ f"expected={expected['strategy']!r} actual={actual.strategy!r}"
+ )