Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): retire python diff_compressor, ship rust-only via pyo3
chopratejasf5f465418bbe
diff --git a/.gitignore b/.gitignoreindex 51e79e0db..af93df6fe 100644--- a/.gitignore+++ b/.gitignore@@ -14,6 +14,7 @@ scripts/*!scripts/fixtures/!scripts/fixtures/*.json!scripts/record_fixtures.py+!scripts/build_rust_extension.sh# Rust / Cargo build artifacts/target/@@ -223,3 +224,11 @@ headroom-managed/# uv lockfile: regenerated locally; not committeduv.lock++# Rust extension `.so` symlinks placed by `scripts/build_rust_extension.sh`+# into the `headroom/` package dir for local development. The real binary+# lives in `crates/headroom-py/python/headroom/`; this is the dev overlay+# that lets `import headroom._core` resolve when the source `headroom/`+# package shadows the maturin overlay on sys.path.+/headroom/_core.*.so+/headroom/_core.sodiff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rsindex 70a693c98..0e51e1ba9 100644--- a/crates/headroom-py/src/lib.rs+++ b/crates/headroom-py/src/lib.rs@@ -1,15 +1,380 @@//! PyO3 bindings for headroom-core. Exposed to Python as `headroom._core`.+//!+//! # Stage 3b — diff_compressor bridge+//!+//! The `DiffCompressor` family is exported here so the Python+//! `ContentRouter` can route to the Rust implementation in-process via+//! PyO3 instead of running the Python port. Backend selection happens in+//! `headroom.transforms._rust_diff_compressor.RustBackedDiffCompressor`,+//! which mirrors the Python `DiffCompressor` API one-for-one (so callers+//! don't notice the swap).+//!+//! Why in-process: ContentRouter compresses on the proxy's hot path. Any+//! IPC / subprocess / RPC bridge would dominate the cost we're trying to+//! save. PyO3 calls cost ~microseconds; staying in-process is ~free.+use std::collections::BTreeMap;++use headroom_core::transforms::{+ DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,+};use pyo3::prelude::*;-/// Return the identity string from headroom-core.+/// Identity stub used by the Python smoke test to verify linkage.#[pyfunction]fn hello() -> &'static str {headroom_core::hello()}+// ─── DiffCompressorConfig ──────────────────────────────────────────────────++/// Mirror of `headroom.transforms.diff_compressor.DiffCompressorConfig`.+/// Defaults match Python; constructor accepts every field as a kwarg with+/// the same name and type as the Python dataclass for drop-in+/// compatibility.+#[pyclass(name = "DiffCompressorConfig", module = "headroom._core")]+#[derive(Clone)]+struct PyDiffCompressorConfig {+ inner: DiffCompressorConfig,+}++#[pymethods]+impl PyDiffCompressorConfig {+ #[new]+ #[pyo3(signature = (+ max_context_lines = 2,+ max_hunks_per_file = 10,+ max_files = 20,+ always_keep_additions = true,+ always_keep_deletions = true,+ enable_ccr = true,+ min_lines_for_ccr = 50,+ min_compression_ratio_for_ccr = 0.8,+ ))]+ #[allow(clippy::too_many_arguments)]+ fn new(+ max_context_lines: usize,+ max_hunks_per_file: usize,+ max_files: usize,+ always_keep_additions: bool,+ always_keep_deletions: bool,+ enable_ccr: bool,+ min_lines_for_ccr: usize,+ min_compression_ratio_for_ccr: f64,+ ) -> Self {+ Self {+ inner: DiffCompressorConfig {+ max_context_lines,+ max_hunks_per_file,+ max_files,+ always_keep_additions,+ always_keep_deletions,+ enable_ccr,+ min_lines_for_ccr,+ min_compression_ratio_for_ccr,+ },+ }+ }++ // Read-only field accessors mirroring the Python dataclass surface.+ #[getter]+ fn max_context_lines(&self) -> usize {+ self.inner.max_context_lines+ }+ #[getter]+ fn max_hunks_per_file(&self) -> usize {+ self.inner.max_hunks_per_file+ }+ #[getter]+ fn max_files(&self) -> usize {+ self.inner.max_files+ }+ #[getter]+ fn always_keep_additions(&self) -> bool {+ self.inner.always_keep_additions+ }+ #[getter]+ fn always_keep_deletions(&self) -> bool {+ self.inner.always_keep_deletions+ }+ #[getter]+ fn enable_ccr(&self) -> bool {+ self.inner.enable_ccr+ }+ #[getter]+ fn min_lines_for_ccr(&self) -> usize {+ self.inner.min_lines_for_ccr+ }+ #[getter]+ fn min_compression_ratio_for_ccr(&self) -> f64 {+ self.inner.min_compression_ratio_for_ccr+ }++ fn __repr__(&self) -> String {+ format!(+ "DiffCompressorConfig(max_context_lines={}, max_hunks_per_file={}, max_files={}, \+ always_keep_additions={}, always_keep_deletions={}, enable_ccr={}, \+ min_lines_for_ccr={}, min_compression_ratio_for_ccr={})",+ self.inner.max_context_lines,+ self.inner.max_hunks_per_file,+ self.inner.max_files,+ self.inner.always_keep_additions,+ self.inner.always_keep_deletions,+ self.inner.enable_ccr,+ self.inner.min_lines_for_ccr,+ self.inner.min_compression_ratio_for_ccr,+ )+ }+}++// ─── DiffCompressionResult ─────────────────────────────────────────────────++/// Mirror of `headroom.transforms.diff_compressor.DiffCompressionResult`.+/// Read-only on the Python side: ContentRouter consumes fields, doesn't+/// mutate. `compression_ratio` and `tokens_saved_estimate` are exposed as+/// methods (not `@property`) — Python callers reach them via `.method()`.+/// The Python adapter wraps and re-exposes them as properties for full+/// dataclass compatibility.+#[pyclass(name = "DiffCompressionResult", module = "headroom._core")]+struct PyDiffCompressionResult {+ inner: DiffCompressionResult,+}++#[pymethods]+impl PyDiffCompressionResult {+ #[getter]+ fn compressed(&self) -> &str {+ &self.inner.compressed+ }+ #[getter]+ fn original_line_count(&self) -> usize {+ self.inner.original_line_count+ }+ #[getter]+ fn compressed_line_count(&self) -> usize {+ self.inner.compressed_line_count+ }+ #[getter]+ fn files_affected(&self) -> usize {+ self.inner.files_affected+ }+ #[getter]+ fn additions(&self) -> usize {+ self.inner.additions+ }+ #[getter]+ fn deletions(&self) -> usize {+ self.inner.deletions+ }+ #[getter]+ fn hunks_kept(&self) -> usize {+ self.inner.hunks_kept+ }+ #[getter]+ fn hunks_removed(&self) -> usize {+ self.inner.hunks_removed+ }+ #[getter]+ fn cache_key(&self) -> Option<String> {+ self.inner.cache_key.clone()+ }++ /// Mirror of Python `@property compression_ratio`. Returns+ /// `compressed_line_count / original_line_count` (1.0 if input was+ /// empty).+ fn compression_ratio(&self) -> f64 {+ if self.inner.original_line_count == 0 {+ 1.0+ } else {+ self.inner.compressed_line_count as f64 / self.inner.original_line_count as f64+ }+ }++ /// Mirror of Python `@property tokens_saved_estimate`. Same `chars *+ /// 40 / 4` heuristic; bytes-equivalent numeric result.+ fn tokens_saved_estimate(&self) -> usize {+ let saved = self+ .inner+ .original_line_count+ .saturating_sub(self.inner.compressed_line_count);+ (saved * 40) / 4+ }++ fn __repr__(&self) -> String {+ format!(+ "DiffCompressionResult(compressed=<{} chars>, original_line_count={}, \+ compressed_line_count={}, files_affected={}, additions={}, deletions={}, \+ hunks_kept={}, hunks_removed={}, cache_key={:?})",+ self.inner.compressed.len(),+ self.inner.original_line_count,+ self.inner.compressed_line_count,+ self.inner.files_affected,+ self.inner.additions,+ self.inner.deletions,+ self.inner.hunks_kept,+ self.inner.hunks_removed,+ self.inner.cache_key,+ )+ }+}++// ─── DiffCompressorStats ───────────────────────────────────────────────────++/// Mirror of Rust `DiffCompressorStats` — sidecar observability not+/// present in the Python dataclass. Returned only from `compress_with_stats`,+/// which the Python adapter exposes as a method on the wrapper. `Vec`s are+/// returned as Python lists; the `BTreeMap` becomes a `dict`.+#[pyclass(name = "DiffCompressorStats", module = "headroom._core")]+struct PyDiffCompressorStats {+ inner: DiffCompressorStats,+}++#[pymethods]+impl PyDiffCompressorStats {+ #[getter]+ fn input_lines(&self) -> usize {+ self.inner.input_lines+ }+ #[getter]+ fn output_lines(&self) -> usize {+ self.inner.output_lines+ }+ #[getter]+ fn compression_ratio(&self) -> f64 {+ self.inner.compression_ratio+ }+ #[getter]+ fn files_total(&self) -> usize {+ self.inner.files_total+ }+ #[getter]+ fn files_kept(&self) -> usize {+ self.inner.files_kept+ }+ #[getter]+ fn files_dropped(&self) -> Vec<String> {+ self.inner.files_dropped.clone()+ }+ #[getter]+ fn hunks_total(&self) -> usize {+ self.inner.hunks_total+ }+ #[getter]+ fn hunks_kept(&self) -> usize {+ self.inner.hunks_kept+ }+ #[getter]+ fn hunks_dropped(&self) -> usize {+ self.inner.hunks_dropped+ }+ #[getter]+ fn hunks_dropped_per_file(&self) -> BTreeMap<String, usize> {+ self.inner.hunks_dropped_per_file.clone()+ }+ #[getter]+ fn context_lines_input(&self) -> usize {+ self.inner.context_lines_input+ }+ #[getter]+ fn context_lines_kept(&self) -> usize {+ self.inner.context_lines_kept+ }+ #[getter]+ fn context_lines_trimmed(&self) -> usize {+ self.inner.context_lines_trimmed+ }+ #[getter]+ fn largest_hunk_kept_lines(&self) -> usize {+ self.inner.largest_hunk_kept_lines+ }+ #[getter]+ fn largest_hunk_dropped_lines(&self) -> usize {+ self.inner.largest_hunk_dropped_lines+ }+ #[getter]+ fn parse_warnings(&self) -> Vec<String> {+ self.inner.parse_warnings.clone()+ }+ #[getter]+ fn processing_duration_us(&self) -> u64 {+ self.inner.processing_duration_us+ }+ #[getter]+ fn cache_key_emitted(&self) -> bool {+ self.inner.cache_key_emitted+ }+ #[getter]+ fn ccr_skipped_reason(&self) -> Option<String> {+ self.inner.ccr_skipped_reason.clone()+ }+ #[getter]+ fn file_mode_normalizations(&self) -> Vec<(String, String)> {+ self.inner.file_mode_normalizations.clone()+ }+ #[getter]+ fn binary_files_simplified(&self) -> Vec<String> {+ self.inner.binary_files_simplified.clone()+ }+}++// ─── DiffCompressor ────────────────────────────────────────────────────────++/// Mirror of `headroom.transforms.diff_compressor.DiffCompressor`. The+/// Python adapter wraps this in `RustBackedDiffCompressor` so+/// `ContentRouter` can swap backends transparently.+#[pyclass(name = "DiffCompressor", module = "headroom._core")]+struct PyDiffCompressor {+ inner: DiffCompressor,+}++#[pymethods]+impl PyDiffCompressor {+ /// `__init__(config: DiffCompressorConfig | None = None)` — matches the+ /// Python constructor signature one-for-one.+ #[new]+ #[pyo3(signature = (config = None))]+ fn new(config: Option<&PyDiffCompressorConfig>) -> Self {+ let cfg = config.map(|c| c.inner.clone()).unwrap_or_default();+ Self {+ inner: DiffCompressor::new(cfg),+ }+ }++ /// `compress(content: str, context: str = "") -> DiffCompressionResult`.+ /// Argument order and keyword names match the Python implementation.+ #[pyo3(signature = (content, context = ""))]+ fn compress(&self, content: &str, context: &str) -> PyDiffCompressionResult {+ PyDiffCompressionResult {+ inner: self.inner.compress(content, context),+ }+ }++ /// `compress_with_stats(content, context="") -> (result, stats)`.+ /// Sidecar API not present in Python — exposes the Rust observability+ /// struct alongside the parity-equal result. Returned as a 2-tuple to+ /// keep the call site Pythonic.+ #[pyo3(signature = (content, context = ""))]+ fn compress_with_stats(+ &self,+ content: &str,+ context: &str,+ ) -> (PyDiffCompressionResult, PyDiffCompressorStats) {+ let (result, stats) = self.inner.compress_with_stats(content, context);+ (+ PyDiffCompressionResult { inner: result },+ PyDiffCompressorStats { inner: stats },+ )+ }+}++// ─── Module init ───────────────────────────────────────────────────────────+#[pymodule]fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {m.add_function(wrap_pyfunction!(hello, m)?)?;+ m.add_class::<PyDiffCompressorConfig>()?;+ m.add_class::<PyDiffCompressionResult>()?;+ m.add_class::<PyDiffCompressorStats>()?;+ m.add_class::<PyDiffCompressor>()?;Ok(())}diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 8c264fc7c..ff7278305 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -1197,14 +1197,13 @@ class ContentRouter(Transform):return self._log_compressordef _get_diff_compressor(self) -> Any:- """Get DiffCompressor (lazy load)."""+ """Get DiffCompressor (lazy load). Rust-only — Python implementation+ retired in Stage 3b. The wheel (`headroom._core`) is a hard import.+ """if self._diff_compressor is None:- try:- from .diff_compressor import DiffCompressor+ from .diff_compressor import DiffCompressor- self._diff_compressor = DiffCompressor()- except ImportError:- logger.debug("DiffCompressor not available")+ self._diff_compressor = DiffCompressor()return self._diff_compressordef _get_html_extractor(self) -> Any:diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.pyindex ba81902e0..7b0418f74 100644--- a/headroom/transforms/diff_compressor.py+++ b/headroom/transforms/diff_compressor.py@@ -1,108 +1,41 @@-"""Git diff output compressor for unified diff format.--This module compresses git diff output which can be very verbose with-many context lines. Typical compression: 3-10x.--Supported formats:-- Unified diff format (git diff, diff -u)-- Combined diff format (merge conflicts)--Compression Strategy:-1. Parse unified diff format into file sections and hunks-2. Always keep file headers (diff --git, ---, +++)-3. Always keep ALL actual changes (+/- lines)-4. Reduce context lines (` ` prefix) to configurable max-5. If too many hunks, keep first N and summarize rest-6. Add summary at end--Key Patterns to Preserve:-- All additions (+)-- All deletions (-)-- Hunk headers (@@ ... @@)-- File headers (diff --git, ---, +++)-- Context around changes (limited)+"""Git diff output compressor — Rust-backed via PyO3.++The Python implementation has been retired (Stage 3b, 2026-04-25). All+diff compression now goes through `headroom._core.DiffCompressor` (built+from `crates/headroom-py`). The byte-equality of the two implementations+was verified against 27 recorded fixtures before the Python source was+removed; the Rust crate has its own test coverage in `crates/headroom-core/`.++This module retains the public surface — `DiffCompressorConfig`,+`DiffCompressionResult`, `DiffCompressor` — so existing call sites+(ContentRouter, parity recorder, integrations, downstream users) keep+working unchanged. The dataclasses are still pure-Python because they+appear in dataclass-aware code paths (`asdict()`, `__dict__`, dataclass+matching). Only the `DiffCompressor` class delegates to Rust.++The `headroom._core` extension is a hard import: there is no Python+fallback. Build it locally with `scripts/build_rust_extension.sh`+(wraps `maturin develop`) or install a prebuilt wheel."""from __future__ import annotationsimport logging-import re-from dataclasses import dataclass, field+from dataclasses import dataclass+from typing import Anylogger = logging.getLogger(__name__)-@dataclass-class DiffHunk:- """A single hunk within a diff file."""-- header: str # @@ -start,count +start,count @@ optional function- lines: list[str] # All lines in the hunk- additions: int = 0- deletions: int = 0- context_lines: int = 0- score: float = 0.0 # Relevance score for context-aware compression-- @property- def change_count(self) -> int:- """Total number of actual changes (additions + deletions)."""- return self.additions + self.deletions---@dataclass-class DiffFile:- """A single file's diff."""-- header: str # diff --git a/... b/...- old_file: str # --- a/...- new_file: str # +++ b/...- hunks: list[DiffHunk] = field(default_factory=list)- is_binary: bool = False- is_new_file: bool = False- is_deleted_file: bool = False- is_renamed: bool = False- # Bug-fix: rename / copy / similarity / dissimilarity marker lines that- # were emitted by git BETWEEN `diff --git` and `--- a/` (e.g. `rename- # from old.py`, `rename to new.py`, `similarity index 95%`). Previously- # we set is_renamed=True and discarded the lines, so the output looked- # like a plain modification. Now captured verbatim and re-emitted in- # `_format_output`.- rename_lines: list[str] = field(default_factory=list)- # Bug-fix: original `new file mode <NNNNNN>` / `deleted file mode- # <NNNNNN>` / `Binary files X and Y differ` lines. Emit normalizes to- # `100644` / bare `Binary files differ`; capturing the originals lets- # the caller observe the loss via logging.- original_new_file_mode_line: str | None = None- original_deleted_file_mode_line: str | None = None- original_binary_line: str | None = None-- @property- def total_additions(self) -> int:- return sum(h.additions for h in self.hunks)-- @property- def total_deletions(self) -> int:- return sum(h.deletions for h in self.hunks)--@dataclassclass DiffCompressorConfig:"""Configuration for diff compression."""- # Context line limits- max_context_lines: int = 2 # Reduce from default 3 lines before/after changes-- # Hunk limits+ max_context_lines: int = 2max_hunks_per_file: int = 10-- # File limitsmax_files: int = 20-- # Change preservation- always_keep_additions: bool = True # Always keep + lines- always_keep_deletions: bool = True # Always keep - lines-- # CCR integration+ always_keep_additions: bool = True+ always_keep_deletions: bool = Trueenable_ccr: bool = Truemin_lines_for_ccr: int = 50@@ -123,666 +56,85 @@ class DiffCompressionResult:@propertydef compression_ratio(self) -> float:- """Ratio of compressed to original (lower is better compression)."""if self.original_line_count == 0:return 1.0return self.compressed_line_count / self.original_line_count@propertydef tokens_saved_estimate(self) -> int:- """Estimate tokens saved (rough: 1 token per 4 chars)."""- # Use line counts as proxy for charslines_saved = self.original_line_count - self.compressed_line_count- # Estimate ~40 chars per line average for diffschars_saved = lines_saved * 40return max(0, chars_saved // 4)class DiffCompressor:- """Compresses git diff output.+ """Rust-backed `DiffCompressor` (via PyO3 / `headroom._core`).- Example:- >>> compressor = DiffCompressor()- >>> result = compressor.compress(git_diff_output)- >>> print(result.compressed) # Reduced diff with summary+ Same `__init__` and `compress` shape as the retired Python class —+ drop-in replacement. Returns Python `DiffCompressionResult` dataclass+ instances so call sites that destructure with `asdict()` or read the+ `@property` fields work unchanged."""- # Pattern for `diff --git a/X b/Y` header (regular diff).- _DIFF_GIT_PATTERN = re.compile(r"^diff --git a/(.+) b/(.+)$")-- # Bug-fix (2026-04-25): merge-commit headers — `diff --combined <path>`- # and `diff --cc <path>`. Both produce a single-path file diff with- # combined-diff hunk syntax (`@@@`+). Previously the parser only- # recognized `diff --git`, so merge-commit diffs from `git log -p`- # of merges fell through to "no files parsed" and were passed- # through unchanged — even though ContentRouter had routed them here- # because `--- a/` triggered the detector.- _DIFF_COMBINED_PATTERN = re.compile(r"^diff --combined (.+)$")- _DIFF_CC_PATTERN = re.compile(r"^diff --cc (.+)$")-- # Pattern for --- a/file or --- /dev/null- _OLD_FILE_PATTERN = re.compile(r"^--- (a/(.+)|/dev/null)$")-- # Pattern for +++ b/file or +++ /dev/null- _NEW_FILE_PATTERN = re.compile(r"^\+\+\+ (b/(.+)|/dev/null)$")-- # Pattern for ANY hunk header — matches both regular `@@ -A,B +C,D @@`- # and combined-diff `@@@ -A,B -C,D +E,F @@@` (and 4-way `@@@@ ... @@@@`).- # Group 1 is the @-prefix (so closing @@ can backreference). Bug-fix:- # previously hardcoded to `@@`, which silently dropped all content from- # combined-diff hunks (merge commits) — `current_hunk` was never set so- # subsequent +/- lines fell through to the no-op branch.- _HUNK_HEADER_PATTERN = re.compile(r"^(@@+) (?:-\d+(?:,\d+)? )+\+\d+(?:,\d+)? \1(.*)$")-- # Used to extract the new-file starting line number for in-order resort- # after middle-hunk selection. Works for both regular and combined diffs.- _HUNK_NEW_RANGE_PATTERN = re.compile(r"\+(\d+)")-- # Pattern for binary file indication- _BINARY_PATTERN = re.compile(r"^Binary files .+ differ$")-- # Patterns for new/deleted file mode- _NEW_FILE_MODE_PATTERN = re.compile(r"^new file mode")- _DELETED_FILE_MODE_PATTERN = re.compile(r"^deleted file mode")- # Bug-fix: extended to include `dissimilarity` (low-similarity rename- # marker, real git output). Previously dropped silently.- _RENAME_PATTERN = re.compile(r"^(rename|similarity|copy|dissimilarity) ")-- # Priority patterns for context-aware hunk selection (centralized)- from headroom.transforms.error_detection import PRIORITY_PATTERNS_DIFF-- _PRIORITY_PATTERNS = PRIORITY_PATTERNS_DIFF-def __init__(self, config: DiffCompressorConfig | None = None):- """Initialize diff compressor.-- Args:- config: Compression configuration.- """- self.config = config or DiffCompressorConfig()-- def compress(self, content: str, context: str = "") -> DiffCompressionResult:- """Compress diff output.-- Args:- content: Raw git diff output.- context: User query context for relevance scoring.-- Returns:- DiffCompressionResult with compressed output and metadata.- """- lines = content.split("\n")- original_line_count = len(lines)-- if original_line_count < self.config.min_lines_for_ccr:- return DiffCompressionResult(- compressed=content,- original_line_count=original_line_count,- compressed_line_count=original_line_count,- files_affected=0,- additions=0,- deletions=0,- hunks_kept=0,- hunks_removed=0,- )-- # Parse diff into structured format. Returns pre-diff content- # (commit headers, email headers from `git format-patch`, etc.)- # alongside the parsed files. Bug-fix: previously this content was- # silently dropped; now it's preserved verbatim before the- # compressed file sections in the output.- pre_diff_lines, diff_files = self._parse_diff(lines)+ # Hard import — no fallback. If the wheel is missing, the user+ # must build it (scripts/build_rust_extension.sh) or install a+ # prebuilt one. Failing loudly here is better than silently+ # degrading; see feedback memory `feedback_no_silent_fallbacks.md`.+ from headroom._core import (+ DiffCompressor as _RustDiffCompressor,+ )+ from headroom._core import (+ DiffCompressorConfig as _RustDiffCompressorConfig,+ )- if not diff_files:- return DiffCompressionResult(- compressed=content,- original_line_count=original_line_count,- compressed_line_count=original_line_count,- files_affected=0,- additions=0,- deletions=0,- hunks_kept=0,- hunks_removed=0,+ cfg = config or DiffCompressorConfig()+ self.config = cfg+ self._rust = _RustDiffCompressor(+ _RustDiffCompressorConfig(+ max_context_lines=cfg.max_context_lines,+ max_hunks_per_file=cfg.max_hunks_per_file,+ max_files=cfg.max_files,+ always_keep_additions=cfg.always_keep_additions,+ always_keep_deletions=cfg.always_keep_deletions,+ enable_ccr=cfg.enable_ccr,+ min_lines_for_ccr=cfg.min_lines_for_ccr,)+ )- # Score hunks by relevance- self._score_hunks(diff_files, context)-- # Compress each file's hunks- compressed_files, stats = self._compress_files(diff_files)-- # Format output, prepending pre-diff content if any.- compressed_output = self._format_output(compressed_files, stats)- if pre_diff_lines:- compressed_output = "\n".join(pre_diff_lines) + "\n" + compressed_output-- # Observability: log lossy-emit signals so prod monitoring can- # alert on outlier diffs. Counts only — no PII / file content.- self._log_loss_signals(diff_files, stats)-- compressed_line_count = len(compressed_output.split("\n"))-- # Store in CCR if significant compression- cache_key = None- if self.config.enable_ccr and compressed_line_count < original_line_count * 0.8:- cache_key = self._store_in_ccr(content, compressed_output, original_line_count)- if cache_key:- compressed_output += f"\n[{original_line_count} lines compressed to {compressed_line_count}. Retrieve full diff: hash={cache_key}]"-+ def compress(self, content: str, context: str = "") -> DiffCompressionResult:+ r = self._rust.compress(content, context)return DiffCompressionResult(- compressed=compressed_output,- original_line_count=original_line_count,- compressed_line_count=compressed_line_count,- files_affected=stats["files_affected"],- additions=stats["total_additions"],- deletions=stats["total_deletions"],- hunks_kept=stats["hunks_kept"],- hunks_removed=stats["hunks_removed"],- cache_key=cache_key,+ compressed=r.compressed,+ original_line_count=r.original_line_count,+ compressed_line_count=r.compressed_line_count,+ files_affected=r.files_affected,+ additions=r.additions,+ deletions=r.deletions,+ hunks_kept=r.hunks_kept,+ hunks_removed=r.hunks_removed,+ cache_key=r.cache_key,)- def _parse_diff(self, lines: list[str]) -> tuple[list[str], list[DiffFile]]:- """Parse diff content into structured format.-- Args:- lines: Lines of diff content.-- Returns:- Tuple of (pre_diff_lines, diff_files). `pre_diff_lines` are any- lines before the first ``diff --git`` (commit headers from- ``git log -p``, email headers from ``git format-patch``, etc.) —- previously dropped, now preserved verbatim and re-emitted.- """- diff_files: list[DiffFile] = []- current_file: DiffFile | None = None- current_hunk: DiffHunk | None = None- pre_diff_lines: list[str] = []- i = 0-- while i < len(lines):- line = lines[i]-- # Check for diff --git header (new file section). Bug-fix:- # `diff --combined <path>` and `diff --cc <path>` (merge- # commits) also start a new file section — single-path,- # combined-diff hunk syntax. Treat them the same as- # `diff --git` for sectioning purposes; the path goes in- # `header` verbatim.- if (- self._DIFF_GIT_PATTERN.match(line)- or self._DIFF_COMBINED_PATTERN.match(line)- or self._DIFF_CC_PATTERN.match(line)- ):- # Save previous hunk and file- if current_hunk and current_file:- current_file.hunks.append(current_hunk)- if current_file:- diff_files.append(current_file)-- current_file = DiffFile(- header=line,- old_file="",- new_file="",- )- current_hunk = None- i += 1- continue-- # Bug-fix: any line before the first `diff --git` is pre-diff- # content (commit headers, email headers). Capture for verbatim- # re-emission rather than dropping.- if current_file is None:- pre_diff_lines.append(line)- i += 1- continue-- # Check for file mode indicators / rename / binary markers.- # Capture the original line in addition to the boolean so the- # logger / sidecar observability can surface emit-time- # normalization losses.- if current_file:- if self._NEW_FILE_MODE_PATTERN.match(line):- current_file.is_new_file = True- current_file.original_new_file_mode_line = line- elif self._DELETED_FILE_MODE_PATTERN.match(line):- current_file.is_deleted_file = True- current_file.original_deleted_file_mode_line = line- elif self._RENAME_PATTERN.match(line):- current_file.is_renamed = True- # Bug-fix: capture rename marker lines so they get- # re-emitted. Previously the boolean was set but the- # actual `rename from` / `rename to` / `similarity index`- # lines were discarded — output looked like a plain- # modification and the LLM had no way to know a file- # was renamed.- current_file.rename_lines.append(line)- elif self._BINARY_PATTERN.match(line):- current_file.is_binary = True- current_file.original_binary_line = line-- # Check for --- a/file- if self._OLD_FILE_PATTERN.match(line):- if current_file:- current_file.old_file = line- i += 1- continue-- # Check for +++ b/file- if self._NEW_FILE_PATTERN.match(line):- if current_file:- current_file.new_file = line- i += 1- continue-- # Check for hunk header (regular `@@` or combined-diff `@@@`+).- # Bug-fix: previously `@@@` headers didn't match, so combined- # diffs (merge commits) had ALL their content silently dropped.- if self._HUNK_HEADER_PATTERN.match(line):- # Save previous hunk- if current_hunk and current_file:- current_file.hunks.append(current_hunk)-- current_hunk = DiffHunk(- header=line,- lines=[],- )- i += 1- continue-- # Process hunk content lines- if current_hunk is not None:- if line.startswith("+") and not line.startswith("+++"):- current_hunk.additions += 1- current_hunk.lines.append(line)- elif line.startswith("-") and not line.startswith("---"):- current_hunk.deletions += 1- current_hunk.lines.append(line)- elif line.startswith(" ") or line == "":- current_hunk.context_lines += 1- current_hunk.lines.append(line)- else:- # Other line (e.g., "\ No newline at end of file" — note- # leading backslash). Preserved here; `_reduce_context`- # force-keeps `\` lines regardless of distance from- # changes, so they survive the context trim.- current_hunk.lines.append(line)-- i += 1-- # Save final hunk and file- if current_hunk and current_file:- current_file.hunks.append(current_hunk)- if current_file:- diff_files.append(current_file)-- return pre_diff_lines, diff_files-- def _score_hunks(self, diff_files: list[DiffFile], context: str) -> None:- """Score hunks by relevance to context.-- Args:- diff_files: Parsed diff files.- context: User query context.+ def compress_with_stats(+ self, content: str, context: str = ""+ ) -> tuple[DiffCompressionResult, Any]:+ """Sidecar API exposing the Rust-only `DiffCompressorStats` struct+ (per-file hunk drops, context lines trimmed, file_mode normalizations,+ etc.) alongside the result. Stats is the raw PyO3 wrapper — no+ Python equivalent to mirror to. Typed as `Any` because the PyO3+ class has no Python type stub."""- context_lower = context.lower()- context_words = set(context_lower.split()) if context else set()-- for diff_file in diff_files:- for hunk in diff_file.hunks:- score = 0.0-- # Base score from change count (more changes = more important)- score += min(0.3, hunk.change_count * 0.03)-- hunk_content = "\n".join(hunk.lines).lower()-- # Score by context word overlap- for word in context_words:- if len(word) > 2 and word in hunk_content:- score += 0.2-- # Boost for priority patterns- for pattern in self._PRIORITY_PATTERNS:- if pattern.search(hunk_content):- score += 0.3- break-- hunk.score = min(1.0, score)-- def _compress_files(self, diff_files: list[DiffFile]) -> tuple[list[DiffFile], dict[str, int]]:- """Compress hunks in each file.-- Args:- diff_files: Parsed diff files.-- Returns:- Tuple of (compressed files, stats dict).- """- stats = {- "files_affected": 0,- "total_additions": 0,- "total_deletions": 0,- "hunks_kept": 0,- "hunks_removed": 0,- }-- # Limit files if too many- if len(diff_files) > self.config.max_files:- # Sort by total changes (most changes first)- diff_files = sorted(- diff_files,- key=lambda f: f.total_additions + f.total_deletions,- reverse=True,- )- diff_files = diff_files[: self.config.max_files]-- compressed_files: list[DiffFile] = []-- for diff_file in diff_files:- stats["files_affected"] += 1- stats["total_additions"] += diff_file.total_additions- stats["total_deletions"] += diff_file.total_deletions-- # Compress hunks within file- compressed_hunks = self._compress_hunks(diff_file.hunks)-- stats["hunks_kept"] += len(compressed_hunks)- stats["hunks_removed"] += len(diff_file.hunks) - len(compressed_hunks)-- # Create compressed file with reduced context in hunks. Bug-fix:- # previously this constructor dropped `rename_lines` and the- # `original_*_line` fields by omission — they were captured in- # the parser but never propagated to `_format_output`, so the- # emit looked exactly like the buggy old behavior. Carry them- # all through.- new_file = DiffFile(- header=diff_file.header,- old_file=diff_file.old_file,- new_file=diff_file.new_file,- hunks=compressed_hunks,- is_binary=diff_file.is_binary,- is_new_file=diff_file.is_new_file,- is_deleted_file=diff_file.is_deleted_file,- is_renamed=diff_file.is_renamed,- rename_lines=diff_file.rename_lines,- original_new_file_mode_line=diff_file.original_new_file_mode_line,- original_deleted_file_mode_line=diff_file.original_deleted_file_mode_line,- original_binary_line=diff_file.original_binary_line,- )- compressed_files.append(new_file)-- return compressed_files, stats-- def _compress_hunks(self, hunks: list[DiffHunk]) -> list[DiffHunk]:- """Compress hunks by reducing context and limiting count.-- Args:- hunks: List of hunks to compress.-- Returns:- Compressed list of hunks.- """- if not hunks:- return []-- # Sort by score if we need to limit- if len(hunks) > self.config.max_hunks_per_file:- # Keep first and last hunks (often important)- first_hunk = hunks[0]- last_hunk = hunks[-1] if len(hunks) > 1 else None-- # Sort middle hunks by score- middle_hunks = sorted(- hunks[1:-1] if last_hunk else [], key=lambda h: h.score, reverse=True- )-- # Take top scoring middle hunks- remaining_slots = (- self.config.max_hunks_per_file - 2- if last_hunk- else self.config.max_hunks_per_file - 1- )- selected_middle = middle_hunks[:remaining_slots]-- # Rebuild list in original order by re-sorting by appearance- selected = [first_hunk] + selected_middle- if last_hunk:- selected.append(last_hunk)-- # Sort back to original order (using header line numbers as proxy)- hunks = sorted(selected, key=lambda h: self._extract_line_number(h.header))-- # Reduce context in each hunk- compressed_hunks = []- for hunk in hunks:- compressed_hunk = self._reduce_context(hunk)- compressed_hunks.append(compressed_hunk)-- return compressed_hunks-- def _extract_line_number(self, header: str) -> int:- """Extract starting `+`-side line number from hunk header for sorting.-- Works for both regular ``@@ -A,B +C,D @@`` and combined-diff- ``@@@ -A,B -C,D +E,F @@@`` formats — finds the first ``+N`` token.- """- match = self._HUNK_NEW_RANGE_PATTERN.search(header)- if match:- return int(match.group(1))- return 0-- def _reduce_context(self, hunk: DiffHunk) -> DiffHunk:- """Reduce context lines while preserving all changes.-- Args:- hunk: Hunk to reduce context in.-- Returns:- New hunk with reduced context.- """- max_context = self.config.max_context_lines-- # Identify change positions- change_positions: list[int] = []- for i, line in enumerate(hunk.lines):- if line.startswith("+") or line.startswith("-"):- change_positions.append(i)-- if not change_positions:- # No changes, just context - keep minimal- return DiffHunk(- header=hunk.header,- lines=hunk.lines[:max_context] if hunk.lines else [],- additions=0,- deletions=0,- context_lines=min(len(hunk.lines), max_context),- score=hunk.score,- )-- # Determine which lines to keep- keep_indices: set[int] = set()-- for pos in change_positions:- # Always keep the change line- keep_indices.add(pos)-- # Keep context before- for i in range(max(0, pos - max_context), pos):- keep_indices.add(i)-- # Keep context after- for i in range(pos + 1, min(len(hunk.lines), pos + max_context + 1)):- keep_indices.add(i)-- # Bug-fix: ALWAYS keep `\ No newline at end of file` markers (and- # any other backslash-prefixed metadata) regardless of distance- # from a change. These are structural patch markers, not context —- # losing them breaks round-trippable patches and changes the- # semantic meaning of the trailing line in the file.- for i, line in enumerate(hunk.lines):- if line.startswith("\\"):- keep_indices.add(i)-- # Build new lines list- new_lines: list[str] = []- additions = 0- deletions = 0- context_lines = 0-- for i in sorted(keep_indices):- line = hunk.lines[i]- new_lines.append(line)- if line.startswith("+"):- additions += 1- elif line.startswith("-"):- deletions += 1- else:- context_lines += 1-- return DiffHunk(- header=hunk.header,- lines=new_lines,- additions=additions,- deletions=deletions,- context_lines=context_lines,- score=hunk.score,+ r, stats = self._rust.compress_with_stats(content, context)+ result = DiffCompressionResult(+ compressed=r.compressed,+ original_line_count=r.original_line_count,+ compressed_line_count=r.compressed_line_count,+ files_affected=r.files_affected,+ additions=r.additions,+ deletions=r.deletions,+ hunks_kept=r.hunks_kept,+ hunks_removed=r.hunks_removed,+ cache_key=r.cache_key,)-- def _format_output(self, diff_files: list[DiffFile], stats: dict[str, int]) -> str:- """Format compressed diff files back to unified diff format.-- Args:- diff_files: Compressed diff files.- stats: Compression statistics.-- Returns:- Formatted diff string.- """- output_lines: list[str] = []-- for diff_file in diff_files:- # File header- output_lines.append(diff_file.header)-- # Bug-fix: emit rename / similarity / dissimilarity / copy- # marker lines immediately after `diff --git`, matching git's- # canonical output order. Previously these were captured as- # `is_renamed=True` and dropped — output looked like a plain- # modification of the old file's path.- if diff_file.rename_lines:- output_lines.extend(diff_file.rename_lines)-- # File mode indicators if present. Note: parity-bound- # normalization to `100644` (the original mode is captured in- # `original_new_file_mode_line` for observability).- if diff_file.is_new_file:- output_lines.append("new file mode 100644")- elif diff_file.is_deleted_file:- output_lines.append("deleted file mode 100644")-- if diff_file.is_binary:- output_lines.append("Binary files differ")- continue-- # Old/new file markers- if diff_file.old_file:- output_lines.append(diff_file.old_file)- if diff_file.new_file:- output_lines.append(diff_file.new_file)-- # Hunks- for hunk in diff_file.hunks:- output_lines.append(hunk.header)- output_lines.extend(hunk.lines)-- # Add summary- if stats["hunks_removed"] > 0 or stats["files_affected"] > 0:- summary_parts = [- f"{stats['files_affected']} files changed",- f"+{stats['total_additions']} -{stats['total_deletions']} lines",- ]- if stats["hunks_removed"] > 0:- summary_parts.append(f"{stats['hunks_removed']} hunks omitted")- output_lines.append(f"[{', '.join(summary_parts)}]")-- return "\n".join(output_lines)-- def _log_loss_signals(- self,- diff_files: list[DiffFile],- stats: dict[str, int],- ) -> None:- """Surface lossy-emit signals to the logger.-- Some normalizations are parity-bound and silent in the compressed- output: file modes flatten to ``100644``, ``Binary files X and Y- differ`` simplifies to ``Binary files differ``, the ``max_files`` /- ``max_hunks_per_file`` caps drop entire files / middle hunks. Stats- about caps live in the result; emit-time normalizations live only- here. Logs let prod monitoring alert on outlier diffs without- changing the compressed bytes.- """- mode_normalizations: list[str] = []- binary_simplifications: list[str] = []- for f in diff_files:- if (- f.original_new_file_mode_line- and f.original_new_file_mode_line != "new file mode 100644"- ):- mode_normalizations.append(- f"{f.old_file} -> {f.new_file}: {f.original_new_file_mode_line}"- )- if (- f.original_deleted_file_mode_line- and f.original_deleted_file_mode_line != "deleted file mode 100644"- ):- mode_normalizations.append(- f"{f.old_file} -> {f.new_file}: {f.original_deleted_file_mode_line}"- )- if f.original_binary_line and f.original_binary_line != "Binary files differ":- binary_simplifications.append(f.original_binary_line)-- if mode_normalizations:- logger.warning(- "DiffCompressor: %d file mode line(s) normalized to 100644 — "- "executable / symlink / submodule signals lost: %s",- len(mode_normalizations),- mode_normalizations,- )- if binary_simplifications:- logger.warning(- "DiffCompressor: %d binary file detail line(s) simplified — filenames lost: %s",- len(binary_simplifications),- binary_simplifications,- )- if stats.get("hunks_removed", 0) > 0:- logger.info(- "DiffCompressor: dropped %d hunks across %d files",- stats["hunks_removed"],- stats.get("files_affected", 0),- )-- def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:- """Store original in CCR for later retrieval.-- Args:- original: Original diff content.- compressed: Compressed diff content.- original_count: Original line count.-- Returns:- Cache key if stored, None otherwise.- """- try:- from ..cache.compression_store import get_compression_store-- store = get_compression_store()- return store.store(- original,- compressed,- original_item_count=original_count,- )- except ImportError:- return None- except Exception:- return None+ return result, statsdiff --git a/scripts/build_rust_extension.sh b/scripts/build_rust_extension.shnew file mode 100755index 000000000..0e678540c--- /dev/null+++ b/scripts/build_rust_extension.sh@@ -0,0 +1,45 @@+#!/usr/bin/env bash+# Build the Rust → Python extension (headroom._core) and link it into the+# in-tree `headroom/` package so `import headroom._core` resolves.+#+# Why a wrapper script: `maturin develop` builds the `.so` and installs it+# into the venv's site-packages, but the in-tree `headroom/` source+# directory (loaded via `pip install -e .`) shadows that on sys.path.+# Python finds `headroom/__init__.py` at the project root before reaching+# the maturin overlay, so `import headroom._core` fails. Symlinking the+# built `.so` into `headroom/` fixes the lookup with zero copies.+#+# Idempotent. Safe to run repeatedly. Requires `maturin` in PATH (i.e.+# inside the project venv).++set -euo pipefail++cd "$(dirname "$0")/.."++if ! command -v maturin >/dev/null 2>&1; then+ echo "error: maturin not found. Activate the venv first:" >&2+ echo " source .venv/bin/activate" >&2+ exit 1+fi++# Build the wheel + install into the venv site-packages.+maturin develop -m crates/headroom-py/Cargo.toml++# Locate the built `.so`. `maturin develop` writes it under+# `crates/headroom-py/python/headroom/_core.cpython-<ver>-<platform>.so`.+SO_FILE=$(find crates/headroom-py/python/headroom -maxdepth 1 \+ -name "_core.cpython-*.so" -o -name "_core.cpython-*.dylib" -o -name "_core.pyd" \+ 2>/dev/null | head -1)++if [[ -z "$SO_FILE" ]]; then+ echo "error: maturin develop succeeded but produced no _core.* binary." >&2+ exit 1+fi++# Symlink into the in-tree package dir.+LINK_NAME="headroom/$(basename "$SO_FILE")"+ln -sf "$(pwd)/$SO_FILE" "$LINK_NAME"+echo "linked: $LINK_NAME -> $SO_FILE"++# Smoke-test the import to fail loudly if anything is misconfigured.+python -c "from headroom._core import DiffCompressor; print('headroom._core OK:', DiffCompressor)"diff --git a/tests/test_transforms/test_diff_compressor.py b/tests/test_transforms/test_diff_compressor.pyindex 5be31cd14..522434767 100644--- a/tests/test_transforms/test_diff_compressor.py+++ b/tests/test_transforms/test_diff_compressor.py@@ -1,142 +1,29 @@-"""Comprehensive tests for diff_compressor.py.+"""Comprehensive tests for the public DiffCompressor API.Tests cover:-1. Parsing of unified diff format-2. Context line reduction-3. Hunk selection and limiting-4. Compression ratios-5. Edge cases+1. Context line reduction+2. Hunk selection and limiting+3. Compression ratios+4. Edge cases+5. Bug-fix regressions and routing-gap fixtures++Stage 3b note (2026-04-25): the Python `DiffCompressor` implementation+was retired in favor of the Rust-backed shim (`headroom._core` via PyO3).+Tests that probed Python-only internals — `_parse_diff`, `_score_hunks`,+the `DiffHunk` / `DiffFile` parser dataclasses — were removed because+the Rust crate has its own parallel coverage in+`crates/headroom-core/tests`. Public-API tests (anything calling+`compressor.compress(...)`) are preserved unchanged: they exercise the+Rust backend through the same import path and assert the same outputs."""from headroom.transforms.diff_compressor import (DiffCompressionResult,DiffCompressor,DiffCompressorConfig,- DiffFile,- DiffHunk,)-class TestDiffParsing:- """Tests for parsing unified diff format."""-- def test_parse_simple_diff(self):- """Simple single-file diff is parsed correctly."""- content = """diff --git a/src/main.py b/src/main.py---- a/src/main.py-+++ b/src/main.py-@@ -10,6 +10,7 @@ def main():- print("hello")-+ print("world")- return 0-"""- compressor = DiffCompressor()- _, diff_files = compressor._parse_diff(content.split("\n"))-- assert len(diff_files) == 1- assert diff_files[0].header == "diff --git a/src/main.py b/src/main.py"- assert diff_files[0].old_file == "--- a/src/main.py"- assert diff_files[0].new_file == "+++ b/src/main.py"- assert len(diff_files[0].hunks) == 1- assert diff_files[0].hunks[0].additions == 1- assert diff_files[0].hunks[0].deletions == 0-- def test_parse_multi_file_diff(self):- """Multi-file diff is parsed into separate DiffFile objects."""- content = """diff --git a/file1.py b/file1.py---- a/file1.py-+++ b/file1.py-@@ -1,3 +1,4 @@- line1-+added line- line2-diff --git a/file2.py b/file2.py---- a/file2.py-+++ b/file2.py-@@ -5,4 +5,3 @@- keep--removed- keep2-"""- compressor = DiffCompressor()- _, diff_files = compressor._parse_diff(content.split("\n"))-- assert len(diff_files) == 2- assert "file1.py" in diff_files[0].header- assert "file2.py" in diff_files[1].header- assert diff_files[0].hunks[0].additions == 1- assert diff_files[0].hunks[0].deletions == 0- assert diff_files[1].hunks[0].additions == 0- assert diff_files[1].hunks[0].deletions == 1-- def test_parse_multi_hunk_file(self):- """File with multiple hunks is parsed correctly."""- content = """diff --git a/src/utils.py b/src/utils.py---- a/src/utils.py-+++ b/src/utils.py-@@ -10,4 +10,5 @@ def helper():- pass-+ # added comment- return True-@@ -50,3 +51,4 @@ def other():- x = 1-+ y = 2- return x-"""- compressor = DiffCompressor()- _, diff_files = compressor._parse_diff(content.split("\n"))-- assert len(diff_files) == 1- assert len(diff_files[0].hunks) == 2- assert diff_files[0].total_additions == 2-- def test_parse_new_file(self):- """New file diff is detected."""- content = """diff --git a/newfile.py b/newfile.py-new file mode 100644---- /dev/null-+++ b/newfile.py-@@ -0,0 +1,3 @@-+def new_func():-+ pass-+ return None-"""- compressor = DiffCompressor()- _, diff_files = compressor._parse_diff(content.split("\n"))-- assert len(diff_files) == 1- assert diff_files[0].is_new_file is True- assert diff_files[0].hunks[0].additions == 3-- def test_parse_deleted_file(self):- """Deleted file diff is detected."""- content = """diff --git a/oldfile.py b/oldfile.py-deleted file mode 100644---- a/oldfile.py-+++ /dev/null-@@ -1,2 +0,0 @@--def old_func():-- pass-"""- compressor = DiffCompressor()- _, diff_files = compressor._parse_diff(content.split("\n"))-- assert len(diff_files) == 1- assert diff_files[0].is_deleted_file is True- assert diff_files[0].hunks[0].deletions == 2-- def test_parse_binary_file(self):- """Binary file diff is detected."""- content = """diff --git a/image.png b/image.png-Binary files a/image.png and b/image.png differ-"""- compressor = DiffCompressor()- _, diff_files = compressor._parse_diff(content.split("\n"))-- assert len(diff_files) == 1- assert diff_files[0].is_binary is True--class TestContextReduction:"""Tests for context line reduction."""@@ -336,58 +223,6 @@ class TestCompressionResult:assert result.tokens_saved_estimate == 900-class TestHunkScoring:- """Tests for context-aware hunk scoring."""-- def test_score_by_context_keywords(self):- """Hunks containing context keywords get higher scores."""- content = """diff --git a/file.py b/file.py---- a/file.py-+++ b/file.py-@@ -1,3 +1,4 @@- normal context-+normal change- more context-@@ -10,3 +11,4 @@- error handling-+fix the bug here- return result-"""- compressor = DiffCompressor()- _, diff_files = compressor._parse_diff(content.split("\n"))- compressor._score_hunks(diff_files, "fix error bug")-- # Second hunk should have higher score (contains "fix" and "bug")- assert len(diff_files[0].hunks) == 2- assert diff_files[0].hunks[1].score > diff_files[0].hunks[0].score-- def test_score_priority_patterns(self):- """Hunks with priority patterns (error, security) score higher."""- compressor = DiffCompressor()-- hunk_normal = DiffHunk(- header="@@ -1,1 +1,2 @@",- lines=["+normal change"],- additions=1,- )- hunk_error = DiffHunk(- header="@@ -10,1 +10,2 @@",- lines=["+fix critical error"],- additions=1,- )-- diff_file = DiffFile(- header="diff --git a/f.py b/f.py",- old_file="--- a/f.py",- new_file="+++ b/f.py",- hunks=[hunk_normal, hunk_error],- )-- compressor._score_hunks([diff_file], "")-- assert hunk_error.score > hunk_normal.score--class TestSmallDiffPassthrough:"""Tests for small diff passthrough behavior."""@@ -539,56 +374,6 @@ class TestEdgeCases:assert result.compressed is not None-class TestDiffHunkDataclass:- """Tests for DiffHunk dataclass."""-- def test_change_count_property(self):- """change_count returns sum of additions and deletions."""- hunk = DiffHunk(- header="@@ -1,5 +1,6 @@",- lines=["+a", "+b", "-c", " ctx"],- additions=2,- deletions=1,- )- assert hunk.change_count == 3-- def test_default_values(self):- """DiffHunk default values are correct."""- hunk = DiffHunk(header="@@", lines=[])- assert hunk.additions == 0- assert hunk.deletions == 0- assert hunk.context_lines == 0- assert hunk.score == 0.0---class TestDiffFileDataclass:- """Tests for DiffFile dataclass."""-- def test_total_additions_property(self):- """total_additions sums across all hunks."""- hunk1 = DiffHunk(header="@@", lines=[], additions=3)- hunk2 = DiffHunk(header="@@", lines=[], additions=5)- diff_file = DiffFile(- header="diff --git",- old_file="---",- new_file="+++",- hunks=[hunk1, hunk2],- )- assert diff_file.total_additions == 8-- def test_total_deletions_property(self):- """total_deletions sums across all hunks."""- hunk1 = DiffHunk(header="@@", lines=[], deletions=2)- hunk2 = DiffHunk(header="@@", lines=[], deletions=4)- diff_file = DiffFile(- header="diff --git",- old_file="---",- new_file="+++",- hunks=[hunk1, hunk2],- )- assert diff_file.total_deletions == 6--class TestConfigOptions:"""Tests for configuration options."""diff --git a/tests/test_transforms/test_diff_compressor_rust_parity.py b/tests/test_transforms/test_diff_compressor_rust_parity.pynew file mode 100644index 000000000..3feac188c--- /dev/null+++ b/tests/test_transforms/test_diff_compressor_rust_parity.py@@ -0,0 +1,119 @@+"""Parity test: Rust-backed `DiffCompressor` vs recorded fixtures.++Stage 3b verification, post-deletion. The Python implementation has+been retired; the only remaining `DiffCompressor` is Rust-backed via+PyO3 (`headroom._core`). These tests guard against regressions by+replaying every recorded fixture in+`tests/parity/fixtures/diff_compressor/` through the Rust backend and+asserting the output matches the recording byte-for-byte.++Skipped automatically when the `headroom._core` wheel isn't installed+(e.g. CI lane without the maturin step). The Rust crate's own+`cargo run -p headroom-parity` covers the same fixtures from the Rust+side; this Python-side test catches PyO3 bridge regressions+(input/output mistranslation) that the Rust-only test cannot.+"""++from __future__ import annotations++import json+from pathlib import Path++import pytest+++def _has_core() -> bool:+ try:+ import headroom._core # noqa: F401++ return True+ except ImportError:+ return False+++pytestmark = pytest.mark.skipif(+ not _has_core(),+ reason="headroom._core wheel not installed (run `scripts/build_rust_extension.sh`)",+)+++_FIXTURES_DIR = Path(__file__).parent.parent / "parity" / "fixtures" / "diff_compressor"+++def _all_fixtures() -> list[Path]:+ return sorted(_FIXTURES_DIR.glob("*.json"))+++def test_at_least_27_fixtures_present():+ """Sanity check: the bug-fix and routing-gap fixtures landed."""+ fixtures = _all_fixtures()+ assert len(fixtures) >= 27, (+ f"expected >= 27 fixtures, found {len(fixtures)}. "+ "If you re-recorded and got fewer, something deleted them."+ )++[email protected]("fixture_path", _all_fixtures(), ids=lambda p: p.name)+def test_rust_backend_matches_recorded_output(fixture_path: Path):+ """Replay each recorded input through the Rust backend; every output+ field must match the recording. Any mismatch is a PyO3 bridge bug or a+ real Rust regression (cross-check with `cargo run -p headroom-parity`+ to localize).+ """+ from headroom.transforms.diff_compressor import DiffCompressor, DiffCompressorConfig++ rec = json.loads(fixture_path.read_text())+ cfg = DiffCompressorConfig(**rec["config"])+ result = DiffCompressor(cfg).compress(rec["input"])++ expected = rec["output"]+ # Field-by-field surfaces a single divergence rather than dumping the+ # whole result on failure.+ assert result.compressed == expected["compressed"], f"{fixture_path.name}: compressed differs"+ assert result.original_line_count == expected["original_line_count"]+ assert result.compressed_line_count == expected["compressed_line_count"]+ assert result.files_affected == expected["files_affected"]+ assert result.additions == expected["additions"]+ assert result.deletions == expected["deletions"]+ assert result.hunks_kept == expected["hunks_kept"]+ assert result.hunks_removed == expected["hunks_removed"]+ assert result.cache_key == expected["cache_key"]+++def test_compress_with_stats_returns_python_dataclass_and_pyo3_stats():+ """The sidecar stats API returns a `(DiffCompressionResult,+ _core.DiffCompressorStats)` tuple. Verify both halves are usable from+ Python — PyO3 doesn't auto-promote the stats class to a dataclass.+ """+ from headroom.transforms.diff_compressor import (+ DiffCompressionResult,+ DiffCompressor,+ DiffCompressorConfig,+ )++ diff = (+ "diff --git a/a.py b/a.py\n--- a/a.py\n+++ b/a.py\n@@ -1 +1 @@\n-old\n+new\n"+ ) + "# pad\n" * 60+ result, stats = DiffCompressor(DiffCompressorConfig()).compress_with_stats(diff)++ assert isinstance(result, DiffCompressionResult)+ assert stats.input_lines >= 60+ assert stats.processing_duration_us >= 0+ assert isinstance(stats.parse_warnings, list)+++def test_content_router_uses_rust_backend():+ """Stage 3b deletion check: ContentRouter._get_diff_compressor must+ return the (now Rust-only) DiffCompressor — there's no other backend+ left, so this guards against an accidental re-introduction of a+ Python-side stub or fallback.+ """+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig+ from headroom.transforms.diff_compressor import DiffCompressor++ router = ContentRouter(ContentRouterConfig())+ compressor = router._get_diff_compressor()+ assert isinstance(compressor, DiffCompressor)+ # The Rust delegation handle must be live — guards against a+ # half-deleted shim that defines the class but not `_rust`.+ assert hasattr(compressor, "_rust")