Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): retire python diff_compressor, ship rust-only via pyo3

chopratejasf5f465418bbe
diff --git a/.gitignore b/.gitignore
index 51e79e0db..af93df6fe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,7 @@ scripts/*
!scripts/fixtures/
!scripts/fixtures/*.json
!scripts/record_fixtures.py
+!scripts/build_rust_extension.sh
# Rust / Cargo build artifacts
/target/
@@ -223,3 +224,11 @@ headroom-managed/
# uv lockfile: regenerated locally; not committed
uv.lock
+
+# Rust extension `.so` symlinks placed by `scripts/build_rust_extension.sh`
+# into the `headroom/` package dir for local development. The real binary
+# lives in `crates/headroom-py/python/headroom/`; this is the dev overlay
+# that lets `import headroom._core` resolve when the source `headroom/`
+# package shadows the maturin overlay on sys.path.
+/headroom/_core.*.so
+/headroom/_core.so
diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs
index 70a693c98..0e51e1ba9 100644
--- a/crates/headroom-py/src/lib.rs
+++ b/crates/headroom-py/src/lib.rs
@@ -1,15 +1,380 @@
//! PyO3 bindings for headroom-core. Exposed to Python as `headroom._core`.
+//!
+//! # Stage 3b — diff_compressor bridge
+//!
+//! The `DiffCompressor` family is exported here so the Python
+//! `ContentRouter` can route to the Rust implementation in-process via
+//! PyO3 instead of running the Python port. Backend selection happens in
+//! `headroom.transforms._rust_diff_compressor.RustBackedDiffCompressor`,
+//! which mirrors the Python `DiffCompressor` API one-for-one (so callers
+//! don't notice the swap).
+//!
+//! Why in-process: ContentRouter compresses on the proxy's hot path. Any
+//! IPC / subprocess / RPC bridge would dominate the cost we're trying to
+//! save. PyO3 calls cost ~microseconds; staying in-process is ~free.
+use std::collections::BTreeMap;
+
+use headroom_core::transforms::{
+ DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,
+};
use pyo3::prelude::*;
-/// Return the identity string from headroom-core.
+/// Identity stub used by the Python smoke test to verify linkage.
#[pyfunction]
fn hello() -> &'static str {
headroom_core::hello()
}
+// ─── DiffCompressorConfig ──────────────────────────────────────────────────
+
+/// Mirror of `headroom.transforms.diff_compressor.DiffCompressorConfig`.
+/// Defaults match Python; constructor accepts every field as a kwarg with
+/// the same name and type as the Python dataclass for drop-in
+/// compatibility.
+#[pyclass(name = "DiffCompressorConfig", module = "headroom._core")]
+#[derive(Clone)]
+struct PyDiffCompressorConfig {
+ inner: DiffCompressorConfig,
+}
+
+#[pymethods]
+impl PyDiffCompressorConfig {
+ #[new]
+ #[pyo3(signature = (
+ max_context_lines = 2,
+ max_hunks_per_file = 10,
+ max_files = 20,
+ always_keep_additions = true,
+ always_keep_deletions = true,
+ enable_ccr = true,
+ min_lines_for_ccr = 50,
+ min_compression_ratio_for_ccr = 0.8,
+ ))]
+ #[allow(clippy::too_many_arguments)]
+ fn new(
+ max_context_lines: usize,
+ max_hunks_per_file: usize,
+ max_files: usize,
+ always_keep_additions: bool,
+ always_keep_deletions: bool,
+ enable_ccr: bool,
+ min_lines_for_ccr: usize,
+ min_compression_ratio_for_ccr: f64,
+ ) -> Self {
+ Self {
+ inner: DiffCompressorConfig {
+ max_context_lines,
+ max_hunks_per_file,
+ max_files,
+ always_keep_additions,
+ always_keep_deletions,
+ enable_ccr,
+ min_lines_for_ccr,
+ min_compression_ratio_for_ccr,
+ },
+ }
+ }
+
+ // Read-only field accessors mirroring the Python dataclass surface.
+ #[getter]
+ fn max_context_lines(&self) -> usize {
+ self.inner.max_context_lines
+ }
+ #[getter]
+ fn max_hunks_per_file(&self) -> usize {
+ self.inner.max_hunks_per_file
+ }
+ #[getter]
+ fn max_files(&self) -> usize {
+ self.inner.max_files
+ }
+ #[getter]
+ fn always_keep_additions(&self) -> bool {
+ self.inner.always_keep_additions
+ }
+ #[getter]
+ fn always_keep_deletions(&self) -> bool {
+ self.inner.always_keep_deletions
+ }
+ #[getter]
+ fn enable_ccr(&self) -> bool {
+ self.inner.enable_ccr
+ }
+ #[getter]
+ fn min_lines_for_ccr(&self) -> usize {
+ self.inner.min_lines_for_ccr
+ }
+ #[getter]
+ fn min_compression_ratio_for_ccr(&self) -> f64 {
+ self.inner.min_compression_ratio_for_ccr
+ }
+
+ fn __repr__(&self) -> String {
+ format!(
+ "DiffCompressorConfig(max_context_lines={}, max_hunks_per_file={}, max_files={}, \
+ always_keep_additions={}, always_keep_deletions={}, enable_ccr={}, \
+ min_lines_for_ccr={}, min_compression_ratio_for_ccr={})",
+ self.inner.max_context_lines,
+ self.inner.max_hunks_per_file,
+ self.inner.max_files,
+ self.inner.always_keep_additions,
+ self.inner.always_keep_deletions,
+ self.inner.enable_ccr,
+ self.inner.min_lines_for_ccr,
+ self.inner.min_compression_ratio_for_ccr,
+ )
+ }
+}
+
+// ─── DiffCompressionResult ─────────────────────────────────────────────────
+
+/// Mirror of `headroom.transforms.diff_compressor.DiffCompressionResult`.
+/// Read-only on the Python side: ContentRouter consumes fields, doesn't
+/// mutate. `compression_ratio` and `tokens_saved_estimate` are exposed as
+/// methods (not `@property`) — Python callers reach them via `.method()`.
+/// The Python adapter wraps and re-exposes them as properties for full
+/// dataclass compatibility.
+#[pyclass(name = "DiffCompressionResult", module = "headroom._core")]
+struct PyDiffCompressionResult {
+ inner: DiffCompressionResult,
+}
+
+#[pymethods]
+impl PyDiffCompressionResult {
+ #[getter]
+ fn compressed(&self) -> &str {
+ &self.inner.compressed
+ }
+ #[getter]
+ fn original_line_count(&self) -> usize {
+ self.inner.original_line_count
+ }
+ #[getter]
+ fn compressed_line_count(&self) -> usize {
+ self.inner.compressed_line_count
+ }
+ #[getter]
+ fn files_affected(&self) -> usize {
+ self.inner.files_affected
+ }
+ #[getter]
+ fn additions(&self) -> usize {
+ self.inner.additions
+ }
+ #[getter]
+ fn deletions(&self) -> usize {
+ self.inner.deletions
+ }
+ #[getter]
+ fn hunks_kept(&self) -> usize {
+ self.inner.hunks_kept
+ }
+ #[getter]
+ fn hunks_removed(&self) -> usize {
+ self.inner.hunks_removed
+ }
+ #[getter]
+ fn cache_key(&self) -> Option<String> {
+ self.inner.cache_key.clone()
+ }
+
+ /// Mirror of Python `@property compression_ratio`. Returns
+ /// `compressed_line_count / original_line_count` (1.0 if input was
+ /// empty).
+ fn compression_ratio(&self) -> f64 {
+ if self.inner.original_line_count == 0 {
+ 1.0
+ } else {
+ self.inner.compressed_line_count as f64 / self.inner.original_line_count as f64
+ }
+ }
+
+ /// Mirror of Python `@property tokens_saved_estimate`. Same `chars *
+ /// 40 / 4` heuristic; bytes-equivalent numeric result.
+ fn tokens_saved_estimate(&self) -> usize {
+ let saved = self
+ .inner
+ .original_line_count
+ .saturating_sub(self.inner.compressed_line_count);
+ (saved * 40) / 4
+ }
+
+ fn __repr__(&self) -> String {
+ format!(
+ "DiffCompressionResult(compressed=<{} chars>, original_line_count={}, \
+ compressed_line_count={}, files_affected={}, additions={}, deletions={}, \
+ hunks_kept={}, hunks_removed={}, cache_key={:?})",
+ self.inner.compressed.len(),
+ self.inner.original_line_count,
+ self.inner.compressed_line_count,
+ self.inner.files_affected,
+ self.inner.additions,
+ self.inner.deletions,
+ self.inner.hunks_kept,
+ self.inner.hunks_removed,
+ self.inner.cache_key,
+ )
+ }
+}
+
+// ─── DiffCompressorStats ───────────────────────────────────────────────────
+
+/// Mirror of Rust `DiffCompressorStats` — sidecar observability not
+/// present in the Python dataclass. Returned only from `compress_with_stats`,
+/// which the Python adapter exposes as a method on the wrapper. `Vec`s are
+/// returned as Python lists; the `BTreeMap` becomes a `dict`.
+#[pyclass(name = "DiffCompressorStats", module = "headroom._core")]
+struct PyDiffCompressorStats {
+ inner: DiffCompressorStats,
+}
+
+#[pymethods]
+impl PyDiffCompressorStats {
+ #[getter]
+ fn input_lines(&self) -> usize {
+ self.inner.input_lines
+ }
+ #[getter]
+ fn output_lines(&self) -> usize {
+ self.inner.output_lines
+ }
+ #[getter]
+ fn compression_ratio(&self) -> f64 {
+ self.inner.compression_ratio
+ }
+ #[getter]
+ fn files_total(&self) -> usize {
+ self.inner.files_total
+ }
+ #[getter]
+ fn files_kept(&self) -> usize {
+ self.inner.files_kept
+ }
+ #[getter]
+ fn files_dropped(&self) -> Vec<String> {
+ self.inner.files_dropped.clone()
+ }
+ #[getter]
+ fn hunks_total(&self) -> usize {
+ self.inner.hunks_total
+ }
+ #[getter]
+ fn hunks_kept(&self) -> usize {
+ self.inner.hunks_kept
+ }
+ #[getter]
+ fn hunks_dropped(&self) -> usize {
+ self.inner.hunks_dropped
+ }
+ #[getter]
+ fn hunks_dropped_per_file(&self) -> BTreeMap<String, usize> {
+ self.inner.hunks_dropped_per_file.clone()
+ }
+ #[getter]
+ fn context_lines_input(&self) -> usize {
+ self.inner.context_lines_input
+ }
+ #[getter]
+ fn context_lines_kept(&self) -> usize {
+ self.inner.context_lines_kept
+ }
+ #[getter]
+ fn context_lines_trimmed(&self) -> usize {
+ self.inner.context_lines_trimmed
+ }
+ #[getter]
+ fn largest_hunk_kept_lines(&self) -> usize {
+ self.inner.largest_hunk_kept_lines
+ }
+ #[getter]
+ fn largest_hunk_dropped_lines(&self) -> usize {
+ self.inner.largest_hunk_dropped_lines
+ }
+ #[getter]
+ fn parse_warnings(&self) -> Vec<String> {
+ self.inner.parse_warnings.clone()
+ }
+ #[getter]
+ fn processing_duration_us(&self) -> u64 {
+ self.inner.processing_duration_us
+ }
+ #[getter]
+ fn cache_key_emitted(&self) -> bool {
+ self.inner.cache_key_emitted
+ }
+ #[getter]
+ fn ccr_skipped_reason(&self) -> Option<String> {
+ self.inner.ccr_skipped_reason.clone()
+ }
+ #[getter]
+ fn file_mode_normalizations(&self) -> Vec<(String, String)> {
+ self.inner.file_mode_normalizations.clone()
+ }
+ #[getter]
+ fn binary_files_simplified(&self) -> Vec<String> {
+ self.inner.binary_files_simplified.clone()
+ }
+}
+
+// ─── DiffCompressor ────────────────────────────────────────────────────────
+
+/// Mirror of `headroom.transforms.diff_compressor.DiffCompressor`. The
+/// Python adapter wraps this in `RustBackedDiffCompressor` so
+/// `ContentRouter` can swap backends transparently.
+#[pyclass(name = "DiffCompressor", module = "headroom._core")]
+struct PyDiffCompressor {
+ inner: DiffCompressor,
+}
+
+#[pymethods]
+impl PyDiffCompressor {
+ /// `__init__(config: DiffCompressorConfig | None = None)` — matches the
+ /// Python constructor signature one-for-one.
+ #[new]
+ #[pyo3(signature = (config = None))]
+ fn new(config: Option<&PyDiffCompressorConfig>) -> Self {
+ let cfg = config.map(|c| c.inner.clone()).unwrap_or_default();
+ Self {
+ inner: DiffCompressor::new(cfg),
+ }
+ }
+
+ /// `compress(content: str, context: str = "") -> DiffCompressionResult`.
+ /// Argument order and keyword names match the Python implementation.
+ #[pyo3(signature = (content, context = ""))]
+ fn compress(&self, content: &str, context: &str) -> PyDiffCompressionResult {
+ PyDiffCompressionResult {
+ inner: self.inner.compress(content, context),
+ }
+ }
+
+ /// `compress_with_stats(content, context="") -> (result, stats)`.
+ /// Sidecar API not present in Python — exposes the Rust observability
+ /// struct alongside the parity-equal result. Returned as a 2-tuple to
+ /// keep the call site Pythonic.
+ #[pyo3(signature = (content, context = ""))]
+ fn compress_with_stats(
+ &self,
+ content: &str,
+ context: &str,
+ ) -> (PyDiffCompressionResult, PyDiffCompressorStats) {
+ let (result, stats) = self.inner.compress_with_stats(content, context);
+ (
+ PyDiffCompressionResult { inner: result },
+ PyDiffCompressorStats { inner: stats },
+ )
+ }
+}
+
+// ─── Module init ───────────────────────────────────────────────────────────
+
#[pymodule]
fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(hello, m)?)?;
+ m.add_class::<PyDiffCompressorConfig>()?;
+ m.add_class::<PyDiffCompressionResult>()?;
+ m.add_class::<PyDiffCompressorStats>()?;
+ m.add_class::<PyDiffCompressor>()?;
Ok(())
}
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 8c264fc7c..ff7278305 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1197,14 +1197,13 @@ class ContentRouter(Transform):
return self._log_compressor
def _get_diff_compressor(self) -> Any:
- """Get DiffCompressor (lazy load)."""
+ """Get DiffCompressor (lazy load). Rust-only — Python implementation
+ retired in Stage 3b. The wheel (`headroom._core`) is a hard import.
+ """
if self._diff_compressor is None:
- try:
- from .diff_compressor import DiffCompressor
+ from .diff_compressor import DiffCompressor
- self._diff_compressor = DiffCompressor()
- except ImportError:
- logger.debug("DiffCompressor not available")
+ self._diff_compressor = DiffCompressor()
return self._diff_compressor
def _get_html_extractor(self) -> Any:
diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.py
index ba81902e0..7b0418f74 100644
--- a/headroom/transforms/diff_compressor.py
+++ b/headroom/transforms/diff_compressor.py
@@ -1,108 +1,41 @@
-"""Git diff output compressor for unified diff format.
-
-This module compresses git diff output which can be very verbose with
-many context lines. Typical compression: 3-10x.
-
-Supported formats:
-- Unified diff format (git diff, diff -u)
-- Combined diff format (merge conflicts)
-
-Compression Strategy:
-1. Parse unified diff format into file sections and hunks
-2. Always keep file headers (diff --git, ---, +++)
-3. Always keep ALL actual changes (+/- lines)
-4. Reduce context lines (` ` prefix) to configurable max
-5. If too many hunks, keep first N and summarize rest
-6. Add summary at end
-
-Key Patterns to Preserve:
-- All additions (+)
-- All deletions (-)
-- Hunk headers (@@ ... @@)
-- File headers (diff --git, ---, +++)
-- Context around changes (limited)
+"""Git diff output compressor — Rust-backed via PyO3.
+
+The Python implementation has been retired (Stage 3b, 2026-04-25). All
+diff compression now goes through `headroom._core.DiffCompressor` (built
+from `crates/headroom-py`). The byte-equality of the two implementations
+was verified against 27 recorded fixtures before the Python source was
+removed; the Rust crate has its own test coverage in `crates/headroom-core/`.
+
+This module retains the public surface — `DiffCompressorConfig`,
+`DiffCompressionResult`, `DiffCompressor` — so existing call sites
+(ContentRouter, parity recorder, integrations, downstream users) keep
+working unchanged. The dataclasses are still pure-Python because they
+appear in dataclass-aware code paths (`asdict()`, `__dict__`, dataclass
+matching). Only the `DiffCompressor` class delegates to Rust.
+
+The `headroom._core` extension is a hard import: there is no Python
+fallback. Build it locally with `scripts/build_rust_extension.sh`
+(wraps `maturin develop`) or install a prebuilt wheel.
"""
from __future__ import annotations
import logging
-import re
-from dataclasses import dataclass, field
+from dataclasses import dataclass
+from typing import Any
logger = logging.getLogger(__name__)
-@dataclass
-class DiffHunk:
- """A single hunk within a diff file."""
-
- header: str # @@ -start,count +start,count @@ optional function
- lines: list[str] # All lines in the hunk
- additions: int = 0
- deletions: int = 0
- context_lines: int = 0
- score: float = 0.0 # Relevance score for context-aware compression
-
- @property
- def change_count(self) -> int:
- """Total number of actual changes (additions + deletions)."""
- return self.additions + self.deletions
-
-
-@dataclass
-class DiffFile:
- """A single file's diff."""
-
- header: str # diff --git a/... b/...
- old_file: str # --- a/...
- new_file: str # +++ b/...
- hunks: list[DiffHunk] = field(default_factory=list)
- is_binary: bool = False
- is_new_file: bool = False
- is_deleted_file: bool = False
- is_renamed: bool = False
- # Bug-fix: rename / copy / similarity / dissimilarity marker lines that
- # were emitted by git BETWEEN `diff --git` and `--- a/` (e.g. `rename
- # from old.py`, `rename to new.py`, `similarity index 95%`). Previously
- # we set is_renamed=True and discarded the lines, so the output looked
- # like a plain modification. Now captured verbatim and re-emitted in
- # `_format_output`.
- rename_lines: list[str] = field(default_factory=list)
- # Bug-fix: original `new file mode <NNNNNN>` / `deleted file mode
- # <NNNNNN>` / `Binary files X and Y differ` lines. Emit normalizes to
- # `100644` / bare `Binary files differ`; capturing the originals lets
- # the caller observe the loss via logging.
- original_new_file_mode_line: str | None = None
- original_deleted_file_mode_line: str | None = None
- original_binary_line: str | None = None
-
- @property
- def total_additions(self) -> int:
- return sum(h.additions for h in self.hunks)
-
- @property
- def total_deletions(self) -> int:
- return sum(h.deletions for h in self.hunks)
-
-
@dataclass
class DiffCompressorConfig:
"""Configuration for diff compression."""
- # Context line limits
- max_context_lines: int = 2 # Reduce from default 3 lines before/after changes
-
- # Hunk limits
+ max_context_lines: int = 2
max_hunks_per_file: int = 10
-
- # File limits
max_files: int = 20
-
- # Change preservation
- always_keep_additions: bool = True # Always keep + lines
- always_keep_deletions: bool = True # Always keep - lines
-
- # CCR integration
+ always_keep_additions: bool = True
+ always_keep_deletions: bool = True
enable_ccr: bool = True
min_lines_for_ccr: int = 50
@@ -123,666 +56,85 @@ class DiffCompressionResult:
@property
def compression_ratio(self) -> float:
- """Ratio of compressed to original (lower is better compression)."""
if self.original_line_count == 0:
return 1.0
return self.compressed_line_count / self.original_line_count
@property
def tokens_saved_estimate(self) -> int:
- """Estimate tokens saved (rough: 1 token per 4 chars)."""
- # Use line counts as proxy for chars
lines_saved = self.original_line_count - self.compressed_line_count
- # Estimate ~40 chars per line average for diffs
chars_saved = lines_saved * 40
return max(0, chars_saved // 4)
class DiffCompressor:
- """Compresses git diff output.
+ """Rust-backed `DiffCompressor` (via PyO3 / `headroom._core`).
- Example:
- >>> compressor = DiffCompressor()
- >>> result = compressor.compress(git_diff_output)
- >>> print(result.compressed) # Reduced diff with summary
+ Same `__init__` and `compress` shape as the retired Python class —
+ drop-in replacement. Returns Python `DiffCompressionResult` dataclass
+ instances so call sites that destructure with `asdict()` or read the
+ `@property` fields work unchanged.
"""
- # Pattern for `diff --git a/X b/Y` header (regular diff).
- _DIFF_GIT_PATTERN = re.compile(r"^diff --git a/(.+) b/(.+)$")
-
- # Bug-fix (2026-04-25): merge-commit headers — `diff --combined <path>`
- # and `diff --cc <path>`. Both produce a single-path file diff with
- # combined-diff hunk syntax (`@@@`+). Previously the parser only
- # recognized `diff --git`, so merge-commit diffs from `git log -p`
- # of merges fell through to "no files parsed" and were passed
- # through unchanged — even though ContentRouter had routed them here
- # because `--- a/` triggered the detector.
- _DIFF_COMBINED_PATTERN = re.compile(r"^diff --combined (.+)$")
- _DIFF_CC_PATTERN = re.compile(r"^diff --cc (.+)$")
-
- # Pattern for --- a/file or --- /dev/null
- _OLD_FILE_PATTERN = re.compile(r"^--- (a/(.+)|/dev/null)$")
-
- # Pattern for +++ b/file or +++ /dev/null
- _NEW_FILE_PATTERN = re.compile(r"^\+\+\+ (b/(.+)|/dev/null)$")
-
- # Pattern for ANY hunk header — matches both regular `@@ -A,B +C,D @@`
- # and combined-diff `@@@ -A,B -C,D +E,F @@@` (and 4-way `@@@@ ... @@@@`).
- # Group 1 is the @-prefix (so closing @@ can backreference). Bug-fix:
- # previously hardcoded to `@@`, which silently dropped all content from
- # combined-diff hunks (merge commits) — `current_hunk` was never set so
- # subsequent +/- lines fell through to the no-op branch.
- _HUNK_HEADER_PATTERN = re.compile(r"^(@@+) (?:-\d+(?:,\d+)? )+\+\d+(?:,\d+)? \1(.*)$")
-
- # Used to extract the new-file starting line number for in-order resort
- # after middle-hunk selection. Works for both regular and combined diffs.
- _HUNK_NEW_RANGE_PATTERN = re.compile(r"\+(\d+)")
-
- # Pattern for binary file indication
- _BINARY_PATTERN = re.compile(r"^Binary files .+ differ$")
-
- # Patterns for new/deleted file mode
- _NEW_FILE_MODE_PATTERN = re.compile(r"^new file mode")
- _DELETED_FILE_MODE_PATTERN = re.compile(r"^deleted file mode")
- # Bug-fix: extended to include `dissimilarity` (low-similarity rename
- # marker, real git output). Previously dropped silently.
- _RENAME_PATTERN = re.compile(r"^(rename|similarity|copy|dissimilarity) ")
-
- # Priority patterns for context-aware hunk selection (centralized)
- from headroom.transforms.error_detection import PRIORITY_PATTERNS_DIFF
-
- _PRIORITY_PATTERNS = PRIORITY_PATTERNS_DIFF
-
def __init__(self, config: DiffCompressorConfig | None = None):
- """Initialize diff compressor.
-
- Args:
- config: Compression configuration.
- """
- self.config = config or DiffCompressorConfig()
-
- def compress(self, content: str, context: str = "") -> DiffCompressionResult:
- """Compress diff output.
-
- Args:
- content: Raw git diff output.
- context: User query context for relevance scoring.
-
- Returns:
- DiffCompressionResult with compressed output and metadata.
- """
- lines = content.split("\n")
- original_line_count = len(lines)
-
- if original_line_count < self.config.min_lines_for_ccr:
- return DiffCompressionResult(
- compressed=content,
- original_line_count=original_line_count,
- compressed_line_count=original_line_count,
- files_affected=0,
- additions=0,
- deletions=0,
- hunks_kept=0,
- hunks_removed=0,
- )
-
- # Parse diff into structured format. Returns pre-diff content
- # (commit headers, email headers from `git format-patch`, etc.)
- # alongside the parsed files. Bug-fix: previously this content was
- # silently dropped; now it's preserved verbatim before the
- # compressed file sections in the output.
- pre_diff_lines, diff_files = self._parse_diff(lines)
+ # Hard import — no fallback. If the wheel is missing, the user
+ # must build it (scripts/build_rust_extension.sh) or install a
+ # prebuilt one. Failing loudly here is better than silently
+ # degrading; see feedback memory `feedback_no_silent_fallbacks.md`.
+ from headroom._core import (
+ DiffCompressor as _RustDiffCompressor,
+ )
+ from headroom._core import (
+ DiffCompressorConfig as _RustDiffCompressorConfig,
+ )
- if not diff_files:
- return DiffCompressionResult(
- compressed=content,
- original_line_count=original_line_count,
- compressed_line_count=original_line_count,
- files_affected=0,
- additions=0,
- deletions=0,
- hunks_kept=0,
- hunks_removed=0,
+ cfg = config or DiffCompressorConfig()
+ self.config = cfg
+ self._rust = _RustDiffCompressor(
+ _RustDiffCompressorConfig(
+ max_context_lines=cfg.max_context_lines,
+ max_hunks_per_file=cfg.max_hunks_per_file,
+ max_files=cfg.max_files,
+ always_keep_additions=cfg.always_keep_additions,
+ always_keep_deletions=cfg.always_keep_deletions,
+ enable_ccr=cfg.enable_ccr,
+ min_lines_for_ccr=cfg.min_lines_for_ccr,
)
+ )
- # Score hunks by relevance
- self._score_hunks(diff_files, context)
-
- # Compress each file's hunks
- compressed_files, stats = self._compress_files(diff_files)
-
- # Format output, prepending pre-diff content if any.
- compressed_output = self._format_output(compressed_files, stats)
- if pre_diff_lines:
- compressed_output = "\n".join(pre_diff_lines) + "\n" + compressed_output
-
- # Observability: log lossy-emit signals so prod monitoring can
- # alert on outlier diffs. Counts only — no PII / file content.
- self._log_loss_signals(diff_files, stats)
-
- compressed_line_count = len(compressed_output.split("\n"))
-
- # Store in CCR if significant compression
- cache_key = None
- if self.config.enable_ccr and compressed_line_count < original_line_count * 0.8:
- cache_key = self._store_in_ccr(content, compressed_output, original_line_count)
- if cache_key:
- compressed_output += f"\n[{original_line_count} lines compressed to {compressed_line_count}. Retrieve full diff: hash={cache_key}]"
-
+ def compress(self, content: str, context: str = "") -> DiffCompressionResult:
+ r = self._rust.compress(content, context)
return DiffCompressionResult(
- compressed=compressed_output,
- original_line_count=original_line_count,
- compressed_line_count=compressed_line_count,
- files_affected=stats["files_affected"],
- additions=stats["total_additions"],
- deletions=stats["total_deletions"],
- hunks_kept=stats["hunks_kept"],
- hunks_removed=stats["hunks_removed"],
- cache_key=cache_key,
+ compressed=r.compressed,
+ original_line_count=r.original_line_count,
+ compressed_line_count=r.compressed_line_count,
+ files_affected=r.files_affected,
+ additions=r.additions,
+ deletions=r.deletions,
+ hunks_kept=r.hunks_kept,
+ hunks_removed=r.hunks_removed,
+ cache_key=r.cache_key,
)
- def _parse_diff(self, lines: list[str]) -> tuple[list[str], list[DiffFile]]:
- """Parse diff content into structured format.
-
- Args:
- lines: Lines of diff content.
-
- Returns:
- Tuple of (pre_diff_lines, diff_files). `pre_diff_lines` are any
- lines before the first ``diff --git`` (commit headers from
- ``git log -p``, email headers from ``git format-patch``, etc.) —
- previously dropped, now preserved verbatim and re-emitted.
- """
- diff_files: list[DiffFile] = []
- current_file: DiffFile | None = None
- current_hunk: DiffHunk | None = None
- pre_diff_lines: list[str] = []
- i = 0
-
- while i < len(lines):
- line = lines[i]
-
- # Check for diff --git header (new file section). Bug-fix:
- # `diff --combined <path>` and `diff --cc <path>` (merge
- # commits) also start a new file section — single-path,
- # combined-diff hunk syntax. Treat them the same as
- # `diff --git` for sectioning purposes; the path goes in
- # `header` verbatim.
- if (
- self._DIFF_GIT_PATTERN.match(line)
- or self._DIFF_COMBINED_PATTERN.match(line)
- or self._DIFF_CC_PATTERN.match(line)
- ):
- # Save previous hunk and file
- if current_hunk and current_file:
- current_file.hunks.append(current_hunk)
- if current_file:
- diff_files.append(current_file)
-
- current_file = DiffFile(
- header=line,
- old_file="",
- new_file="",
- )
- current_hunk = None
- i += 1
- continue
-
- # Bug-fix: any line before the first `diff --git` is pre-diff
- # content (commit headers, email headers). Capture for verbatim
- # re-emission rather than dropping.
- if current_file is None:
- pre_diff_lines.append(line)
- i += 1
- continue
-
- # Check for file mode indicators / rename / binary markers.
- # Capture the original line in addition to the boolean so the
- # logger / sidecar observability can surface emit-time
- # normalization losses.
- if current_file:
- if self._NEW_FILE_MODE_PATTERN.match(line):
- current_file.is_new_file = True
- current_file.original_new_file_mode_line = line
- elif self._DELETED_FILE_MODE_PATTERN.match(line):
- current_file.is_deleted_file = True
- current_file.original_deleted_file_mode_line = line
- elif self._RENAME_PATTERN.match(line):
- current_file.is_renamed = True
- # Bug-fix: capture rename marker lines so they get
- # re-emitted. Previously the boolean was set but the
- # actual `rename from` / `rename to` / `similarity index`
- # lines were discarded — output looked like a plain
- # modification and the LLM had no way to know a file
- # was renamed.
- current_file.rename_lines.append(line)
- elif self._BINARY_PATTERN.match(line):
- current_file.is_binary = True
- current_file.original_binary_line = line
-
- # Check for --- a/file
- if self._OLD_FILE_PATTERN.match(line):
- if current_file:
- current_file.old_file = line
- i += 1
- continue
-
- # Check for +++ b/file
- if self._NEW_FILE_PATTERN.match(line):
- if current_file:
- current_file.new_file = line
- i += 1
- continue
-
- # Check for hunk header (regular `@@` or combined-diff `@@@`+).
- # Bug-fix: previously `@@@` headers didn't match, so combined
- # diffs (merge commits) had ALL their content silently dropped.
- if self._HUNK_HEADER_PATTERN.match(line):
- # Save previous hunk
- if current_hunk and current_file:
- current_file.hunks.append(current_hunk)
-
- current_hunk = DiffHunk(
- header=line,
- lines=[],
- )
- i += 1
- continue
-
- # Process hunk content lines
- if current_hunk is not None:
- if line.startswith("+") and not line.startswith("+++"):
- current_hunk.additions += 1
- current_hunk.lines.append(line)
- elif line.startswith("-") and not line.startswith("---"):
- current_hunk.deletions += 1
- current_hunk.lines.append(line)
- elif line.startswith(" ") or line == "":
- current_hunk.context_lines += 1
- current_hunk.lines.append(line)
- else:
- # Other line (e.g., "\ No newline at end of file" — note
- # leading backslash). Preserved here; `_reduce_context`
- # force-keeps `\` lines regardless of distance from
- # changes, so they survive the context trim.
- current_hunk.lines.append(line)
-
- i += 1
-
- # Save final hunk and file
- if current_hunk and current_file:
- current_file.hunks.append(current_hunk)
- if current_file:
- diff_files.append(current_file)
-
- return pre_diff_lines, diff_files
-
- def _score_hunks(self, diff_files: list[DiffFile], context: str) -> None:
- """Score hunks by relevance to context.
-
- Args:
- diff_files: Parsed diff files.
- context: User query context.
+ def compress_with_stats(
+ self, content: str, context: str = ""
+ ) -> tuple[DiffCompressionResult, Any]:
+ """Sidecar API exposing the Rust-only `DiffCompressorStats` struct
+ (per-file hunk drops, context lines trimmed, file_mode normalizations,
+ etc.) alongside the result. Stats is the raw PyO3 wrapper — no
+ Python equivalent to mirror to. Typed as `Any` because the PyO3
+ class has no Python type stub.
"""
- context_lower = context.lower()
- context_words = set(context_lower.split()) if context else set()
-
- for diff_file in diff_files:
- for hunk in diff_file.hunks:
- score = 0.0
-
- # Base score from change count (more changes = more important)
- score += min(0.3, hunk.change_count * 0.03)
-
- hunk_content = "\n".join(hunk.lines).lower()
-
- # Score by context word overlap
- for word in context_words:
- if len(word) > 2 and word in hunk_content:
- score += 0.2
-
- # Boost for priority patterns
- for pattern in self._PRIORITY_PATTERNS:
- if pattern.search(hunk_content):
- score += 0.3
- break
-
- hunk.score = min(1.0, score)
-
- def _compress_files(self, diff_files: list[DiffFile]) -> tuple[list[DiffFile], dict[str, int]]:
- """Compress hunks in each file.
-
- Args:
- diff_files: Parsed diff files.
-
- Returns:
- Tuple of (compressed files, stats dict).
- """
- stats = {
- "files_affected": 0,
- "total_additions": 0,
- "total_deletions": 0,
- "hunks_kept": 0,
- "hunks_removed": 0,
- }
-
- # Limit files if too many
- if len(diff_files) > self.config.max_files:
- # Sort by total changes (most changes first)
- diff_files = sorted(
- diff_files,
- key=lambda f: f.total_additions + f.total_deletions,
- reverse=True,
- )
- diff_files = diff_files[: self.config.max_files]
-
- compressed_files: list[DiffFile] = []
-
- for diff_file in diff_files:
- stats["files_affected"] += 1
- stats["total_additions"] += diff_file.total_additions
- stats["total_deletions"] += diff_file.total_deletions
-
- # Compress hunks within file
- compressed_hunks = self._compress_hunks(diff_file.hunks)
-
- stats["hunks_kept"] += len(compressed_hunks)
- stats["hunks_removed"] += len(diff_file.hunks) - len(compressed_hunks)
-
- # Create compressed file with reduced context in hunks. Bug-fix:
- # previously this constructor dropped `rename_lines` and the
- # `original_*_line` fields by omission — they were captured in
- # the parser but never propagated to `_format_output`, so the
- # emit looked exactly like the buggy old behavior. Carry them
- # all through.
- new_file = DiffFile(
- header=diff_file.header,
- old_file=diff_file.old_file,
- new_file=diff_file.new_file,
- hunks=compressed_hunks,
- is_binary=diff_file.is_binary,
- is_new_file=diff_file.is_new_file,
- is_deleted_file=diff_file.is_deleted_file,
- is_renamed=diff_file.is_renamed,
- rename_lines=diff_file.rename_lines,
- original_new_file_mode_line=diff_file.original_new_file_mode_line,
- original_deleted_file_mode_line=diff_file.original_deleted_file_mode_line,
- original_binary_line=diff_file.original_binary_line,
- )
- compressed_files.append(new_file)
-
- return compressed_files, stats
-
- def _compress_hunks(self, hunks: list[DiffHunk]) -> list[DiffHunk]:
- """Compress hunks by reducing context and limiting count.
-
- Args:
- hunks: List of hunks to compress.
-
- Returns:
- Compressed list of hunks.
- """
- if not hunks:
- return []
-
- # Sort by score if we need to limit
- if len(hunks) > self.config.max_hunks_per_file:
- # Keep first and last hunks (often important)
- first_hunk = hunks[0]
- last_hunk = hunks[-1] if len(hunks) > 1 else None
-
- # Sort middle hunks by score
- middle_hunks = sorted(
- hunks[1:-1] if last_hunk else [], key=lambda h: h.score, reverse=True
- )
-
- # Take top scoring middle hunks
- remaining_slots = (
- self.config.max_hunks_per_file - 2
- if last_hunk
- else self.config.max_hunks_per_file - 1
- )
- selected_middle = middle_hunks[:remaining_slots]
-
- # Rebuild list in original order by re-sorting by appearance
- selected = [first_hunk] + selected_middle
- if last_hunk:
- selected.append(last_hunk)
-
- # Sort back to original order (using header line numbers as proxy)
- hunks = sorted(selected, key=lambda h: self._extract_line_number(h.header))
-
- # Reduce context in each hunk
- compressed_hunks = []
- for hunk in hunks:
- compressed_hunk = self._reduce_context(hunk)
- compressed_hunks.append(compressed_hunk)
-
- return compressed_hunks
-
- def _extract_line_number(self, header: str) -> int:
- """Extract starting `+`-side line number from hunk header for sorting.
-
- Works for both regular ``@@ -A,B +C,D @@`` and combined-diff
- ``@@@ -A,B -C,D +E,F @@@`` formats — finds the first ``+N`` token.
- """
- match = self._HUNK_NEW_RANGE_PATTERN.search(header)
- if match:
- return int(match.group(1))
- return 0
-
- def _reduce_context(self, hunk: DiffHunk) -> DiffHunk:
- """Reduce context lines while preserving all changes.
-
- Args:
- hunk: Hunk to reduce context in.
-
- Returns:
- New hunk with reduced context.
- """
- max_context = self.config.max_context_lines
-
- # Identify change positions
- change_positions: list[int] = []
- for i, line in enumerate(hunk.lines):
- if line.startswith("+") or line.startswith("-"):
- change_positions.append(i)
-
- if not change_positions:
- # No changes, just context - keep minimal
- return DiffHunk(
- header=hunk.header,
- lines=hunk.lines[:max_context] if hunk.lines else [],
- additions=0,
- deletions=0,
- context_lines=min(len(hunk.lines), max_context),
- score=hunk.score,
- )
-
- # Determine which lines to keep
- keep_indices: set[int] = set()
-
- for pos in change_positions:
- # Always keep the change line
- keep_indices.add(pos)
-
- # Keep context before
- for i in range(max(0, pos - max_context), pos):
- keep_indices.add(i)
-
- # Keep context after
- for i in range(pos + 1, min(len(hunk.lines), pos + max_context + 1)):
- keep_indices.add(i)
-
- # Bug-fix: ALWAYS keep `\ No newline at end of file` markers (and
- # any other backslash-prefixed metadata) regardless of distance
- # from a change. These are structural patch markers, not context —
- # losing them breaks round-trippable patches and changes the
- # semantic meaning of the trailing line in the file.
- for i, line in enumerate(hunk.lines):
- if line.startswith("\\"):
- keep_indices.add(i)
-
- # Build new lines list
- new_lines: list[str] = []
- additions = 0
- deletions = 0
- context_lines = 0
-
- for i in sorted(keep_indices):
- line = hunk.lines[i]
- new_lines.append(line)
- if line.startswith("+"):
- additions += 1
- elif line.startswith("-"):
- deletions += 1
- else:
- context_lines += 1
-
- return DiffHunk(
- header=hunk.header,
- lines=new_lines,
- additions=additions,
- deletions=deletions,
- context_lines=context_lines,
- score=hunk.score,
+ r, stats = self._rust.compress_with_stats(content, context)
+ result = DiffCompressionResult(
+ compressed=r.compressed,
+ original_line_count=r.original_line_count,
+ compressed_line_count=r.compressed_line_count,
+ files_affected=r.files_affected,
+ additions=r.additions,
+ deletions=r.deletions,
+ hunks_kept=r.hunks_kept,
+ hunks_removed=r.hunks_removed,
+ cache_key=r.cache_key,
)
-
- def _format_output(self, diff_files: list[DiffFile], stats: dict[str, int]) -> str:
- """Format compressed diff files back to unified diff format.
-
- Args:
- diff_files: Compressed diff files.
- stats: Compression statistics.
-
- Returns:
- Formatted diff string.
- """
- output_lines: list[str] = []
-
- for diff_file in diff_files:
- # File header
- output_lines.append(diff_file.header)
-
- # Bug-fix: emit rename / similarity / dissimilarity / copy
- # marker lines immediately after `diff --git`, matching git's
- # canonical output order. Previously these were captured as
- # `is_renamed=True` and dropped — output looked like a plain
- # modification of the old file's path.
- if diff_file.rename_lines:
- output_lines.extend(diff_file.rename_lines)
-
- # File mode indicators if present. Note: parity-bound
- # normalization to `100644` (the original mode is captured in
- # `original_new_file_mode_line` for observability).
- if diff_file.is_new_file:
- output_lines.append("new file mode 100644")
- elif diff_file.is_deleted_file:
- output_lines.append("deleted file mode 100644")
-
- if diff_file.is_binary:
- output_lines.append("Binary files differ")
- continue
-
- # Old/new file markers
- if diff_file.old_file:
- output_lines.append(diff_file.old_file)
- if diff_file.new_file:
- output_lines.append(diff_file.new_file)
-
- # Hunks
- for hunk in diff_file.hunks:
- output_lines.append(hunk.header)
- output_lines.extend(hunk.lines)
-
- # Add summary
- if stats["hunks_removed"] > 0 or stats["files_affected"] > 0:
- summary_parts = [
- f"{stats['files_affected']} files changed",
- f"+{stats['total_additions']} -{stats['total_deletions']} lines",
- ]
- if stats["hunks_removed"] > 0:
- summary_parts.append(f"{stats['hunks_removed']} hunks omitted")
- output_lines.append(f"[{', '.join(summary_parts)}]")
-
- return "\n".join(output_lines)
-
- def _log_loss_signals(
- self,
- diff_files: list[DiffFile],
- stats: dict[str, int],
- ) -> None:
- """Surface lossy-emit signals to the logger.
-
- Some normalizations are parity-bound and silent in the compressed
- output: file modes flatten to ``100644``, ``Binary files X and Y
- differ`` simplifies to ``Binary files differ``, the ``max_files`` /
- ``max_hunks_per_file`` caps drop entire files / middle hunks. Stats
- about caps live in the result; emit-time normalizations live only
- here. Logs let prod monitoring alert on outlier diffs without
- changing the compressed bytes.
- """
- mode_normalizations: list[str] = []
- binary_simplifications: list[str] = []
- for f in diff_files:
- if (
- f.original_new_file_mode_line
- and f.original_new_file_mode_line != "new file mode 100644"
- ):
- mode_normalizations.append(
- f"{f.old_file} -> {f.new_file}: {f.original_new_file_mode_line}"
- )
- if (
- f.original_deleted_file_mode_line
- and f.original_deleted_file_mode_line != "deleted file mode 100644"
- ):
- mode_normalizations.append(
- f"{f.old_file} -> {f.new_file}: {f.original_deleted_file_mode_line}"
- )
- if f.original_binary_line and f.original_binary_line != "Binary files differ":
- binary_simplifications.append(f.original_binary_line)
-
- if mode_normalizations:
- logger.warning(
- "DiffCompressor: %d file mode line(s) normalized to 100644 — "
- "executable / symlink / submodule signals lost: %s",
- len(mode_normalizations),
- mode_normalizations,
- )
- if binary_simplifications:
- logger.warning(
- "DiffCompressor: %d binary file detail line(s) simplified — filenames lost: %s",
- len(binary_simplifications),
- binary_simplifications,
- )
- if stats.get("hunks_removed", 0) > 0:
- logger.info(
- "DiffCompressor: dropped %d hunks across %d files",
- stats["hunks_removed"],
- stats.get("files_affected", 0),
- )
-
- def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:
- """Store original in CCR for later retrieval.
-
- Args:
- original: Original diff content.
- compressed: Compressed diff content.
- original_count: Original line count.
-
- Returns:
- Cache key if stored, None otherwise.
- """
- try:
- from ..cache.compression_store import get_compression_store
-
- store = get_compression_store()
- return store.store(
- original,
- compressed,
- original_item_count=original_count,
- )
- except ImportError:
- return None
- except Exception:
- return None
+ return result, stats
diff --git a/scripts/build_rust_extension.sh b/scripts/build_rust_extension.sh
new file mode 100755
index 000000000..0e678540c
--- /dev/null
+++ b/scripts/build_rust_extension.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Build the Rust → Python extension (headroom._core) and link it into the
+# in-tree `headroom/` package so `import headroom._core` resolves.
+#
+# Why a wrapper script: `maturin develop` builds the `.so` and installs it
+# into the venv's site-packages, but the in-tree `headroom/` source
+# directory (loaded via `pip install -e .`) shadows that on sys.path.
+# Python finds `headroom/__init__.py` at the project root before reaching
+# the maturin overlay, so `import headroom._core` fails. Symlinking the
+# built `.so` into `headroom/` fixes the lookup with zero copies.
+#
+# Idempotent. Safe to run repeatedly. Requires `maturin` in PATH (i.e.
+# inside the project venv).
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+if ! command -v maturin >/dev/null 2>&1; then
+ echo "error: maturin not found. Activate the venv first:" >&2
+ echo " source .venv/bin/activate" >&2
+ exit 1
+fi
+
+# Build the wheel + install into the venv site-packages.
+maturin develop -m crates/headroom-py/Cargo.toml
+
+# Locate the built `.so`. `maturin develop` writes it under
+# `crates/headroom-py/python/headroom/_core.cpython-<ver>-<platform>.so`.
+SO_FILE=$(find crates/headroom-py/python/headroom -maxdepth 1 \
+ -name "_core.cpython-*.so" -o -name "_core.cpython-*.dylib" -o -name "_core.pyd" \
+ 2>/dev/null | head -1)
+
+if [[ -z "$SO_FILE" ]]; then
+ echo "error: maturin develop succeeded but produced no _core.* binary." >&2
+ exit 1
+fi
+
+# Symlink into the in-tree package dir.
+LINK_NAME="headroom/$(basename "$SO_FILE")"
+ln -sf "$(pwd)/$SO_FILE" "$LINK_NAME"
+echo "linked: $LINK_NAME -> $SO_FILE"
+
+# Smoke-test the import to fail loudly if anything is misconfigured.
+python -c "from headroom._core import DiffCompressor; print('headroom._core OK:', DiffCompressor)"
diff --git a/tests/test_transforms/test_diff_compressor.py b/tests/test_transforms/test_diff_compressor.py
index 5be31cd14..522434767 100644
--- a/tests/test_transforms/test_diff_compressor.py
+++ b/tests/test_transforms/test_diff_compressor.py
@@ -1,142 +1,29 @@
-"""Comprehensive tests for diff_compressor.py.
+"""Comprehensive tests for the public DiffCompressor API.
Tests cover:
-1. Parsing of unified diff format
-2. Context line reduction
-3. Hunk selection and limiting
-4. Compression ratios
-5. Edge cases
+1. Context line reduction
+2. Hunk selection and limiting
+3. Compression ratios
+4. Edge cases
+5. Bug-fix regressions and routing-gap fixtures
+
+Stage 3b note (2026-04-25): the Python `DiffCompressor` implementation
+was retired in favor of the Rust-backed shim (`headroom._core` via PyO3).
+Tests that probed Python-only internals — `_parse_diff`, `_score_hunks`,
+the `DiffHunk` / `DiffFile` parser dataclasses — were removed because
+the Rust crate has its own parallel coverage in
+`crates/headroom-core/tests`. Public-API tests (anything calling
+`compressor.compress(...)`) are preserved unchanged: they exercise the
+Rust backend through the same import path and assert the same outputs.
"""
from headroom.transforms.diff_compressor import (
DiffCompressionResult,
DiffCompressor,
DiffCompressorConfig,
- DiffFile,
- DiffHunk,
)
-class TestDiffParsing:
- """Tests for parsing unified diff format."""
-
- def test_parse_simple_diff(self):
- """Simple single-file diff is parsed correctly."""
- content = """diff --git a/src/main.py b/src/main.py
---- a/src/main.py
-+++ b/src/main.py
-@@ -10,6 +10,7 @@ def main():
- print("hello")
-+ print("world")
- return 0
-"""
- compressor = DiffCompressor()
- _, diff_files = compressor._parse_diff(content.split("\n"))
-
- assert len(diff_files) == 1
- assert diff_files[0].header == "diff --git a/src/main.py b/src/main.py"
- assert diff_files[0].old_file == "--- a/src/main.py"
- assert diff_files[0].new_file == "+++ b/src/main.py"
- assert len(diff_files[0].hunks) == 1
- assert diff_files[0].hunks[0].additions == 1
- assert diff_files[0].hunks[0].deletions == 0
-
- def test_parse_multi_file_diff(self):
- """Multi-file diff is parsed into separate DiffFile objects."""
- content = """diff --git a/file1.py b/file1.py
---- a/file1.py
-+++ b/file1.py
-@@ -1,3 +1,4 @@
- line1
-+added line
- line2
-diff --git a/file2.py b/file2.py
---- a/file2.py
-+++ b/file2.py
-@@ -5,4 +5,3 @@
- keep
--removed
- keep2
-"""
- compressor = DiffCompressor()
- _, diff_files = compressor._parse_diff(content.split("\n"))
-
- assert len(diff_files) == 2
- assert "file1.py" in diff_files[0].header
- assert "file2.py" in diff_files[1].header
- assert diff_files[0].hunks[0].additions == 1
- assert diff_files[0].hunks[0].deletions == 0
- assert diff_files[1].hunks[0].additions == 0
- assert diff_files[1].hunks[0].deletions == 1
-
- def test_parse_multi_hunk_file(self):
- """File with multiple hunks is parsed correctly."""
- content = """diff --git a/src/utils.py b/src/utils.py
---- a/src/utils.py
-+++ b/src/utils.py
-@@ -10,4 +10,5 @@ def helper():
- pass
-+ # added comment
- return True
-@@ -50,3 +51,4 @@ def other():
- x = 1
-+ y = 2
- return x
-"""
- compressor = DiffCompressor()
- _, diff_files = compressor._parse_diff(content.split("\n"))
-
- assert len(diff_files) == 1
- assert len(diff_files[0].hunks) == 2
- assert diff_files[0].total_additions == 2
-
- def test_parse_new_file(self):
- """New file diff is detected."""
- content = """diff --git a/newfile.py b/newfile.py
-new file mode 100644
---- /dev/null
-+++ b/newfile.py
-@@ -0,0 +1,3 @@
-+def new_func():
-+ pass
-+ return None
-"""
- compressor = DiffCompressor()
- _, diff_files = compressor._parse_diff(content.split("\n"))
-
- assert len(diff_files) == 1
- assert diff_files[0].is_new_file is True
- assert diff_files[0].hunks[0].additions == 3
-
- def test_parse_deleted_file(self):
- """Deleted file diff is detected."""
- content = """diff --git a/oldfile.py b/oldfile.py
-deleted file mode 100644
---- a/oldfile.py
-+++ /dev/null
-@@ -1,2 +0,0 @@
--def old_func():
-- pass
-"""
- compressor = DiffCompressor()
- _, diff_files = compressor._parse_diff(content.split("\n"))
-
- assert len(diff_files) == 1
- assert diff_files[0].is_deleted_file is True
- assert diff_files[0].hunks[0].deletions == 2
-
- def test_parse_binary_file(self):
- """Binary file diff is detected."""
- content = """diff --git a/image.png b/image.png
-Binary files a/image.png and b/image.png differ
-"""
- compressor = DiffCompressor()
- _, diff_files = compressor._parse_diff(content.split("\n"))
-
- assert len(diff_files) == 1
- assert diff_files[0].is_binary is True
-
-
class TestContextReduction:
"""Tests for context line reduction."""
@@ -336,58 +223,6 @@ class TestCompressionResult:
assert result.tokens_saved_estimate == 900
-class TestHunkScoring:
- """Tests for context-aware hunk scoring."""
-
- def test_score_by_context_keywords(self):
- """Hunks containing context keywords get higher scores."""
- content = """diff --git a/file.py b/file.py
---- a/file.py
-+++ b/file.py
-@@ -1,3 +1,4 @@
- normal context
-+normal change
- more context
-@@ -10,3 +11,4 @@
- error handling
-+fix the bug here
- return result
-"""
- compressor = DiffCompressor()
- _, diff_files = compressor._parse_diff(content.split("\n"))
- compressor._score_hunks(diff_files, "fix error bug")
-
- # Second hunk should have higher score (contains "fix" and "bug")
- assert len(diff_files[0].hunks) == 2
- assert diff_files[0].hunks[1].score > diff_files[0].hunks[0].score
-
- def test_score_priority_patterns(self):
- """Hunks with priority patterns (error, security) score higher."""
- compressor = DiffCompressor()
-
- hunk_normal = DiffHunk(
- header="@@ -1,1 +1,2 @@",
- lines=["+normal change"],
- additions=1,
- )
- hunk_error = DiffHunk(
- header="@@ -10,1 +10,2 @@",
- lines=["+fix critical error"],
- additions=1,
- )
-
- diff_file = DiffFile(
- header="diff --git a/f.py b/f.py",
- old_file="--- a/f.py",
- new_file="+++ b/f.py",
- hunks=[hunk_normal, hunk_error],
- )
-
- compressor._score_hunks([diff_file], "")
-
- assert hunk_error.score > hunk_normal.score
-
-
class TestSmallDiffPassthrough:
"""Tests for small diff passthrough behavior."""
@@ -539,56 +374,6 @@ class TestEdgeCases:
assert result.compressed is not None
-class TestDiffHunkDataclass:
- """Tests for DiffHunk dataclass."""
-
- def test_change_count_property(self):
- """change_count returns sum of additions and deletions."""
- hunk = DiffHunk(
- header="@@ -1,5 +1,6 @@",
- lines=["+a", "+b", "-c", " ctx"],
- additions=2,
- deletions=1,
- )
- assert hunk.change_count == 3
-
- def test_default_values(self):
- """DiffHunk default values are correct."""
- hunk = DiffHunk(header="@@", lines=[])
- assert hunk.additions == 0
- assert hunk.deletions == 0
- assert hunk.context_lines == 0
- assert hunk.score == 0.0
-
-
-class TestDiffFileDataclass:
- """Tests for DiffFile dataclass."""
-
- def test_total_additions_property(self):
- """total_additions sums across all hunks."""
- hunk1 = DiffHunk(header="@@", lines=[], additions=3)
- hunk2 = DiffHunk(header="@@", lines=[], additions=5)
- diff_file = DiffFile(
- header="diff --git",
- old_file="---",
- new_file="+++",
- hunks=[hunk1, hunk2],
- )
- assert diff_file.total_additions == 8
-
- def test_total_deletions_property(self):
- """total_deletions sums across all hunks."""
- hunk1 = DiffHunk(header="@@", lines=[], deletions=2)
- hunk2 = DiffHunk(header="@@", lines=[], deletions=4)
- diff_file = DiffFile(
- header="diff --git",
- old_file="---",
- new_file="+++",
- hunks=[hunk1, hunk2],
- )
- assert diff_file.total_deletions == 6
-
-
class TestConfigOptions:
"""Tests for configuration options."""
diff --git a/tests/test_transforms/test_diff_compressor_rust_parity.py b/tests/test_transforms/test_diff_compressor_rust_parity.py
new file mode 100644
index 000000000..3feac188c
--- /dev/null
+++ b/tests/test_transforms/test_diff_compressor_rust_parity.py
@@ -0,0 +1,119 @@
+"""Parity test: Rust-backed `DiffCompressor` vs recorded fixtures.
+
+Stage 3b verification, post-deletion. The Python implementation has
+been retired; the only remaining `DiffCompressor` is Rust-backed via
+PyO3 (`headroom._core`). These tests guard against regressions by
+replaying every recorded fixture in
+`tests/parity/fixtures/diff_compressor/` through the Rust backend and
+asserting the output matches the recording byte-for-byte.
+
+Skipped automatically when the `headroom._core` wheel isn't installed
+(e.g. CI lane without the maturin step). The Rust crate's own
+`cargo run -p headroom-parity` covers the same fixtures from the Rust
+side; this Python-side test catches PyO3 bridge regressions
+(input/output mistranslation) that the Rust-only test cannot.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+
+def _has_core() -> bool:
+ try:
+ import headroom._core # noqa: F401
+
+ return True
+ except ImportError:
+ return False
+
+
+pytestmark = pytest.mark.skipif(
+ not _has_core(),
+ reason="headroom._core wheel not installed (run `scripts/build_rust_extension.sh`)",
+)
+
+
+_FIXTURES_DIR = Path(__file__).parent.parent / "parity" / "fixtures" / "diff_compressor"
+
+
+def _all_fixtures() -> list[Path]:
+ return sorted(_FIXTURES_DIR.glob("*.json"))
+
+
+def test_at_least_27_fixtures_present():
+ """Sanity check: the bug-fix and routing-gap fixtures landed."""
+ fixtures = _all_fixtures()
+ assert len(fixtures) >= 27, (
+ f"expected >= 27 fixtures, found {len(fixtures)}. "
+ "If you re-recorded and got fewer, something deleted them."
+ )
+
+
[email protected]("fixture_path", _all_fixtures(), ids=lambda p: p.name)
+def test_rust_backend_matches_recorded_output(fixture_path: Path):
+ """Replay each recorded input through the Rust backend; every output
+ field must match the recording. Any mismatch is a PyO3 bridge bug or a
+ real Rust regression (cross-check with `cargo run -p headroom-parity`
+ to localize).
+ """
+ from headroom.transforms.diff_compressor import DiffCompressor, DiffCompressorConfig
+
+ rec = json.loads(fixture_path.read_text())
+ cfg = DiffCompressorConfig(**rec["config"])
+ result = DiffCompressor(cfg).compress(rec["input"])
+
+ expected = rec["output"]
+ # Field-by-field surfaces a single divergence rather than dumping the
+ # whole result on failure.
+ assert result.compressed == expected["compressed"], f"{fixture_path.name}: compressed differs"
+ assert result.original_line_count == expected["original_line_count"]
+ assert result.compressed_line_count == expected["compressed_line_count"]
+ assert result.files_affected == expected["files_affected"]
+ assert result.additions == expected["additions"]
+ assert result.deletions == expected["deletions"]
+ assert result.hunks_kept == expected["hunks_kept"]
+ assert result.hunks_removed == expected["hunks_removed"]
+ assert result.cache_key == expected["cache_key"]
+
+
+def test_compress_with_stats_returns_python_dataclass_and_pyo3_stats():
+ """The sidecar stats API returns a `(DiffCompressionResult,
+ _core.DiffCompressorStats)` tuple. Verify both halves are usable from
+ Python — PyO3 doesn't auto-promote the stats class to a dataclass.
+ """
+ from headroom.transforms.diff_compressor import (
+ DiffCompressionResult,
+ DiffCompressor,
+ DiffCompressorConfig,
+ )
+
+ diff = (
+ "diff --git a/a.py b/a.py\n--- a/a.py\n+++ b/a.py\n@@ -1 +1 @@\n-old\n+new\n"
+ ) + "# pad\n" * 60
+ result, stats = DiffCompressor(DiffCompressorConfig()).compress_with_stats(diff)
+
+ assert isinstance(result, DiffCompressionResult)
+ assert stats.input_lines >= 60
+ assert stats.processing_duration_us >= 0
+ assert isinstance(stats.parse_warnings, list)
+
+
+def test_content_router_uses_rust_backend():
+ """Stage 3b deletion check: ContentRouter._get_diff_compressor must
+ return the (now Rust-only) DiffCompressor — there's no other backend
+ left, so this guards against an accidental re-introduction of a
+ Python-side stub or fallback.
+ """
+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
+ from headroom.transforms.diff_compressor import DiffCompressor
+
+ router = ContentRouter(ContentRouterConfig())
+ compressor = router._get_diff_compressor()
+ assert isinstance(compressor, DiffCompressor)
+ # The Rust delegation handle must be live — guards against a
+ # half-deleted shim that defines the class but not `_rust`.
+ assert hasattr(compressor, "_rust")