Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor(rust): diff_compressor — surface hidden cutoffs + lossy-emit stats

chopratejas255b4295dc24
diff --git a/crates/headroom-core/src/transforms/diff_compressor.rs b/crates/headroom-core/src/transforms/diff_compressor.rs
index d1dd32d6a..bfb16a991 100644
--- a/crates/headroom-core/src/transforms/diff_compressor.rs
+++ b/crates/headroom-core/src/transforms/diff_compressor.rs
@@ -42,18 +42,71 @@ use std::time::Instant;
use md5::{Digest, Md5};
use regex::Regex;
+// ─── Score-weight constants ────────────────────────────────────────────────
+//
+// These knobs tune the relevance scorer (used only when `max_hunks_per_file`
+// fires and we have to rank middle hunks). Promoted from inline magic numbers
+// so a future tuning PR can move them with full visibility, and so reviewers
+// can see exactly what bias the scorer encodes. Defaults match the Python
+// implementation byte-for-byte.
+
+/// Per-line-change weight in the change-density base term. The base score is
+/// `min(CHANGE_DENSITY_CAP, change_count * CHANGE_DENSITY_WEIGHT)`.
+pub const SCORE_CHANGE_DENSITY_WEIGHT: f64 = 0.03;
+/// Cap on the change-density base term; beyond this, additional changes
+/// don't keep raising the score.
+pub const SCORE_CHANGE_DENSITY_CAP: f64 = 0.3;
+/// Boost added per matching word from the user-query context that appears in
+/// the hunk content (case-insensitive substring match).
+pub const SCORE_CONTEXT_WORD_WEIGHT: f64 = 0.2;
+/// Minimum word length (exclusive of) for context-word matching. Words of
+/// length ≤ this are skipped (matches Python's `len(word) > 2`). Filters out
+/// stop-words like "is", "to", "a".
+pub const SCORE_CONTEXT_MIN_WORD_LEN: usize = 2;
+/// Boost added when ANY priority pattern matches (only one boost per hunk —
+/// matches Python's `break` after first match).
+pub const SCORE_PRIORITY_PATTERN_BOOST: f64 = 0.3;
+/// Cap on the total hunk score after all boosts.
+pub const SCORE_TOTAL_CAP: f64 = 1.0;
+
// ─── Public API ─────────────────────────────────────────────────────────────
/// Configuration. All defaults match Python `DiffCompressorConfig`.
#[derive(Debug, Clone)]
pub struct DiffCompressorConfig {
+ /// How many context lines (` ` prefix) to keep on either side of each
+ /// `+`/`-` change line. Python default: 2.
pub max_context_lines: usize,
+ /// Cap on the number of hunks kept per file. When exceeded, keeps first
+ /// + last + top-scored middle. Python default: 10.
pub max_hunks_per_file: usize,
+ /// Cap on the number of files kept across the whole diff. When exceeded,
+ /// sorts files by total changes (desc) and keeps top N. Files beyond
+ /// the cap are silently dropped from the output (their names appear in
+ /// [`DiffCompressorStats::files_dropped`] for observability). Python
+ /// default: 20.
pub max_files: usize,
+ /// Reserved — Python config exposes this but the algorithm always keeps
+ /// `+` lines. Kept for fixture-config schema compatibility.
pub always_keep_additions: bool,
+ /// Reserved — same as `always_keep_additions` but for `-` lines.
pub always_keep_deletions: bool,
+ /// If true, attach an MD5-based CCR retrieval marker to the compressed
+ /// output when compression met the savings threshold. Python default: true.
pub enable_ccr: bool,
+ /// **Misnomer alert.** This actually gates the entire compression path,
+ /// not just the CCR marker: when `original_line_count <
+ /// min_lines_for_ccr`, the input is returned unchanged, no parsing, no
+ /// summary, no CCR. The name comes from the Python implementation; we
+ /// keep it to maintain fixture-config compatibility. Treat it as
+ /// "minimum diff size before we bother compressing." Python default: 50.
pub min_lines_for_ccr: usize,
+ /// CCR retrieval marker is emitted only when
+ /// `compressed_line_count < original_line_count *
+ /// min_compression_ratio_for_ccr`. Lower values demand more aggressive
+ /// compression before we bother emitting the marker. **Rust-only knob**
+ /// — Python hardcodes 0.8. Default: 0.8 (matches Python).
+ pub min_compression_ratio_for_ccr: f64,
}
impl Default for DiffCompressorConfig {
@@ -66,6 +119,7 @@ impl Default for DiffCompressorConfig {
always_keep_deletions: true,
enable_ccr: true,
min_lines_for_ccr: 50,
+ min_compression_ratio_for_ccr: 0.8,
}
}
}
@@ -119,6 +173,30 @@ pub struct DiffCompressorStats {
/// Lines in the largest hunk we dropped (per-file cap). 0 if none dropped.
pub largest_hunk_dropped_lines: usize,
+ /// Files whose original `new file mode` / `deleted file mode` line was
+ /// normalized to `100644` on output. Each entry is `(file_label,
+ /// original_mode_line)` — e.g. `("a/foo.sh -> b/foo.sh", "new file
+ /// mode 100755")`. Empty when no normalization occurred (mode was
+ /// already 100644, or no mode line was present).
+ ///
+ /// Why this matters: parity with Python forces us to hardcode `100644`
+ /// in the emit path regardless of what the input said. An input with
+ /// executable bit `100755` becomes a non-executable `100644` on output —
+ /// silent information loss. Surfacing this lets prod monitoring catch
+ /// real cases where it bites.
+ pub file_mode_normalizations: Vec<(String, String)>,
+
+ /// Binary file marker lines whose original detail (e.g. `Binary files
+ /// a/x.png and b/x.png differ`) was simplified to `Binary files differ`
+ /// on output. Each entry is the full original line. Empty when no
+ /// simplification occurred (input was already `Binary files differ`,
+ /// or the file wasn't binary).
+ ///
+ /// Same parity-loss pattern as `file_mode_normalizations`: Python's
+ /// emitter hardcodes `Binary files differ`, dropping the filename
+ /// detail. This stat surfaces what was lost.
+ pub binary_files_simplified: Vec<String>,
+
/// Non-fatal parser hiccups: unrecognized line patterns, malformed
/// hunk headers, etc. Surfacing them rather than dropping silently.
pub parse_warnings: Vec<String>,
@@ -243,6 +321,40 @@ impl DiffCompressor {
}
stats.files_kept = diff_files.len();
+ // Capture lossy-emit signals on the files that survived the file cap.
+ // These cases are parity-bound (Python's emit hardcodes `100644` and
+ // `Binary files differ` regardless of input), so the only honest move
+ // is to surface the loss via observability rather than fix it.
+ for file in diff_files.iter() {
+ let label = format!("{} -> {}", file.old_file, file.new_file);
+ // File-mode normalization: any original mode line not literally
+ // `new file mode 100644` / `deleted file mode 100644` is lost on
+ // emit. Includes `100755` (executable), `100600` (private),
+ // `120000` (symlink), `160000` (gitlink/submodule), etc.
+ if let Some(orig) = &file.original_new_file_mode_line {
+ if orig != "new file mode 100644" {
+ stats
+ .file_mode_normalizations
+ .push((label.clone(), orig.clone()));
+ }
+ }
+ if let Some(orig) = &file.original_deleted_file_mode_line {
+ if orig != "deleted file mode 100644" {
+ stats
+ .file_mode_normalizations
+ .push((label.clone(), orig.clone()));
+ }
+ }
+ // Binary detail: any line richer than the bare `Binary files
+ // differ` (which is virtually all of them — git emits filenames)
+ // gets simplified on emit.
+ if let Some(orig) = &file.original_binary_line {
+ if orig != "Binary files differ" {
+ stats.binary_files_simplified.push(orig.clone());
+ }
+ }
+ }
+
// Compress each file's hunks: cap count, then trim context.
let mut compressed_files: Vec<DiffFile> = Vec::with_capacity(diff_files.len());
let mut total_additions = 0usize;
@@ -314,7 +426,9 @@ impl DiffCompressor {
let compressed_line_count = count_split_lines(&compressed_output);
// CCR layer: hash original with MD5[:24], append retrieval marker
- // *only* if we saved >20% of lines (matches Python's threshold).
+ // *only* if compression met `min_compression_ratio_for_ccr`. Python
+ // hardcodes 0.8 (>20% savings); we expose it as a config knob with
+ // the same default.
//
// CRITICAL: `compressed_line_count` is captured BEFORE the CCR marker
// is appended, both for the marker's own text ("compressed to N")
@@ -322,9 +436,10 @@ impl DiffCompressor {
// line than `compressed_line_count` reports, by design — Python
// does the same. Mismatching this by recounting after the append
// breaks parity by 1.
+ let savings_threshold = self.config.min_compression_ratio_for_ccr;
let mut cache_key: Option<String> = None;
if self.config.enable_ccr
- && (compressed_line_count as f64) < (original_line_count as f64) * 0.8
+ && (compressed_line_count as f64) < (original_line_count as f64) * savings_threshold
{
let key = md5_hex_24(content);
compressed_output.push('\n');
@@ -337,7 +452,15 @@ impl DiffCompressor {
} else if !self.config.enable_ccr {
stats.ccr_skipped_reason = Some("ccr disabled".into());
} else {
- stats.ccr_skipped_reason = Some("compression below 20% threshold".into());
+ stats.ccr_skipped_reason = Some(format!(
+ "compression ratio {:.3} above threshold {:.3}",
+ if original_line_count == 0 {
+ 1.0
+ } else {
+ compressed_line_count as f64 / original_line_count as f64
+ },
+ savings_threshold
+ ));
}
stats.output_lines = compressed_line_count;
@@ -387,6 +510,15 @@ struct DiffFile {
is_new_file: bool,
is_deleted_file: bool,
is_renamed: bool,
+ /// Full original `new file mode <NNNNNN>` line if present. Captured so
+ /// we can detect when emit-time normalization to `100644` lost the
+ /// executable bit (or any other mode signal).
+ original_new_file_mode_line: Option<String>,
+ /// Full original `deleted file mode <NNNNNN>` line if present.
+ original_deleted_file_mode_line: Option<String>,
+ /// Full original `Binary files X and Y differ` line if present.
+ /// Captured so we can detect when emit simplifies to `Binary files differ`.
+ original_binary_line: Option<String>,
}
impl DiffFile {
@@ -457,16 +589,24 @@ fn parse_diff(lines: &[&str]) -> (Vec<DiffFile>, Vec<String>) {
is_new_file: false,
is_deleted_file: false,
is_renamed: false,
+ original_new_file_mode_line: None,
+ original_deleted_file_mode_line: None,
+ original_binary_line: None,
});
continue;
}
- // File-level mode/binary/rename markers.
+ // File-level mode/binary/rename markers. Capture the full original
+ // line in addition to the boolean — Python only sets the flag and
+ // discards the actual mode/detail, but we want the original around
+ // so we can surface emit-time normalizations as observability.
if let Some(f) = current_file.as_mut() {
if line.starts_with("new file mode") {
f.is_new_file = true;
+ f.original_new_file_mode_line = Some(line.to_string());
} else if line.starts_with("deleted file mode") {
f.is_deleted_file = true;
+ f.original_deleted_file_mode_line = Some(line.to_string());
} else if line.starts_with("rename ")
|| line.starts_with("similarity ")
|| line.starts_with("copy ")
@@ -474,6 +614,7 @@ fn parse_diff(lines: &[&str]) -> (Vec<DiffFile>, Vec<String>) {
f.is_renamed = true;
} else if binary_regex().is_match(line) {
f.is_binary = true;
+ f.original_binary_line = Some(line.to_string());
}
}
@@ -569,28 +710,28 @@ fn score_hunks(files: &mut [DiffFile], context: &str) {
for hunk in file.hunks.iter_mut() {
let mut score: f64 = 0.0;
// Base score from change density (capped).
- score += (hunk.additions as f64 + hunk.deletions as f64) * 0.03;
- if score > 0.3 {
- score = 0.3;
+ score += (hunk.additions as f64 + hunk.deletions as f64) * SCORE_CHANGE_DENSITY_WEIGHT;
+ if score > SCORE_CHANGE_DENSITY_CAP {
+ score = SCORE_CHANGE_DENSITY_CAP;
}
let hunk_content_lower = hunk.lines.join("\n").to_lowercase();
for word in &context_words {
- if word.len() > 2 && hunk_content_lower.contains(word) {
- score += 0.2;
+ if word.len() > SCORE_CONTEXT_MIN_WORD_LEN && hunk_content_lower.contains(word) {
+ score += SCORE_CONTEXT_WORD_WEIGHT;
}
}
for pat in priority_patterns() {
if pat.is_match(&hunk_content_lower) {
- score += 0.3;
+ score += SCORE_PRIORITY_PATTERN_BOOST;
break;
}
}
- if score > 1.0 {
- score = 1.0;
+ if score > SCORE_TOTAL_CAP {
+ score = SCORE_TOTAL_CAP;
}
hunk.score = score;
}
@@ -858,6 +999,8 @@ fn emit_span_and_return(stats: DiffCompressorStats) -> DiffCompressorStats {
parse_warnings = stats.parse_warnings.len(),
processing_duration_us = stats.processing_duration_us,
cache_key_emitted = stats.cache_key_emitted,
+ file_mode_normalizations = stats.file_mode_normalizations.len(),
+ binary_files_simplified = stats.binary_files_simplified.len(),
"diff_compressor finished"
);
stats
@@ -960,4 +1103,154 @@ mod tests {
assert_eq!(r.compressed_line_count, 129);
assert!(r.cache_key.is_some());
}
+
+ // ─── Lossy-path tests ───────────────────────────────────────────────────
+
+ /// Build a single-file diff with N hunks. Each hunk has 2 context lines,
+ /// 1 deletion, 1 addition, 2 context lines. Hunk headers use distinct
+ /// start lines so the in-order resort after middle-hunk selection works.
+ fn build_n_hunk_diff(n: usize) -> String {
+ let mut s = String::from("diff --git a/big.py b/big.py\n--- a/big.py\n+++ b/big.py\n");
+ for i in 0..n {
+ // 100 lines apart per hunk so they're independent.
+ let start = i * 100 + 1;
+ s.push_str(&format!("@@ -{0},6 +{0},6 @@\n", start));
+ s.push_str(&format!(" ctx_a_{i}\n"));
+ s.push_str(&format!(" ctx_b_{i}\n"));
+ s.push_str(&format!("-old_{i}\n"));
+ s.push_str(&format!("+new_{i}\n"));
+ s.push_str(&format!(" ctx_c_{i}\n"));
+ s.push_str(&format!(" ctx_d_{i}\n"));
+ }
+ s
+ }
+
+ #[test]
+ fn max_hunks_per_file_cap_drops_excess_and_records_stats() {
+ // 15 hunks, cap = 10 → 5 dropped. First + last + 8 top-scored middle kept.
+ let cfg = DiffCompressorConfig {
+ max_hunks_per_file: 10,
+ ..Default::default()
+ };
+ let input = build_n_hunk_diff(15);
+ let (result, stats) = DiffCompressor::new(cfg).compress_with_stats(&input, "");
+
+ assert_eq!(result.hunks_kept, 10, "kept 10 hunks");
+ assert_eq!(result.hunks_removed, 5, "dropped 5");
+ assert_eq!(stats.hunks_total, 15);
+ assert_eq!(stats.hunks_dropped, 5);
+ // Per-file accounting must match overall.
+ let per_file_total: usize = stats.hunks_dropped_per_file.values().sum();
+ assert_eq!(per_file_total, 5);
+ // The dropped hunks have 6 lines each (after parsing); largest_dropped
+ // should reflect that.
+ assert!(stats.largest_hunk_dropped_lines >= 6);
+ }
+
+ #[test]
+ fn max_files_cap_drops_files_and_records_names_in_stats() {
+ // 25 files, cap = 20 → 5 dropped. files_dropped should carry the names.
+ let cfg = DiffCompressorConfig {
+ max_files: 20,
+ ..Default::default()
+ };
+ let input = build_synthetic_diff(25);
+ let (_result, stats) = DiffCompressor::new(cfg).compress_with_stats(&input, "");
+
+ assert_eq!(stats.files_total, 25);
+ assert_eq!(stats.files_kept, 20);
+ assert_eq!(
+ stats.files_dropped.len(),
+ 5,
+ "expected 5 dropped file labels"
+ );
+ // Each label should be the `old_file -> new_file` form.
+ for label in &stats.files_dropped {
+ assert!(
+ label.contains("-> "),
+ "label `{label}` should contain ` -> `"
+ );
+ }
+ }
+
+ #[test]
+ fn file_mode_normalization_is_recorded_for_executable_bit() {
+ // Construct a long-enough diff that introduces an executable file.
+ // Mode 100755 != 100644, so emit will silently normalize and stats
+ // must capture the original.
+ let mut input = String::from(
+ "diff --git a/script.sh b/script.sh\n\
+ new file mode 100755\n\
+ --- /dev/null\n\
+ +++ b/script.sh\n\
+ @@ -0,0 +1,3 @@\n\
+ +#!/bin/sh\n\
+ +echo hi\n\
+ +exit 0\n",
+ );
+ // Pad to clear `min_lines_for_ccr` so compression runs.
+ for _ in 0..50 {
+ input.push_str("# pad\n");
+ }
+ let (_r, stats) = DiffCompressor::default().compress_with_stats(&input, "");
+ assert_eq!(stats.file_mode_normalizations.len(), 1, "{stats:?}");
+ let (label, original) = &stats.file_mode_normalizations[0];
+ assert!(label.contains("script.sh"));
+ assert_eq!(original, "new file mode 100755");
+ }
+
+ #[test]
+ fn binary_files_simplification_is_recorded() {
+ let mut input = String::from(
+ "diff --git a/img.png b/img.png\n\
+ Binary files a/img.png and b/img.png differ\n",
+ );
+ // Pad to clear min_lines_for_ccr.
+ for _ in 0..60 {
+ input.push_str("# pad\n");
+ }
+ let (_r, stats) = DiffCompressor::default().compress_with_stats(&input, "");
+ assert_eq!(stats.binary_files_simplified.len(), 1, "{stats:?}");
+ assert_eq!(
+ stats.binary_files_simplified[0],
+ "Binary files a/img.png and b/img.png differ"
+ );
+ }
+
+ #[test]
+ fn min_compression_ratio_for_ccr_is_configurable() {
+ // With default 0.8, the 8-file synthetic compresses 177→129 (ratio
+ // 0.729) which beats the threshold → CCR marker emitted.
+ let r = DiffCompressor::default().compress(&build_synthetic_diff(8), "");
+ assert!(r.cache_key.is_some(), "default 0.8 should emit CCR");
+
+ // With 0.5, the same compression (0.729 ratio) does NOT beat
+ // 0.5 → no CCR marker, no cache_key.
+ let cfg = DiffCompressorConfig {
+ min_compression_ratio_for_ccr: 0.5,
+ ..Default::default()
+ };
+ let (r2, stats) =
+ DiffCompressor::new(cfg).compress_with_stats(&build_synthetic_diff(8), "");
+ assert!(
+ r2.cache_key.is_none(),
+ "0.5 threshold should suppress CCR for 0.729-ratio compression"
+ );
+ assert!(!stats.cache_key_emitted);
+ assert!(stats.ccr_skipped_reason.is_some());
+ }
+
+ #[test]
+ fn score_constants_match_inline_values() {
+ // Pin the constants so a future tuning PR has to update both.
+ // (If you're updating these, also update the docs in the parity
+ // contract — the scorer only fires when max_hunks_per_file caps,
+ // so the impact is limited but observable.)
+ assert_eq!(SCORE_CHANGE_DENSITY_WEIGHT, 0.03);
+ assert_eq!(SCORE_CHANGE_DENSITY_CAP, 0.3);
+ assert_eq!(SCORE_CONTEXT_WORD_WEIGHT, 0.2);
+ assert_eq!(SCORE_CONTEXT_MIN_WORD_LEN, 2);
+ assert_eq!(SCORE_PRIORITY_PATTERN_BOOST, 0.3);
+ assert_eq!(SCORE_TOTAL_CAP, 1.0);
+ }
}
diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rs
index 8b45ca011..43720d32f 100644
--- a/crates/headroom-parity/src/lib.rs
+++ b/crates/headroom-parity/src/lib.rs
@@ -220,6 +220,12 @@ impl TransformComparator for DiffCompressorComparator {
.and_then(|v| v.as_u64())
.map(|v| v as usize)
.unwrap_or(50),
+ // Rust-only knob; Python fixtures don't carry this field. The
+ // 0.8 default reproduces Python's hardcoded 20%-savings gate.
+ min_compression_ratio_for_ccr: config
+ .get("min_compression_ratio_for_ccr")
+ .and_then(|v| v.as_f64())
+ .unwrap_or(0.8),
};
let compressor = DiffCompressor::new(cfg);