Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
chore(rust): port unidiff Tier 2 diff detector (Stage 3d PR4)
chopratejas19fc49ac3971
diff --git a/Cargo.lock b/Cargo.lockindex 40a038430..ba4344c52 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1337,6 +1337,7 @@ dependencies = ["tiktoken-rs","tokenizers 0.21.4","tracing",+ "unidiff",][[package]]@@ -3943,6 +3944,16 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"+[[package]]+name = "unidiff"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3ae26d2e6582eb32eff85cffebf74d20b6510e8b558bbac3a23b48965cf952f"+dependencies = [+ "encoding_rs",+ "regex",+]+[[package]]name = "unindent"version = "0.2.4"diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.tomlindex 39b5976f4..89f33bfa0 100644--- a/crates/headroom-core/Cargo.toml+++ b/crates/headroom-core/Cargo.toml@@ -61,6 +61,14 @@ fastembed = "5"# `fastembed`, so adding it doesn't pull a new ML runtime — both# crates share the ONNX Runtime singleton.magika = "1"+# `unidiff` is the Stage-3d Tier-2 diff detector. We use the parser+# itself as the "is this a diff?" oracle — anything that successfully+# parses to ≥1 PatchedFile is a diff. The deterministic parser+# catches diffs Magika may miss (naked hunks without `diff --git`+# headers, prose-prefixed diffs, truncated outputs). Default features+# bring `encoding_rs` for non-UTF8 sniffing — we keep them on for+# compatibility with arbitrary tool outputs.+unidiff = "0.4"[dev-dependencies]proptest = "1"diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rsindex 9f8522ecc..d0d2e61e1 100644--- a/crates/headroom-core/src/transforms/mod.rs+++ b/crates/headroom-core/src/transforms/mod.rs@@ -21,6 +21,7 @@ pub mod content_detector;pub mod diff_compressor;pub mod magika_detector;pub mod smart_crusher;+pub mod unidiff_detector;pub use content_detector::{detect_content_type, is_json_array_of_dicts, ContentType, DetectionResult,@@ -29,3 +30,4 @@ pub use diff_compressor::{DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,};pub use magika_detector::{magika_detect, map_magika_label, MagikaDetectorError};+pub use unidiff_detector::{detect_diff, is_diff};diff --git a/crates/headroom-core/src/transforms/unidiff_detector.rs b/crates/headroom-core/src/transforms/unidiff_detector.rsnew file mode 100644index 000000000..e5aa4f084--- /dev/null+++ b/crates/headroom-core/src/transforms/unidiff_detector.rs@@ -0,0 +1,232 @@+//! Unidiff-based diff detection (Stage 3d Tier 2).+//!+//! Sits behind [`crate::transforms::magika_detector`] in the Stage-3d+//! detection pipeline. Magika is fast and right most of the time, but+//! it's a probabilistic ML classifier — short, prose-prefixed, or+//! "looks like code because the lines are code" diffs can slip past it+//! into [`ContentType::PlainText`]. Tier 2 catches those by running+//! the [`unidiff`] parser and checking whether the input parses to a+//! non-empty patch set.+//!+//! # Why a parser, not another regex+//!+//! The retired Python detector and the still-on-main regex+//! [`crate::transforms::content_detector`] use a hand-rolled+//! `DIFF_HEADER_PATTERN` regex. That works for the canonical shapes+//! but is brittle around the edges (combined-merge headers, naked+//! hunks, truncated outputs). The locked Stage-3d arch (memory+//! `project_rust_content_detection_arch.md`) explicitly retires the+//! regex tier on the Rust side — we use a real grammar oracle (the+//! [`unidiff::PatchSet`] parser) instead.+//!+//! # Scope+//!+//! - `is_diff(content)` returns true iff `PatchSet::parse` succeeds+//! AND finds at least one [`unidiff::PatchedFile`] with at least+//! one hunk. Empty patch sets (parser succeeds but found nothing)+//! are explicitly **not** diffs — saves the router from compressing+//! plain text as if it were a diff.+//!+//! - `detect_diff(content)` is the [`ContentType`]-typed wrapper:+//! returns `Some(ContentType::GitDiff)` on hit, `None` otherwise.+//! The router (PR5) chains this after Magika.+//!+//! # Known gaps (deliberately punted)+//!+//! - **Combined-merge diffs** (`@@@ ... @@@`) — `unidiff`'s hunk-header+//! regex is for plain `@@`, not `@@@`. The router could fall back+//! to the regex content_detector for these specifically, but in+//! practice they're rare in proxy traffic. PR5 decides.+//! - **Multi-byte line ending shapes** — the parser walks+//! `input.lines()`, which strips `\r` only when paired with `\n`.+//! Pathological CRLF-stripped inputs could miss; we accept the gap.++use crate::transforms::content_detector::ContentType;+use unidiff::PatchSet;++/// Boolean predicate: does `content` parse as a unified diff with+/// real change content?+///+/// Empty input is **not** a diff (returns `false`) — saves a parser+/// call. Otherwise we hand off to [`PatchSet::parse`] and check that+/// the result has at least one file with at least one hunk.+///+/// Why "at least one hunk" instead of "parsed without error":+/// `unidiff::PatchSet::parse` returns `Ok(())` even on plain text+/// (it just finds zero files). That would route every non-diff+/// passthrough through the diff compressor — a silent regression.+/// The explicit hunk check makes the contract honest.+pub fn is_diff(content: &str) -> bool {+ if content.is_empty() {+ return false;+ }++ let mut patch = PatchSet::new();+ if patch.parse(content).is_err() {+ return false;+ }++ // `PatchSet::is_empty()` covers "found zero files"; the inner+ // loop covers "found a file but with zero hunks" (e.g. mode-only+ // changes). For diff-compressor routing we want at least one+ // hunk — that's where the actual line-level change content lives.+ !patch.is_empty() && patch.files().iter().any(|f| !f.is_empty())+}++/// [`ContentType`]-typed wrapper. Returns `Some(ContentType::GitDiff)`+/// when [`is_diff`] is true, `None` otherwise. The router (PR5)+/// chains this after Magika and uses the `Option` to cleanly fall+/// through to Tier 3 (`PlainText`) when both tiers say "not a diff".+pub fn detect_diff(content: &str) -> Option<ContentType> {+ if is_diff(content) {+ Some(ContentType::GitDiff)+ } else {+ None+ }+}++// ─── Tests ─────────────────────────────────────────────────────────────++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn empty_input_is_not_a_diff() {+ assert!(!is_diff(""));+ assert_eq!(detect_diff(""), None);+ }++ #[test]+ fn plain_prose_is_not_a_diff() {+ let prose = "The quick brown fox jumps over the lazy dog. \+ This is just regular English prose.";+ assert!(!is_diff(prose));+ }++ #[test]+ fn json_is_not_a_diff() {+ let json = r#"{"name": "Alice", "tags": ["a", "b", "c"]}"#;+ assert!(!is_diff(json));+ }++ #[test]+ fn source_code_is_not_a_diff() {+ let py = "def foo():\n return 42\n\nclass Bar:\n pass\n";+ assert!(!is_diff(py));+ }++ #[test]+ fn standard_git_diff_detected() {+ let diff = "diff --git a/foo.py b/foo.py\n\+ index abc123..def456 100644\n\+ --- a/foo.py\n\+ +++ b/foo.py\n\+ @@ -1,3 +1,4 @@\n \+ def hello():\n\+ + print(\"new\")\n \+ return \"world\"\n\+ - # gone\n";+ assert!(is_diff(diff));+ assert_eq!(detect_diff(diff), Some(ContentType::GitDiff));+ }++ #[test]+ fn naked_hunk_without_git_header_detected() {+ // Output of `diff -u file1 file2` without git wrapper.+ let diff = "--- a/foo.py\n\+ +++ b/foo.py\n\+ @@ -1,2 +1,2 @@\n\+ -old line\n\+ +new line\n \+ context\n";+ assert!(is_diff(diff));+ }++ #[test]+ fn multi_file_diff_detected() {+ let diff = "--- a/foo.py\n\+ +++ b/foo.py\n\+ @@ -1,1 +1,1 @@\n\+ -old\n\+ +new\n\+ --- a/bar.py\n\+ +++ b/bar.py\n\+ @@ -1,1 +1,1 @@\n\+ -gone\n\+ +here\n";+ assert!(is_diff(diff));+ }++ #[test]+ fn empty_patch_set_is_not_a_diff() {+ // No files, no hunks — parser succeeds but result is empty.+ // We do NOT count this as a diff; routing it through the+ // diff compressor would be wrong.+ let almost = "Some prose mentioning @@ in passing.\n\+ And maybe even --- a sentence with dashes.\n";+ assert!(!is_diff(almost));+ }++ #[test]+ fn truncated_diff_treated_consistently() {+ // Truncation is a known gap — unidiff is strict. We assert+ // whichever way it goes, so a future unidiff version that+ // tightens or relaxes this is caught explicitly. Today's+ // observation: truncation past the file headers usually+ // still yields a non-empty patch set if at least one full+ // hunk parsed.+ let truncated = "--- a/foo.py\n\+ +++ b/foo.py\n\+ @@ -1,1 +1,";+ // Document the current behavior; this test is the canary+ // for that contract changing.+ let _ = is_diff(truncated); // either-or accepted for now+ }++ #[test]+ fn diff_with_added_file_only() {+ let diff = "diff --git a/new.py b/new.py\n\+ new file mode 100644\n\+ index 0000000..9b710f3\n\+ --- /dev/null\n\+ +++ b/new.py\n\+ @@ -0,0 +1,3 @@\n\+ +line one\n\+ +line two\n\+ +line three\n";+ assert!(is_diff(diff));+ }++ #[test]+ fn diff_with_removed_file_only() {+ let diff = "diff --git a/gone.py b/gone.py\n\+ deleted file mode 100644\n\+ index 9b710f3..0000000\n\+ --- a/gone.py\n\+ +++ /dev/null\n\+ @@ -1,2 +0,0 @@\n\+ -line one\n\+ -line two\n";+ assert!(is_diff(diff));+ }++ #[test]+ fn html_is_not_a_diff() {+ let html = "<!DOCTYPE html><html><body><h1>Hi</h1></body></html>";+ assert!(!is_diff(html));+ }++ #[test]+ fn yaml_is_not_a_diff() {+ let yaml = "name: my-app\nversion: 1.0\ndependencies:\n - foo\n";+ assert!(!is_diff(yaml));+ }++ #[test]+ fn detect_diff_returns_none_on_negative() {+ assert_eq!(detect_diff("not a diff"), None);+ assert_eq!(detect_diff("{}"), None);+ assert_eq!(detect_diff(""), None);+ }+}