Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(diff): close ContentRouter routing gaps for merge diffs and long preambles
chopratejas48c13245c537
diff --git a/crates/headroom-core/src/transforms/diff_compressor.rs b/crates/headroom-core/src/transforms/diff_compressor.rsindex f15ac9a56..e1a1c39fc 100644--- a/crates/headroom-core/src/transforms/diff_compressor.rs+++ b/crates/headroom-core/src/transforms/diff_compressor.rs@@ -593,6 +593,29 @@ fn diff_git_regex() -> &'static Regex {RE.get_or_init(|| Regex::new(r"^diff --git a/(.+) b/(.+)$").expect("static regex compiles"))}+/// Bug-fix (2026-04-25): merge-commit headers `diff --combined <path>` and+/// `diff --cc <path>`. Single-path file diffs paired with combined-diff+/// hunk syntax (`@@@`+). Previously not recognized — merge diffs from+/// `git log -p` were treated as a single non-diff blob because the header+/// didn't match `diff --git`, so they fell into pre-diff content.+fn diff_combined_regex() -> &'static Regex {+ static RE: OnceLock<Regex> = OnceLock::new();+ RE.get_or_init(|| Regex::new(r"^diff --combined (.+)$").expect("static regex compiles"))+}++fn diff_cc_regex() -> &'static Regex {+ static RE: OnceLock<Regex> = OnceLock::new();+ RE.get_or_init(|| Regex::new(r"^diff --cc (.+)$").expect("static regex compiles"))+}++/// Returns true if `line` is any kind of `diff --…` file-section header+/// (regular `--git`, or the combined-diff `--combined` / `--cc` variants).+fn is_diff_header(line: &str) -> bool {+ diff_git_regex().is_match(line)+ || diff_combined_regex().is_match(line)+ || diff_cc_regex().is_match(line)+}+fn old_file_regex() -> &'static Regex {static RE: OnceLock<Regex> = OnceLock::new();RE.get_or_init(|| Regex::new(r"^--- (a/(.+)|/dev/null)$").expect("static regex compiles"))@@ -626,8 +649,12 @@ fn parse_diff(lines: &[&str]) -> ParsedDiff {let warnings: Vec<String> = Vec::new();for &line in lines {- // New file section.- if diff_git_regex().is_match(line) {+ // New file section. Includes regular `diff --git` AND merge-commit+ // `diff --combined <path>` / `diff --cc <path>` (bug-fix+ // 2026-04-25). Without these, merge diffs from `git log -p` got+ // treated as one giant pre-diff blob and never reached the+ // hunk-parsing path.+ if is_diff_header(line) {if let Some(h) = current_hunk.take() {if let Some(f) = current_file.as_mut() {f.hunks.push(h);@@ -1477,6 +1504,57 @@ mod tests {);}+ /// Routing-gap test: `diff --combined <path>` (merge-commit header)+ /// must start a new file section. Before the fix, only `diff --git`+ /// was recognized — merge diffs from `git log -p` got treated as one+ /// big pre-diff blob and passed through unchanged.+ #[test]+ fn gap_diff_combined_header_starts_a_file() {+ let input = "diff --combined merge.py\n\+ index abc..def..ghi 100644\n\+ --- a/merge.py\n\+ +++ b/merge.py\n\+ @@@ -1,3 -1,3 +1,4 @@@\n\+ ctx_a\n\+ - removed_p1\n\+ -removed_p2\n\+ ++added_in_merge\n\+ ctx_b\n";+ let cfg = DiffCompressorConfig {+ min_lines_for_ccr: 5,+ ..Default::default()+ };+ let r = DiffCompressor::new(cfg).compress(input, "");+ assert_eq!(r.files_affected, 1);+ assert!(r.compressed.contains("diff --combined merge.py"));+ assert!(r.compressed.contains("@@@ -1,3 -1,3 +1,4 @@@"));+ assert!(r.compressed.contains("++added_in_merge"));+ }++ /// Routing-gap test: `diff --cc <path>` (alternate merge-commit form).+ /// Same reasoning as `diff_combined`.+ #[test]+ fn gap_diff_cc_header_starts_a_file() {+ let input = "diff --cc cc_target.py\n\+ index abc..def..ghi\n\+ --- a/cc_target.py\n\+ +++ b/cc_target.py\n\+ @@@ -1,3 -1,3 +1,4 @@@\n\+ ctx\n\+ - p1_removed\n\+ -p2_removed\n\+ ++merge_added\n\+ more_ctx\n";+ let cfg = DiffCompressorConfig {+ min_lines_for_ccr: 5,+ ..Default::default()+ };+ let r = DiffCompressor::new(cfg).compress(input, "");+ assert_eq!(r.files_affected, 1);+ assert!(r.compressed.contains("diff --cc cc_target.py"));+ assert!(r.compressed.contains("++merge_added"));+ }+/// Bug-fix test: pre-diff content (commit headers, email-style/// metadata) is preserved verbatim before the diff sections. Before/// the fix, anything before the first `diff --git` was silentlydiff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.pyindex 68e15055f..cca6f0e93 100644--- a/headroom/transforms/content_detector.py+++ b/headroom/transforms/content_detector.py@@ -47,7 +47,21 @@ _SEARCH_RESULT_PATTERN = re.compile(r"^[^\s:]+:\d+:" # file:line: format (grep -n style))-_DIFF_HEADER_PATTERN = re.compile(r"^(diff --git|--- a/|@@\s+-\d+,\d+\s+\+\d+,\d+\s+@@)")+# Bug-fix (2026-04-25): extended to recognize merge-commit headers+# (`diff --combined <path>`, `diff --cc <path>`) and combined-diff hunk+# headers (`@@@`+ ranges). Previously only `git diff` shape was detected,+# so merge-commit diffs from `git log -p` got misrouted away from+# DiffCompressor entirely.+_DIFF_HEADER_PATTERN = re.compile(+ r"^("+ r"diff --git"+ r"|diff --combined "+ r"|diff --cc "+ r"|--- a/"+ r"|@@\s+-\d+,\d+\s+\+\d+,\d+\s+@@"+ r"|@@@+\s+-\d+(?:,\d+)?\s+(?:-\d+(?:,\d+)?\s+)+\+\d+(?:,\d+)?\s+@@@+"+ r")"+)_DIFF_CHANGE_PATTERN = re.compile(r"^[+-][^+-]")@@ -184,8 +198,18 @@ def _try_detect_json(content: str) -> DetectionResult | None:def _try_detect_diff(content: str) -> DetectionResult | None:- """Try to detect git diff format."""- lines = content.split("\n")[:50] # Check first 50 lines+ """Try to detect git diff format.++ Bug-fix (2026-04-25): widened the scan window from 50 to 500 lines.+ `git log -p` and `git format-patch` outputs commonly have multi-line+ commit messages or email headers ahead of the actual diff; with the+ 50-line cap, those long preambles pushed the `diff --git` header out+ of the detection window, and the input was misrouted to a+ plain-text/code compressor instead of DiffCompressor. 500 lines+ covers commit messages of ~500 lines (rare; if longer, you've got+ bigger problems).+ """+ lines = content.split("\n")[:500]header_matches = 0change_matches = 0diff --git a/headroom/transforms/diff_compressor.py b/headroom/transforms/diff_compressor.pyindex 5802784f4..ba81902e0 100644--- a/headroom/transforms/diff_compressor.py+++ b/headroom/transforms/diff_compressor.py@@ -147,9 +147,19 @@ class DiffCompressor:>>> print(result.compressed) # Reduced diff with summary"""- # Pattern for diff --git header+ # Pattern for `diff --git a/X b/Y` header (regular diff)._DIFF_GIT_PATTERN = re.compile(r"^diff --git a/(.+) b/(.+)$")+ # Bug-fix (2026-04-25): merge-commit headers — `diff --combined <path>`+ # and `diff --cc <path>`. Both produce a single-path file diff with+ # combined-diff hunk syntax (`@@@`+). Previously the parser only+ # recognized `diff --git`, so merge-commit diffs from `git log -p`+ # of merges fell through to "no files parsed" and were passed+ # through unchanged — even though ContentRouter had routed them here+ # because `--- a/` triggered the detector.+ _DIFF_COMBINED_PATTERN = re.compile(r"^diff --combined (.+)$")+ _DIFF_CC_PATTERN = re.compile(r"^diff --cc (.+)$")+# Pattern for --- a/file or --- /dev/null_OLD_FILE_PATTERN = re.compile(r"^--- (a/(.+)|/dev/null)$")@@ -292,8 +302,17 @@ class DiffCompressor:while i < len(lines):line = lines[i]- # Check for diff --git header (new file section)- if self._DIFF_GIT_PATTERN.match(line):+ # Check for diff --git header (new file section). Bug-fix:+ # `diff --combined <path>` and `diff --cc <path>` (merge+ # commits) also start a new file section — single-path,+ # combined-diff hunk syntax. Treat them the same as+ # `diff --git` for sectioning purposes; the path goes in+ # `header` verbatim.+ if (+ self._DIFF_GIT_PATTERN.match(line)+ or self._DIFF_COMBINED_PATTERN.match(line)+ or self._DIFF_CC_PATTERN.match(line)+ ):# Save previous hunk and fileif current_hunk and current_file:current_file.hunks.append(current_hunk)diff --git a/tests/parity/fixtures/diff_compressor/066bc82007dc2dd4.json b/tests/parity/fixtures/diff_compressor/066bc82007dc2dd4.jsonnew file mode 100644index 000000000..74931bfe5--- /dev/null+++ b/tests/parity/fixtures/diff_compressor/066bc82007dc2dd4.json@@ -0,0 +1,26 @@+{+ "config": {+ "always_keep_additions": true,+ "always_keep_deletions": true,+ "enable_ccr": true,+ "max_context_lines": 2,+ "max_files": 20,+ "max_hunks_per_file": 10,+ "min_lines_for_ccr": 50+ },+ "input": "diff --combined merge_target.py\nindex abc..def..ghi 100644\n--- a/merge_target.py\n+++ b/merge_target.py\n@@@ -1,4 -1,4 +1,5 @@@\n unchanged_a\n- old_branch_1\n -old_branch_2\n++new_in_merge\n +new_added\n unchanged_b\n\n# bugfix:diff-combined",+ "input_sha256": "066bc82007dc2dd4d0083ba09e9d37ecfc2783f631891241dd7fd22003f0f282",+ "output": {+ "additions": 0,+ "cache_key": null,+ "compressed": "diff --combined merge_target.py\nindex abc..def..ghi 100644\n--- a/merge_target.py\n+++ b/merge_target.py\n@@@ -1,4 -1,4 +1,5 @@@\n unchanged_a\n- old_branch_1\n -old_branch_2\n++new_in_merge\n +new_added\n unchanged_b\n\n# bugfix:diff-combined",+ "compressed_line_count": 13,+ "deletions": 0,+ "files_affected": 0,+ "hunks_kept": 0,+ "hunks_removed": 0,+ "original_line_count": 13+ },+ "recorded_at": "2026-04-26T04:02:31.295649+00:00",+ "transform": "diff_compressor"+}diff --git a/tests/parity/fixtures/diff_compressor/5d950a94b8c22480.json b/tests/parity/fixtures/diff_compressor/5d950a94b8c22480.jsonnew file mode 100644index 000000000..4b66ba0e7--- /dev/null+++ b/tests/parity/fixtures/diff_compressor/5d950a94b8c22480.json@@ -0,0 +1,26 @@+{+ "config": {+ "always_keep_additions": true,+ "always_keep_deletions": true,+ "enable_ccr": true,+ "max_context_lines": 2,+ "max_files": 20,+ "max_hunks_per_file": 10,+ "min_lines_for_ccr": 50+ },+ "input": "commit 0123456789abcdef\nAuthor: Tester <[email protected]>\nDate: Mon Apr 25 12:00:00 2026\n\n msg line 0\n msg line 1\n msg line 2\n msg line 3\n msg line 4\n msg line 5\n msg line 6\n msg line 7\n msg line 8\n msg line 9\n msg line 10\n msg line 11\n msg line 12\n msg line 13\n msg line 14\n msg line 15\n msg line 16\n msg line 17\n msg line 18\n msg line 19\n msg line 20\n msg line 21\n msg line 22\n msg line 23\n msg line 24\n msg line 25\n msg line 26\n msg line 27\n msg line 28\n msg line 29\n msg line 30\n msg line 31\n msg line 32\n msg line 33\n msg line 34\n msg line 35\n msg line 36\n msg line 37\n msg line 38\n msg line 39\n msg line 40\n msg line 41\n msg line 42\n msg line 43\n msg line 44\n msg line 45\n msg line 46\n msg line 47\n msg line 48\n msg line 49\n msg line 50\n msg line 51\n msg line 52\n msg line 53\n msg line 54\n msg line 55\n msg line 56\n msg line 57\n msg line 58\n msg line 59\n\ndiff --git a/auth/old_handler.py b/auth/new_handler.py\nsimilarity index 92%\nrename from auth/old_handler.py\nrename to auth/new_handler.py\n--- a/auth/old_handler.py\n+++ b/auth/new_handler.py\n@@ -1,8 +1,8 @@\n import os\n import sys\n-from auth import legacy\n+from auth import modern\n\n def authenticate(user):\n return user.is_valid()\n\n# bugfix:long-pre-diff",+ "input_sha256": "5d950a94b8c224808ce16afdb7abc84eecdc94539d829e6113a7fa06be8584c3",+ "output": {+ "additions": 1,+ "cache_key": null,+ "compressed": "commit 0123456789abcdef\nAuthor: Tester <[email protected]>\nDate: Mon Apr 25 12:00:00 2026\n\n msg line 0\n msg line 1\n msg line 2\n msg line 3\n msg line 4\n msg line 5\n msg line 6\n msg line 7\n msg line 8\n msg line 9\n msg line 10\n msg line 11\n msg line 12\n msg line 13\n msg line 14\n msg line 15\n msg line 16\n msg line 17\n msg line 18\n msg line 19\n msg line 20\n msg line 21\n msg line 22\n msg line 23\n msg line 24\n msg line 25\n msg line 26\n msg line 27\n msg line 28\n msg line 29\n msg line 30\n msg line 31\n msg line 32\n msg line 33\n msg line 34\n msg line 35\n msg line 36\n msg line 37\n msg line 38\n msg line 39\n msg line 40\n msg line 41\n msg line 42\n msg line 43\n msg line 44\n msg line 45\n msg line 46\n msg line 47\n msg line 48\n msg line 49\n msg line 50\n msg line 51\n msg line 52\n msg line 53\n msg line 54\n msg line 55\n msg line 56\n msg line 57\n msg line 58\n msg line 59\n\ndiff --git a/auth/old_handler.py b/auth/new_handler.py\nsimilarity index 92%\nrename from auth/old_handler.py\nrename to auth/new_handler.py\n--- a/auth/old_handler.py\n+++ b/auth/new_handler.py\n@@ -1,8 +1,8 @@\n import os\n import sys\n-from auth import legacy\n+from auth import modern\n\n def authenticate(user):\n[1 files changed, +1 -1 lines]",+ "compressed_line_count": 79,+ "deletions": 1,+ "files_affected": 1,+ "hunks_kept": 1,+ "hunks_removed": 0,+ "original_line_count": 81+ },+ "recorded_at": "2026-04-26T04:02:31.296040+00:00",+ "transform": "diff_compressor"+}diff --git a/tests/parity/fixtures/diff_compressor/66c86f64baebfc3f.json b/tests/parity/fixtures/diff_compressor/66c86f64baebfc3f.jsonnew file mode 100644index 000000000..e571bbba2--- /dev/null+++ b/tests/parity/fixtures/diff_compressor/66c86f64baebfc3f.json@@ -0,0 +1,26 @@+{+ "config": {+ "always_keep_additions": true,+ "always_keep_deletions": true,+ "enable_ccr": true,+ "max_context_lines": 2,+ "max_files": 20,+ "max_hunks_per_file": 10,+ "min_lines_for_ccr": 50+ },+ "input": "diff --cc cc_target.py\nindex abc..def..ghi\n--- a/cc_target.py\n+++ b/cc_target.py\n@@@ -1,3 -1,3 +1,4 @@@\n ctx\n- removed_p1\n -removed_p2\n++added_in_merge\n more_ctx\n\n# bugfix:diff-cc",+ "input_sha256": "66c86f64baebfc3f802ce7a3ff7369889f2551f8895b735a08e0eeee86d7c3d9",+ "output": {+ "additions": 0,+ "cache_key": null,+ "compressed": "diff --cc cc_target.py\nindex abc..def..ghi\n--- a/cc_target.py\n+++ b/cc_target.py\n@@@ -1,3 -1,3 +1,4 @@@\n ctx\n- removed_p1\n -removed_p2\n++added_in_merge\n more_ctx\n\n# bugfix:diff-cc",+ "compressed_line_count": 12,+ "deletions": 0,+ "files_affected": 0,+ "hunks_kept": 0,+ "hunks_removed": 0,+ "original_line_count": 12+ },+ "recorded_at": "2026-04-26T04:02:31.295801+00:00",+ "transform": "diff_compressor"+}diff --git a/tests/parity/recorder.py b/tests/parity/recorder.pyindex 367926f0a..b231bdf97 100644--- a/tests/parity/recorder.py+++ b/tests/parity/recorder.py@@ -478,6 +478,46 @@ Date: Mon Apr 25 12:00:00 2026out.append(f"{combined_diff}\n# bugfix:combined-diff-3way")out.append(f"{no_newline_diff}\n# bugfix:no-newline-marker")out.append(f"{pre_diff_content}\n# bugfix:pre-diff-content")++ # Routing-gap path coverage (2026-04-25 follow-up). These exercise the+ # ContentRouter→DiffCompressor pipeline gaps:+ # 1. `diff --combined <path>` merge-commit header (parser had hardcoded+ # `diff --git`, so the whole input fell into pre-diff blob).+ # 2. `diff --cc <path>` (alternate merge-commit form).+ # 3. Long pre-diff content (>50 lines) that previously slipped past+ # the detector's first-50-lines scan window.+ merge_combined_diff = """diff --combined merge_target.py+index abc..def..ghi 100644+--- a/merge_target.py++++ b/merge_target.py+@@@ -1,4 -1,4 +1,5 @@@+ unchanged_a+- old_branch_1+ -old_branch_2+++new_in_merge+ +new_added+ unchanged_b+"""+ merge_cc_diff = """diff --cc cc_target.py+index abc..def..ghi+--- a/cc_target.py++++ b/cc_target.py+@@@ -1,3 -1,3 +1,4 @@@+ ctx+- removed_p1+ -removed_p2+++added_in_merge+ more_ctx+"""+ long_pre_diff = (+ "commit 0123456789abcdef\n"+ "Author: Tester <[email protected]>\n"+ "Date: Mon Apr 25 12:00:00 2026\n"+ "\n" + "\n".join(f" msg line {i}" for i in range(60)) + "\n\n" + rename_diff+ )+ out.append(f"{merge_combined_diff}\n# bugfix:diff-combined")+ out.append(f"{merge_cc_diff}\n# bugfix:diff-cc")+ out.append(f"{long_pre_diff}\n# bugfix:long-pre-diff")return outdiff --git a/tests/test_transforms/test_diff_compressor.py b/tests/test_transforms/test_diff_compressor.pyindex f16af5173..5be31cd14 100644--- a/tests/test_transforms/test_diff_compressor.py+++ b/tests/test_transforms/test_diff_compressor.py@@ -837,3 +837,101 @@ class TestBugfixPreDiffContent:diff = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -1 +1 @@\n-a\n+b\n"result = DiffCompressor(_cfg_below_threshold()).compress(diff)assert result.compressed.startswith("diff --git a/x.py b/x.py")+++class TestRoutingGapMergeDiffs:+ """Routing gap (2026-04-25 follow-up): ContentRouter detects diff inputs+ and routes them to DiffCompressor, but the parser previously only knew+ the `diff --git` shape. Merge-commit diffs from `git log -p` use+ `diff --combined <path>` or `diff --cc <path>` and were treated as+ non-diff blobs and passed through unchanged.+ """++ def test_diff_combined_header_starts_a_file_section(self):+ from headroom.transforms.diff_compressor import DiffCompressor++ diff = (+ "diff --combined merge_target.py\n"+ "index abc..def..ghi 100644\n"+ "--- a/merge_target.py\n"+ "+++ b/merge_target.py\n"+ "@@@ -1,3 -1,3 +1,4 @@@\n"+ " unchanged_a\n"+ "- old_p1\n"+ " -old_p2\n"+ "++new_in_merge\n"+ " unchanged_b\n"+ )+ result = DiffCompressor(_cfg_below_threshold()).compress(diff)+ assert result.files_affected == 1+ assert "diff --combined merge_target.py" in result.compressed+ assert "@@@ -1,3 -1,3 +1,4 @@@" in result.compressed+ assert "++new_in_merge" in result.compressed++ def test_diff_cc_header_starts_a_file_section(self):+ from headroom.transforms.diff_compressor import DiffCompressor++ diff = (+ "diff --cc cc_target.py\n"+ "index abc..def..ghi\n"+ "--- a/cc_target.py\n"+ "+++ b/cc_target.py\n"+ "@@@ -1,3 -1,3 +1,4 @@@\n"+ " ctx\n"+ "- removed_p1\n"+ " -removed_p2\n"+ "++added_in_merge\n"+ " more_ctx\n"+ )+ result = DiffCompressor(_cfg_below_threshold()).compress(diff)+ assert result.files_affected == 1+ assert "diff --cc cc_target.py" in result.compressed+ assert "++added_in_merge" in result.compressed+++class TestRoutingGapDetectorScanWindow:+ """Routing gap (2026-04-25 follow-up): `_try_detect_diff` only scanned+ the first 50 lines, so `git log -p` outputs with long commit messages+ pushed the diff past the detection window — input was misrouted away+ from DiffCompressor entirely. Window widened to 500 lines.+ """++ def test_detect_picks_up_diff_after_long_commit_message(self):+ from headroom.transforms.content_detector import (+ ContentType,+ detect_content_type,+ )++ # 60 lines of commit message before the diff. Old 50-line cap+ # would have missed the `diff --git` header entirely.+ msg_lines = [+ "commit abc123",+ "Author: Tester <[email protected]>",+ "Date: Mon Apr 25 12:00:00 2026",+ "",+ ] + [f" msg line {i}" for i in range(60)]+ diff = (+ "\n".join(msg_lines)+ + "\n\n"+ + "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -1 +1 @@\n-old\n+new\n"+ )+ result = detect_content_type(diff)+ assert result.content_type == ContentType.GIT_DIFF+ assert result.confidence >= 0.7++ def test_detect_recognizes_combined_diff_headers(self):+ """The detector also gained recognition for combined-diff hunk+ headers (`@@@`+) — useful when the only signal in a snippet is+ the merge-style hunk."""+ from headroom.transforms.content_detector import (+ ContentType,+ detect_content_type,+ )++ # Full merge diff (with `--- a/` shared with regular diffs as a+ # belt-and-suspenders signal).+ diff = (+ "diff --combined m.py\n--- a/m.py\n+++ b/m.py\n@@@ -1,2 -1,2 +1,3 @@@\n ctx\n++added\n"+ )+ result = detect_content_type(diff)+ assert result.content_type == ContentType.GIT_DIFF