Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(diff-compressor): CJK-aware relevance scoring for hunk selection (#2220)
Zhenjia ZHOU528517cff86a
diff --git a/crates/headroom-core/src/transforms/diff_compressor.rs b/crates/headroom-core/src/transforms/diff_compressor.rsindex 959addff5..510240ce0 100644--- a/crates/headroom-core/src/transforms/diff_compressor.rs+++ b/crates/headroom-core/src/transforms/diff_compressor.rs@@ -35,7 +35,7 @@//! method returns it alongside the parity-equal result; `compress` is the//! parity-only API that just emits a `tracing::info_span`.-use std::collections::BTreeMap;+use std::collections::{BTreeMap, BTreeSet};use std::sync::OnceLock;use std::time::Instant;@@ -840,9 +840,42 @@ fn priority_patterns() -> &'static [Regex] {})}+/// True for CJK ideographs, kana, and Hangul.+fn is_cjk_char(c: char) -> bool {+ matches!(+ c as u32,+ 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xAC00..=0xD7AF | 0xF900..=0xFAFF+ )+}++/// CJK character bigrams from the CJK runs of a (lowercased) query. A spaceless+/// CJK query rarely appears verbatim in a hunk, but its bigrams do, so these let+/// a CJK query still boost the hunks it partially overlaps.+fn cjk_bigrams(text: &str) -> BTreeSet<String> {+ let mut out = BTreeSet::new();+ let mut run: Vec<char> = Vec::new();+ for c in text.chars() {+ if is_cjk_char(c) {+ run.push(c);+ } else {+ for w in run.windows(2) {+ out.insert(w.iter().collect::<String>());+ }+ run.clear();+ }+ }+ for w in run.windows(2) {+ out.insert(w.iter().collect::<String>());+ }+ out+}+fn score_hunks(files: &mut [DiffFile], context: &str) {let context_lower = context.to_lowercase();let context_words: Vec<&str> = context_lower.split_whitespace().collect();+ // Spaceless CJK queries don't survive whitespace-splitting into matchable+ // words; add CJK character bigrams so they can still boost overlapping hunks.+ let cjk_bg = cjk_bigrams(&context_lower);for file in files.iter_mut() {for hunk in file.hunks.iter_mut() {@@ -860,6 +893,11 @@ fn score_hunks(files: &mut [DiffFile], context: &str) {score += SCORE_CONTEXT_WORD_WEIGHT;}}+ for bg in &cjk_bg {+ if hunk_content_lower.contains(bg.as_str()) {+ score += SCORE_CONTEXT_WORD_WEIGHT;+ }+ }for pat in priority_patterns() {if pat.is_match(&hunk_content_lower) {@@ -1518,6 +1556,58 @@ mod tests {);}+ #[test]+ fn cjk_bigrams_from_query_runs() {+ let b = cjk_bigrams("数据库连接");+ assert!(+ b.contains("数据") && b.contains("据库") && b.contains("库连") && b.contains("连接")+ );+ assert_eq!(b.len(), 4);+ assert!(cjk_bigrams("hello world").is_empty()); // ASCII -> no CJK bigrams+ assert!(cjk_bigrams("a数b据").is_empty()); // isolated CJK chars -> no pair+ }++ #[test]+ fn cjk_query_boosts_matching_hunk_into_kept_set() {+ // Two competing middle hunks, one middle slot. The plain hunk has the+ // higher change-density base score, so WITHOUT a query it takes the+ // slot; a CJK query overlapping the CJK hunk boosts it past the plain one.+ let input = "diff --git a/svc.py b/svc.py\n\+ --- a/svc.py\n\+ +++ b/svc.py\n\+ @@ -1,2 +1,2 @@\n\+ -first_old\n\+ +first_new\n\+ @@ -10,4 +10,4 @@\n\+ -plain_a\n\+ +plain_b\n\+ -plain_c\n\+ +plain_d\n\+ @@ -20,2 +20,2 @@\n\+ -数据库连接失败重试\n\+ +数据库连接成功\n\+ @@ -30,2 +30,2 @@\n\+ -last_old\n\+ +last_new\n";+ let mk = || DiffCompressorConfig {+ max_hunks_per_file: 3,+ min_lines_for_ccr: 5,+ ..Default::default()+ };+ let with_cjk = DiffCompressor::new(mk()).compress(input, "数据库连接超时排查");+ let no_query = DiffCompressor::new(mk()).compress(input, "");+ assert!(+ with_cjk.compressed.contains("数据库连接失败重试"),+ "CJK query should boost the overlapping hunk into the kept set:\n{}",+ with_cjk.compressed+ );+ assert!(+ !no_query.compressed.contains("数据库连接失败重试"),+ "without a query the higher-density plain hunk should take the middle slot:\n{}",+ no_query.compressed+ );+ }+/// Bug-fix test: combined-diff (`@@@`) hunk content must NOT be/// silently dropped. Before the fix the hunk-header regex only/// matched `@@`, so 3-way merge hunks had `current_hunk` never set