Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(relevance): order BM25 query terms once, not once per document (#3468)
trs-806880984b4a96
diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.tomlindex 76d79e59e..4b0d04cbb 100644--- a/crates/headroom-core/Cargo.toml+++ b/crates/headroom-core/Cargo.toml@@ -210,3 +210,7 @@ harness = false[[bench]]name = "auth_mode"harness = false++[[bench]]+name = "bm25_scoring"+harness = falsediff --git a/crates/headroom-core/benches/bm25_scoring.rs b/crates/headroom-core/benches/bm25_scoring.rsnew file mode 100644index 000000000..b23626a3b--- /dev/null+++ b/crates/headroom-core/benches/bm25_scoring.rs@@ -0,0 +1,63 @@+//! Criterion benchmark for `BM25Scorer::score_batch`.+//!+//! Query preparation used to happen once per document: every `bm25_score`+//! call collected and sorted the same query keys. Cost therefore scales with+//! documents x query length, so both are varied here.++use std::hint::black_box;++use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};+use headroom_core::relevance::{BM25Scorer, RelevanceScorer};++const VOCAB: [&str; 16] = [+ "handler",+ "parse",+ "timeout",+ "cache",+ "retry",+ "index",+ "error",+ "550e8400-e29b-41d4-a716-446655440000",+ "9f8e7d6c-1234-4321-abcd-0123456789ab",+ "12345",+ "987654",+ "resolve",+ "connection",+ "serialize",+ "upstream",+ "budget",+];++fn text(seed: usize, words: usize) -> String {+ let mut parts: Vec<&str> = Vec::with_capacity(words);+ for i in 0..words {+ parts.push(VOCAB[(seed * 7 + i * 3) % VOCAB.len()]);+ }+ parts.join(" ")+}++fn bench_score_batch(c: &mut Criterion) {+ let scorer = BM25Scorer::default();+ let mut group = c.benchmark_group("bm25/score_batch");++ for docs in [10usize, 100, 1000] {+ for query_words in [4usize, 16, 64] {+ let items: Vec<String> = (0..docs).map(|d| text(d, 60)).collect();+ let refs: Vec<&str> = items.iter().map(|s| s.as_str()).collect();+ let context = text(9_999, query_words);++ group.throughput(Throughput::Elements(docs as u64));+ group.bench_with_input(+ BenchmarkId::from_parameter(format!("{docs}d_q{query_words}w")),+ &(refs, context),+ |b, (refs, context)| {+ b.iter(|| black_box(scorer.score_batch(black_box(refs), black_box(context))))+ },+ );+ }+ }+ group.finish();+}++criterion_group!(benches, bench_score_batch);+criterion_main!(benches);diff --git a/crates/headroom-core/examples/bm25_query_prep_parity.rs b/crates/headroom-core/examples/bm25_query_prep_parity.rsnew file mode 100644index 000000000..7dbba1008--- /dev/null+++ b/crates/headroom-core/examples/bm25_query_prep_parity.rs@@ -0,0 +1,188 @@+//! Dump BM25 scores, rankings and matched terms so the query-preparation+//! change can be diffed byte-for-byte against its baseline.+//!+//! Scores print as raw IEEE-754 bits, not decimals: the change alters when+//! query terms are ordered, not the order itself, so accumulation must stay+//! bit-identical rather than merely close.+//!+//! `cargo run --release --example bm25_query_prep_parity > out.txt` on each+//! side, then `diff`.++use headroom_core::relevance::{BM25Scorer, RelevanceScorer};++/// xorshift64*, so both sides generate the identical corpus without+/// depending on a rand version.+struct Rng(u64);++impl Rng {+ fn next(&mut self) -> u64 {+ let mut x = self.0;+ x ^= x >> 12;+ x ^= x << 25;+ x ^= x >> 27;+ self.0 = x;+ x.wrapping_mul(0x2545_F491_4F6C_DD1D)+ }++ fn below(&mut self, n: u64) -> u64 {+ if n == 0 {+ 0+ } else {+ self.next() % n+ }+ }+}++const VOCAB: [&str; 16] = [+ "handler",+ "parse",+ "timeout",+ "cache",+ "retry",+ "index",+ "error",+ "550e8400-e29b-41d4-a716-446655440000",+ "9f8e7d6c-1234-4321-abcd-0123456789ab",+ "12345",+ "987654",+ "a",+ "zz",+ "Mixed_Case_Token",+ "UPPER",+ "repeated",+];++fn text(rng: &mut Rng, words: usize) -> String {+ let mut parts: Vec<&str> = Vec::with_capacity(words);+ for _ in 0..words {+ parts.push(VOCAB[rng.below(VOCAB.len() as u64) as usize]);+ }+ parts.join(" ")+}++fn dump(name: &str, scorer: &BM25Scorer, items: &[String], context: &str) {+ let refs: Vec<&str> = items.iter().map(|s| s.as_str()).collect();+ let batch = scorer.score_batch(&refs, context);++ println!(+ "===== {name} (n={}, context={:?}) =====",+ items.len(),+ context+ );+ for (i, s) in batch.iter().enumerate() {+ // Raw bits: a reordered summation would change these even when the+ // rounded decimal looks identical.+ println!(+ " batch[{i}] bits={:016x} reason={:?} matched={:?}",+ s.score.to_bits(),+ s.reason,+ s.matched_terms+ );+ }++ // Ranking is what callers actually consume, so compare it explicitly.+ let mut ranked: Vec<usize> = (0..batch.len()).collect();+ ranked.sort_by(|a, b| {+ batch[*b]+ .score+ .partial_cmp(&batch[*a].score)+ .unwrap_or(std::cmp::Ordering::Equal)+ .then_with(|| a.cmp(b))+ });+ println!(" ranking={ranked:?}");++ // The single-item path shares bm25_score, so verify it agrees with the+ // batch path item for item (reasons and caps differ by design).+ for (i, item) in items.iter().enumerate().take(20) {+ let single = scorer.score(item, context);+ println!(+ " single[{i}] bits={:016x} reason={:?} matched={:?}",+ single.score.to_bits(),+ single.reason,+ single.matched_terms+ );+ }+ println!();+}++fn main() {+ let scorer = BM25Scorer::default();+ let unnormalized = BM25Scorer::new(1.5, 0.75, false, 10.0);+ let tuned = BM25Scorer::new(0.5, 0.0, true, 3.0);++ // Degenerate shapes first.+ dump("empty-batch", &scorer, &[], "error handler");+ dump("empty-context", &scorer, &["error here".into()], "");+ dump(+ "empty-docs",+ &scorer,+ &[String::new(), String::new()],+ "error",+ );+ dump("single-doc", &scorer, &["error handler".into()], "error");+ dump(+ "no-overlap",+ &scorer,+ &["aaa bbb ccc".into(), "ddd eee".into()],+ "zzz yyy",+ );++ // Repeated query terms: query frequency > 1 multiplies each term score,+ // so a dropped or double-counted key shows up immediately.+ dump(+ "repeated-query-terms",+ &scorer,+ &[+ "error error handler".into(),+ "handler parse".into(),+ "error".into(),+ ],+ "error error error handler",+ );++ // Exact ties: identical documents must keep identical scores and a+ // stable ranking.+ dump(+ "exact-ties",+ &scorer,+ &vec!["error handler parse".to_string(); 6],+ "error parse",+ );++ // Long tokens trip the >=8 char bonus in finalize_score.+ dump(+ "long-token-bonus",+ &scorer,+ &[+ "550e8400-e29b-41d4-a716-446655440000 tail".into(),+ "short a b".into(),+ ],+ "550e8400-e29b-41d4-a716-446655440000 short",+ );++ // Vary document count and query length across configs.+ for (label, s) in [+ ("default", &scorer),+ ("unnormalized", &unnormalized),+ ("tuned", &tuned),+ ] {+ for docs in [1usize, 2, 17, 64, 250] {+ for query_words in [1usize, 4, 32] {+ let mut rng = Rng(0x9E37_79B9_7F4A_7C15 ^ (docs as u64) << 8 ^ query_words as u64);+ let items: Vec<String> = (0..docs)+ .map(|_| {+ let words = 1 + rng.below(40) as usize;+ text(&mut rng, words)+ })+ .collect();+ let context = text(&mut rng, query_words);+ dump(+ &format!("{label}-d{docs}-q{query_words}"),+ s,+ &items,+ &context,+ );+ }+ }+ }+}diff --git a/crates/headroom-core/src/relevance/bm25.rs b/crates/headroom-core/src/relevance/bm25.rsindex 1a722c157..062f2313a 100644--- a/crates/headroom-core/src/relevance/bm25.rs+++ b/crates/headroom-core/src/relevance/bm25.rs@@ -13,7 +13,7 @@//! high-signal matches). Final score is clamped to `[0, 1]` via//! `RelevanceScore::new`.-use std::collections::HashMap;+use std::collections::{BTreeMap, HashMap};use std::sync::LazyLock;use regex::Regex;@@ -87,7 +87,7 @@ impl BM25Scorer {fn bm25_score(&self,doc_tokens: &[String],- query_freq: &HashMap<String, usize>,+ query_freq: &BTreeMap<String, usize>,avg_doc_len: f64,) -> (f64, Vec<String>) {if doc_tokens.is_empty() || query_freq.is_empty() {@@ -113,16 +113,13 @@ impl BM25Scorer {let mut matched: Vec<String> = Vec::new();let idf = 2.0_f64.ln();- // Iterate query_freq in HashMap order — Python iterates dict- // order (insertion order in 3.7+). For matched_terms we only- // care about MEMBERSHIP not ordering downstream, but we sort- // tokens alphabetically here for deterministic test output- // when multiple terms match.- let mut keys: Vec<&String> = query_freq.keys().collect();- keys.sort();-- for term in keys {- let qf = query_freq[term];+ // Python iterates dict order (insertion order in 3.7+). For+ // matched_terms we only care about MEMBERSHIP not ordering+ // downstream, but terms are visited alphabetically here for+ // deterministic test output when multiple terms match. A+ // `BTreeMap` already iterates in key order, so batch scoring no+ // longer re-sorts the same query keys once per document.+ for (term, qf) in query_freq {let Some(&f) = doc_freq.get(term.as_str()) else {continue;};@@ -132,7 +129,7 @@ impl BM25Scorer {let numerator = f * (self.k1 + 1.0);let denominator = f + self.k1 * (1.0 - self.b + self.b * doc_len / avgdl);let term_score = idf * numerator / denominator;- score += term_score * qf as f64;+ score += term_score * *qf as f64;}(score, matched)@@ -158,7 +155,7 @@ impl RelevanceScorer for BM25Scorer {let context_tokens = self.tokenize(context);// Build query frequency map from context.- let mut query_freq: HashMap<String, usize> = HashMap::new();+ let mut query_freq: BTreeMap<String, usize> = BTreeMap::new();for t in &context_tokens {*query_freq.entry(t.clone()).or_insert(0) += 1;}@@ -197,7 +194,7 @@ impl RelevanceScorer for BM25Scorer {.collect();}- let mut query_freq: HashMap<String, usize> = HashMap::new();+ let mut query_freq: BTreeMap<String, usize> = BTreeMap::new();for t in &context_tokens {*query_freq.entry(t.clone()).or_insert(0) += 1;}@@ -351,6 +348,27 @@ mod tests {assert_eq!(scores.len(), 50);}+ #[test]+ fn matched_terms_are_lexicographic_regardless_of_context_order() {+ // Query preparation is what makes matched_terms deterministic. The+ // context here is deliberately in reverse lexicographic order, so a+ // map that iterates in insertion or hash order would fail this.+ let doc = "zulu yankee mike delta alpha";+ let context = "zulu yankee mike delta alpha";+ let expected = ["alpha", "delta", "mike", "yankee", "zulu"];++ let single = scorer().score(doc, context);+ assert_eq!(single.matched_terms, expected);++ // Every document in a batch must get the same ordering, not just+ // the first one: preparation is shared across the whole batch.+ let items = vec![doc; 4];+ for score in scorer().score_batch(&items, context) {+ // The batch path caps matched_terms at 5.+ assert_eq!(score.matched_terms, expected);+ }+ }+// ---------- BM25 formula ----------#[test]