Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(relevance): order BM25 query terms once, not once per document (#3468)

trs-806880984b4a96
diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml
index 76d79e59e..4b0d04cbb 100644
--- a/crates/headroom-core/Cargo.toml
+++ b/crates/headroom-core/Cargo.toml
@@ -210,3 +210,7 @@ harness = false
[[bench]]
name = "auth_mode"
harness = false
+
+[[bench]]
+name = "bm25_scoring"
+harness = false
diff --git a/crates/headroom-core/benches/bm25_scoring.rs b/crates/headroom-core/benches/bm25_scoring.rs
new file mode 100644
index 000000000..b23626a3b
--- /dev/null
+++ b/crates/headroom-core/benches/bm25_scoring.rs
@@ -0,0 +1,63 @@
+//! Criterion benchmark for `BM25Scorer::score_batch`.
+//!
+//! Query preparation used to happen once per document: every `bm25_score`
+//! call collected and sorted the same query keys. Cost therefore scales with
+//! documents x query length, so both are varied here.
+
+use std::hint::black_box;
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use headroom_core::relevance::{BM25Scorer, RelevanceScorer};
+
+const VOCAB: [&str; 16] = [
+ "handler",
+ "parse",
+ "timeout",
+ "cache",
+ "retry",
+ "index",
+ "error",
+ "550e8400-e29b-41d4-a716-446655440000",
+ "9f8e7d6c-1234-4321-abcd-0123456789ab",
+ "12345",
+ "987654",
+ "resolve",
+ "connection",
+ "serialize",
+ "upstream",
+ "budget",
+];
+
+fn text(seed: usize, words: usize) -> String {
+ let mut parts: Vec<&str> = Vec::with_capacity(words);
+ for i in 0..words {
+ parts.push(VOCAB[(seed * 7 + i * 3) % VOCAB.len()]);
+ }
+ parts.join(" ")
+}
+
+fn bench_score_batch(c: &mut Criterion) {
+ let scorer = BM25Scorer::default();
+ let mut group = c.benchmark_group("bm25/score_batch");
+
+ for docs in [10usize, 100, 1000] {
+ for query_words in [4usize, 16, 64] {
+ let items: Vec<String> = (0..docs).map(|d| text(d, 60)).collect();
+ let refs: Vec<&str> = items.iter().map(|s| s.as_str()).collect();
+ let context = text(9_999, query_words);
+
+ group.throughput(Throughput::Elements(docs as u64));
+ group.bench_with_input(
+ BenchmarkId::from_parameter(format!("{docs}d_q{query_words}w")),
+ &(refs, context),
+ |b, (refs, context)| {
+ b.iter(|| black_box(scorer.score_batch(black_box(refs), black_box(context))))
+ },
+ );
+ }
+ }
+ group.finish();
+}
+
+criterion_group!(benches, bench_score_batch);
+criterion_main!(benches);
diff --git a/crates/headroom-core/examples/bm25_query_prep_parity.rs b/crates/headroom-core/examples/bm25_query_prep_parity.rs
new file mode 100644
index 000000000..7dbba1008
--- /dev/null
+++ b/crates/headroom-core/examples/bm25_query_prep_parity.rs
@@ -0,0 +1,188 @@
+//! Dump BM25 scores, rankings and matched terms so the query-preparation
+//! change can be diffed byte-for-byte against its baseline.
+//!
+//! Scores print as raw IEEE-754 bits, not decimals: the change alters when
+//! query terms are ordered, not the order itself, so accumulation must stay
+//! bit-identical rather than merely close.
+//!
+//! `cargo run --release --example bm25_query_prep_parity > out.txt` on each
+//! side, then `diff`.
+
+use headroom_core::relevance::{BM25Scorer, RelevanceScorer};
+
+/// xorshift64*, so both sides generate the identical corpus without
+/// depending on a rand version.
+struct Rng(u64);
+
+impl Rng {
+ fn next(&mut self) -> u64 {
+ let mut x = self.0;
+ x ^= x >> 12;
+ x ^= x << 25;
+ x ^= x >> 27;
+ self.0 = x;
+ x.wrapping_mul(0x2545_F491_4F6C_DD1D)
+ }
+
+ fn below(&mut self, n: u64) -> u64 {
+ if n == 0 {
+ 0
+ } else {
+ self.next() % n
+ }
+ }
+}
+
+const VOCAB: [&str; 16] = [
+ "handler",
+ "parse",
+ "timeout",
+ "cache",
+ "retry",
+ "index",
+ "error",
+ "550e8400-e29b-41d4-a716-446655440000",
+ "9f8e7d6c-1234-4321-abcd-0123456789ab",
+ "12345",
+ "987654",
+ "a",
+ "zz",
+ "Mixed_Case_Token",
+ "UPPER",
+ "repeated",
+];
+
+fn text(rng: &mut Rng, words: usize) -> String {
+ let mut parts: Vec<&str> = Vec::with_capacity(words);
+ for _ in 0..words {
+ parts.push(VOCAB[rng.below(VOCAB.len() as u64) as usize]);
+ }
+ parts.join(" ")
+}
+
+fn dump(name: &str, scorer: &BM25Scorer, items: &[String], context: &str) {
+ let refs: Vec<&str> = items.iter().map(|s| s.as_str()).collect();
+ let batch = scorer.score_batch(&refs, context);
+
+ println!(
+ "===== {name} (n={}, context={:?}) =====",
+ items.len(),
+ context
+ );
+ for (i, s) in batch.iter().enumerate() {
+ // Raw bits: a reordered summation would change these even when the
+ // rounded decimal looks identical.
+ println!(
+ " batch[{i}] bits={:016x} reason={:?} matched={:?}",
+ s.score.to_bits(),
+ s.reason,
+ s.matched_terms
+ );
+ }
+
+ // Ranking is what callers actually consume, so compare it explicitly.
+ let mut ranked: Vec<usize> = (0..batch.len()).collect();
+ ranked.sort_by(|a, b| {
+ batch[*b]
+ .score
+ .partial_cmp(&batch[*a].score)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ .then_with(|| a.cmp(b))
+ });
+ println!(" ranking={ranked:?}");
+
+ // The single-item path shares bm25_score, so verify it agrees with the
+ // batch path item for item (reasons and caps differ by design).
+ for (i, item) in items.iter().enumerate().take(20) {
+ let single = scorer.score(item, context);
+ println!(
+ " single[{i}] bits={:016x} reason={:?} matched={:?}",
+ single.score.to_bits(),
+ single.reason,
+ single.matched_terms
+ );
+ }
+ println!();
+}
+
+fn main() {
+ let scorer = BM25Scorer::default();
+ let unnormalized = BM25Scorer::new(1.5, 0.75, false, 10.0);
+ let tuned = BM25Scorer::new(0.5, 0.0, true, 3.0);
+
+ // Degenerate shapes first.
+ dump("empty-batch", &scorer, &[], "error handler");
+ dump("empty-context", &scorer, &["error here".into()], "");
+ dump(
+ "empty-docs",
+ &scorer,
+ &[String::new(), String::new()],
+ "error",
+ );
+ dump("single-doc", &scorer, &["error handler".into()], "error");
+ dump(
+ "no-overlap",
+ &scorer,
+ &["aaa bbb ccc".into(), "ddd eee".into()],
+ "zzz yyy",
+ );
+
+ // Repeated query terms: query frequency > 1 multiplies each term score,
+ // so a dropped or double-counted key shows up immediately.
+ dump(
+ "repeated-query-terms",
+ &scorer,
+ &[
+ "error error handler".into(),
+ "handler parse".into(),
+ "error".into(),
+ ],
+ "error error error handler",
+ );
+
+ // Exact ties: identical documents must keep identical scores and a
+ // stable ranking.
+ dump(
+ "exact-ties",
+ &scorer,
+ &vec!["error handler parse".to_string(); 6],
+ "error parse",
+ );
+
+ // Long tokens trip the >=8 char bonus in finalize_score.
+ dump(
+ "long-token-bonus",
+ &scorer,
+ &[
+ "550e8400-e29b-41d4-a716-446655440000 tail".into(),
+ "short a b".into(),
+ ],
+ "550e8400-e29b-41d4-a716-446655440000 short",
+ );
+
+ // Vary document count and query length across configs.
+ for (label, s) in [
+ ("default", &scorer),
+ ("unnormalized", &unnormalized),
+ ("tuned", &tuned),
+ ] {
+ for docs in [1usize, 2, 17, 64, 250] {
+ for query_words in [1usize, 4, 32] {
+ let mut rng = Rng(0x9E37_79B9_7F4A_7C15 ^ (docs as u64) << 8 ^ query_words as u64);
+ let items: Vec<String> = (0..docs)
+ .map(|_| {
+ let words = 1 + rng.below(40) as usize;
+ text(&mut rng, words)
+ })
+ .collect();
+ let context = text(&mut rng, query_words);
+ dump(
+ &format!("{label}-d{docs}-q{query_words}"),
+ s,
+ &items,
+ &context,
+ );
+ }
+ }
+ }
+}
diff --git a/crates/headroom-core/src/relevance/bm25.rs b/crates/headroom-core/src/relevance/bm25.rs
index 1a722c157..062f2313a 100644
--- a/crates/headroom-core/src/relevance/bm25.rs
+++ b/crates/headroom-core/src/relevance/bm25.rs
@@ -13,7 +13,7 @@
//! high-signal matches). Final score is clamped to `[0, 1]` via
//! `RelevanceScore::new`.
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
use std::sync::LazyLock;
use regex::Regex;
@@ -87,7 +87,7 @@ impl BM25Scorer {
fn bm25_score(
&self,
doc_tokens: &[String],
- query_freq: &HashMap<String, usize>,
+ query_freq: &BTreeMap<String, usize>,
avg_doc_len: f64,
) -> (f64, Vec<String>) {
if doc_tokens.is_empty() || query_freq.is_empty() {
@@ -113,16 +113,13 @@ impl BM25Scorer {
let mut matched: Vec<String> = Vec::new();
let idf = 2.0_f64.ln();
- // Iterate query_freq in HashMap order — Python iterates dict
- // order (insertion order in 3.7+). For matched_terms we only
- // care about MEMBERSHIP not ordering downstream, but we sort
- // tokens alphabetically here for deterministic test output
- // when multiple terms match.
- let mut keys: Vec<&String> = query_freq.keys().collect();
- keys.sort();
-
- for term in keys {
- let qf = query_freq[term];
+ // Python iterates dict order (insertion order in 3.7+). For
+ // matched_terms we only care about MEMBERSHIP not ordering
+ // downstream, but terms are visited alphabetically here for
+ // deterministic test output when multiple terms match. A
+ // `BTreeMap` already iterates in key order, so batch scoring no
+ // longer re-sorts the same query keys once per document.
+ for (term, qf) in query_freq {
let Some(&f) = doc_freq.get(term.as_str()) else {
continue;
};
@@ -132,7 +129,7 @@ impl BM25Scorer {
let numerator = f * (self.k1 + 1.0);
let denominator = f + self.k1 * (1.0 - self.b + self.b * doc_len / avgdl);
let term_score = idf * numerator / denominator;
- score += term_score * qf as f64;
+ score += term_score * *qf as f64;
}
(score, matched)
@@ -158,7 +155,7 @@ impl RelevanceScorer for BM25Scorer {
let context_tokens = self.tokenize(context);
// Build query frequency map from context.
- let mut query_freq: HashMap<String, usize> = HashMap::new();
+ let mut query_freq: BTreeMap<String, usize> = BTreeMap::new();
for t in &context_tokens {
*query_freq.entry(t.clone()).or_insert(0) += 1;
}
@@ -197,7 +194,7 @@ impl RelevanceScorer for BM25Scorer {
.collect();
}
- let mut query_freq: HashMap<String, usize> = HashMap::new();
+ let mut query_freq: BTreeMap<String, usize> = BTreeMap::new();
for t in &context_tokens {
*query_freq.entry(t.clone()).or_insert(0) += 1;
}
@@ -351,6 +348,27 @@ mod tests {
assert_eq!(scores.len(), 50);
}
+ #[test]
+ fn matched_terms_are_lexicographic_regardless_of_context_order() {
+ // Query preparation is what makes matched_terms deterministic. The
+ // context here is deliberately in reverse lexicographic order, so a
+ // map that iterates in insertion or hash order would fail this.
+ let doc = "zulu yankee mike delta alpha";
+ let context = "zulu yankee mike delta alpha";
+ let expected = ["alpha", "delta", "mike", "yankee", "zulu"];
+
+ let single = scorer().score(doc, context);
+ assert_eq!(single.matched_terms, expected);
+
+ // Every document in a batch must get the same ordering, not just
+ // the first one: preparation is shared across the whole batch.
+ let items = vec![doc; 4];
+ for score in scorer().score_batch(&items, context) {
+ // The batch path caps matched_terms at 5.
+ assert_eq!(score.matched_terms, expected);
+ }
+ }
+
// ---------- BM25 formula ----------
#[test]