Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): port relevance/{base,bm25} — keyword scorer for SmartCrusher

chopratejas64f1092a890b
diff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rs
index 53502062a..6071c860c 100644
--- a/crates/headroom-core/src/lib.rs
+++ b/crates/headroom-core/src/lib.rs
@@ -1,5 +1,6 @@
//! headroom-core: foundation crate for the Rust port of Headroom.
+pub mod relevance;
pub mod tokenizer;
pub mod transforms;
diff --git a/crates/headroom-core/src/relevance/base.rs b/crates/headroom-core/src/relevance/base.rs
new file mode 100644
index 000000000..36674bc3e
--- /dev/null
+++ b/crates/headroom-core/src/relevance/base.rs
@@ -0,0 +1,136 @@
+//! Base trait and types for relevance scoring.
+//!
+//! Direct port of `headroom/relevance/base.py`. The Python version uses
+//! ABC + abstractmethod; we use a Rust trait with the same two
+//! required methods. `default_batch_score` is provided as a free
+//! function so concrete scorers without an optimized batch impl can
+//! delegate to it.
+
+/// Relevance score with explainability fields.
+///
+/// Mirrors Python's `RelevanceScore` dataclass. The `__post_init__`
+/// score-clamp is enforced via the `new` constructor.
+#[derive(Debug, Clone)]
+pub struct RelevanceScore {
+ pub score: f64,
+ pub reason: String,
+ pub matched_terms: Vec<String>,
+}
+
+impl RelevanceScore {
+ /// Build a score, clamping to `[0.0, 1.0]` to mirror Python's
+ /// `__post_init__` behavior.
+ pub fn new(score: f64, reason: impl Into<String>, matched_terms: Vec<String>) -> Self {
+ RelevanceScore {
+ score: score.clamp(0.0, 1.0),
+ reason: reason.into(),
+ matched_terms,
+ }
+ }
+
+ /// Convenience for "no match" scores.
+ pub fn empty(reason: impl Into<String>) -> Self {
+ RelevanceScore::new(0.0, reason, Vec::new())
+ }
+}
+
+impl Default for RelevanceScore {
+ fn default() -> Self {
+ RelevanceScore::new(0.0, "", Vec::new())
+ }
+}
+
+/// Trait that every relevance scorer implements.
+///
+/// Mirrors Python's `RelevanceScorer` ABC: required `score` for single
+/// items, required `score_batch` for collections (subclasses override
+/// for vectorized impls; otherwise delegate to `default_batch_score`).
+pub trait RelevanceScorer {
+ /// Score a single item against the context.
+ fn score(&self, item: &str, context: &str) -> RelevanceScore;
+
+ /// Score a batch of items. Default impl delegates to per-item
+ /// `score` — override when the scorer can amortize work across
+ /// items (BM25 pre-tokenizes context once, embeddings batch the
+ /// matrix multiplication, etc.).
+ fn score_batch(&self, items: &[&str], context: &str) -> Vec<RelevanceScore> {
+ items.iter().map(|item| self.score(item, context)).collect()
+ }
+
+ /// Whether this scorer is available in the current environment.
+ /// Override for scorers with optional deps (e.g. ONNX embeddings).
+ fn is_available(&self) -> bool {
+ true
+ }
+}
+
+/// Default batch implementation as a free function — convenient for
+/// tests that want to verify the fall-back behavior without
+/// constructing a trait object.
+pub fn default_batch_score<S: RelevanceScorer>(
+ scorer: &S,
+ items: &[&str],
+ context: &str,
+) -> Vec<RelevanceScore> {
+ items.iter().map(|item| scorer.score(item, context)).collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn relevance_score_clamps_above_one() {
+ let s = RelevanceScore::new(1.5, "", Vec::new());
+ assert_eq!(s.score, 1.0);
+ }
+
+ #[test]
+ fn relevance_score_clamps_below_zero() {
+ let s = RelevanceScore::new(-0.5, "", Vec::new());
+ assert_eq!(s.score, 0.0);
+ }
+
+ #[test]
+ fn relevance_score_passes_through_valid_range() {
+ let s = RelevanceScore::new(0.5, "test", vec!["term".to_string()]);
+ assert_eq!(s.score, 0.5);
+ assert_eq!(s.reason, "test");
+ assert_eq!(s.matched_terms, vec!["term"]);
+ }
+
+ #[test]
+ fn empty_score_zero_with_reason() {
+ let s = RelevanceScore::empty("no match");
+ assert_eq!(s.score, 0.0);
+ assert_eq!(s.reason, "no match");
+ assert!(s.matched_terms.is_empty());
+ }
+
+ // Trivial scorer to test the default batch fallback.
+ struct StubScorer;
+ impl RelevanceScorer for StubScorer {
+ fn score(&self, _item: &str, _context: &str) -> RelevanceScore {
+ RelevanceScore::new(0.42, "stub", Vec::new())
+ }
+ }
+
+ #[test]
+ fn default_batch_calls_score_per_item() {
+ let scorer = StubScorer;
+ let items = ["a", "b", "c"];
+ let scores = default_batch_score(&scorer, &items, "ctx");
+ assert_eq!(scores.len(), 3);
+ for s in scores {
+ assert_eq!(s.score, 0.42);
+ }
+ }
+
+ #[test]
+ fn trait_default_batch_uses_per_item_score() {
+ let scorer = StubScorer;
+ let items = ["x", "y"];
+ let scores = scorer.score_batch(&items, "ctx");
+ assert_eq!(scores.len(), 2);
+ }
+}
diff --git a/crates/headroom-core/src/relevance/bm25.rs b/crates/headroom-core/src/relevance/bm25.rs
new file mode 100644
index 000000000..883f3bd3b
--- /dev/null
+++ b/crates/headroom-core/src/relevance/bm25.rs
@@ -0,0 +1,380 @@
+//! BM25 keyword relevance scorer.
+//!
+//! Direct port of `headroom/relevance/bm25.py`. Zero ML dependencies —
+//! pure-Rust regex tokenization + integer arithmetic. Excellent for
+//! exact-match cases (UUIDs, numeric IDs, tool-call argument values
+//! that appear verbatim in the response).
+//!
+//! # Score post-processing
+//!
+//! The raw BM25 score is normalized to `[0, 1]` by dividing by
+//! `max_score` (default 10.0). Items with at least one matched token
+//! of length 8 or more get a `+0.3` long-token bonus (UUIDs, long IDs are
+//! high-signal matches). Final score is clamped to `[0, 1]` via
+//! `RelevanceScore::new`.
+
+use std::collections::HashMap;
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+use super::base::{RelevanceScore, RelevanceScorer};
+
+/// Tokenization regex. Order matters — UUID first so that hex-string
+/// IDs aren't broken into pieces by the alphanumeric arm:
+/// 1. UUID — 8-4-4-4-12 hex with dashes.
+/// 2. Numeric ID — 4+ digits with word boundaries.
+/// 3. Alphanumeric (incl. underscore) — fallback.
+///
+/// Note: Python's `\b\d{4,}\b` uses word boundaries. The Rust `regex`
+/// crate supports `\b`, so the pattern translates literally.
+static TOKEN_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
+ // Single-line alternation. Order matters: UUID first so hex IDs
+ // aren't broken into 8/4/4/4/12 alphanumeric pieces.
+ Regex::new(
+ r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}|\b\d{4,}\b|[a-zA-Z0-9_]+",
+ )
+ .expect("BM25 token regex must compile")
+});
+
+pub struct BM25Scorer {
+ pub k1: f64,
+ pub b: f64,
+ pub normalize_score: bool,
+ pub max_score: f64,
+}
+
+impl Default for BM25Scorer {
+ fn default() -> Self {
+ BM25Scorer {
+ k1: 1.5,
+ b: 0.75,
+ normalize_score: true,
+ max_score: 10.0,
+ }
+ }
+}
+
+impl BM25Scorer {
+ pub fn new(k1: f64, b: f64, normalize_score: bool, max_score: f64) -> Self {
+ BM25Scorer {
+ k1,
+ b,
+ normalize_score,
+ max_score,
+ }
+ }
+
+ /// Tokenize text per Python's `_tokenize`: lowercase + regex
+ /// `findall`. Returns lowercase tokens in document order.
+ fn tokenize(&self, text: &str) -> Vec<String> {
+ if text.is_empty() {
+ return Vec::new();
+ }
+ let lower = text.to_lowercase();
+ TOKEN_PATTERN
+ .find_iter(&lower)
+ .map(|m| m.as_str().to_string())
+ .collect()
+ }
+
+ /// BM25 score for a single (doc, query) pair. Returns
+ /// `(raw_score, matched_terms)` matching Python's `_bm25_score`.
+ ///
+ /// The `idf = ln(2)` constant mirrors Python's simplified
+ /// single-document IDF — same formula whether scoring one item
+ /// or batch-scoring many.
+ fn bm25_score(
+ &self,
+ doc_tokens: &[String],
+ query_freq: &HashMap<String, usize>,
+ avg_doc_len: f64,
+ ) -> (f64, Vec<String>) {
+ if doc_tokens.is_empty() || query_freq.is_empty() {
+ return (0.0, Vec::new());
+ }
+
+ let doc_len = doc_tokens.len() as f64;
+ let avgdl = if avg_doc_len > 0.0 {
+ avg_doc_len
+ } else if doc_len > 0.0 {
+ doc_len
+ } else {
+ 1.0
+ };
+
+ // Doc-side term frequency.
+ let mut doc_freq: HashMap<&str, usize> = HashMap::new();
+ for t in doc_tokens {
+ *doc_freq.entry(t.as_str()).or_insert(0) += 1;
+ }
+
+ let mut score = 0.0;
+ let mut matched: Vec<String> = Vec::new();
+ let idf = 2.0_f64.ln();
+
+ // Iterate query_freq in HashMap order — Python iterates dict
+ // order (insertion order in 3.7+). For matched_terms we only
+ // care about MEMBERSHIP not ordering downstream, but we sort
+ // tokens alphabetically here for deterministic test output
+ // when multiple terms match.
+ let mut keys: Vec<&String> = query_freq.keys().collect();
+ keys.sort();
+
+ for term in keys {
+ let qf = query_freq[term];
+ let Some(&f) = doc_freq.get(term.as_str()) else {
+ continue;
+ };
+ matched.push(term.clone());
+
+ let f = f as f64;
+ let numerator = f * (self.k1 + 1.0);
+ let denominator = f + self.k1 * (1.0 - self.b + self.b * doc_len / avgdl);
+ let term_score = idf * numerator / denominator;
+ score += term_score * qf as f64;
+ }
+
+ (score, matched)
+ }
+
+ /// Compute the final normalized score with the long-match bonus.
+ fn finalize_score(&self, raw: f64, matched: &[String]) -> f64 {
+ let mut normalized = if self.normalize_score {
+ (raw / self.max_score).min(1.0)
+ } else {
+ raw
+ };
+ if matched.iter().any(|t| t.len() >= 8) {
+ normalized = (normalized + 0.3).min(1.0);
+ }
+ normalized
+ }
+}
+
+impl RelevanceScorer for BM25Scorer {
+ fn score(&self, item: &str, context: &str) -> RelevanceScore {
+ let item_tokens = self.tokenize(item);
+ let context_tokens = self.tokenize(context);
+
+ // Build query frequency map from context.
+ let mut query_freq: HashMap<String, usize> = HashMap::new();
+ for t in &context_tokens {
+ *query_freq.entry(t.clone()).or_insert(0) += 1;
+ }
+
+ let (raw, matched) = self.bm25_score(&item_tokens, &query_freq, 0.0);
+ let normalized = self.finalize_score(raw, &matched);
+
+ let reason = match matched.len() {
+ 0 => "BM25: no term matches".to_string(),
+ 1 => format!("BM25: matched '{}'", matched[0]),
+ n => {
+ let preview: Vec<&str> = matched.iter().take(3).map(|s| s.as_str()).collect();
+ let suffix = if n > 3 { "..." } else { "" };
+ format!("BM25: matched {} terms ({}{})", n, preview.join(", "), suffix)
+ }
+ };
+
+ // Limit matched_terms field for readability (Python: matched[:10]).
+ let matched_capped: Vec<String> = matched.iter().take(10).cloned().collect();
+
+ RelevanceScore::new(normalized, reason, matched_capped)
+ }
+
+ fn score_batch(&self, items: &[&str], context: &str) -> Vec<RelevanceScore> {
+ let context_tokens = self.tokenize(context);
+
+ if context_tokens.is_empty() {
+ return items
+ .iter()
+ .map(|_| RelevanceScore::empty("BM25: empty context"))
+ .collect();
+ }
+
+ let mut query_freq: HashMap<String, usize> = HashMap::new();
+ for t in &context_tokens {
+ *query_freq.entry(t.clone()).or_insert(0) += 1;
+ }
+
+ // Pre-tokenize all items.
+ let all_tokens: Vec<Vec<String>> = items.iter().map(|item| self.tokenize(item)).collect();
+
+ // Average document length, matching Python:
+ // avg_len = sum(len(t) for t in all_tokens) / max(len(items), 1)
+ let total_len: usize = all_tokens.iter().map(|t| t.len()).sum();
+ let avg_len = total_len as f64 / items.len().max(1) as f64;
+
+ all_tokens
+ .into_iter()
+ .map(|item_tokens| {
+ let (raw, matched) = self.bm25_score(&item_tokens, &query_freq, avg_len);
+ let normalized = self.finalize_score(raw, &matched);
+
+ let reason = match matched.len() {
+ 0 => "BM25: no matches".to_string(),
+ n => format!("BM25: {} terms", n),
+ };
+
+ // Python uses matched[:5] for the batch path.
+ let matched_capped: Vec<String> = matched.iter().take(5).cloned().collect();
+ RelevanceScore::new(normalized, reason, matched_capped)
+ })
+ .collect()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn scorer() -> BM25Scorer {
+ BM25Scorer::default()
+ }
+
+ // ---------- tokenization ----------
+
+ #[test]
+ fn tokenize_empty_returns_empty() {
+ assert!(scorer().tokenize("").is_empty());
+ }
+
+ #[test]
+ fn tokenize_lowercases() {
+ let toks = scorer().tokenize("Hello WORLD");
+ assert_eq!(toks, vec!["hello", "world"]);
+ }
+
+ #[test]
+ fn tokenize_uuid_as_single_token() {
+ let toks = scorer().tokenize("find 550e8400-e29b-41d4-a716-446655440000 fast");
+ assert!(
+ toks.contains(&"550e8400-e29b-41d4-a716-446655440000".to_string()),
+ "got {:?}",
+ toks
+ );
+ }
+
+ #[test]
+ fn tokenize_numeric_id_when_4_plus_digits() {
+ let toks = scorer().tokenize("user 12345 logged in 99 times");
+ // 12345 = 5 digits (>=4) → kept as numeric ID via second arm.
+ // 99 = 2 digits → falls to alphanumeric arm and matches.
+ // Both end up as tokens.
+ assert!(toks.contains(&"12345".to_string()));
+ assert!(toks.contains(&"99".to_string()));
+ }
+
+ #[test]
+ fn tokenize_strips_punctuation() {
+ let toks = scorer().tokenize("hello, world!");
+ assert_eq!(toks, vec!["hello", "world"]);
+ }
+
+ // ---------- score ----------
+
+ #[test]
+ fn score_no_match_returns_zero() {
+ let s = scorer().score(r#"{"id": 1, "name": "alice"}"#, "completely unrelated query");
+ assert_eq!(s.score, 0.0);
+ assert_eq!(s.reason, "BM25: no term matches");
+ assert!(s.matched_terms.is_empty());
+ }
+
+ #[test]
+ fn score_uuid_match_gets_long_token_bonus() {
+ let item = r#"{"id": "550e8400-e29b-41d4-a716-446655440000", "name": "Alice"}"#;
+ let s = scorer().score(
+ item,
+ "find record 550e8400-e29b-41d4-a716-446655440000",
+ );
+ // Long-match bonus is +0.3, applied after normalization.
+ // Even a low raw score should clear 0.3 with the bonus.
+ assert!(
+ s.score >= 0.3,
+ "UUID match should clear 0.3 from long-token bonus: got {}",
+ s.score
+ );
+ assert!(s.matched_terms.iter().any(|t| t.contains("550e8400")));
+ }
+
+ #[test]
+ fn score_explainability_reason_shape() {
+ let s = scorer().score("alice bob", "alice");
+ assert!(s.reason.starts_with("BM25:"));
+ assert!(s.reason.contains("alice"));
+ }
+
+ // ---------- score_batch ----------
+
+ #[test]
+ fn score_batch_empty_context_zero_scores() {
+ let items = ["foo", "bar"];
+ let scores = scorer().score_batch(&items, "");
+ assert_eq!(scores.len(), 2);
+ for s in scores {
+ assert_eq!(s.score, 0.0);
+ assert_eq!(s.reason, "BM25: empty context");
+ }
+ }
+
+ #[test]
+ fn score_batch_ranks_by_relevance() {
+ let items = [
+ r#"{"id": 1, "msg": "user alice logged in"}"#,
+ r#"{"id": 2, "msg": "system started"}"#,
+ r#"{"id": 3, "msg": "user bob logged out"}"#,
+ ];
+ let scores = scorer().score_batch(&items, "alice login");
+ // Item 0 (alice + logged) should outrank item 1 (no match).
+ assert!(
+ scores[0].score > scores[1].score,
+ "alice match should outrank: got {} vs {}",
+ scores[0].score,
+ scores[1].score
+ );
+ }
+
+ #[test]
+ fn score_batch_amortizes_context_tokenization() {
+ // Sanity: large batch doesn't crash and returns correct length.
+ let items: Vec<String> = (0..50)
+ .map(|i| format!(r#"{{"id": {}, "name": "user{}"}}"#, i, i))
+ .collect();
+ let refs: Vec<&str> = items.iter().map(|s| s.as_str()).collect();
+ let scores = scorer().score_batch(&refs, "user42");
+ assert_eq!(scores.len(), 50);
+ }
+
+ // ---------- BM25 formula ----------
+
+ #[test]
+ fn higher_term_frequency_increases_score() {
+ // Item that mentions "alice" three times should score higher
+ // than item mentioning it once.
+ let single = r#"{"name": "alice"}"#;
+ let triple = r#"{"a": "alice", "b": "alice", "c": "alice"}"#;
+ let s_single = scorer().score(single, "alice");
+ let s_triple = scorer().score(triple, "alice");
+ assert!(
+ s_triple.score >= s_single.score,
+ "more matches should not decrease score: triple={} single={}",
+ s_triple.score,
+ s_single.score
+ );
+ }
+
+ #[test]
+ fn long_match_bonus_applied_only_for_8plus_chars() {
+ let short = scorer().score(r#"{"x": "ab"}"#, "ab");
+ let long = scorer().score(r#"{"x": "abcdefgh"}"#, "abcdefgh");
+ // Even with similar BM25 raw scores, long_match adds +0.3.
+ // We just assert long >= short here (bonus applies to long only).
+ assert!(long.score >= short.score);
+ }
+
+ #[test]
+ fn is_available_is_true() {
+ assert!(scorer().is_available());
+ }
+}
diff --git a/crates/headroom-core/src/relevance/mod.rs b/crates/headroom-core/src/relevance/mod.rs
new file mode 100644
index 000000000..faa52a92c
--- /dev/null
+++ b/crates/headroom-core/src/relevance/mod.rs
@@ -0,0 +1,27 @@
+//! Relevance scoring — Rust port of `headroom/relevance/`.
+//!
+//! Used by SmartCrusher's planning layer to decide which items in a tool
+//! output match the user's query (the user's recent prompts plus the
+//! assistant's tool-call argument JSON, joined). Items above a relevance
+//! threshold are pinned into `keep_indices`.
+//!
+//! # Scorer ladder
+//!
+//! 1. **BM25** (`bm25`): keyword overlap with TF-IDF + length
+//! normalization. No ML deps. Excellent for exact-match cases (UUIDs,
+//! field=value filters). Tool-call arguments are usually literal
+//! keywords that appear verbatim in the response, so BM25 catches
+//! most cases.
+//! 2. **Embedding** (future commit): sentence-transformer ONNX model
+//! for semantic matching when query and items use different
+//! vocabularies.
+//! 3. **Hybrid** (future commit): combines BM25 and embedding signals.
+//!
+//! Each scorer implements the `RelevanceScorer` trait — same surface
+//! as Python's abstract base class.
+
+mod base;
+mod bm25;
+
+pub use base::{default_batch_score, RelevanceScore, RelevanceScorer};
+pub use bm25::BM25Scorer;