Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(relevance): cap embedding batch size so ORT's arena stays small (#4507)
gglucassa2d4fe72381e
diff --git a/crates/headroom-core/src/relevance/embedding.rs b/crates/headroom-core/src/relevance/embedding.rsindex 96f3fb447..dc61fd33e 100644--- a/crates/headroom-core/src/relevance/embedding.rs+++ b/crates/headroom-core/src/relevance/embedding.rs@@ -30,10 +30,23 @@use std::sync::Mutex;#[cfg(feature = "ml")]-use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};+use fastembed::{EmbeddingModel, InitOptions, QuantizationMode, TextEmbedding};use super::base::{RelevanceScore, RelevanceScorer};+/// Texts per ONNX Runtime run, matching Python's `EMBED_BATCH_SIZE`: fastembed's+/// default of 256 grows ORT's never-shrinking CPU arena to the biggest batch scored.+#[cfg(feature = "ml")]+const EMBED_BATCH_SIZE: usize = 16;++/// fastembed refuses to split a dynamic-quantized model's input across runs (each+/// run calibrates its own int8 range, so the embeddings would not compare): one batch.+#[cfg(feature = "ml")]+fn embed_batch_size(model_kind: &EmbeddingModel) -> Option<usize> {+ (TextEmbedding::get_quantization_mode(model_kind) != QuantizationMode::Dynamic)+ .then_some(EMBED_BATCH_SIZE)+}+/// fastembed-backed semantic relevance scorer.////// Construct via `EmbeddingScorer::try_new()` to handle the model-load@@ -56,6 +69,8 @@ pub struct EmbeddingScorer {/// dominant cost so contention is bounded by inference latency,/// not lock latency.model: Option<Mutex<TextEmbedding>>,+ /// `embed` batch size for this model, from `embed_batch_size`.+ batch_size: Option<usize>,}#[cfg(feature = "ml")]@@ -75,6 +90,7 @@ impl Default for EmbeddingScorer {EmbeddingScorer {model_name: "BAAI/bge-small-en-v1.5".to_string(),model: None,+ batch_size: Some(EMBED_BATCH_SIZE),}}}@@ -129,11 +145,13 @@ impl EmbeddingScorer {// `dynamic_ort_loader_ready`).crate::transforms::magika_detector::dynamic_ort_loader_ready().map_err(|e| format!("EmbeddingScorer: ONNX Runtime unavailable: {e}"))?;+ let batch_size = embed_batch_size(&model_kind);let model = TextEmbedding::try_new(InitOptions::new(model_kind)).map_err(|e| format!("EmbeddingScorer model load failed: {}", e))?;Ok(EmbeddingScorer {model_name: name,model: Some(Mutex::new(model)),+ batch_size,})}}@@ -192,11 +210,11 @@ impl RelevanceScorer for EmbeddingScorer {}};- // Encode items + context in one batch — saves model dispatch- // overhead. Mirrors Python fastembed batch encoding.+ // Encode items + context in one `embed` call, which runs them in+ // `self.batch_size` chunks. Mirrors Python fastembed batch encoding.let mut all_texts: Vec<String> = items.iter().map(|s| s.to_string()).collect();all_texts.push(context.to_string());- let embeddings = match guard.embed(all_texts, None) {+ let embeddings = match guard.embed(all_texts, self.batch_size) {Ok(e) => e,Err(e) => {return items@@ -352,6 +370,7 @@ mod tests {EmbeddingScorer {model_name: "test".to_string(),model: None,+ batch_size: Some(EMBED_BATCH_SIZE),}}@@ -462,6 +481,15 @@ mod tests {}}+ #[cfg(feature = "ml")]+ #[test]+ fn dynamic_quantized_models_embed_in_one_batch() {+ let capped = Some(EMBED_BATCH_SIZE);+ assert_eq!(embed_batch_size(&EmbeddingModel::BGESmallENV15), capped);+ assert_eq!(embed_batch_size(&EmbeddingModel::BGESmallENV15Q), capped);+ assert_eq!(embed_batch_size(&EmbeddingModel::AllMiniLML6V2Q), None);+ }+// ---------- model-backed tests (gated on RUN_FASTEMBED_TESTS) ----------#[cfg(feature = "ml")]@@ -506,4 +534,71 @@ mod tests {assert!((0.0..=1.0).contains(&sc.score));}}++ #[cfg(feature = "ml")]+ #[test]+ fn fastembed_batches_past_the_cap_keep_each_score_with_its_item() {+ if !fastembed_enabled() {+ return;+ }+ // Three EMBED_BATCH_SIZE batches of mixed lengths, the one related item+ // in the last: a misaligned batch boundary would hand its score away.+ let s = EmbeddingScorer::try_new().expect("model loads");+ let mut items: Vec<String> = (0..40)+ .map(|i| {+ format!(+ "inventory row {i}: {}",+ "pallets of garden furniture ".repeat(i % 5 + 1)+ )+ })+ .collect();+ items[37] = "authentication failed for user admin: invalid password".to_string();+ let refs: Vec<&str> = items.iter().map(String::as_str).collect();+ let batch = s.score_batch(&refs, "login error");+ assert_eq!(batch.len(), 40);+ let top = (0..40)+ .max_by(|&a, &b| batch[a].score.total_cmp(&batch[b].score))+ .unwrap();+ assert_eq!(top, 37);+ for (item, sc) in refs.iter().zip(&batch) {+ let single = s.score(item, "login error").score;+ assert!(+ (sc.score - single).abs() < 1e-3,+ "{item}: batched {} vs single {single}",+ sc.score+ );+ }+ }++ #[cfg(feature = "ml")]+ #[test]+ fn fastembed_dynamic_quantized_model_scores_on_both_sides_of_the_cap() {+ if !fastembed_enabled() {+ return;+ }+ // 15 items plus the context fill exactly one EMBED_BATCH_SIZE run; 16+ // and 40 go past it, where fastembed refuses to split this model.+ let s = EmbeddingScorer::try_new_with_model(EmbeddingModel::AllMiniLML6V2Q)+ .expect("model loads");+ for n in [15, 16, 40] {+ let mut items: Vec<String> = (0..n)+ .map(|i| format!("inventory row {i}: pallets of garden furniture"))+ .collect();+ items[n - 3] = "authentication failed for user admin: invalid password".to_string();+ let refs: Vec<&str> = items.iter().map(String::as_str).collect();+ let batch = s.score_batch(&refs, "login error");+ assert_eq!(batch.len(), n);+ for sc in &batch {+ assert!(+ !sc.reason.contains("inference failed"),+ "{n} items: {}",+ sc.reason+ );+ }+ let top = (0..n)+ .max_by(|&a, &b| batch[a].score.total_cmp(&batch[b].score))+ .unwrap();+ assert_eq!(top, n - 3, "{n} items");+ }+ }}diff --git a/headroom/relevance/embedding.py b/headroom/relevance/embedding.pyindex d10e1b728..06b1cc5b9 100644--- a/headroom/relevance/embedding.py+++ b/headroom/relevance/embedding.py@@ -37,6 +37,12 @@ from headroom.onnx_runtime import _resolve_revision, hf_hub_download_local_firstfrom .base import RelevanceScore, RelevanceScorer+# Texts per ONNX Runtime run. fastembed pads each batch to its longest text, and+# its default batch of 256 grows ORT's CPU arena (which never shrinks) to fit+# the biggest batch a proxy ever scores: 2.5 GB retained in a long-lived macOS+# proxy. 16 holds ~10x less and runs faster, with less padding per batch.+EMBED_BATCH_SIZE = 16+# numpy is an optional dependency - import lazily_numpy = None@@ -329,7 +335,7 @@ class EmbeddingScorer(RelevanceScorer):"""np = _get_numpy()model = self._get_model()- embeddings = list(model.embed(texts))+ embeddings = list(model.embed(texts, batch_size=EMBED_BATCH_SIZE))return np.array(embeddings)def score(self, item: str, context: str) -> RelevanceScore:diff --git a/tests/test_relevance.py b/tests/test_relevance.pyindex 162b5c39d..0563eda25 100644--- a/tests/test_relevance.py+++ b/tests/test_relevance.py@@ -186,6 +186,20 @@ class TestEmbeddingScorer:# "failed"/"error" semantically relates to "errors"assert scores[0].score > scores[1].score+ def test_batches_past_the_cap_keep_each_score_with_its_item(self, scorer):+ """Three EMBED_BATCH_SIZE batches of mixed lengths, the related item in+ the last: a misaligned batch boundary would hand its score away."""+ items = [+ f"inventory row {i}: " + "pallets of garden furniture " * (i % 5 + 1) for i in range(40)+ ]+ items[37] = "authentication failed for user admin: invalid password"++ scores = [s.score for s in scorer.score_batch(items, "login error")]+ assert len(scores) == 40+ assert max(range(40), key=scores.__getitem__) == 37+ for item, batched in zip(items, scores):+ assert batched == pytest.approx(scorer.score(item, "login error").score, abs=1e-3)+def test_paraphrase_match(self, scorer):"""Embeddings match paraphrases."""items = [diff --git a/tests/test_relevance_extra.py b/tests/test_relevance_extra.pyindex 90b6d34e0..f6cae3b2a 100644--- a/tests/test_relevance_extra.py+++ b/tests/test_relevance_extra.py@@ -124,6 +124,22 @@ def test_embedding_score_and_batch_with_fake_model(monkeypatch) -> None:assert [item.reason for item in batch] == ["Embedding: 0.75", "Embedding: 0.25"]+def test_embedding_encode_caps_the_onnx_batch(monkeypatch) -> None:+ # The default fastembed batch (256) sizes ORT's never-shrinking arena to the+ # largest relevance split ever scored; every encode must pass the cap.+ calls = []++ class FakeModel:+ def embed(self, texts, **kwargs):+ calls.append(kwargs)+ return [[1.0, 0.0] for _ in texts]++ scorer = EmbeddingScorer()+ monkeypatch.setattr(scorer, "_get_model", lambda: FakeModel())+ assert len(scorer._encode([f"record {i}" for i in range(300)])) == 300+ assert calls == [{"batch_size": embedding.EMBED_BATCH_SIZE}]++def test_hybrid_constructor_alpha_variants_and_single_score_paths(monkeypatch) -> None:bm25_result = RelevanceScore(score=0.1, reason="bm25", matched_terms=["term"])emb_result = RelevanceScore(score=0.9, reason="emb", matched_terms=[])