Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(relevance): cap embedding batch size so ORT's arena stays small (#4507)

gglucassa2d4fe72381e
diff --git a/crates/headroom-core/src/relevance/embedding.rs b/crates/headroom-core/src/relevance/embedding.rs
index 96f3fb447..dc61fd33e 100644
--- a/crates/headroom-core/src/relevance/embedding.rs
+++ b/crates/headroom-core/src/relevance/embedding.rs
@@ -30,10 +30,23 @@
use std::sync::Mutex;
#[cfg(feature = "ml")]
-use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
+use fastembed::{EmbeddingModel, InitOptions, QuantizationMode, TextEmbedding};
use super::base::{RelevanceScore, RelevanceScorer};
+/// Texts per ONNX Runtime run, matching Python's `EMBED_BATCH_SIZE`: fastembed's
+/// default of 256 grows ORT's never-shrinking CPU arena to the biggest batch scored.
+#[cfg(feature = "ml")]
+const EMBED_BATCH_SIZE: usize = 16;
+
+/// fastembed refuses to split a dynamic-quantized model's input across runs (each
+/// run calibrates its own int8 range, so the embeddings would not compare): one batch.
+#[cfg(feature = "ml")]
+fn embed_batch_size(model_kind: &EmbeddingModel) -> Option<usize> {
+ (TextEmbedding::get_quantization_mode(model_kind) != QuantizationMode::Dynamic)
+ .then_some(EMBED_BATCH_SIZE)
+}
+
/// fastembed-backed semantic relevance scorer.
///
/// Construct via `EmbeddingScorer::try_new()` to handle the model-load
@@ -56,6 +69,8 @@ pub struct EmbeddingScorer {
/// dominant cost so contention is bounded by inference latency,
/// not lock latency.
model: Option<Mutex<TextEmbedding>>,
+ /// `embed` batch size for this model, from `embed_batch_size`.
+ batch_size: Option<usize>,
}
#[cfg(feature = "ml")]
@@ -75,6 +90,7 @@ impl Default for EmbeddingScorer {
EmbeddingScorer {
model_name: "BAAI/bge-small-en-v1.5".to_string(),
model: None,
+ batch_size: Some(EMBED_BATCH_SIZE),
}
}
}
@@ -129,11 +145,13 @@ impl EmbeddingScorer {
// `dynamic_ort_loader_ready`).
crate::transforms::magika_detector::dynamic_ort_loader_ready()
.map_err(|e| format!("EmbeddingScorer: ONNX Runtime unavailable: {e}"))?;
+ let batch_size = embed_batch_size(&model_kind);
let model = TextEmbedding::try_new(InitOptions::new(model_kind))
.map_err(|e| format!("EmbeddingScorer model load failed: {}", e))?;
Ok(EmbeddingScorer {
model_name: name,
model: Some(Mutex::new(model)),
+ batch_size,
})
}
}
@@ -192,11 +210,11 @@ impl RelevanceScorer for EmbeddingScorer {
}
};
- // Encode items + context in one batch — saves model dispatch
- // overhead. Mirrors Python fastembed batch encoding.
+ // Encode items + context in one `embed` call, which runs them in
+ // `self.batch_size` chunks. Mirrors Python fastembed batch encoding.
let mut all_texts: Vec<String> = items.iter().map(|s| s.to_string()).collect();
all_texts.push(context.to_string());
- let embeddings = match guard.embed(all_texts, None) {
+ let embeddings = match guard.embed(all_texts, self.batch_size) {
Ok(e) => e,
Err(e) => {
return items
@@ -352,6 +370,7 @@ mod tests {
EmbeddingScorer {
model_name: "test".to_string(),
model: None,
+ batch_size: Some(EMBED_BATCH_SIZE),
}
}
@@ -462,6 +481,15 @@ mod tests {
}
}
+ #[cfg(feature = "ml")]
+ #[test]
+ fn dynamic_quantized_models_embed_in_one_batch() {
+ let capped = Some(EMBED_BATCH_SIZE);
+ assert_eq!(embed_batch_size(&EmbeddingModel::BGESmallENV15), capped);
+ assert_eq!(embed_batch_size(&EmbeddingModel::BGESmallENV15Q), capped);
+ assert_eq!(embed_batch_size(&EmbeddingModel::AllMiniLML6V2Q), None);
+ }
+
// ---------- model-backed tests (gated on RUN_FASTEMBED_TESTS) ----------
#[cfg(feature = "ml")]
@@ -506,4 +534,71 @@ mod tests {
assert!((0.0..=1.0).contains(&sc.score));
}
}
+
+ #[cfg(feature = "ml")]
+ #[test]
+ fn fastembed_batches_past_the_cap_keep_each_score_with_its_item() {
+ if !fastembed_enabled() {
+ return;
+ }
+ // Three EMBED_BATCH_SIZE batches of mixed lengths, the one related item
+ // in the last: a misaligned batch boundary would hand its score away.
+ let s = EmbeddingScorer::try_new().expect("model loads");
+ let mut items: Vec<String> = (0..40)
+ .map(|i| {
+ format!(
+ "inventory row {i}: {}",
+ "pallets of garden furniture ".repeat(i % 5 + 1)
+ )
+ })
+ .collect();
+ items[37] = "authentication failed for user admin: invalid password".to_string();
+ let refs: Vec<&str> = items.iter().map(String::as_str).collect();
+ let batch = s.score_batch(&refs, "login error");
+ assert_eq!(batch.len(), 40);
+ let top = (0..40)
+ .max_by(|&a, &b| batch[a].score.total_cmp(&batch[b].score))
+ .unwrap();
+ assert_eq!(top, 37);
+ for (item, sc) in refs.iter().zip(&batch) {
+ let single = s.score(item, "login error").score;
+ assert!(
+ (sc.score - single).abs() < 1e-3,
+ "{item}: batched {} vs single {single}",
+ sc.score
+ );
+ }
+ }
+
+ #[cfg(feature = "ml")]
+ #[test]
+ fn fastembed_dynamic_quantized_model_scores_on_both_sides_of_the_cap() {
+ if !fastembed_enabled() {
+ return;
+ }
+ // 15 items plus the context fill exactly one EMBED_BATCH_SIZE run; 16
+ // and 40 go past it, where fastembed refuses to split this model.
+ let s = EmbeddingScorer::try_new_with_model(EmbeddingModel::AllMiniLML6V2Q)
+ .expect("model loads");
+ for n in [15, 16, 40] {
+ let mut items: Vec<String> = (0..n)
+ .map(|i| format!("inventory row {i}: pallets of garden furniture"))
+ .collect();
+ items[n - 3] = "authentication failed for user admin: invalid password".to_string();
+ let refs: Vec<&str> = items.iter().map(String::as_str).collect();
+ let batch = s.score_batch(&refs, "login error");
+ assert_eq!(batch.len(), n);
+ for sc in &batch {
+ assert!(
+ !sc.reason.contains("inference failed"),
+ "{n} items: {}",
+ sc.reason
+ );
+ }
+ let top = (0..n)
+ .max_by(|&a, &b| batch[a].score.total_cmp(&batch[b].score))
+ .unwrap();
+ assert_eq!(top, n - 3, "{n} items");
+ }
+ }
}
diff --git a/headroom/relevance/embedding.py b/headroom/relevance/embedding.py
index d10e1b728..06b1cc5b9 100644
--- a/headroom/relevance/embedding.py
+++ b/headroom/relevance/embedding.py
@@ -37,6 +37,12 @@ from headroom.onnx_runtime import _resolve_revision, hf_hub_download_local_first
from .base import RelevanceScore, RelevanceScorer
+# Texts per ONNX Runtime run. fastembed pads each batch to its longest text, and
+# its default batch of 256 grows ORT's CPU arena (which never shrinks) to fit
+# the biggest batch a proxy ever scores: 2.5 GB retained in a long-lived macOS
+# proxy. 16 holds ~10x less and runs faster, with less padding per batch.
+EMBED_BATCH_SIZE = 16
+
# numpy is an optional dependency - import lazily
_numpy = None
@@ -329,7 +335,7 @@ class EmbeddingScorer(RelevanceScorer):
"""
np = _get_numpy()
model = self._get_model()
- embeddings = list(model.embed(texts))
+ embeddings = list(model.embed(texts, batch_size=EMBED_BATCH_SIZE))
return np.array(embeddings)
def score(self, item: str, context: str) -> RelevanceScore:
diff --git a/tests/test_relevance.py b/tests/test_relevance.py
index 162b5c39d..0563eda25 100644
--- a/tests/test_relevance.py
+++ b/tests/test_relevance.py
@@ -186,6 +186,20 @@ class TestEmbeddingScorer:
# "failed"/"error" semantically relates to "errors"
assert scores[0].score > scores[1].score
+ def test_batches_past_the_cap_keep_each_score_with_its_item(self, scorer):
+ """Three EMBED_BATCH_SIZE batches of mixed lengths, the related item in
+ the last: a misaligned batch boundary would hand its score away."""
+ items = [
+ f"inventory row {i}: " + "pallets of garden furniture " * (i % 5 + 1) for i in range(40)
+ ]
+ items[37] = "authentication failed for user admin: invalid password"
+
+ scores = [s.score for s in scorer.score_batch(items, "login error")]
+ assert len(scores) == 40
+ assert max(range(40), key=scores.__getitem__) == 37
+ for item, batched in zip(items, scores):
+ assert batched == pytest.approx(scorer.score(item, "login error").score, abs=1e-3)
+
def test_paraphrase_match(self, scorer):
"""Embeddings match paraphrases."""
items = [
diff --git a/tests/test_relevance_extra.py b/tests/test_relevance_extra.py
index 90b6d34e0..f6cae3b2a 100644
--- a/tests/test_relevance_extra.py
+++ b/tests/test_relevance_extra.py
@@ -124,6 +124,22 @@ def test_embedding_score_and_batch_with_fake_model(monkeypatch) -> None:
assert [item.reason for item in batch] == ["Embedding: 0.75", "Embedding: 0.25"]
+def test_embedding_encode_caps_the_onnx_batch(monkeypatch) -> None:
+ # The default fastembed batch (256) sizes ORT's never-shrinking arena to the
+ # largest relevance split ever scored; every encode must pass the cap.
+ calls = []
+
+ class FakeModel:
+ def embed(self, texts, **kwargs):
+ calls.append(kwargs)
+ return [[1.0, 0.0] for _ in texts]
+
+ scorer = EmbeddingScorer()
+ monkeypatch.setattr(scorer, "_get_model", lambda: FakeModel())
+ assert len(scorer._encode([f"record {i}" for i in range(300)])) == 300
+ assert calls == [{"batch_size": embedding.EMBED_BATCH_SIZE}]
+
+
def test_hybrid_constructor_alpha_variants_and_single_score_paths(monkeypatch) -> None:
bm25_result = RelevanceScore(score=0.1, reason="bm25", matched_terms=["term"])
emb_result = RelevanceScore(score=0.9, reason="emb", matched_terms=[])