Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(embedding): add bounded ONNX throughput controls

bugere5c2e7bca41b
diff --git a/docs/EXPERIMENTAL_ONNX.md b/docs/EXPERIMENTAL_ONNX.md
index ded78cf75..50f7bb8d6 100644
--- a/docs/EXPERIMENTAL_ONNX.md
+++ b/docs/EXPERIMENTAL_ONNX.md
@@ -146,16 +146,56 @@ The error tells the operator to restore the original assets or re-run
autoindex with `--fresh` and a new prefix. XERJ never silently mixes vector
spaces.
+## Throughput controls
+
+The default remains one ONNX Runtime session and a 64-passage caller window:
+
+```toml
+[embedding]
+mode = "onnx-experimental"
+onnx_model_path = "/models/all-MiniLM-L6-v2/model.onnx"
+onnx_tokenizer_path = "/models/all-MiniLM-L6-v2/tokenizer.json"
+onnx_scheduling_window = 64
+onnx_session_pool_size = 1
+```
+
+Larger windows give the existing length-aware scheduler more passages to group
+without changing the model, vectors, output dimensions, or internal
+`onnx_max_batch` cap. Valid values are 1 through 4096.
+
+Two independent sessions allow two complete scheduling windows to run at once:
+
+```toml
+[embedding]
+onnx_scheduling_window = 512
+onnx_session_pool_size = 2
+onnx_intra_threads = 8
+```
+
+This is opt-in because a second session retains another copy of ONNX Runtime's
+session state and therefore changes the memory/concurrency tradeoff. XERJ
+constructs the requested pool atomically and never publishes a partial pool.
+It runs at most two windows concurrently, drains both native calls, and applies
+their results in original input order. Cancelling an HTTP waiter does not
+release admission permits or return a session to the pool before its native
+call actually finishes.
+
+These controls do not select a smaller or quantized model. Model choice remains
+the operator's responsibility through the explicit ONNX assets (or another
+embedding backend).
+
## Admission and errors
-One safe ONNX Runtime session is shared per complete model configuration.
-Length-aware microbatches are serialized through it. Before model loading or
-tokenization, shared admission enforces:
+One bounded ONNX Runtime session pool is shared per complete model
+configuration. Length-aware microbatches are serialized through each member.
+Before model loading or tokenization, shared admission enforces:
- `onnx_max_inflight_calls` (default 8);
- `onnx_max_input_bytes_per_call` (default 8 MiB);
- `onnx_max_inflight_input_bytes` (default 32 MiB);
- `onnx_max_pending`, microbatch size, and padded-token limits.
+- `onnx_scheduling_window` (default 64, range 1 through 4096);
+- `onnx_session_pool_size` (default 1, range 1 through 2).
Overload returns retryable HTTP 429. Autoindex does not call that junk and does
not journal the affected source file complete; correct the pressure/config and
diff --git a/engine/crates/xerj-ai/src/embedder.rs b/engine/crates/xerj-ai/src/embedder.rs
index 5f86d0751..247881d83 100644
--- a/engine/crates/xerj-ai/src/embedder.rs
+++ b/engine/crates/xerj-ai/src/embedder.rs
@@ -172,7 +172,7 @@ impl<T: Send + Sync + 'static> CancellationSafeInit<T> {
#[cfg(feature = "onnx-experimental")]
struct OnnxShared {
- init: std::sync::Arc<CancellationSafeInit<crate::onnx::OnnxEmbedder>>,
+ init: std::sync::Arc<CancellationSafeInit<crate::onnx::OnnxPool>>,
calls: std::sync::Arc<tokio::sync::Semaphore>,
bytes: std::sync::Arc<tokio::sync::Semaphore>,
}
@@ -292,6 +292,8 @@ pub struct OnnxConfig {
pub model_sha256: String,
pub tokenizer_sha256: String,
pub intra_threads: usize,
+ /// Number of independently constructed ONNX Runtime sessions.
+ pub session_pool_size: usize,
pub microbatch: crate::onnx::MicrobatchConfig,
pub max_inflight_calls: usize,
pub max_input_bytes_per_call: usize,
@@ -336,7 +338,7 @@ impl OnnxHandle {
Self { cfg, shared }
}
- async fn get(&self) -> Result<std::sync::Arc<crate::onnx::OnnxEmbedder>> {
+ async fn get(&self) -> Result<std::sync::Arc<crate::onnx::OnnxPool>> {
let cfg = self.cfg.clone();
self.shared
.init
@@ -362,15 +364,17 @@ impl OnnxHandle {
actual_tokenizer
));
}
- let embedder = crate::onnx::OnnxEmbedder::load_bytes(
+ let embedder = crate::onnx::OnnxPool::load_bytes(
&model_bytes,
&tokenizer_bytes,
cfg.intra_threads,
+ cfg.session_pool_size,
)?;
tracing::info!(
model_sha256 = %cfg.model_sha256,
tokenizer_sha256 = %cfg.tokenizer_sha256,
dimensions = crate::onnx::DIMS,
+ session_pool_size = embedder.len(),
"experimental ONNX embedding backend active; first semantic inference loaded the verified model"
);
Ok(embedder)
@@ -400,12 +404,18 @@ impl OnnxHandle {
reason: "ONNX input byte count overflowed; split the request".into(),
})
})?;
- let (_call_permit, _byte_permits) = self.try_admit(input_bytes)?;
+ let (call_permit, byte_permits) = self.try_admit(input_bytes)?;
let model = self.get().await?;
let limits = self.cfg.microbatch;
- tokio::task::spawn_blocking(move || model.embed_scheduled_blocking(&texts, limits))
- .await
- .map_err(|e| anyhow!("ONNX embed task panicked: {e}"))?
+ tokio::task::spawn_blocking(move || {
+ // Admission remains charged until native inference returns even if
+ // the async waiter is cancelled.
+ let _call_permit = call_permit;
+ let _byte_permits = byte_permits;
+ model.embed_scheduled_blocking(&texts, limits)
+ })
+ .await
+ .map_err(|e| anyhow!("ONNX embed task panicked: {e}"))?
}
fn try_admit(
@@ -487,6 +497,7 @@ mod onnx_handle_tests {
model_sha256: model_sha256.into(),
tokenizer_sha256: "tokenizer-hash".into(),
intra_threads: 4,
+ session_pool_size: 2,
microbatch: MicrobatchConfig::default(),
max_inflight_calls: 2,
max_input_bytes_per_call: 10,
@@ -553,6 +564,38 @@ mod onnx_handle_tests {
assert!(error.contains("per-call limit"), "{error}");
}
+ #[tokio::test]
+ async fn cancelled_waiter_keeps_admission_charged_until_blocking_work_returns() {
+ let handle = OnnxHandle::new(cfg("cancelled-native-admission"));
+ let (call, bytes) = handle.try_admit(8).unwrap();
+ let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+ let (release_tx, release_rx) = std::sync::mpsc::channel();
+ let blocking = tokio::task::spawn_blocking(move || {
+ let _call = call;
+ let _bytes = bytes;
+ let _ = started_tx.send(());
+ release_rx.recv().unwrap();
+ });
+ started_rx.await.unwrap();
+ blocking.abort();
+ let error = handle.try_admit(5).unwrap_err().to_string();
+ assert!(
+ error.contains("byte budget full") || error.contains("admission full"),
+ "{error}"
+ );
+ release_tx.send(()).unwrap();
+ tokio::time::timeout(std::time::Duration::from_secs(2), async {
+ loop {
+ if handle.try_admit(5).is_ok() {
+ break;
+ }
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("native completion must release admission");
+ }
+
#[tokio::test]
async fn document_cap_rejects_before_model_load_or_tokenization() {
let mut config = cfg("admission-docs");
diff --git a/engine/crates/xerj-ai/src/onnx.rs b/engine/crates/xerj-ai/src/onnx.rs
index 65af3fbeb..231ae46e9 100644
--- a/engine/crates/xerj-ai/src/onnx.rs
+++ b/engine/crates/xerj-ai/src/onnx.rs
@@ -17,7 +17,7 @@ use anyhow::{anyhow, Context, Result};
use ort::session::{builder::GraphOptimizationLevel, Session};
use ort::value::Tensor;
use std::path::Path;
-use std::sync::Mutex;
+use std::sync::{Arc, Condvar, Mutex};
use tokenizers::{PaddingParams, PaddingStrategy, Tokenizer, TruncationParams};
pub const MAX_TOKENS: usize = 512;
@@ -64,6 +64,113 @@ pub struct OnnxEmbedder {
tokenizer: Tokenizer,
}
+/// A fixed-width pool of independent ONNX Runtime sessions.
+///
+/// Construction is atomic: callers either receive the requested number of
+/// sessions or an error, never a partially initialized pool.
+pub(crate) struct OnnxPool {
+ size: usize,
+ sessions: Mutex<Vec<Arc<OnnxEmbedder>>>,
+ available: Condvar,
+}
+
+struct SessionLease<'a> {
+ pool: &'a OnnxPool,
+ session: Option<Arc<OnnxEmbedder>>,
+}
+
+impl OnnxPool {
+ pub(crate) fn load_bytes(
+ model_bytes: &[u8],
+ tokenizer_bytes: &[u8],
+ intra_threads: usize,
+ pool_size: usize,
+ ) -> Result<Self> {
+ if pool_size == 0 {
+ return Err(anyhow!("ONNX session pool size must be non-zero"));
+ }
+ let sessions = build_exact_pool(pool_size, |ordinal| {
+ OnnxEmbedder::load_bytes(model_bytes, tokenizer_bytes, intra_threads).with_context(
+ || {
+ format!(
+ "construct ONNX session {} of {}; no partial pool was published",
+ ordinal + 1,
+ pool_size
+ )
+ },
+ )
+ })?
+ .into_iter()
+ .map(Arc::new)
+ .collect();
+ Ok(Self {
+ size: pool_size,
+ sessions: Mutex::new(sessions),
+ available: Condvar::new(),
+ })
+ }
+
+ pub(crate) fn len(&self) -> usize {
+ self.size
+ }
+
+ fn lease(&self) -> Result<SessionLease<'_>> {
+ let mut sessions = self
+ .sessions
+ .lock()
+ .map_err(|_| anyhow!("ONNX session pool mutex poisoned"))?;
+ while sessions.is_empty() {
+ sessions = self
+ .available
+ .wait(sessions)
+ .map_err(|_| anyhow!("ONNX session pool mutex poisoned"))?;
+ }
+ let session = sessions.pop().expect("non-empty pool checked");
+ Ok(SessionLease {
+ pool: self,
+ session: Some(session),
+ })
+ }
+
+ pub(crate) fn embed_scheduled_blocking(
+ &self,
+ texts: &[String],
+ config: MicrobatchConfig,
+ ) -> Result<Vec<Vec<f32>>> {
+ let lease = self.lease()?;
+ lease
+ .session
+ .as_ref()
+ .expect("leased session present")
+ .embed_scheduled_blocking(texts, config)
+ }
+}
+
+fn build_exact_pool<T>(
+ pool_size: usize,
+ mut construct: impl FnMut(usize) -> Result<T>,
+) -> Result<Vec<T>> {
+ let mut values = Vec::with_capacity(pool_size);
+ for ordinal in 0..pool_size {
+ values.push(construct(ordinal)?);
+ }
+ Ok(values)
+}
+
+impl Drop for SessionLease<'_> {
+ fn drop(&mut self) {
+ if let Some(session) = self.session.take() {
+ let mut sessions = self
+ .pool
+ .sessions
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ sessions.push(session);
+ self.pool.available.notify_one();
+ }
+ }
+}
+
impl OnnxEmbedder {
pub fn load(model_path: &Path, tokenizer_path: &Path, intra_threads: usize) -> Result<Self> {
let model = std::fs::read(model_path)
@@ -290,6 +397,79 @@ pub fn plan_microbatches(
mod tests {
use super::*;
+ #[test]
+ fn pool_initialization_discards_partial_members_on_failure() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ #[derive(Debug)]
+ struct Member<'a>(&'a AtomicUsize);
+ impl Drop for Member<'_> {
+ fn drop(&mut self) {
+ self.0.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+
+ let dropped = AtomicUsize::new(0);
+ let error = build_exact_pool(3, |ordinal| {
+ if ordinal == 1 {
+ Err(anyhow!("synthetic second-session failure"))
+ } else {
+ Ok(Member(&dropped))
+ }
+ })
+ .expect_err("a member failure must reject the whole pool")
+ .to_string();
+ assert!(error.contains("second-session failure"), "{error}");
+ assert_eq!(dropped.load(Ordering::Relaxed), 1);
+ }
+
+ /// Run with explicit matching assets:
+ ///
+ /// `XERJ_ONNX_TEST_MODEL=/abs/model.onnx
+ /// XERJ_ONNX_TEST_TOKENIZER=/abs/tokenizer.json
+ /// cargo test -p xerj-ai --features onnx-experimental
+ /// independent_pool_sessions_are_bit_identical -- --ignored`
+ #[test]
+ #[ignore = "requires explicit matching ONNX model and tokenizer assets"]
+ fn independent_pool_sessions_are_bit_identical() {
+ let model = std::fs::read(
+ std::env::var_os("XERJ_ONNX_TEST_MODEL").expect("XERJ_ONNX_TEST_MODEL is required"),
+ )
+ .unwrap();
+ let tokenizer = std::fs::read(
+ std::env::var_os("XERJ_ONNX_TEST_TOKENIZER")
+ .expect("XERJ_ONNX_TEST_TOKENIZER is required"),
+ )
+ .unwrap();
+ let texts = vec![
+ "quarterly revenue increased year over year".to_string(),
+ "operating margin declined in the fourth quarter".to_string(),
+ ];
+ let config = MicrobatchConfig::default();
+ let expected = OnnxEmbedder::load_bytes(&model, &tokenizer, 2)
+ .unwrap()
+ .embed_scheduled_blocking(&texts, config)
+ .unwrap();
+ let expected_bits = expected
+ .iter()
+ .flatten()
+ .map(|value| value.to_bits())
+ .collect::<Vec<_>>();
+ let pool = OnnxPool::load_bytes(&model, &tokenizer, 2, 2).unwrap();
+ let sessions = pool.sessions.lock().unwrap();
+ assert_eq!(sessions.len(), 2);
+ for session in sessions.iter() {
+ let actual = session.embed_scheduled_blocking(&texts, config).unwrap();
+ assert_eq!(
+ actual
+ .iter()
+ .flatten()
+ .map(|value| value.to_bits())
+ .collect::<Vec<_>>(),
+ expected_bits
+ );
+ }
+ }
+
#[test]
fn planner_bounds_every_batch_and_preserves_all_positions() {
let lengths = [512, 14, 200, 16, 400, 15, 90, 300];
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 9e7a91208..139c178e5 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -177,6 +177,17 @@ impl Config {
return Err(XerjError::config("vector.max_dimensions must be > 0"));
}
+ if !(1..=4096).contains(&self.embedding.onnx_scheduling_window) {
+ return Err(XerjError::config(
+ "embedding.onnx_scheduling_window must be in 1..=4096",
+ ));
+ }
+ if !(1..=2).contains(&self.embedding.onnx_session_pool_size) {
+ return Err(XerjError::config(
+ "embedding.onnx_session_pool_size must be in 1..=2",
+ ));
+ }
+
// ── Config honesty guards ────────────────────────────────────────────
// Some config knobs exist in the schema but are not wired into any code
// path in this build. Silently ignoring them is worse than failing: an
@@ -831,8 +842,16 @@ pub struct EmbeddingConfig {
pub onnx_model_path: String,
/// Experimental ONNX backend: tokenizer.json from the same model/export.
pub onnx_tokenizer_path: String,
+ /// Maximum passages collected before one ONNX scheduling call
+ /// (default: `64`, range: `1..=4096`). This is independent of the
+ /// backend's internal `onnx_max_batch` microbatch cap.
+ pub onnx_scheduling_window: usize,
/// Experimental ONNX Runtime intra-op threads (default: available CPUs).
pub onnx_intra_threads: usize,
+ /// Number of independent ONNX Runtime sessions (default: `1`, range:
+ /// `1..=2`). Two sessions permit two scheduling windows to run in
+ /// parallel, at the cost of another model session's memory.
+ pub onnx_session_pool_size: usize,
/// Maximum texts accepted by one ONNX scheduling window.
pub onnx_max_pending: usize,
/// Maximum documents in one ONNX inference microbatch.
@@ -860,9 +879,11 @@ impl Default for EmbeddingConfig {
local_model_dir: String::new(),
onnx_model_path: String::new(),
onnx_tokenizer_path: String::new(),
+ onnx_scheduling_window: 64,
onnx_intra_threads: std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1),
+ onnx_session_pool_size: 1,
onnx_max_pending: 4096,
onnx_max_batch: 64,
onnx_padded_token_budget: 4096,
@@ -1358,6 +1379,35 @@ mod tests {
.expect("shipped xerj.default.toml must pass Config::validate");
}
+ #[test]
+ fn onnx_throughput_controls_preserve_single_session_defaults() {
+ let cfg = Config::from_toml_str("[embedding]\n").unwrap();
+ assert_eq!(cfg.embedding.onnx_scheduling_window, 64);
+ assert_eq!(cfg.embedding.onnx_session_pool_size, 1);
+ }
+
+ #[test]
+ fn onnx_throughput_controls_accept_only_bounded_values() {
+ for (key, valid) in [
+ ("onnx_scheduling_window", [1, 64, 4096].as_slice()),
+ ("onnx_session_pool_size", [1, 2].as_slice()),
+ ] {
+ for value in valid {
+ Config::from_toml_str(&format!("[embedding]\n{key} = {value}\n"))
+ .unwrap_or_else(|error| panic!("{key}={value} must be valid: {error}"));
+ }
+ }
+ for (key, invalid) in [
+ ("onnx_scheduling_window", [0, 4097].as_slice()),
+ ("onnx_session_pool_size", [0, 3].as_slice()),
+ ] {
+ for value in invalid {
+ Config::from_toml_str(&format!("[embedding]\n{key} = {value}\n"))
+ .expect_err(&format!("{key}={value} must be rejected"));
+ }
+ }
+ }
+
#[test]
fn count_user_facing_settings() {
// 50 user-facing settings:
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index e0019ba00..c20c978bc 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -6961,22 +6961,53 @@ impl Index {
// Bound caller-side windows as well as backend-internal batches.
// This avoids an arbitrarily large proxy payload and gives ONNX a
// useful cross-document scheduling window without giant padding.
- const MAX_PASSAGES_PER_WINDOW: usize = 64;
+ let max_passages_per_window = self.embedding_config.onnx_scheduling_window;
+ let passage_counts = jobs.iter().map(|job| job.texts.len()).collect::<Vec<_>>();
+ let mut windows = Vec::new();
let mut start = 0;
while start < jobs.len() {
- let mut end = start;
- let mut passages = 0;
- while end < jobs.len()
- && (end == start || passages + jobs[end].texts.len() <= MAX_PASSAGES_PER_WINDOW)
- {
- passages += jobs[end].texts.len();
- end += 1;
- }
+ let (end, passages) =
+ semantic_embedding_window_end(&passage_counts, start, max_passages_per_window);
let texts: Vec<String> = jobs[start..end]
.iter()
.flat_map(|job| job.texts.iter().cloned())
.collect();
- match embedder.embed_batch(texts).await {
+ windows.push((start, end, passages, texts));
+ start = end;
+ }
+
+ // Two sessions are an explicit opt-in. Launch at most two complete
+ // windows together, drain both, then consume results in input
+ // order. This bounds native work and keeps retry/publication
+ // behavior deterministic when one sibling fails.
+ let mut window_results = Vec::with_capacity(windows.len());
+ if dual_session_scheduler_enabled(
+ onnx_pinned,
+ self.embedding_config.onnx_session_pool_size,
+ ) {
+ let results = collect_ordinal_buffered_two(windows.len(), |ordinal| {
+ embedder.embed_batch(windows[ordinal].3.clone())
+ })
+ .await;
+ window_results.extend(
+ windows
+ .iter()
+ .zip(results)
+ .map(|(window, result)| (window.0, window.1, window.2, result)),
+ );
+ } else {
+ for (start, end, passages, texts) in &windows {
+ window_results.push((
+ *start,
+ *end,
+ *passages,
+ embedder.embed_batch(texts.clone()).await,
+ ));
+ }
+ }
+
+ for (start, end, passages, batch_result) in window_results {
+ match batch_result {
Ok(vectors) if vectors.len() == passages => {
let mut offset = 0;
for job_idx in start..end {
@@ -7048,7 +7079,6 @@ impl Index {
}
}
}
- start = end;
}
} else {
for (job_idx, job) in jobs.iter().enumerate() {
@@ -19443,6 +19473,7 @@ fn make_onnx_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj
model_path,
tokenizer_path,
intra_threads: cfg.onnx_intra_threads.max(1),
+ session_pool_size: cfg.onnx_session_pool_size,
microbatch: xerj_ai::onnx::MicrobatchConfig {
max_pending: cfg.onnx_max_pending,
max_batch: cfg.onnx_max_batch,
@@ -19455,6 +19486,8 @@ fn make_onnx_embedder(cfg: &xerj_common::config::EmbeddingConfig) -> Result<xerj
info!(
model = %onnx_cfg.model_path.display(),
tokenizer = %onnx_cfg.tokenizer_path.display(),
+ scheduling_window = cfg.onnx_scheduling_window,
+ session_pool_size = onnx_cfg.session_pool_size,
"embedding backend: experimental ONNX Runtime (loads on first use)"
);
Ok(xerj_ai::Embedder::onnx(onnx_cfg))
@@ -27900,6 +27933,116 @@ const SEMANTIC_CHUNK_SIZE: usize = 512;
/// Default overlap (characters) between consecutive chunks.
const SEMANTIC_CHUNK_OVERLAP: usize = 64;
+/// Select one order-preserving caller-side embedding window. One oversized
+/// document-field job remains whole so the planner always makes progress.
+fn semantic_embedding_window_end(
+ passage_counts: &[usize],
+ start: usize,
+ max_passages: usize,
+) -> (usize, usize) {
+ debug_assert!(max_passages > 0);
+ let mut end = start;
+ let mut passages = 0usize;
+ while end < passage_counts.len()
+ && (end == start || passages.saturating_add(passage_counts[end]) <= max_passages)
+ {
+ passages = passages.saturating_add(passage_counts[end]);
+ end += 1;
+ }
+ (end, passages)
+}
+
+/// Run no more than two futures concurrently and return every result in input
+/// order. `join!` waits for both siblings even when one fails.
+async fn collect_ordinal_buffered_two<T, F, Fut>(count: usize, launch: F) -> Vec<T>
+where
+ F: Fn(usize) -> Fut,
+ Fut: std::future::Future<Output = T>,
+{
+ let mut results = Vec::with_capacity(count);
+ let mut ordinal = 0;
+ while ordinal < count {
+ if ordinal + 1 < count {
+ let (first, second) = tokio::join!(launch(ordinal), launch(ordinal + 1));
+ results.push(first);
+ results.push(second);
+ ordinal += 2;
+ } else {
+ results.push(launch(ordinal).await);
+ ordinal += 1;
+ }
+ }
+ results
+}
+
+fn dual_session_scheduler_enabled(onnx_pinned: bool, pool_size: usize) -> bool {
+ onnx_pinned && pool_size == 2
+}
+
+#[cfg(test)]
+mod semantic_embedding_window_tests {
+ use super::{
+ collect_ordinal_buffered_two, dual_session_scheduler_enabled, semantic_embedding_window_end,
+ };
+
+ fn windows(counts: &[usize], limit: usize) -> Vec<(std::ops::Range<usize>, usize)> {
+ let mut result = Vec::new();
+ let mut start = 0;
+ while start < counts.len() {
+ let (end, passages) = semantic_embedding_window_end(counts, start, limit);
+ result.push((start..end, passages));
+ start = end;
+ }
+ result
+ }
+
+ #[test]
+ fn preserves_order_and_keeps_oversized_jobs_whole() {
+ let counts = [20, 44, 1, 70, 32, 32];
+ assert_eq!(
+ windows(&counts, 64),
+ vec![(0..2, 64), (2..3, 1), (3..4, 70), (4..6, 64)]
+ );
+ assert_eq!(windows(&[64, 64, 128, 256], 512), vec![(0..4, 512)]);
+ }
+
+ #[test]
+ fn dual_scheduler_is_unreachable_for_defaults_and_non_onnx_backends() {
+ assert!(!dual_session_scheduler_enabled(true, 1));
+ assert!(!dual_session_scheduler_enabled(false, 1));
+ assert!(!dual_session_scheduler_enabled(false, 2));
+ assert!(dual_session_scheduler_enabled(true, 2));
+ }
+
+ #[tokio::test]
+ async fn buffered_pair_drains_reversed_completion_in_ordinal_order() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
+ let completed = Arc::new(AtomicUsize::new(0));
+ let second_finished = Arc::new(tokio::sync::Notify::new());
+ let results = collect_ordinal_buffered_two(3, |ordinal| {
+ let completed = Arc::clone(&completed);
+ let second_finished = Arc::clone(&second_finished);
+ async move {
+ if ordinal == 0 {
+ second_finished.notified().await;
+ } else if ordinal == 1 {
+ second_finished.notify_one();
+ }
+ completed.fetch_add(1, Ordering::SeqCst);
+ if ordinal == 0 {
+ Err("first failed")
+ } else {
+ Ok(ordinal)
+ }
+ }
+ })
+ .await;
+ assert_eq!(completed.load(Ordering::SeqCst), 3);
+ assert_eq!(results, vec![Err("first failed"), Ok(1), Ok(2)]);
+ }
+}
+
/// The chunker used by the auto-embed-on-ingest path.
fn semantic_chunker() -> xerj_ai::TextChunker {
xerj_ai::TextChunker::new(SEMANTIC_CHUNK_SIZE, SEMANTIC_CHUNK_OVERLAP)