Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(search): multi-index fan-out executes concurrently (#875) (#877)

xerj-orgc439476022fa
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 480a42eb9..e68bbc6cb 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -6915,7 +6915,7 @@ type PendingSearchSignals = (
);
#[cfg(test)]
static TEST_PENDING_SEARCH_SIGNALS: std::sync::OnceLock<
- std::sync::Mutex<Option<PendingSearchSignals>>,
+ std::sync::Mutex<Vec<PendingSearchSignals>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
@@ -7009,10 +7009,11 @@ mod search_task_lifetime_tests {
let app = crate::router::build_es_compat_router(state);
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel();
- *TEST_PENDING_SEARCH_SIGNALS
- .get_or_init(|| std::sync::Mutex::new(None))
+ TEST_PENDING_SEARCH_SIGNALS
+ .get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
- .unwrap() = Some((started_tx, dropped_tx));
+ .unwrap()
+ .push((started_tx, dropped_tx));
let request = tokio::spawn(
app.oneshot(
@@ -7034,6 +7035,65 @@ mod search_task_lifetime_tests {
.expect("drop signal");
}
+ /// #875: a wildcard search over N indices must run its per-index searches
+ /// CONCURRENTLY, not one at a time. Proven without timing: two indices
+ /// whose injected search futures signal `started` and then pend forever.
+ /// Under the old serial loop the second search never even started — the
+ /// loop was parked awaiting the first — so its `started` signal never
+ /// fired and this test times out. Under the two-phase fan-out both
+ /// searches start while neither has completed.
+ #[tokio::test]
+ async fn wildcard_fanout_runs_per_index_searches_concurrently() {
+ let state = test_state();
+ for name in ["test-pending-search-task", "test-pending-search-task2"] {
+ state
+ .engine
+ .create_index(name, xerj_common::types::Schema::empty())
+ .unwrap();
+ }
+ let app = crate::router::build_es_compat_router(state);
+ let (started_tx_a, started_rx_a) = tokio::sync::oneshot::channel();
+ let (dropped_tx_a, dropped_rx_a) = tokio::sync::oneshot::channel();
+ let (started_tx_b, started_rx_b) = tokio::sync::oneshot::channel();
+ let (dropped_tx_b, dropped_rx_b) = tokio::sync::oneshot::channel();
+ {
+ let mut slot = TEST_PENDING_SEARCH_SIGNALS
+ .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+ .lock()
+ .unwrap();
+ slot.push((started_tx_a, dropped_tx_a));
+ slot.push((started_tx_b, dropped_tx_b));
+ }
+
+ let request = tokio::spawn(
+ app.oneshot(
+ Request::post("/test-pending-search-task*/_search")
+ .header("content-type", "application/json")
+ .body(axum::body::Body::from(r#"{"query":{"match_all":{}}}"#))
+ .unwrap(),
+ ),
+ );
+ // BOTH per-index searches must start while NEITHER can complete —
+ // exactly what a serial fan-out cannot do. The generous timeout only
+ // bounds the failure mode; a passing run signals immediately.
+ tokio::time::timeout(std::time::Duration::from_secs(10), async {
+ started_rx_a.await.expect("first start signal");
+ started_rx_b.await.expect("second start signal");
+ })
+ .await
+ .expect("both fan-out searches must start concurrently (#875)");
+
+ // Dropping the request aborts every spawned per-index search.
+ request.abort();
+ assert!(request.await.unwrap_err().is_cancelled());
+ tokio::time::timeout(std::time::Duration::from_secs(2), async {
+ dropped_rx_a.await.expect("first drop signal");
+ dropped_rx_b.await.expect("second drop signal");
+ })
+ .await
+ .expect("aborting the request must abort all fan-out searches");
+ }
+
#[tokio::test]
async fn dropping_request_owner_aborts_and_drains_child() {
struct Dropped(tokio::sync::oneshot::Sender<()>);
@@ -11110,6 +11170,17 @@ async fn search_impl(
.map(|v| v == "true")
.unwrap_or(search_selector_has_wildcard);
+ // #875: this fan-out used to execute serially — spawn one search task,
+ // await it, merge, then move to the next index — so a wildcard search
+ // over N datasets cost ~N × single-index latency (measured: 701 ms over
+ // 41 autoindex datasets vs 11.7 ms against one of them). Phase 1 below
+ // spawns every per-index search up front; per-index request prep stays
+ // serial because it is sync map lookups. Actual concurrency is bounded
+ // by the engine's global search permit (`Index::search` acquires
+ // `governor.acquire_search`, sized by `limits.max_concurrent_searches`),
+ // and excess searches queue on that semaphore deadline-aware exactly as
+ // they did when this loop issued them one at a time.
+ let mut spawned_searches = Vec::with_capacity(index_names.len());
for idx_name in &index_names {
let idx = match state.engine.get_index(idx_name) {
Ok(i) => i,
@@ -11195,8 +11266,8 @@ async fn search_impl(
#[cfg(test)]
let inject_task_panic = *idx_name == "test-panic-search-task";
#[cfg(test)]
- let inject_pending_task = *idx_name == "test-pending-search-task";
- let mut search_task = AbortOnDrop(tokio::spawn(async move {
+ let inject_pending_task = idx_name.starts_with("test-pending-search-task");
+ let search_task = AbortOnDrop(tokio::spawn(async move {
#[cfg(test)]
if inject_task_panic {
panic!("injected search task panic");
@@ -11204,10 +11275,10 @@ async fn search_impl(
#[cfg(test)]
if inject_pending_task {
let (started, dropped) = TEST_PENDING_SEARCH_SIGNALS
- .get_or_init(|| std::sync::Mutex::new(None))
+ .get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("pending-search signals lock")
- .take()
+ .pop()
.expect("pending-search signals");
let _ = started.send(());
let _drop_signal = SignalOnDrop(Some(dropped));
@@ -11215,6 +11286,14 @@ async fn search_impl(
}
idx.search(&req_clone).await
}));
+ spawned_searches.push((idx_name, search_task));
+ }
+
+ // #875 phase 2 — await and merge in the SAME deterministic index order
+ // the serial loop used, so hit ordering and agg-merge order are
+ // unchanged. An early error return drops the remaining `AbortOnDrop`
+ // handles, aborting their still-running searches.
+ for (idx_name, mut search_task) in spawned_searches {
let search_result = (&mut search_task.0).await;
match search_result {