AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(search): bound semantic work to request lifetime
bugerd07ab46f04a3
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 4aad9f178..63833617c 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -5386,6 +5386,168 @@ pub(crate) fn top_level_query_type(body_query: &Option<serde_json::Value>) -> &'}}+/// Request-owned child task: dropping the HTTP handler aborts its search+/// instead of leaving detached work running after a client disconnect.+struct AbortOnDrop<T>(tokio::task::JoinHandle<T>);++impl<T> Drop for AbortOnDrop<T> {+ fn drop(&mut self) {+ self.0.abort();+ }+}++#[cfg(test)]+type PendingSearchSignals = (+ tokio::sync::oneshot::Sender<()>,+ tokio::sync::oneshot::Sender<()>,+);+#[cfg(test)]+static TEST_PENDING_SEARCH_SIGNALS: std::sync::OnceLock<+ std::sync::Mutex<Option<PendingSearchSignals>>,+> = std::sync::OnceLock::new();++#[cfg(test)]+struct SignalOnDrop(Option<tokio::sync::oneshot::Sender<()>>);+#[cfg(test)]+impl Drop for SignalOnDrop {+ fn drop(&mut self) {+ if let Some(sender) = self.0.take() {+ let _ = sender.send(());+ }+ }+}++fn search_task_panic_response(join_err: tokio::task::JoinError, index: &str) -> Response {+ tracing::error!(error = %join_err, index, "search task panicked");+ ApiError::new(xerj_common::XerjError::internal(+ "search panicked; check server logs for details",+ ))+ .into_response()+}++#[cfg(test)]+mod search_task_lifetime_tests {+ use super::{search_task_panic_response, AbortOnDrop, TEST_PENDING_SEARCH_SIGNALS};+ use axum::http::{Request, StatusCode};+ use tower::ServiceExt;++ fn test_state() -> crate::state::AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ crate::state::AppState::new(config, engine, metrics)+ }++ #[tokio::test]+ async fn executor_panic_becomes_http_500() {+ let join_err = tokio::spawn(async { panic!("injected search panic") })+ .await+ .expect_err("task must panic");+ let response = search_task_panic_response(join_err, "panic-test");+ assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);+ }++ #[tokio::test]+ async fn es_search_route_maps_child_panic_to_json_http_500() {+ let state = test_state();+ state+ .engine+ .create_index(+ "test-panic-search-task",+ xerj_common::types::Schema::empty(),+ )+ .unwrap();+ let app = crate::router::build_es_compat_router(state);+ let response = app+ .oneshot(+ Request::post("/test-panic-search-task/_search")+ .header("content-type", "application/json")+ .body(axum::body::Body::from(r#"{"query":{"match_all":{}}}"#))+ .unwrap(),+ )+ .await+ .expect("route response");+ assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap();+ assert_eq!(body["status"], 500);+ assert!(+ body["error"]["reason"]+ .as_str()+ .is_some_and(|reason| reason.contains("search panicked")),+ "unexpected error body: {body}"+ );+ }++ #[tokio::test]+ async fn dropping_es_search_route_future_aborts_owned_child() {+ let state = test_state();+ state+ .engine+ .create_index(+ "test-pending-search-task",+ xerj_common::types::Schema::empty(),+ )+ .unwrap();+ let app = crate::router::build_es_compat_router(state);+ let (started_tx, started_rx) = tokio::sync::oneshot::channel();+ let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel();+ *TEST_PENDING_SEARCH_SIGNALS+ .get_or_init(|| std::sync::Mutex::new(None))+ .lock()+ .unwrap() = Some((started_tx, dropped_tx));++ let request = tokio::spawn(+ app.oneshot(+ Request::post("/test-pending-search-task/_search")+ .header("content-type", "application/json")+ .body(axum::body::Body::from(r#"{"query":{"match_all":{}}}"#))+ .unwrap(),+ ),+ );+ tokio::time::timeout(std::time::Duration::from_secs(2), started_rx)+ .await+ .expect("route child did not start")+ .expect("start signal");+ request.abort();+ assert!(request.await.unwrap_err().is_cancelled());+ tokio::time::timeout(std::time::Duration::from_secs(2), dropped_rx)+ .await+ .expect("route-owned child did not drain after request drop")+ .expect("drop signal");+ }++ #[tokio::test]+ async fn dropping_request_owner_aborts_and_drains_child() {+ struct Dropped(tokio::sync::oneshot::Sender<()>);+ impl Drop for Dropped {+ fn drop(&mut self) {+ let (replacement, _) = tokio::sync::oneshot::channel();+ let sender = std::mem::replace(&mut self.0, replacement);+ let _ = sender.send(());+ }+ }++ let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel();+ let task = tokio::spawn(async move {+ let _dropped = Dropped(dropped_tx);+ std::future::pending::<()>().await;+ });+ let owner = AbortOnDrop(task);+ tokio::task::yield_now().await;+ drop(owner);+ tokio::time::timeout(std::time::Duration::from_secs(1), dropped_rx)+ .await+ .expect("aborted child did not drain")+ .expect("drop signal");+ }+}+pub async fn search(State(state): State<AppState>,Path(index): Path<String>,@@ -7471,8 +7633,6 @@ pub async fn search(.with_label_values(&[idx_name]).inc();- // Spawn on a new task so that a panic is caught by the JoinHandle rather- // than propagating up and crashing the server.let mut req_clone = search_req.clone();// PIT search: fetch more hits per-index so the post-merge PIT// filter can drop post-snapshot docs without starving the page.@@ -7506,15 +7666,38 @@ pub async fn search(}}}- let search_result = tokio::task::spawn(async move { idx.search(&req_clone).await }).await;+ // Keep search work owned by the request future. Dropping the handler+ // (for example when the client disconnects) aborts the child task;+ // awaiting its JoinHandle also converts executor panics into HTTP 500+ // instead of unwinding through axum.+ #[cfg(test)]+ let inject_task_panic = *idx_name == "test-panic-search-task";+ #[cfg(test)]+ let inject_pending_task = *idx_name == "test-pending-search-task";+ let mut search_task = AbortOnDrop(tokio::spawn(async move {+ #[cfg(test)]+ if inject_task_panic {+ panic!("injected search task panic");+ }+ #[cfg(test)]+ if inject_pending_task {+ let (started, dropped) = TEST_PENDING_SEARCH_SIGNALS+ .get_or_init(|| std::sync::Mutex::new(None))+ .lock()+ .expect("pending-search signals lock")+ .take()+ .expect("pending-search signals");+ let _ = started.send(());+ let _drop_signal = SignalOnDrop(Some(dropped));+ std::future::pending().await+ }+ idx.search(&req_clone).await+ }));+ let search_result = (&mut search_task.0).await;match search_result {Err(join_err) => {- tracing::error!(error = %join_err, index = idx_name, "search task panicked");- let err = xerj_common::XerjError::internal(- "search panicked; check server logs for details",- );- return ApiError::new(err).into_response();+ return search_task_panic_response(join_err, idx_name);}Ok(Err(e)) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),Ok(Ok(result)) => {diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex f8e2de221..901d31efd 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -44,6 +44,351 @@ impl<'a> Drop for MergeFlagClear<'a> {}}+#[cfg(test)]+mod semantic_deadline_regression_tests {+ use super::*;+ use crate::Engine;+ use tempfile::TempDir;++ fn engine(dir: &TempDir) -> Engine {+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ Engine::new(config).expect("engine")+ }++ fn match_all(timeout_ms: u64) -> SearchRequest {+ SearchRequest {+ timeout_ms: Some(timeout_ms),+ ..SearchRequest::default()+ }+ }++ async fn wait_for_inflight(idx: &Index, expected: usize) {+ tokio::time::timeout(std::time::Duration::from_secs(2), async {+ while idx.query_inflight.len() != expected {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("single-flight state did not converge");+ }++ async fn wait_for_singleflight_follower(idx: &Index) {+ tokio::time::timeout(std::time::Duration::from_secs(2), async {+ loop {+ let subscribed = idx+ .query_inflight+ .iter()+ .any(|entry| entry.value().receiver_count() > 0);+ if subscribed {+ break;+ }+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("follower never subscribed to leader");+ }++ async fn seed_vectors(idx: &Index) {+ for n in 0..256 {+ idx.index_document(+ Some(format!("v{n}")),+ serde_json::json!({"embedding": [1.0, 0.0]}),+ )+ .await+ .unwrap();+ }+ idx.test_scan_checkpoint_delay_ms+ .store(20, Ordering::Relaxed);+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ }++ #[tokio::test]+ async fn timed_out_leader_is_not_cached() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("cache-timeout", Schema::empty())+ .unwrap();+ let idx = engine.get_index("cache-timeout").unwrap();++ let permits = idx+ .max_concurrent_queries+ .acquire_many(64)+ .await+ .expect("query semaphore");+ let request = match_all(1);+ let timed_out = idx.search(&request).await.unwrap();+ assert!(timed_out.timed_out);+ assert!(+ idx.query_cache.is_empty(),+ "partial timeout must never enter the result cache"+ );++ drop(permits);+ let completed = idx.search(&request).await.unwrap();+ assert!(!completed.timed_out);+ assert_eq!(idx.query_cache.len(), 1);+ let cached = idx.search(&request).await.unwrap();+ assert!(!cached.timed_out);+ assert_eq!(cached.took_ms, 0);+ }++ #[tokio::test]+ async fn cancelled_singleflight_leader_drains_and_follower_recomputes() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("cancel-leader", Schema::empty())+ .unwrap();+ let idx = engine.get_index("cancel-leader").unwrap();++ let permits = idx+ .max_concurrent_queries+ .acquire_many(64)+ .await+ .expect("query semaphore");+ let request = match_all(250);++ let leader_idx = Arc::clone(&idx);+ let leader_request = request.clone();+ let leader = tokio::spawn(async move { leader_idx.search(&leader_request).await });+ wait_for_inflight(&idx, 1).await;++ let follower_idx = Arc::clone(&idx);+ let follower_request = request.clone();+ let follower = tokio::spawn(async move { follower_idx.search(&follower_request).await });+ wait_for_singleflight_follower(&idx).await;++ leader.abort();+ assert!(leader.await.unwrap_err().is_cancelled());+ drop(permits);++ let result = tokio::time::timeout(std::time::Duration::from_secs(2), follower)+ .await+ .expect("follower remained stuck on cancelled leader")+ .expect("follower task")+ .expect("follower search");+ assert!(+ !result.timed_out,+ "follower must recompute within its budget"+ );+ wait_for_inflight(&idx, 0).await;+ assert_eq!(idx.query_cache.len(), 1);+ }++ #[tokio::test]+ async fn expired_multi_knn_preserves_partial_timeout_state() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("multi-timeout", Schema::empty())+ .unwrap();+ let idx = engine.get_index("multi-timeout").unwrap();+ seed_vectors(&idx).await;+ let clause = PeeledKnn {+ field: "embedding".into(),+ vector: vec![1.0, 0.0],+ k: 10,+ num_candidates: None,+ filter: None,+ boost: None,+ similarity: None,+ };+ let result = idx+ .run_multi_knn_brute_force(+ &match_all(250),+ std::time::Instant::now() + std::time::Duration::from_millis(5),+ vec![clause],+ )+ .await+ .unwrap();+ assert!(result.timed_out);+ assert_eq!(result.total.relation, TotalHitsRelation::Gte);+ assert_eq!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed),+ 2,+ "single child must do partial work before parent propagation"+ );+ }++ #[tokio::test]+ async fn timed_out_multi_knn_child_stops_later_clause_scheduling() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine.create_index("multi-stop", Schema::empty()).unwrap();+ let idx = engine.get_index("multi-stop").unwrap();+ seed_vectors(&idx).await;+ let clause = PeeledKnn {+ field: "embedding".into(),+ vector: vec![1.0, 0.0],+ k: 10,+ num_candidates: None,+ filter: None,+ boost: None,+ similarity: None,+ };+ let result = idx+ .run_multi_knn_brute_force(+ &match_all(250),+ std::time::Instant::now() + std::time::Duration::from_millis(5),+ vec![clause.clone(), clause],+ )+ .await+ .unwrap();+ assert!(result.timed_out);+ assert_eq!(result.total.relation, TotalHitsRelation::Gte);+ assert_eq!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed),+ 2,+ "second clause must not execute after the first child times out"+ );+ }++ #[tokio::test]+ async fn real_future_deadlines_stop_after_nonzero_partial_scan() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("future-deadline", Schema::empty())+ .unwrap();+ let idx = engine.get_index("future-deadline").unwrap();+ seed_vectors(&idx).await;+ let request = match_all(250);++ for timeout_ms in [1, 10, 250] {+ idx.test_scan_checkpoint_delay_ms+ .store(timeout_ms + 10, Ordering::Relaxed);+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);+ let started = std::time::Instant::now();+ let result = idx+ .run_knn_brute_force_with_deadline(+ &request,+ started + std::time::Duration::from_millis(timeout_ms),+ "embedding",+ &[1.0, 0.0],+ 10,+ None,+ "cosine",+ None,+ None,+ )+ .await+ .unwrap();+ assert!(result.timed_out);+ assert_eq!(result.total.relation, TotalHitsRelation::Gte);+ assert_eq!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed),+ 2,+ "scan must pass checkpoint zero and stop at the later checkpoint"+ );+ assert!(started.elapsed() >= std::time::Duration::from_millis(timeout_ms));+ }+ }++ #[tokio::test]+ async fn expired_hybrid_preserves_partial_timeout_state() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("hybrid-timeout", Schema::empty())+ .unwrap();+ let idx = engine.get_index("hybrid-timeout").unwrap();+ seed_vectors(&idx).await;+ let knn = QueryNode::Knn {+ field: "embedding".into(),+ vector: vec![1.0, 0.0],+ k: 10,+ num_candidates: None,+ filter: None,+ boost: None,+ similarity: None,+ };+ let result = idx+ .run_hybrid_with_deadline(+ &match_all(250),+ std::time::Instant::now() + std::time::Duration::from_millis(5),+ vec![xerj_query::ast::WeightedQuery {+ query: knn,+ weight: 1.0,+ }],+ xerj_query::ast::FusionStrategy::Rrf { k: 60 },+ )+ .await+ .unwrap();+ assert!(result.timed_out);+ assert_eq!(result.total.relation, TotalHitsRelation::Gte);+ assert_eq!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed),+ 2,+ "hybrid child must do partial work before propagating timeout"+ );+ }++ #[tokio::test]+ async fn concurrent_unique_semantic_requests_share_bounded_deadlines() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ let mut schema = Schema::empty();+ let mut field = FieldConfig::new("body", FieldType::Text);+ field.options.dimensions = Some(16);+ field.options.similarity = Some("cosine".into());+ field.embedding = Some(xerj_common::types::EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("body_vector".into()),+ });+ schema.fields.push(field);+ engine.create_index("semantic-concurrent", schema).unwrap();+ let idx = engine.get_index("semantic-concurrent").unwrap();+ for n in 0..256 {+ idx.index_document(+ Some(format!("s{n}")),+ serde_json::json!({"body": format!("semantic document number {n}")}),+ )+ .await+ .unwrap();+ }+ idx.test_scan_checkpoint_delay_ms+ .store(200, Ordering::Relaxed);+ idx.test_scan_checkpoint_count.store(0, Ordering::Relaxed);++ let started = std::time::Instant::now();+ let mut tasks = Vec::new();+ for n in 0..8 {+ let idx = Arc::clone(&idx);+ tasks.push(tokio::spawn(async move {+ let request = SearchRequest {+ query: QueryNode::SemanticSearch {+ field: "body".into(),+ text: format!("unique concurrent deadline {n}"),+ k: 10,+ filter: None,+ boost: None,+ },+ timeout_ms: Some(100),+ ..SearchRequest::default()+ };+ idx.search(&request).await+ }));+ }+ for task in tasks {+ let result = task.await.unwrap().unwrap();+ assert!(result.timed_out);+ assert_eq!(result.total.relation, TotalHitsRelation::Gte);+ }+ assert!(+ idx.test_scan_checkpoint_count.load(Ordering::Relaxed) >= 16,+ "all semantic requests must reach multiple cooperative scan checkpoints"+ );+ assert!(+ started.elapsed() < std::time::Duration::from_millis(500),+ "concurrent semantic scans exceeded their own deadlines"+ );+ }+}+// Doc-values (columnar) fast path for size:0 + match_all + aggs — child// module so it can reach Index's private fields/methods via `super::`.#[path = "fast_aggs.rs"]@@ -525,6 +870,10 @@ pub struct Index {/// from starving other indices in a multi-tenant deployment — a noisy/// neighbour cannot exhaust all available query slots.max_concurrent_queries: Arc<Semaphore>,+ #[cfg(test)]+ test_scan_checkpoint_delay_ms: Arc<AtomicU64>,+ #[cfg(test)]+ test_scan_checkpoint_count: Arc<AtomicU64>,// ── Per-index enrich lookup tables ────────────────────────────────────────/// Named enrich tables: each table maps a key to a JSON object of extra/// fields to merge into matching documents at ingest time.@@ -922,6 +1271,10 @@ impl Index {schema_hash_epoch: Arc::new(AtomicU64::new(0)),last_flush_doc_count: Arc::new(AtomicU64::new(0)),max_concurrent_queries: Arc::new(Semaphore::new(64)),+ #[cfg(test)]+ test_scan_checkpoint_delay_ms: Arc::new(AtomicU64::new(0)),+ #[cfg(test)]+ test_scan_checkpoint_count: Arc::new(AtomicU64::new(0)),enrichments: Arc::new(RwLock::new(HashMap::new())),hnsw: Arc::new(RwLock::new(None)),hnsw_id_map: Arc::new(RwLock::new(HashMap::new())),@@ -1152,6 +1505,10 @@ impl Index {name,schema: Arc::new(RwLock::new(schema)),max_concurrent_queries: Arc::new(Semaphore::new(64)),+ #[cfg(test)]+ test_scan_checkpoint_delay_ms: Arc::new(AtomicU64::new(0)),+ #[cfg(test)]+ test_scan_checkpoint_count: Arc::new(AtomicU64::new(0)),enrichments: Arc::new(RwLock::new(HashMap::new())),store,memtable: Arc::new(memtable),@@ -4458,7 +4815,6 @@ impl Index {similarity: &str,) -> Option<SearchResult> {let started = std::time::Instant::now();-if similarity != "cosine" {return None;}@@ -4602,6 +4958,20 @@ impl Index {/// behaviour (BBQ disk quantisation, IVF clustering) are not/// necessary for wire-correctness on these tests because the/// expected doc order is deterministic given exact scoring.+ async fn exact_scan_checkpoint(&self, _position: usize, deadline: std::time::Instant) -> bool {+ tokio::task::yield_now().await;+ #[cfg(test)]+ {+ self.test_scan_checkpoint_count+ .fetch_add(1, Ordering::Relaxed);+ let delay_ms = self.test_scan_checkpoint_delay_ms.load(Ordering::Relaxed);+ if _position > 0 && delay_ms > 0 {+ tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;+ }+ }+ std::time::Instant::now() >= deadline+ }+#[allow(clippy::too_many_arguments)]pub async fn run_knn_brute_force(&self,@@ -4613,8 +4983,42 @@ impl Index {similarity: &str,boost: Option<f32>,min_similarity: Option<f32>,+ ) -> Result<SearchResult> {+ let deadline = std::time::Instant::now()+ + std::time::Duration::from_millis(request.timeout_ms.unwrap_or(30_000));+ self.run_knn_brute_force_with_deadline(+ request,+ deadline,+ field,+ query_vec,+ k,+ filter,+ similarity,+ boost,+ min_similarity,+ )+ .await+ }++ #[allow(clippy::too_many_arguments)]+ async fn run_knn_brute_force_with_deadline(+ &self,+ request: &SearchRequest,+ deadline: std::time::Instant,+ field: &str,+ query_vec: &[f32],+ k: usize,+ filter: Option<Box<QueryNode>>,+ similarity: &str,+ boost: Option<f32>,+ min_similarity: Option<f32>,) -> Result<SearchResult> {let started = std::time::Instant::now();+ let mut timed_out = false;+ let trace_phases = std::env::var_os("XERJ_TRACE_SEMANTIC_PHASES").is_some();+ if trace_phases {+ tracing::info!(index=%self.name, field, k, "semantic_phase=start_brute");+ }// ── Collect all candidate (doc_id, source) pairs ──────────────let mut candidates: Vec<(String, Value)> = Vec::new();@@ -4627,23 +5031,40 @@ impl Index {let snap = self.store.snapshot();// Track seen IDs so later-segment copies don't duplicate memtable entries.let mut seen: HashSet<String> = candidates.iter().map(|(id, _)| id.clone()).collect();- for meta in snap.segments.iter() {+ 'segments: for meta in snap.segments.iter() {+ let segment_started = std::time::Instant::now();// Cache-backed: first KNN against this segment pays the// I/O + decompress + simd_json parse, every subsequent// query reads from `stored_value_cache`. For a 100-segment// index this turns repeated KNN from O(seg_count * 100MB)// copy work into Arc clone + iterate.+ // `stored_values_for` may synchronously read, decompress, and+ // parse one segment on a cold cache. That operation cannot be+ // interrupted internally; the deadline is checked immediately+ // before and after it, bounding the overrun to one segment load.+ if std::time::Instant::now() >= deadline {+ timed_out = true;+ break;+ }let docs_arc = match self.stored_values_for(&meta.id) {Some(a) => a,None => continue,};+ if std::time::Instant::now() >= deadline {+ timed_out = true;+ break;+ }// Cache-backed iteration: stored_values_for() handles the// I/O + decode + parse; subsequent KNN over the same segment// is an Arc clone. The underlying parse uses serde_json (not// simd_json) per ffd49ac — simd_json silently corrupts some// raw-bytes-flush payloads, the per-doc alloc cost is// irrelevant on the brute-force similarity scan.- for doc in docs_arc.iter() {+ for (position, doc) in docs_arc.iter().enumerate() {+ if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {+ timed_out = true;+ break 'segments;+ }let id = match doc.get("_id").and_then(Value::as_str) {Some(s) => s.to_string(),None => continue,@@ -4687,7 +5108,11 @@ impl Index {});candidates.push((id, src));}+ if trace_phases {+ tracing::info!(index=%self.name, segment=%meta.id, elapsed_ms=segment_started.elapsed().as_millis() as u64, candidates=candidates.len(), "semantic_phase=load_segment");+ }}+ let collect_elapsed = started.elapsed();// ── Determine whether this field opts into SQ8 (scalar8) ──────// Default fields keep the exact f32 brute-force scan below,@@ -4711,7 +5136,11 @@ impl Index {// Post-filter candidate vectors for this field: (id, src, doc_vec).let mut cand: Vec<(String, Value, Vec<f32>)> = Vec::with_capacity(candidates.len());- for (id, src) in candidates {+ for (position, (id, src)) in candidates.into_iter().enumerate() {+ if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {+ timed_out = true;+ break;+ }if let Some(ref f) = filter {let mut src_with_id = src.clone();if let Some(obj) = src_with_id.as_object_mut() {@@ -4779,7 +5208,11 @@ impl Index {let stores = self.sq8_stores.read().await;if let Some(store) = stores.get(field) {let mut decoded = vec![0.0f32; store.dim];- for (id, src, _v) in cand {+ for (position, (id, src, _v)) in cand.into_iter().enumerate() {+ if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {+ timed_out = true;+ break;+ }let codes = match store.codes.get(&id) {Some(c) if c.len() == store.dim => c,_ => continue,@@ -4799,7 +5232,11 @@ impl Index {// kNN and short single-chunk docs have no such companion, so they// fall through to the exact single-vector scan below (unchanged).let chunk_field = format!("{field}_chunks");- for (id, src) in candidates {+ for (position, (id, src)) in candidates.into_iter().enumerate() {+ if position & 127 == 0 && self.exact_scan_checkpoint(position, deadline).await {+ timed_out = true;+ break;+ }// Apply filter: if the filter doesn't match this doc, skip.if let Some(ref f) = filter {let mut src_with_id = src.clone();@@ -4814,7 +5251,13 @@ impl Index {{// Best-matching passage over the stored chunk vectors.let mut best: Option<f32> = None;- for cv in &chunks {+ for (chunk_position, cv) in chunks.iter().enumerate() {+ if chunk_position & 31 == 0+ && self.exact_scan_checkpoint(chunk_position, deadline).await+ {+ timed_out = true;+ break;+ }let dv: Vec<f32> = match cv {Value::Array(a) => a.iter()@@ -4852,6 +5295,9 @@ impl Index {scored.push((id, score, src));}}+ if trace_phases {+ tracing::info!(index=%self.name, collect_ms=collect_elapsed.as_millis() as u64, elapsed_ms=started.elapsed().as_millis() as u64, scored=scored.len(), "semantic_phase=scored");+ }// ── ES `knn.similarity` cutoff ─────────────────────────────────// Convert the RAW-metric threshold into the equivalent@@ -4878,7 +5324,15 @@ impl Index {// ── Rank, cap the candidate pool at k, then paginate ──────────// (shared with the HNSW path so hits format / total semantics// cannot drift between the exact and approximate executors)- Ok(knn_result_from_scored(request, scored, k, started))+ let mut result = knn_result_from_scored(request, scored, k, started);+ result.timed_out = timed_out;+ if timed_out {+ result.total.relation = TotalHitsRelation::Gte;+ }+ if trace_phases {+ tracing::info!(index=%self.name, elapsed_ms=started.elapsed().as_millis() as u64, hits=result.hits.len(), "semantic_phase=complete");+ }+ Ok(result)}/// PURE multi-KNN executor (the ES top-level `knn: [...]` array form,@@ -4892,13 +5346,19 @@ impl Index {async fn run_multi_knn_brute_force(&self,request: &SearchRequest,+ deadline: std::time::Instant,clauses: Vec<PeeledKnn>,) -> Result<SearchResult> {let started = std::time::Instant::now();let mut merged: Vec<(String, f32, Value)> = Vec::new();let mut slot_by_id: HashMap<String, usize> = HashMap::new();let mut k_sum: usize = 0;+ let mut any_timed_out = false;for clause in clauses {+ if std::time::Instant::now() >= deadline {+ any_timed_out = true;+ break;+ }k_sum = k_sum.saturating_add(clause.k);let similarity = {let schema = self.schema.read().await;@@ -4912,8 +5372,9 @@ impl Index {sub_request.from = 0;sub_request.size = clause.k;let sub = self- .run_knn_brute_force(+ .run_knn_brute_force_with_deadline(&sub_request,+ deadline,&clause.field,&clause.vector,clause.k,@@ -4923,6 +5384,7 @@ impl Index {clause.similarity,).await?;+ any_timed_out |= sub.timed_out;for hit in sub.hits {match slot_by_id.get(&hit.id) {Some(&i) => merged[i].1 += hit.score,@@ -4932,8 +5394,16 @@ impl Index {}}}+ if any_timed_out {+ break;+ }}- Ok(knn_result_from_scored(request, merged, k_sum, started))+ let mut result = knn_result_from_scored(request, merged, k_sum, started);+ result.timed_out = any_timed_out;+ if any_timed_out {+ result.total.relation = TotalHitsRelation::Gte;+ }+ Ok(result)}/// Brute-force nested KNN: score each parent by the best (max)@@ -4961,6 +5431,21 @@ impl Index {text: &str,k: usize,filter: Option<Box<QueryNode>>,+ ) -> Result<SearchResult> {+ let deadline = std::time::Instant::now()+ + std::time::Duration::from_millis(request.timeout_ms.unwrap_or(30_000));+ self.run_semantic_with_deadline(request, deadline, field, text, k, filter)+ .await+ }++ async fn run_semantic_with_deadline(+ &self,+ request: &SearchRequest,+ deadline: std::time::Instant,+ field: &str,+ text: &str,+ k: usize,+ filter: Option<Box<QueryNode>>,) -> Result<SearchResult> {// Resolve whether `field` is a `semantic_text` field (has an embedding// config). If so, the vector lives in the companion `target_field` and@@ -5004,7 +5489,12 @@ impl Index {// ingest. A `semantic` query against a plain dense_vector field// with no active backend still returns the original 400 (there// is no comparable stored vector to match against).+ let semantic_started = std::time::Instant::now();+ let trace_phases = std::env::var_os("XERJ_TRACE_SEMANTIC_PHASES").is_some();let embedder = self.embedder.read().await;+ if trace_phases {+ tracing::info!(index=%self.name, field, k, "semantic_phase=embed_start");+ }let query_vec = if embedder.is_active() {// `embed_batch` takes Vec<String>; we only have one text but// batching keeps the wire format stable for callers.@@ -5038,18 +5528,28 @@ impl Index {model and tokenizer paths).",)));};+ drop(embedder);+ if trace_phases {+ tracing::info!(index=%self.name, elapsed_ms=semantic_started.elapsed().as_millis() as u64, "semantic_phase=embed_complete");+ }- self.run_knn_brute_force(- request,- &knn_field,- &query_vec,- k,- filter,- &similarity,- None,- None,- )- .await+ let result = self+ .run_knn_brute_force_with_deadline(+ request,+ deadline,+ &knn_field,+ &query_vec,+ k,+ filter,+ &similarity,+ None,+ None,+ )+ .await;+ if trace_phases {+ tracing::info!(index=%self.name, elapsed_ms=semantic_started.elapsed().as_millis() as u64, ok=result.is_ok(), "semantic_phase=request_complete");+ }+ result}/// Auto-embed `semantic_text` fields on ingest.@@ -5344,6 +5844,19 @@ impl Index {request: &SearchRequest,sub_queries: Vec<xerj_query::ast::WeightedQuery>,fusion: xerj_query::ast::FusionStrategy,+ ) -> Result<SearchResult> {+ let deadline = std::time::Instant::now()+ + std::time::Duration::from_millis(request.timeout_ms.unwrap_or(30_000));+ self.run_hybrid_with_deadline(request, deadline, sub_queries, fusion)+ .await+ }++ async fn run_hybrid_with_deadline(+ &self,+ request: &SearchRequest,+ deadline: std::time::Instant,+ sub_queries: Vec<xerj_query::ast::WeightedQuery>,+ fusion: xerj_query::ast::FusionStrategy,) -> Result<SearchResult> {let started = std::time::Instant::now();@@ -5370,7 +5883,12 @@ impl Index {// for v0.7-P1 — the per-query latency is dominated by the kNN// / FTS scan, not the await ordering.)let mut sub_results: Vec<(Vec<Hit>, f32)> = Vec::with_capacity(sub_queries.len());+ let mut any_timed_out = false;for wq in sub_queries {+ if std::time::Instant::now() >= deadline {+ any_timed_out = true;+ break;+ }let sub_request = SearchRequest {query: wq.query.clone(),from: 0,@@ -5393,8 +5911,12 @@ impl Index {};// Box::pin to break the type-recursion (search_inner ↔// run_hybrid both async fn).- let sub_result = Box::pin(self.search_inner(&sub_request)).await?;+ let sub_result = Box::pin(self.search_inner(&sub_request, deadline)).await?;+ any_timed_out |= sub_result.timed_out;sub_results.push((sub_result.hits, wq.weight));+ if any_timed_out {+ break;+ }}// Apply fusion.@@ -5426,11 +5948,15 @@ impl Index {hits: page,total: TotalHits {value: total_value,- relation: TotalHitsRelation::Eq,+ relation: if any_timed_out {+ TotalHitsRelation::Gte+ } else {+ TotalHitsRelation::Eq+ },},took_ms,aggs: None,- timed_out: false,+ timed_out: any_timed_out,profile: None,max_score: None,})@@ -6266,6 +6792,21 @@ impl Index {/// Execute a search request against this index.pub async fn search(&self, request: &SearchRequest) -> Result<SearchResult> {+ let request_started = std::time::Instant::now();+ let request_deadline = request_started+ + std::time::Duration::from_millis(request.timeout_ms.unwrap_or(30_000));+ let timed_out_result = || SearchResult {+ hits: vec![],+ total: xerj_query::executor::TotalHits {+ value: 0,+ relation: TotalHitsRelation::Gte,+ },+ took_ms: request_started.elapsed().as_millis() as u64,+ aggs: None,+ timed_out: true,+ profile: None,+ max_score: None,+ };// M3 framework: response cache. Hash the request shape and the// current dataset version; on a hit return the cloned `Arc<SearchResult>`// immediately, skipping the semaphore, search_inner, locks, and@@ -6393,7 +6934,17 @@ impl Index {}return Ok(cloned);}- if rx.changed().await.is_err() {+ match tokio::time::timeout_at(+ tokio::time::Instant::from_std(request_deadline),+ rx.changed(),+ )+ .await+ {+ Err(_) => return Ok(timed_out_result()),+ Ok(Ok(())) => continue,+ Ok(Err(_closed)) => {}+ }+ {// Leader dropped its sender. If it succeeded it// inserted the result into `query_cache` BEFORE// dropping, so serve that; otherwise recompute.@@ -6432,7 +6983,15 @@ impl Index {// like ES's search thread pool bounds active workers. Held (via// `_global_permit`) for the whole call; released on drop.let _global_permit = match crate::governor::global() {- Some(g) => Some(g.acquire_search().await?),+ Some(g) => match tokio::time::timeout_at(+ tokio::time::Instant::from_std(request_deadline),+ g.acquire_search(),+ )+ .await+ {+ Ok(result) => Some(result?),+ Err(_) => return Ok(timed_out_result()),+ },None => None,};@@ -6440,11 +6999,19 @@ impl Index {// against a single index, preventing one hot index from monopolising// the global pool. The permit is automatically released when// `_permit` is dropped at the end of this call.- let _permit = self.max_concurrent_queries.acquire().await.map_err(|_| {- EngineError::Common(xerj_common::XerjError::internal(- "query semaphore closed — index is shutting down",- ))- })?;+ let _permit = match tokio::time::timeout_at(+ tokio::time::Instant::from_std(request_deadline),+ self.max_concurrent_queries.acquire(),+ )+ .await+ {+ Ok(result) => result.map_err(|_| {+ EngineError::Common(xerj_common::XerjError::internal(+ "query semaphore closed — index is shutting down",+ ))+ })?,+ Err(_) => return Ok(timed_out_result()),+ };let search_start = std::time::Instant::now();@@ -6474,7 +7041,6 @@ impl Index {// Determine the timeout: use the request-level timeout if set, otherwise// fall back to the default of 30 seconds.let timeout_ms = request.timeout_ms.unwrap_or(30_000);- let timeout_duration = std::time::Duration::from_millis(timeout_ms);// M5.21 — run the CPU-heavy search body inside `block_in_place`// on multi-thread runtimes; fall back to plain await on@@ -6483,18 +7049,22 @@ impl Index {// Pre-M5.21 a concurrent QPS bench collapsed to <1 QPS. See// the commit body for `perf/search-block-in-place` for the// full root-cause analysis.+ let cooperative_vector_query = peel_knn_query(&request.query).is_some()+ || peel_multi_knn_query(&request.query).is_some()+ || peel_semantic_query(&request.query).is_some();let is_multi_thread = tokio::runtime::Handle::current().runtime_flavor()== tokio::runtime::RuntimeFlavor::MultiThread;- let search_fut = self.search_inner(request);- let search_result = if is_multi_thread {+ let search_fut = self.search_inner(request, request_deadline);+ let search_result = if is_multi_thread && !cooperative_vector_query {let fut = async move {tokio::task::block_in_place(|| {tokio::runtime::Handle::current().block_on(search_fut)})};- tokio::time::timeout(timeout_duration, fut).await+ tokio::time::timeout_at(tokio::time::Instant::from_std(request_deadline), fut).await} else {- tokio::time::timeout(timeout_duration, search_fut).await+ tokio::time::timeout_at(tokio::time::Instant::from_std(request_deadline), search_fut)+ .await};match search_result {Ok(result) => {@@ -6584,7 +7154,11 @@ impl Index {}/// Inner search implementation (without timeout wrapper).- async fn search_inner(&self, request: &SearchRequest) -> Result<SearchResult> {+ async fn search_inner(+ &self,+ request: &SearchRequest,+ search_deadline: std::time::Instant,+ ) -> Result<SearchResult> {// Check read block.if self.is_read_blocked().await {return Err(EngineError::Common(xerj_common::XerjError::index_blocked(@@ -6650,8 +7224,6 @@ impl Index {// stopped. Instead we compute a wall-clock deadline here and have// the O(N) loops below check it cooperatively, returning partial// results with `timed_out: true` (ES semantics).- let search_deadline: std::time::Instant = std::time::Instant::now()- + std::time::Duration::from_millis(request.timeout_ms.unwrap_or(30_000));let mut deadline_exceeded = false;// Set when a field-sorted non-match_all scan was narrowed to the// per-segment top-cap sort candidates: the scan's own total tally is@@ -6730,8 +7302,9 @@ impl Index {}}return self- .run_knn_brute_force(+ .run_knn_brute_force_with_deadline(request,+ search_deadline,&field,&query_vec,k,@@ -6747,7 +7320,9 @@ impl Index {// one clause, and `hits.total` is the size of the deduplicated union// — matching ES 8.13 multi-kNN semantics (live-verified 2026-07-12).if let Some(clauses) = peel_multi_knn_query(query) {- return self.run_multi_knn_brute_force(request, clauses).await;+ return self+ .run_multi_knn_brute_force(request, search_deadline, clauses)+ .await;}// Nested `knn` query: `nested { path: P, query: { knn { field: P.vec } } }`.// ES scores each parent by the best-matching nested element and@@ -6800,7 +7375,9 @@ impl Index {// - proxy timeout / 5xx → propagate the proxy's error// - dim mismatch → caught by run_knn_brute_force / HNSWif let Some((field, text, k, filter)) = peel_semantic_query(query) {- return self.run_semantic(request, &field, &text, k, filter).await;+ return self+ .run_semantic_with_deadline(request, search_deadline, &field, &text, k, filter)+ .await;}// ── Hybrid (RRF / Linear / Learned) short-circuit ─────────────────────@@ -6825,7 +7402,9 @@ impl Index {//// Learned: not yet implemented — falls back to RRF with a warn.if let Some((sub_queries, fusion)) = peel_hybrid_query(query) {- return self.run_hybrid(request, sub_queries, fusion).await;+ return self+ .run_hybrid_with_deadline(request, search_deadline, sub_queries, fusion)+ .await;}// ── Max result window enforcement ──────────────────────────────────────diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex cc3b2f81a..49b2e9917 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -3440,6 +3440,59 @@ async fn test_knn_size_windows_into_k() {assert_eq!(res0.total.value, 4, "size=0 still reports the pool total");}+#[tokio::test]+async fn test_public_vector_executor_signatures_remain_compatible() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine+ .create_index("deadline-vectors", Schema::empty())+ .unwrap();+ let idx = engine.get_index("deadline-vectors").unwrap();++ for n in 0..256 {+ idx.index_document(+ Some(format!("d{n}")),+ json!({"embedding": [1.0, 0.0, 0.0, 0.0]}),+ )+ .await+ .unwrap();+ }++ let mut request = parse_request(&json!({+ "query": {"knn": {+ "field": "embedding",+ "query_vector": [1.0, 0.0, 0.0, 0.0],+ "k": 10+ }},+ "size": 10+ }))+ .unwrap();+ request.timeout_ms = Some(250);+ let result = idx+ .run_knn_brute_force(+ &request,+ "embedding",+ &[1.0, 0.0, 0.0, 0.0],+ 10,+ None,+ "cosine",+ None,+ None,+ )+ .await+ .unwrap();+ assert!(!result.timed_out);++ let semantic_future = idx.run_semantic(&request, "body", "query text", 10, None);+ drop(semantic_future);+ let hybrid_future = idx.run_hybrid(+ &request,+ vec![],+ xerj_query::ast::FusionStrategy::Rrf { k: 60 },+ );+ drop(hybrid_future);+}+// ── SQL parser unit tests (inline) ────────────────────────────────────────────#[test]