AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
Stop the memtable scan at the search deadline (Refs #2431) (#2830)
xerj-orgda474292400f
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 3c3adae9c..5f062cfa3 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -26213,7 +26213,16 @@ impl Index {// direction, the same argument the segment walk's `page_worst`// relies on.let mut scan_worst: Option<(f32, u64, String)> = None;- for (seq_no, doc_id, source) in docs {+ for (scanned, (seq_no, doc_id, source)) in docs.into_iter().enumerate() {+ // Cooperative deadline (#2431): this per-document walk+ // of the whole memtable is the `mem_admit` phase that+ // ran 115 s on rc.97 with no check at all. Stop like the+ // segment walk does: partial hits, `timed_out: true`.+ // Checked on the first document and every 256 after.+ if scanned % 256 == 0 && std::time::Instant::now() >= search_deadline {+ deadline_exceeded = true;+ break;+ }let matched = if let QueryNode::Ids { values } = query {values.iter().any(|v| v == doc_id.as_str())} else if !query_may_read_id || source.get("_id").is_some() {diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex e7d2be0cc..3d58a2121 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -5774,6 +5774,53 @@ async fn test_knn_size_windows_into_k() {assert_eq!(res0.total.value, 4, "size=0 still reports the pool total");}+/// #2431: on rc.97 one search spent 115 s in `mem_admit`, the per-document+/// walk of the unflushed memtable, which never looked at the request+/// deadline. A memtable-only index has no segment loop to stop it, so an+/// already-expired request used to scan every buffered document and answer+/// `timed_out: false`. The walk now checks the deadline like the segment loop.+#[tokio::test]+async fn test_memtable_scan_stops_at_the_search_deadline() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine+ .create_index("mem-deadline", Schema::empty())+ .unwrap();+ let idx = engine.get_index("mem-deadline").unwrap();+ for n in 0..1500u64 {+ idx.index_document(Some(format!("d{n}")), json!({"a": n % 7, "b": n}))+ .await+ .unwrap();+ }+ // A two-key sort is outside the bounded memtable sort path, so the+ // memtable is offered to the per-document scan (no flush: no segments).+ let body = json!({+ "query": {"match_all": {}},+ "sort": [{"a": "asc"}, {"b": "asc"}],+ "size": 10+ });++ // Expired first, so no cached whole answer can stand in for the scan.+ let mut expired = parse_request(&body).unwrap();+ expired.timeout_ms = Some(0);+ let partial = idx.search(&expired).await.unwrap();+ assert!(+ partial.timed_out,+ "an expired deadline must stop the memtable scan and report timed_out"+ );+ let mut live = parse_request(&body).unwrap();+ live.timeout_ms = Some(60_000);+ let full = idx.search(&live).await.unwrap();+ assert!(!full.timed_out, "a live deadline must not trip");+ assert_eq!(full.total.value, 1500);+ assert_eq!(full.hits.len(), 10);++ // A timed-out answer is never cached: the next live request is whole.+ let again = idx.search(&live).await.unwrap();+ assert!(!again.timed_out);+ assert_eq!(again.hits.len(), 10);+}+#[tokio::test]async fn test_public_vector_executor_signatures_remain_compatible() {let dir = TempDir::new().unwrap();