AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(xerj-autoindex): never read a timed-out partial search page as end-of-data (#1212)
xerj-org09d1ca72fc8b
diff --git a/engine/crates/xerj-autoindex/src/detect/mod.rs b/engine/crates/xerj-autoindex/src/detect/mod.rsindex d8cb2da1f..a51a6dbaa 100644--- a/engine/crates/xerj-autoindex/src/detect/mod.rs+++ b/engine/crates/xerj-autoindex/src/detect/mod.rs@@ -803,7 +803,12 @@ pub fn scan_by_id(if let Some(after) = &search_after {body["search_after"] = after.clone();}- let Some(resp) = es.search_present(index, &body)? else {+ // #1212: `search_page_present`, not `search_present` — the engine+ // answers a missed default search deadline with a partial 200+ // (`timed_out`), and this walk's short-page return read one partial+ // page as the end of the index. The page variant retries a timed-out+ // page instead of returning it; a missing index is still an answer.+ let Some(resp) = es.search_page_present(index, &body)? else {return Ok(seen);};let hits = respdiff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 72b964452..c7375cc2f 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -383,6 +383,22 @@ const BULK_MAX_ACTIONS: usize = 10_000;/// passes this is repeating itself and the count on `xerj-done` is the record.const SPLIT_ANNOUNCE_CAP: u64 = 16;+/// Page timeout every [`Es::search_page`] request carries when its caller set+/// no `timeout` of its own (#1212). The engine's default — a 30 s cooperative+/// deadline on any body without one — answers a slow node with HTTP 200,+/// partial hits and `timed_out: true`, which a page walker cannot tell from+/// the last page. 120 s sits above the slowest sorted page measured on a+/// breaker-drained node (40 s); the bounded retry in `search_page` covers+/// anything slower.+pub const SEARCH_PAGE_TIMEOUT: &str = "120s";++/// How many times [`Es::search_page`] re-asks the SAME page when the node+/// keeps answering it partial. Six attempts at a 120 s page timeout is a+/// 12-minute ceiling per page — enough to ride out a merge or a breaker+/// drain, short enough to fail a genuinely wedged node within a run's+/// patience (the bulk patience is 600 s, `THROTTLE_PATIENCE`).+const PARTIAL_PAGE_ATTEMPTS: usize = 6;+/// What this run has learned about how large a `_bulk` request the server/// takes, shared by every clone of a client like [`BulkAdmission`].struct RequestSizeLedger {@@ -1827,6 +1843,71 @@ impl Es {Ok(Some(v))}+ /// One page of a paged walk, complete or not at all.+ ///+ /// The engine applies a 30 s cooperative deadline to any search whose+ /// body carries no `timeout`, and answers a missed deadline with HTTP+ /// 200, PARTIAL hits and `timed_out: true` — legal on the wire, and+ /// lethal to a walker: a page that comes back short looks exactly like+ /// the last page. That truncated the xerj-search catalog read-back at+ /// 56,441 of 58,568 documents (#1212; the index held all 58,568 the+ /// whole time). This helper sends the body with an explicit page+ /// timeout (added only when the caller set none — 120 s covers the+ /// slowest sorted page measured on a breaker-drained node, 40 s) and+ /// re-asks the SAME page when the node answers `timed_out`, bounded by+ /// [`PARTIAL_PAGE_ATTEMPTS`]. A timed-out page costs the node its whole+ /// page budget, so consecutive attempts are naturally spaced; there is+ /// no extra backoff to add. Exhaustion is a loud error naming the+ /// index: rerunning the same command resumes, and every walker above+ /// this layer is idempotent per page.+ pub fn search_page(&self, index: &str, body: &Value) -> Result<Value> {+ Ok(self+ .search_page_inner(index, body, false)?+ .expect("strict search page never returns None"))+ }++ /// [`Es::search_page`] for walkers over an index that may legitimately+ /// not exist: a 404 is `Ok(None)`, every other non-success is an error,+ /// and a `timed_out` page is retried exactly like `search_page`'s.+ pub fn search_page_present(&self, index: &str, body: &Value) -> Result<Option<Value>> {+ self.search_page_inner(index, body, true)+ }++ fn search_page_inner(+ &self,+ index: &str,+ body: &Value,+ allow_missing: bool,+ ) -> Result<Option<Value>> {+ let mut body = body.clone();+ body.as_object_mut()+ .context("search page body must be a JSON object")?+ .entry("timeout")+ .or_insert_with(|| Value::String(SEARCH_PAGE_TIMEOUT.to_owned()));+ for attempt in 1..=PARTIAL_PAGE_ATTEMPTS {+ let (status, response) = self.search_raw(index, &body)?;+ if allow_missing && status == reqwest::StatusCode::NOT_FOUND {+ return Ok(None);+ }+ if !status.is_success() {+ return Err(anyhow!("search /{index} HTTP {status}: {response}"));+ }+ if response.get("timed_out") != Some(&Value::Bool(true)) {+ return Ok(Some(response));+ }+ if attempt == PARTIAL_PAGE_ATTEMPTS {+ anyhow::bail!(+ "search /{index} answered this page partial (timed_out) on \+ {PARTIAL_PAGE_ATTEMPTS} consecutive attempts at timeout \+ {SEARCH_PAGE_TIMEOUT}: the node cannot complete one page of \+ the walk. Nothing from this page was acted on — reduce load \+ on the node and rerun the same command, which resumes"+ );+ }+ }+ unreachable!("the attempt loop returns on its final iteration")+ }+pub fn count(&self, index: &str) -> Result<u64> {// `_count` first: it is the endpoint this call always meant, and a// node audits it as `count.post` — a size:0 search here lands in the@@ -2018,6 +2099,122 @@ mod tests {server.join().unwrap();}+ /// The fake page server for the `search_page` tests: answers each request+ /// with the next body in `answers`, recording every request it saw.+ fn page_server(answers: Vec<&'static [u8]>) -> (std::net::SocketAddr, Arc<Mutex<Vec<String>>>) {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let seen = Arc::new(Mutex::new(Vec::new()));+ let recorded = Arc::clone(&seen);+ std::thread::spawn(move || {+ for answer in answers {+ let (mut stream, _) = listener.accept().unwrap();+ let request = read_request(&mut stream);+ recorded+ .lock()+ .unwrap()+ .push(String::from_utf8_lossy(&request).into_owned());+ respond_json(&mut stream, answer);+ }+ });+ (address, seen)+ }++ /// #1212: the engine legally answers a missed search deadline with HTTP+ /// 200, partial hits and `timed_out: true`. A page walker that took that+ /// answer at face value read one partial page as the end of the catalog+ /// (56,441 of 58,568 documents on the xerj-search rebuild). `search_page`+ /// must retry the SAME request until the node answers a complete page,+ /// and must put an explicit page timeout on the wire so the engine's 30 s+ /// default deadline is not what decides.+ #[test]+ fn search_page_retries_a_partial_answer_and_returns_the_complete_page() {+ let (address, seen) = page_server(vec![+ br#"{"timed_out":true,"hits":{"hits":[]}}"#,+ br#"{"timed_out":true,"hits":{"hits":[]}}"#,+ br#"{"timed_out":false,"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_id":"a","_source":{},"sort":["a"]}]}}"#,+ ]);+ let es = Es::new(&format!("http://{address}"), None).unwrap();+ let page = es+ .search_page(+ "autoindex-catalog",+ &serde_json::json!({"size": 1000, "sort": [{"_id": "asc"}]}),+ )+ .unwrap();+ assert_eq!(page.pointer("/hits/hits").unwrap().as_array().unwrap().len(), 1);+ let seen = seen.lock().unwrap();+ assert_eq!(seen.len(), 3, "two partial answers, one retry each: {seen:?}");+ assert!(+ seen.iter().all(|request| request.contains("\"timeout\":\"120s\"")),+ "every attempt must carry the explicit page timeout: {seen:?}"+ );+ }++ /// #1212: a node that cannot complete one page at all must end the walk+ /// in a named error, never in a short page the caller would read as+ /// end-of-data — after exactly the bounded attempt count.+ #[test]+ fn search_page_fails_loudly_when_every_attempt_answers_partial() {+ let answers: Vec<&'static [u8]> = (0..super::PARTIAL_PAGE_ATTEMPTS)+ .map(|_| br#"{"timed_out":true,"hits":{"hits":[]}}"# as &'static [u8])+ .collect();+ let (address, seen) = page_server(answers);+ let es = Es::new(&format!("http://{address}"), None).unwrap();+ let error = es+ .search_page("autoindex-catalog", &serde_json::json!({"size": 1000}))+ .unwrap_err();+ let message = format!("{error:#}");+ assert!(message.contains("partial"), "{message}");+ assert!(message.contains("autoindex-catalog"), "{message}");+ assert_eq!(seen.lock().unwrap().len(), super::PARTIAL_PAGE_ATTEMPTS);+ }++ /// #1212: a caller's own `timeout` is honoured, not overridden — the page+ /// budget belongs to the walk that knows how slow its node is.+ #[test]+ fn search_page_keeps_a_caller_supplied_timeout() {+ let (address, seen) = page_server(vec![+ br#"{"timed_out":false,"hits":{"hits":[]}}"#,+ ]);+ let es = Es::new(&format!("http://{address}"), None).unwrap();+ es.search_page(+ "autoindex-catalog",+ &serde_json::json!({"size": 1000, "timeout": "5s"}),+ )+ .unwrap();+ let seen = seen.lock().unwrap();+ assert!(+ seen[0].contains("\"timeout\":\"5s\"") && !seen[0].contains("120s"),+ "caller timeout must survive: {seen:?}"+ );+ }++ /// #1212: the present variant keeps `search_present`'s contract — a 404+ /// is an answer (`Ok(None)`), and it is not retried as if it were a+ /// partial page.+ #[test]+ fn search_page_present_treats_a_missing_index_as_an_answer() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let seen = Arc::new(Mutex::new(Vec::new()));+ let recorded = Arc::clone(&seen);+ std::thread::spawn(move || {+ let (mut stream, _) = listener.accept().unwrap();+ let request = read_request(&mut stream);+ recorded+ .lock()+ .unwrap()+ .push(String::from_utf8_lossy(&request).into_owned());+ respond_status(&mut stream, "404 Not Found", br#"{"error":"index_missing"}"#);+ });+ let es = Es::new(&format!("http://{address}"), None).unwrap();+ assert!(es+ .search_page_present("autoindex-catalog", &serde_json::json!({"size": 1000}))+ .unwrap()+ .is_none());+ assert_eq!(seen.lock().unwrap().len(), 1);+ }+fn read_request(stream: &mut std::net::TcpStream) -> Vec<u8> {stream.set_read_timeout(Some(Duration::from_secs(2)))diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 58632907e..cfed512f8 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -4981,7 +4981,18 @@ fn delete_catalog_hits_where(let mut ids: Vec<String> = Vec::new();let mut from = 0usize;for _ in 0..SCOPED_SWEEP_MAX_PAGES {- let page = es.search(+ // #1212: `search_page`, not `search` — the engine answers a missed+ // default search deadline with a partial 200 (`timed_out`), and a+ // partial page here would end the sweep early holding a prefix of+ // the candidate set: the deletes below would remove only what the+ // prefix contained while `Ok(())` reported the sweep complete. (The+ // `relation` check below cannot catch it either — a timed-out scan+ // can still answer `relation: "eq"` with an under-counted total,+ // live-observed against 58,568 real documents answering+ // `{value: 0, relation: "eq"}`.) The page variant retries a timed-out+ // page instead of returning it, so every page this loop sees is+ // complete.+ let page = es.search_page(catalog::CATALOG_INDEX,&json!({"query": query.clone(), "from": from, "size": SCOPED_SWEEP_PAGE}),)?;diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex ec6269fb5..ac4edfe49 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -648,7 +648,14 @@ impl<'a> EsSyncBackend<'a> {if let Some(after) = &search_after {body["search_after"] = after.clone();}- let response = self.es.search(crate::catalog::CATALOG_INDEX, &body)?;+ // #1212: a `search_page`, not a `search` — the engine answers a+ // missed 30 s default deadline with a partial 200 (`timed_out`),+ // and this walk's short-page break read one partial page as the+ // end of the catalog: 56,441 documents observed against 58,568+ // that were all there, on a node whose sorted pages took 11-40 s+ // while its breaker drained. The page variant retries a timed-out+ // page at the same continuation key instead of returning it.+ let response = self.es.search_page(crate::catalog::CATALOG_INDEX, &body)?;let hits = response.pointer("/hits/hits").and_then(Value::as_array)