AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): retry a timed_out finalize-verify window instead of exiting 1 (#2419)
sabrimete33ea2cbd1d89
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 89b896999..91e143266 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -726,7 +726,7 @@ pub const SEARCH_PAGE_TIMEOUT: &str = "120s";/// 12-minute ceiling per page — enough to ride out a merge or a breaker/// drain, short enough to fail a genuinely wedged node within a run's/// patience (the bulk patience is 600 s, `THROTTLE_PATIENCE`).-const PARTIAL_PAGE_ATTEMPTS: usize = 6;+pub(crate) const PARTIAL_PAGE_ATTEMPTS: usize = 6;/// What this run has learned about how large a `_bulk` request the server/// takes, shared by every clone of a client like [`BulkAdmission`].@@ -2414,6 +2414,43 @@ impl Es {}}+ /// [`Es::search_for_verify`] carrying [`Es::search_page`]'s deadline and+ /// bounded retry. The finalize-verify windows treated the first partial+ /// `timed_out` page as fatal — exit 1 on a 92-dataset corpus whose+ /// documents were all indexed (#2405) — while the same answer on a+ /// walked page has been re-asked since #1212. The body gets the page+ /// timeout when the caller set none and the SAME window is re-asked+ /// while the node answers `timed_out`, bounded by+ /// [`PARTIAL_PAGE_ATTEMPTS`]. Every attempt still goes through+ /// `search_for_verify`, so a target deleted mid-window (#2222) ends the+ /// wait at once, and a window that never completes bails loud rather+ /// than being counted as zero (#1761).+ pub(crate) fn search_page_for_verify(&self, index: &str, body: &Value) -> Result<Value> {+ let mut body = body.clone();+ body.as_object_mut()+ .context("search page body must be a JSON object")?+ .entry("timeout")+ .or_insert_with(|| Value::String(SEARCH_PAGE_TIMEOUT.to_owned()));+ for attempt in 1..=PARTIAL_PAGE_ATTEMPTS {+ let response = self.search_for_verify(index, &body)?;+ if response.get("timed_out") != Some(&Value::Bool(true)) {+ return Ok(response);+ }+ if attempt == PARTIAL_PAGE_ATTEMPTS {+ anyhow::bail!(+ "finalize-verify search on {index} answered partial (timed_out) on \+ {PARTIAL_PAGE_ATTEMPTS} consecutive attempts at timeout \+ {SEARCH_PAGE_TIMEOUT}: a partial page is not a live count \+ (a concurrent writer on this index can hold publication \+ across the deadline). Nothing from this window was \+ counted — reduce load on the node and rerun the same \+ command, which resumes"+ );+ }+ }+ unreachable!("the attempt loop returns on its final iteration")+ }+/// `Some(true)` the index is there, `Some(false)` the node answered 404,/// `None` when this probe learned nothing (timeout, transport error, or/// any other status). `None` must not be read as gone.diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex 52990d528..e1adfbb62 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -950,8 +950,9 @@ impl<'a> EsSyncBackend<'a> {/// `credit` ticks the current phase once per window after that window's/// answer is in. The capability probe passes `false`: it is one sample,/// not one of the phase's windows, and a tick there would walk the bar- /// past its denominator. A `timed_out` page bails before the tick, so a- /// partial window is not counted as zero and is not credited (#1761).+ /// past its denominator. A `timed_out` window is re-asked, bounded, and+ /// only its exhaustion bails before the tick, so a partial window is+ /// not counted as zero and is not credited (#1761, #2405).fn windowed_value_counts(&self,index: &str,@@ -966,11 +967,18 @@ impl<'a> EsSyncBackend<'a> {let mut filter = Vec::with_capacity(1 + extra_filters.len());filter.push(serde_json::json!({"terms": {field: window}}));filter.extend(extra_filters.iter().cloned());- // #2222: `search_for_verify`, not `search`. A DELETE of this- // index while the window is in flight used to leave the caller- // parked on it (`waiting_on` the deleted name) until the- // transport budget ran out, and the generation never committed.- let response = self.es.search_for_verify(+ // #2222 + #2405: `search_page_for_verify`, not `search`. A+ // DELETE of this index while the window is in flight used to+ // leave the caller parked on it (`waiting_on` the deleted name)+ // until the transport budget ran out, and the generation never+ // committed; and a `timed_out` answer used to exit 1 on the+ // first partial page. The call re-asks the same window, bounded,+ // and still fails fast on a deleted index. A deadline page is+ // HTTP 200 with `timed_out` and an empty aggregation — counting+ // it as "every digest is absent" would abort a generation whose+ // rows are on the node, so exhaustion stays a loud error and is+ // never counted as zero (#1761).+ let response = self.es.search_page_for_verify(index,&serde_json::json!({"size": 0,@@ -980,18 +988,6 @@ impl<'a> EsSyncBackend<'a> {"aggs": {"values": {"terms": {"field": field, "size": window.len()}}}}),)?;- // A deadline page is HTTP 200 with `timed_out` and an empty- // aggregation. Counting it as "every digest is absent" aborts a- // generation whose rows are on the node (#1761's concurrent- // catalog writer hits this deadline). Say so; rerunning resumes.- if response.get("timed_out") == Some(&Value::Bool(true)) {- anyhow::bail!(- "finalize-verify search on {index} returned timed_out: the page is partial \- and is not a live count (a concurrent writer on this index can hold \- publication across the deadline). Nothing from this window was counted — \- rerun the same command, which resumes"- );- }for bucket in response.pointer("/aggregations/values/buckets").and_then(Value::as_array)@@ -1049,7 +1045,7 @@ impl<'a> EsSyncBackend<'a> {"catalog read-back window needed more than {MAX_PAGES} pages");pages += 1;- let response = self.es.search(+ let response = self.es.search_page(crate::catalog::CATALOG_INDEX,&serde_json::json!({"size": PAGE,@@ -1062,16 +1058,6 @@ impl<'a> EsSyncBackend<'a> {]}}}),)?;- if response.get("timed_out") == Some(&Value::Bool(true)) {- anyhow::bail!(- "finalize-verify search on {} returned timed_out: the page is partial \- and is not a live catalog count (a concurrent corpus build publishing \- to this shared index can hold the publication bracket across the \- deadline). Nothing from this window was counted — rerun the same \- command, which resumes",- crate::catalog::CATALOG_INDEX- );- }let page_total = response.pointer("/hits/total/value").and_then(Value::as_u64)@@ -1110,9 +1096,10 @@ impl<'a> EsSyncBackend<'a> {);// One tick per window, after the window's pages have all come// back. The phase denominator is windows, not pages (#1267).- // A `timed_out` page bails above, so a partial catalog window is- // not counted as zero and the phase does not sit at 0/N while- // another corpus publishes to this shared index (#1761).+ // `search_page` re-asks a `timed_out` page and bails only on+ // exhaustion, so a partial catalog window is not counted as+ // zero and the phase does not sit at 0/N while another corpus+ // publishes to this shared index (#1761, #2405).credit_verify_items(self.pr, 1);}Ok(counts)@@ -6566,6 +6553,241 @@ mod tests {server.join().unwrap();}+ /// A scripted node for the finalize-verify retry tests (#2405). Every+ /// connection gets its own handler thread so a `_search` in flight+ /// cannot starve the `HEAD` probe `search_for_verify` may send while a+ /// search is unanswered. `HEAD` always answers 200 (the index lives);+ /// each `_search` records its request body and answers with the next+ /// scripted response — the last one repeats when the script runs out,+ /// so a caller that keeps re-asking past the script is still answered.+ struct ScriptedSearch {+ address: std::net::SocketAddr,+ search_bodies: std::sync::Arc<std::sync::Mutex<Vec<Value>>>,+ stop: std::sync::Arc<std::sync::atomic::AtomicBool>,+ server: Option<std::thread::JoinHandle<()>>,+ }++ impl ScriptedSearch {+ fn start(script: &[&str]) -> Self {+ use std::io::{Read, Write};+ use std::sync::atomic::{AtomicBool, Ordering};+ use std::sync::{Arc, Mutex};+ use std::time::Duration;++ fn read_request(stream: &mut std::net::TcpStream) -> (String, Vec<u8>) {+ stream+ .set_read_timeout(Some(Duration::from_secs(5)))+ .unwrap();+ let mut buf = Vec::new();+ let mut tmp = [0u8; 4096];+ loop {+ match stream.read(&mut tmp) {+ Ok(0) => break,+ Ok(n) => {+ buf.extend_from_slice(&tmp[..n]);+ if let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") {+ let head = String::from_utf8_lossy(&buf[..end]).into_owned();+ let length = head+ .lines()+ .find_map(|line| {+ line.to_ascii_lowercase()+ .strip_prefix("content-length:")+ .and_then(|v| v.trim().parse::<usize>().ok())+ })+ .unwrap_or(0);+ let have = buf.len() - (end + 4);+ if have >= length {+ return (head, buf[end + 4..end + 4 + length].to_vec());+ }+ }+ }+ Err(_) => break,+ }+ }+ let head = String::from_utf8_lossy(&buf).into_owned();+ (head, Vec::new())+ }++ fn write_json(stream: &mut std::net::TcpStream, body: &str) {+ write!(+ stream,+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",+ body.len()+ )+ .unwrap();+ stream.write_all(body.as_bytes()).unwrap();+ }++ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ listener.set_nonblocking(true).unwrap();+ let search_bodies = Arc::new(Mutex::new(Vec::new()));+ let stop = Arc::new(AtomicBool::new(false));+ let script: Arc<Mutex<std::collections::VecDeque<String>>> =+ Arc::new(Mutex::new(script.iter().map(|s| s.to_string()).collect()));+ let bodies_srv = Arc::clone(&search_bodies);+ let stop_srv = Arc::clone(&stop);+ let script_srv = Arc::clone(&script);+ let server = std::thread::spawn(move || {+ let mut handlers = Vec::new();+ while !stop_srv.load(Ordering::SeqCst) {+ match listener.accept() {+ Ok((stream, _)) => {+ let bodies_one = Arc::clone(&bodies_srv);+ let script_one = Arc::clone(&script_srv);+ handlers.push(std::thread::spawn(move || {+ let mut stream = stream;+ stream.set_nonblocking(false).unwrap();+ let (head, body) = read_request(&mut stream);+ if head.starts_with("HEAD ") {+ write_json(&mut stream, "{}");+ } else if head.contains("_search") {+ let parsed =+ serde_json::from_slice(&body).unwrap_or(Value::Null);+ bodies_one.lock().unwrap().push(parsed);+ let answer = {+ let mut scripted = script_one.lock().unwrap();+ if scripted.len() > 1 {+ scripted.pop_front().unwrap()+ } else {+ scripted+ .front()+ .cloned()+ .unwrap_or_else(|| "{}".to_string())+ }+ };+ write_json(&mut stream, &answer);+ } else {+ write_json(&mut stream, "{}");+ }+ }));+ }+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {+ std::thread::sleep(Duration::from_millis(10));+ }+ Err(_) => break,+ }+ }+ for handler in handlers {+ let _ = handler.join();+ }+ });+ ScriptedSearch {+ address,+ search_bodies,+ stop,+ server: Some(server),+ }+ }++ fn searches(&self) -> Vec<Value> {+ self.search_bodies.lock().unwrap().clone()+ }++ fn es(&self) -> crate::esclient::Es {+ crate::esclient::Es::new(&format!("http://{}", self.address), None).unwrap()+ }++ fn shutdown(&mut self) {+ self.stop.store(true, std::sync::atomic::Ordering::SeqCst);+ // Unblock `accept` if it is parked.+ let _ = std::net::TcpStream::connect(self.address);+ if let Some(server) = self.server.take() {+ server.join().unwrap();+ }+ }+ }++ impl Drop for ScriptedSearch {+ fn drop(&mut self) {+ self.shutdown();+ }+ }++ /// #2405: a `timed_out` window used to exit 1 on the spot. The window+ /// is re-asked — with the page timeout the body lacked — and the second+ /// answer counts.+ #[test]+ fn finalize_verify_window_retries_a_timed_out_answer() {+ let mut server = ScriptedSearch::start(&[+ r#"{"timed_out":true,"hits":{"hits":[]}}"#,+ r#"{"aggregations":{"values":{"buckets":[{"key":"digest-a","doc_count":3}]}}}"#,+ ]);+ let es = server.es();+ let dir = tempfile::tempdir().unwrap();+ let pr = crate::progress::Progress::silent();+ let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);+ let counts = backend+ .windowed_value_counts(+ "xc-retry",+ "ax_file",+ &["digest-a".into(), "digest-b".into()],+ &[],+ false,+ )+ .unwrap();+ assert_eq!(counts.get("digest-a"), Some(&3));+ assert_eq!(counts.get("digest-b"), Some(&0));+ let searches = server.searches();+ assert_eq!(+ searches.len(),+ 2,+ "one retry, not a page storm: {searches:?}"+ );+ for body in &searches {+ assert_eq!(+ body.get("timeout").and_then(Value::as_str),+ Some("120s"),+ "the retried window must carry the page timeout: {body}"+ );+ }+ server.shutdown();+ }++ /// The retry is bounded: a node that answers every attempt partial+ /// gets the loud error, not an infinite wait, and the window was never+ /// counted.+ #[test]+ fn finalize_verify_window_fails_loud_when_every_attempt_times_out() {+ let mut server = ScriptedSearch::start(&[r#"{"timed_out":true,"hits":{"hits":[]}}"#]);+ let es = server.es();+ let dir = tempfile::tempdir().unwrap();+ let pr = crate::progress::Progress::silent();+ let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);+ let error = backend+ .windowed_value_counts("xc-wedged", "ax_file", &["digest-a".into()], &[], false)+ .unwrap_err()+ .to_string();+ assert!(error.contains("xc-wedged"), "{error}");+ assert!(error.contains("timed_out"), "{error}");+ assert!(error.contains("consecutive attempts"), "{error}");+ assert!(error.contains("rerun the same command"), "{error}");+ assert_eq!(+ server.searches().len(),+ crate::esclient::PARTIAL_PAGE_ATTEMPTS,+ );+ server.shutdown();+ }++ /// The catalog read-back's `search_page` leg retries the same way.+ #[test]+ fn catalog_read_back_window_retries_a_timed_out_page() {+ let mut server = ScriptedSearch::start(&[+ r#"{"timed_out":true,"hits":{"hits":[]}}"#,+ r#"{"timed_out":false,"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_id":"x","_source":{"file_key":"k1"}}]}}"#,+ ]);+ let es = server.es();+ let dir = tempfile::tempdir().unwrap();+ let pr = crate::progress::Progress::silent();+ let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);+ let counts = backend+ .windowed_catalog_counts(&["k1".into()], "run-1")+ .unwrap();+ assert_eq!(counts.get("k1"), Some(&1));+ assert_eq!(server.searches().len(), 2);+ server.shutdown();+ }+#[test]fn a_zero_exists_total_skips_the_per_digest_windows() {// #1267 measured 2,619 term+exists probes, each a brute scan with zero