Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(xerj-autoindex): never read a timed-out partial search page as end-of-data (#1212)

xerj-org09d1ca72fc8b
diff --git a/engine/crates/xerj-autoindex/src/detect/mod.rs b/engine/crates/xerj-autoindex/src/detect/mod.rs
index d8cb2da1f..a51a6dbaa 100644
--- a/engine/crates/xerj-autoindex/src/detect/mod.rs
+++ b/engine/crates/xerj-autoindex/src/detect/mod.rs
@@ -803,7 +803,12 @@ pub fn scan_by_id(
if let Some(after) = &search_after {
body["search_after"] = after.clone();
}
- let Some(resp) = es.search_present(index, &body)? else {
+ // #1212: `search_page_present`, not `search_present` — the engine
+ // answers a missed default search deadline with a partial 200
+ // (`timed_out`), and this walk's short-page return read one partial
+ // page as the end of the index. The page variant retries a timed-out
+ // page instead of returning it; a missing index is still an answer.
+ let Some(resp) = es.search_page_present(index, &body)? else {
return Ok(seen);
};
let hits = resp
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 72b964452..c7375cc2f 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -383,6 +383,22 @@ const BULK_MAX_ACTIONS: usize = 10_000;
/// passes this is repeating itself and the count on `xerj-done` is the record.
const SPLIT_ANNOUNCE_CAP: u64 = 16;
+/// Page timeout every [`Es::search_page`] request carries when its caller set
+/// no `timeout` of its own (#1212). The engine's default — a 30 s cooperative
+/// deadline on any body without one — answers a slow node with HTTP 200,
+/// partial hits and `timed_out: true`, which a page walker cannot tell from
+/// the last page. 120 s sits above the slowest sorted page measured on a
+/// breaker-drained node (40 s); the bounded retry in `search_page` covers
+/// anything slower.
+pub const SEARCH_PAGE_TIMEOUT: &str = "120s";
+
+/// How many times [`Es::search_page`] re-asks the SAME page when the node
+/// keeps answering it partial. Six attempts at a 120 s page timeout is a
+/// 12-minute ceiling per page — enough to ride out a merge or a breaker
+/// drain, short enough to fail a genuinely wedged node within a run's
+/// patience (the bulk patience is 600 s, `THROTTLE_PATIENCE`).
+const PARTIAL_PAGE_ATTEMPTS: usize = 6;
+
/// What this run has learned about how large a `_bulk` request the server
/// takes, shared by every clone of a client like [`BulkAdmission`].
struct RequestSizeLedger {
@@ -1827,6 +1843,71 @@ impl Es {
Ok(Some(v))
}
+ /// One page of a paged walk, complete or not at all.
+ ///
+ /// The engine applies a 30 s cooperative deadline to any search whose
+ /// body carries no `timeout`, and answers a missed deadline with HTTP
+ /// 200, PARTIAL hits and `timed_out: true` — legal on the wire, and
+ /// lethal to a walker: a page that comes back short looks exactly like
+ /// the last page. That truncated the xerj-search catalog read-back at
+ /// 56,441 of 58,568 documents (#1212; the index held all 58,568 the
+ /// whole time). This helper sends the body with an explicit page
+ /// timeout (added only when the caller set none — 120 s covers the
+ /// slowest sorted page measured on a breaker-drained node, 40 s) and
+ /// re-asks the SAME page when the node answers `timed_out`, bounded by
+ /// [`PARTIAL_PAGE_ATTEMPTS`]. A timed-out page costs the node its whole
+ /// page budget, so consecutive attempts are naturally spaced; there is
+ /// no extra backoff to add. Exhaustion is a loud error naming the
+ /// index: rerunning the same command resumes, and every walker above
+ /// this layer is idempotent per page.
+ pub fn search_page(&self, index: &str, body: &Value) -> Result<Value> {
+ Ok(self
+ .search_page_inner(index, body, false)?
+ .expect("strict search page never returns None"))
+ }
+
+ /// [`Es::search_page`] for walkers over an index that may legitimately
+ /// not exist: a 404 is `Ok(None)`, every other non-success is an error,
+ /// and a `timed_out` page is retried exactly like `search_page`'s.
+ pub fn search_page_present(&self, index: &str, body: &Value) -> Result<Option<Value>> {
+ self.search_page_inner(index, body, true)
+ }
+
+ fn search_page_inner(
+ &self,
+ index: &str,
+ body: &Value,
+ allow_missing: bool,
+ ) -> Result<Option<Value>> {
+ let mut body = body.clone();
+ body.as_object_mut()
+ .context("search page body must be a JSON object")?
+ .entry("timeout")
+ .or_insert_with(|| Value::String(SEARCH_PAGE_TIMEOUT.to_owned()));
+ for attempt in 1..=PARTIAL_PAGE_ATTEMPTS {
+ let (status, response) = self.search_raw(index, &body)?;
+ if allow_missing && status == reqwest::StatusCode::NOT_FOUND {
+ return Ok(None);
+ }
+ if !status.is_success() {
+ return Err(anyhow!("search /{index} HTTP {status}: {response}"));
+ }
+ if response.get("timed_out") != Some(&Value::Bool(true)) {
+ return Ok(Some(response));
+ }
+ if attempt == PARTIAL_PAGE_ATTEMPTS {
+ anyhow::bail!(
+ "search /{index} answered this page partial (timed_out) on \
+ {PARTIAL_PAGE_ATTEMPTS} consecutive attempts at timeout \
+ {SEARCH_PAGE_TIMEOUT}: the node cannot complete one page of \
+ the walk. Nothing from this page was acted on — reduce load \
+ on the node and rerun the same command, which resumes"
+ );
+ }
+ }
+ unreachable!("the attempt loop returns on its final iteration")
+ }
+
pub fn count(&self, index: &str) -> Result<u64> {
// `_count` first: it is the endpoint this call always meant, and a
// node audits it as `count.post` — a size:0 search here lands in the
@@ -2018,6 +2099,122 @@ mod tests {
server.join().unwrap();
}
+ /// The fake page server for the `search_page` tests: answers each request
+ /// with the next body in `answers`, recording every request it saw.
+ fn page_server(answers: Vec<&'static [u8]>) -> (std::net::SocketAddr, Arc<Mutex<Vec<String>>>) {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let seen = Arc::new(Mutex::new(Vec::new()));
+ let recorded = Arc::clone(&seen);
+ std::thread::spawn(move || {
+ for answer in answers {
+ let (mut stream, _) = listener.accept().unwrap();
+ let request = read_request(&mut stream);
+ recorded
+ .lock()
+ .unwrap()
+ .push(String::from_utf8_lossy(&request).into_owned());
+ respond_json(&mut stream, answer);
+ }
+ });
+ (address, seen)
+ }
+
+ /// #1212: the engine legally answers a missed search deadline with HTTP
+ /// 200, partial hits and `timed_out: true`. A page walker that took that
+ /// answer at face value read one partial page as the end of the catalog
+ /// (56,441 of 58,568 documents on the xerj-search rebuild). `search_page`
+ /// must retry the SAME request until the node answers a complete page,
+ /// and must put an explicit page timeout on the wire so the engine's 30 s
+ /// default deadline is not what decides.
+ #[test]
+ fn search_page_retries_a_partial_answer_and_returns_the_complete_page() {
+ let (address, seen) = page_server(vec![
+ br#"{"timed_out":true,"hits":{"hits":[]}}"#,
+ br#"{"timed_out":true,"hits":{"hits":[]}}"#,
+ br#"{"timed_out":false,"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_id":"a","_source":{},"sort":["a"]}]}}"#,
+ ]);
+ let es = Es::new(&format!("http://{address}"), None).unwrap();
+ let page = es
+ .search_page(
+ "autoindex-catalog",
+ &serde_json::json!({"size": 1000, "sort": [{"_id": "asc"}]}),
+ )
+ .unwrap();
+ assert_eq!(page.pointer("/hits/hits").unwrap().as_array().unwrap().len(), 1);
+ let seen = seen.lock().unwrap();
+ assert_eq!(seen.len(), 3, "two partial answers, one retry each: {seen:?}");
+ assert!(
+ seen.iter().all(|request| request.contains("\"timeout\":\"120s\"")),
+ "every attempt must carry the explicit page timeout: {seen:?}"
+ );
+ }
+
+ /// #1212: a node that cannot complete one page at all must end the walk
+ /// in a named error, never in a short page the caller would read as
+ /// end-of-data — after exactly the bounded attempt count.
+ #[test]
+ fn search_page_fails_loudly_when_every_attempt_answers_partial() {
+ let answers: Vec<&'static [u8]> = (0..super::PARTIAL_PAGE_ATTEMPTS)
+ .map(|_| br#"{"timed_out":true,"hits":{"hits":[]}}"# as &'static [u8])
+ .collect();
+ let (address, seen) = page_server(answers);
+ let es = Es::new(&format!("http://{address}"), None).unwrap();
+ let error = es
+ .search_page("autoindex-catalog", &serde_json::json!({"size": 1000}))
+ .unwrap_err();
+ let message = format!("{error:#}");
+ assert!(message.contains("partial"), "{message}");
+ assert!(message.contains("autoindex-catalog"), "{message}");
+ assert_eq!(seen.lock().unwrap().len(), super::PARTIAL_PAGE_ATTEMPTS);
+ }
+
+ /// #1212: a caller's own `timeout` is honoured, not overridden — the page
+ /// budget belongs to the walk that knows how slow its node is.
+ #[test]
+ fn search_page_keeps_a_caller_supplied_timeout() {
+ let (address, seen) = page_server(vec![
+ br#"{"timed_out":false,"hits":{"hits":[]}}"#,
+ ]);
+ let es = Es::new(&format!("http://{address}"), None).unwrap();
+ es.search_page(
+ "autoindex-catalog",
+ &serde_json::json!({"size": 1000, "timeout": "5s"}),
+ )
+ .unwrap();
+ let seen = seen.lock().unwrap();
+ assert!(
+ seen[0].contains("\"timeout\":\"5s\"") && !seen[0].contains("120s"),
+ "caller timeout must survive: {seen:?}"
+ );
+ }
+
+ /// #1212: the present variant keeps `search_present`'s contract — a 404
+ /// is an answer (`Ok(None)`), and it is not retried as if it were a
+ /// partial page.
+ #[test]
+ fn search_page_present_treats_a_missing_index_as_an_answer() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let seen = Arc::new(Mutex::new(Vec::new()));
+ let recorded = Arc::clone(&seen);
+ std::thread::spawn(move || {
+ let (mut stream, _) = listener.accept().unwrap();
+ let request = read_request(&mut stream);
+ recorded
+ .lock()
+ .unwrap()
+ .push(String::from_utf8_lossy(&request).into_owned());
+ respond_status(&mut stream, "404 Not Found", br#"{"error":"index_missing"}"#);
+ });
+ let es = Es::new(&format!("http://{address}"), None).unwrap();
+ assert!(es
+ .search_page_present("autoindex-catalog", &serde_json::json!({"size": 1000}))
+ .unwrap()
+ .is_none());
+ assert_eq!(seen.lock().unwrap().len(), 1);
+ }
+
fn read_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
stream
.set_read_timeout(Some(Duration::from_secs(2)))
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 58632907e..cfed512f8 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -4981,7 +4981,18 @@ fn delete_catalog_hits_where(
let mut ids: Vec<String> = Vec::new();
let mut from = 0usize;
for _ in 0..SCOPED_SWEEP_MAX_PAGES {
- let page = es.search(
+ // #1212: `search_page`, not `search` — the engine answers a missed
+ // default search deadline with a partial 200 (`timed_out`), and a
+ // partial page here would end the sweep early holding a prefix of
+ // the candidate set: the deletes below would remove only what the
+ // prefix contained while `Ok(())` reported the sweep complete. (The
+ // `relation` check below cannot catch it either — a timed-out scan
+ // can still answer `relation: "eq"` with an under-counted total,
+ // live-observed against 58,568 real documents answering
+ // `{value: 0, relation: "eq"}`.) The page variant retries a timed-out
+ // page instead of returning it, so every page this loop sees is
+ // complete.
+ let page = es.search_page(
catalog::CATALOG_INDEX,
&json!({"query": query.clone(), "from": from, "size": SCOPED_SWEEP_PAGE}),
)?;
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index ec6269fb5..ac4edfe49 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -648,7 +648,14 @@ impl<'a> EsSyncBackend<'a> {
if let Some(after) = &search_after {
body["search_after"] = after.clone();
}
- let response = self.es.search(crate::catalog::CATALOG_INDEX, &body)?;
+ // #1212: a `search_page`, not a `search` — the engine answers a
+ // missed 30 s default deadline with a partial 200 (`timed_out`),
+ // and this walk's short-page break read one partial page as the
+ // end of the catalog: 56,441 documents observed against 58,568
+ // that were all there, on a node whose sorted pages took 11-40 s
+ // while its breaker drained. The page variant retries a timed-out
+ // page at the same continuation key instead of returning it.
+ let response = self.es.search_page(crate::catalog::CATALOG_INDEX, &body)?;
let hits = response
.pointer("/hits/hits")
.and_then(Value::as_array)