AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): do not abort finalize when a count does not answer (#1639)
xerj-orgf0af7aecb8f1
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 7e3d3e1a5..6db4e3d14 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -1961,15 +1961,13 @@ impl Es {match self.count_endpoint(index) {Count::Number(n) => Ok(n),Count::Zero => Ok(0),- Count::Unknown(_) => {+ Count::Unknown(reason) => {let v = self.search(index,&serde_json::json!({"size": 0, "track_total_hits": true}),)?;- v.pointer("/hits/total/value")- .and_then(|t| t.as_u64())- .or_else(|| v.pointer("/hits/total").and_then(|t| t.as_u64()))- .ok_or_else(|| anyhow!("no total in search response"))+ search_hit_total(&v)+ .with_context(|| format!("count /{index}: _count did not answer ({reason})"))}}}@@ -2083,6 +2081,25 @@ impl Es {}}+/// The `hits.total` of a size:0 search used as a `_count` fallback.+///+/// A deadline answer is HTTP 200 with `timed_out: true` and `total.value: 0`+/// (`relation: gte`). Reading that 0 as the live count is what made+/// finalize abort a finished run: the journal had the records, the node+/// had not finished counting, and the zero-live gate treated the partial+/// page as an empty index.+pub(crate) fn search_hit_total(v: &Value) -> Result<u64> {+ if v.get("timed_out") == Some(&Value::Bool(true)) {+ anyhow::bail!(+ "search count fallback timed out; the total is partial and is not a live count"+ );+ }+ v.pointer("/hits/total/value")+ .and_then(|t| t.as_u64())+ .or_else(|| v.pointer("/hits/total").and_then(|t| t.as_u64()))+ .ok_or_else(|| anyhow!("no total in search response"))+}+/// The tri-state answer of [`Es::count_endpoint`].#[derive(Debug, Clone, PartialEq, Eq)]pub enum Count {@@ -2107,6 +2124,23 @@ mod tests {use std::sync::{mpsc, Arc, Mutex};use std::time::{Duration, Instant};+ #[test]+ fn a_timed_out_search_is_not_a_zero_count() {+ let partial = serde_json::json!({+ "timed_out": true,+ "hits": {"total": {"value": 0, "relation": "gte"}, "hits": []}+ });+ let err = super::search_hit_total(&partial).unwrap_err();+ assert!(+ err.to_string().contains("timed out"),+ "a partial total must not become 0: {err}"+ );+ let complete = serde_json::json!({"hits": {"total": {"value": 3, "relation": "eq"}}});+ assert_eq!(super::search_hit_total(&complete).unwrap(), 3);+ let bare = serde_json::json!({"hits": {"total": 4}});+ assert_eq!(super::search_hit_total(&bare).unwrap(), 4);+ }+#[test]fn bulk_request_uses_custom_timeout_and_drops_timed_out_connection() {let listener = TcpListener::bind("127.0.0.1:0").unwrap();diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 1d3554463..b6e049151 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -133,6 +133,10 @@ struct MockState {/// Accept every data bulk (`errors: false`) but persist nothing — the/// shape of any rejection path the client-side classifier misses.swallow_data_bulks: bool,+ /// `_count` answers 200 with no `count`, and a size:0 `_search` answers+ /// `timed_out`. Models a node that accepted the writes and then could+ /// not finish a count (merge, deadline). Must not look like zero docs.+ fail_counts: bool,/// Answer the catalog's duplicate-alias sweep — and ONLY that request —/// with a 500 whose body names the condition. Scoped to the one call so/// the fixture proves the sweep is what wedged the run (#345), not that a@@ -422,7 +426,26 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {}(200, json!({"deleted": ids.len(), "failures": []}))} else if method == "GET" && path.ends_with("/_count") {- (200, json!({"count": state.lock().unwrap().docs.len()}))+ if state.lock().unwrap().fail_counts {+ (200, json!({"error": "count unavailable"}))+ } else {+ (200, json!({"count": state.lock().unwrap().docs.len()}))+ }+ } else if method == "POST"+ && path.ends_with("/_search")+ && serde_json::from_slice::<Value>(&body)+ .ok()+ .is_some_and(|query| query.get("size").and_then(Value::as_u64) == Some(0))+ && state.lock().unwrap().fail_counts+ {+ (+ 200,+ json!({+ "took": 1,+ "timed_out": true,+ "hits": {"total": {"value": 0, "relation": "gte"}, "hits": []}+ }),+ )} else if method == "POST" && path.ends_with("/_search") {// Report the REAL number of stored docs: `run_index` verifies live// counts against the journal (#195), so a mock that always answered 0@@ -3033,6 +3056,37 @@ fn zero_live_documents_after_journaled_records_fails_the_run() {);}+/// A node that accepted the bulks and then could not count (the size:0+/// fallback comes back `timed_out` with total 0) must not abort finalize.+/// Treating that 0 as "nothing is live" is the count-mismatch abort: the+/// journal is right, the count is not a measurement.+#[test]+fn a_count_that_does_not_answer_does_not_fail_a_run_that_wrote_records() {+ let _guard = FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("records.csv"),+ "id,value\n0,kept\n1,kept\n",+ )+ .unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ endpoint.state.lock().unwrap().fail_counts = true;+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);++ let code = run_index(config).expect("an unanswered count is not an empty index");+ assert_eq!(code, 0, "the records were accepted");+ assert!(+ endpoint.state.lock().unwrap().docs.len() >= 2,+ "the mock kept the documents the count could not see"+ );+}+/// #241 regression: a run must never be silent.////// Before this fix `autoindex` printed a handful of phase banners, one linediff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 474bd215f..6d3331a55 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -8607,10 +8607,22 @@ fn run_index_report_inner(pr.phase("finalize-count", plan.datasets.len() as u64, 0);let mut ds_counts: HashMap<String, u64> = HashMap::new();let mut ds_timerange: HashMap<String, (Option<String>, Option<String>)> = HashMap::new();+ // A count that did not answer is not a zero. `unwrap_or(0)` used to+ // turn a timeout, a 500, or a `timed_out` size:0 page into "0 live+ // documents", and the gate below then aborted a run whose records+ // were already journaled (#195's last resort, fired on the wrong+ // input). Unknown counts skip the gate and are named in the summary.+ let mut count_unknown: Vec<String> = Vec::new();for d in &plan.datasets {let _counted = pr.file(&d.index, 0);- let cnt = es.count(&d.index).unwrap_or(0);- ds_counts.insert(d.slug.clone(), cnt);+ match es.count(&d.index) {+ Ok(cnt) => {+ ds_counts.insert(d.slug.clone(), cnt);+ }+ Err(error) => {+ count_unknown.push(format!("{}: {error:#}", d.index));+ }+ }if let Some(t) = &d.time_field {let body = json!({"size":0,"aggs":{"mn":{"min":{"field":t}},"mx":{"max":{"field":t}}}});@@ -8650,7 +8662,15 @@ fn run_index_report_inner()};let live_records: u64 = ds_counts.values().sum();- if journal_records > 0 && live_records == 0 {+ if !count_unknown.is_empty() {+ pr.warn(&format!(+ "finalize-count: {} dataset index(es) did not answer a count, so the \+ zero-live check was not applied. The journal records {journal_records} \+ record(s). {}",+ count_unknown.len(),+ count_unknown.join(" | ")+ ));+ } else if journal_records > 0 && live_records == 0 {// Typed, not `bail!`: a caller that composes this pipeline can only// tell "the journal outlived its destination" from "the server// rejected our writes" if the condition carries a type. `xerj brain`@@ -8866,6 +8886,12 @@ fn run_index_report_inner(formats.sort();formats.dedup();let (tmin, tmax) = ds_timerange.get(&d.slug).cloned().unwrap_or((None, None));+ if !ds_counts.contains_key(&d.slug) {+ notes.push(+ "record_count unavailable: the node did not answer a count, so 0 here is not a measurement"+ .to_string(),+ );+ }let (id, doc) = catalog::dataset_doc(&catalog::DatasetDocInput {prefix: &cfg.prefix,pd: d,