Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): do not abort finalize when a count does not answer (#1639)

xerj-orgf0af7aecb8f1
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 7e3d3e1a5..6db4e3d14 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -1961,15 +1961,13 @@ impl Es {
match self.count_endpoint(index) {
Count::Number(n) => Ok(n),
Count::Zero => Ok(0),
- Count::Unknown(_) => {
+ Count::Unknown(reason) => {
let v = self.search(
index,
&serde_json::json!({"size": 0, "track_total_hits": true}),
)?;
- v.pointer("/hits/total/value")
- .and_then(|t| t.as_u64())
- .or_else(|| v.pointer("/hits/total").and_then(|t| t.as_u64()))
- .ok_or_else(|| anyhow!("no total in search response"))
+ search_hit_total(&v)
+ .with_context(|| format!("count /{index}: _count did not answer ({reason})"))
}
}
}
@@ -2083,6 +2081,25 @@ impl Es {
}
}
+/// The `hits.total` of a size:0 search used as a `_count` fallback.
+///
+/// A deadline answer is HTTP 200 with `timed_out: true` and `total.value: 0`
+/// (`relation: gte`). Reading that 0 as the live count is what made
+/// finalize abort a finished run: the journal had the records, the node
+/// had not finished counting, and the zero-live gate treated the partial
+/// page as an empty index.
+pub(crate) fn search_hit_total(v: &Value) -> Result<u64> {
+ if v.get("timed_out") == Some(&Value::Bool(true)) {
+ anyhow::bail!(
+ "search count fallback timed out; the total is partial and is not a live count"
+ );
+ }
+ v.pointer("/hits/total/value")
+ .and_then(|t| t.as_u64())
+ .or_else(|| v.pointer("/hits/total").and_then(|t| t.as_u64()))
+ .ok_or_else(|| anyhow!("no total in search response"))
+}
+
/// The tri-state answer of [`Es::count_endpoint`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Count {
@@ -2107,6 +2124,23 @@ mod tests {
use std::sync::{mpsc, Arc, Mutex};
use std::time::{Duration, Instant};
+ #[test]
+ fn a_timed_out_search_is_not_a_zero_count() {
+ let partial = serde_json::json!({
+ "timed_out": true,
+ "hits": {"total": {"value": 0, "relation": "gte"}, "hits": []}
+ });
+ let err = super::search_hit_total(&partial).unwrap_err();
+ assert!(
+ err.to_string().contains("timed out"),
+ "a partial total must not become 0: {err}"
+ );
+ let complete = serde_json::json!({"hits": {"total": {"value": 3, "relation": "eq"}}});
+ assert_eq!(super::search_hit_total(&complete).unwrap(), 3);
+ let bare = serde_json::json!({"hits": {"total": 4}});
+ assert_eq!(super::search_hit_total(&bare).unwrap(), 4);
+ }
+
#[test]
fn bulk_request_uses_custom_timeout_and_drops_timed_out_connection() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 1d3554463..b6e049151 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -133,6 +133,10 @@ struct MockState {
/// Accept every data bulk (`errors: false`) but persist nothing — the
/// shape of any rejection path the client-side classifier misses.
swallow_data_bulks: bool,
+ /// `_count` answers 200 with no `count`, and a size:0 `_search` answers
+ /// `timed_out`. Models a node that accepted the writes and then could
+ /// not finish a count (merge, deadline). Must not look like zero docs.
+ fail_counts: bool,
/// Answer the catalog's duplicate-alias sweep — and ONLY that request —
/// with a 500 whose body names the condition. Scoped to the one call so
/// the fixture proves the sweep is what wedged the run (#345), not that a
@@ -422,7 +426,26 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {
}
(200, json!({"deleted": ids.len(), "failures": []}))
} else if method == "GET" && path.ends_with("/_count") {
- (200, json!({"count": state.lock().unwrap().docs.len()}))
+ if state.lock().unwrap().fail_counts {
+ (200, json!({"error": "count unavailable"}))
+ } else {
+ (200, json!({"count": state.lock().unwrap().docs.len()}))
+ }
+ } else if method == "POST"
+ && path.ends_with("/_search")
+ && serde_json::from_slice::<Value>(&body)
+ .ok()
+ .is_some_and(|query| query.get("size").and_then(Value::as_u64) == Some(0))
+ && state.lock().unwrap().fail_counts
+ {
+ (
+ 200,
+ json!({
+ "took": 1,
+ "timed_out": true,
+ "hits": {"total": {"value": 0, "relation": "gte"}, "hits": []}
+ }),
+ )
} else if method == "POST" && path.ends_with("/_search") {
// Report the REAL number of stored docs: `run_index` verifies live
// counts against the journal (#195), so a mock that always answered 0
@@ -3033,6 +3056,37 @@ fn zero_live_documents_after_journaled_records_fails_the_run() {
);
}
+/// A node that accepted the bulks and then could not count (the size:0
+/// fallback comes back `timed_out` with total 0) must not abort finalize.
+/// Treating that 0 as "nothing is live" is the count-mismatch abort: the
+/// journal is right, the count is not a measurement.
+#[test]
+fn a_count_that_does_not_answer_does_not_fail_a_run_that_wrote_records() {
+ let _guard = FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("records.csv"),
+ "id,value\n0,kept\n1,kept\n",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ endpoint.state.lock().unwrap().fail_counts = true;
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+
+ let code = run_index(config).expect("an unanswered count is not an empty index");
+ assert_eq!(code, 0, "the records were accepted");
+ assert!(
+ endpoint.state.lock().unwrap().docs.len() >= 2,
+ "the mock kept the documents the count could not see"
+ );
+}
+
/// #241 regression: a run must never be silent.
///
/// Before this fix `autoindex` printed a handful of phase banners, one line
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 474bd215f..6d3331a55 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -8607,10 +8607,22 @@ fn run_index_report_inner(
pr.phase("finalize-count", plan.datasets.len() as u64, 0);
let mut ds_counts: HashMap<String, u64> = HashMap::new();
let mut ds_timerange: HashMap<String, (Option<String>, Option<String>)> = HashMap::new();
+ // A count that did not answer is not a zero. `unwrap_or(0)` used to
+ // turn a timeout, a 500, or a `timed_out` size:0 page into "0 live
+ // documents", and the gate below then aborted a run whose records
+ // were already journaled (#195's last resort, fired on the wrong
+ // input). Unknown counts skip the gate and are named in the summary.
+ let mut count_unknown: Vec<String> = Vec::new();
for d in &plan.datasets {
let _counted = pr.file(&d.index, 0);
- let cnt = es.count(&d.index).unwrap_or(0);
- ds_counts.insert(d.slug.clone(), cnt);
+ match es.count(&d.index) {
+ Ok(cnt) => {
+ ds_counts.insert(d.slug.clone(), cnt);
+ }
+ Err(error) => {
+ count_unknown.push(format!("{}: {error:#}", d.index));
+ }
+ }
if let Some(t) = &d.time_field {
let body = json!({"size":0,"aggs":{
"mn":{"min":{"field":t}},"mx":{"max":{"field":t}}}});
@@ -8650,7 +8662,15 @@ fn run_index_report_inner(
)
};
let live_records: u64 = ds_counts.values().sum();
- if journal_records > 0 && live_records == 0 {
+ if !count_unknown.is_empty() {
+ pr.warn(&format!(
+ "finalize-count: {} dataset index(es) did not answer a count, so the \
+ zero-live check was not applied. The journal records {journal_records} \
+ record(s). {}",
+ count_unknown.len(),
+ count_unknown.join(" | ")
+ ));
+ } else if journal_records > 0 && live_records == 0 {
// Typed, not `bail!`: a caller that composes this pipeline can only
// tell "the journal outlived its destination" from "the server
// rejected our writes" if the condition carries a type. `xerj brain`
@@ -8866,6 +8886,12 @@ fn run_index_report_inner(
formats.sort();
formats.dedup();
let (tmin, tmax) = ds_timerange.get(&d.slug).cloned().unwrap_or((None, None));
+ if !ds_counts.contains_key(&d.slug) {
+ notes.push(
+ "record_count unavailable: the node did not answer a count, so 0 here is not a measurement"
+ .to_string(),
+ );
+ }
let (id, doc) = catalog::dataset_doc(&catalog::DatasetDocInput {
prefix: &cfg.prefix,
pd: d,