AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): probe server generation before finalize-verify #1267
xerj-org47196b60f94d
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex 405416531..20671190e 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -1064,6 +1064,90 @@ impl<'a> EsSyncBackend<'a> {Ok(documents)}+ /// A warm verify window slower than this means the server is answering+ /// the window shape on its brute path. Measured on the #1267 resume: the+ /// `term ax_file` + `exists` probe took 3.4 s warm on a server without the+ /// #1260 exists arm and 2.7 ms on one with it — three orders of magnitude+ /// either side of this line, so it separates the two generations without+ /// being sensitive to node load.+ const SERVER_PROBE_SLOW: std::time::Duration = std::time::Duration::from_secs(1);++ /// The operational half of #1267: before `finalize-verify` issues its+ /// windows, time ONE representative window and, if it is slow, say so.+ ///+ /// The resume protocol cannot otherwise tell that the server predates the+ /// client's fast-agg fix generation (#1260/#1270). On the cve-records+ /// resume that cost four hours of ~3.4 s probes before anyone looked —+ /// the client was a #1259 binary, the server an older measure build, and+ /// nothing in the handshake differs between them. `GET /` cannot help:+ /// its `version.number` is the ES-compat version (`8.13.0` or the+ /// `--compat-version` override), not XERJ's. `/v1/health` does carry the+ /// XERJ version and is quoted in the warning when it differs from the+ /// client's, but a branch or measure build reports the same string as+ /// the release it forked from — exactly the #1267 shape — so latency is+ /// the signal and the version only context.+ ///+ /// `window_query` runs the window and returns how long it took (the caller+ /// times it, which keeps this pinnable without a node or a clock). A slow+ /// first answer is re-asked once before warning: the first touch after a+ /// server boot pages the index in cold (8.8 s on the FIXED binary in+ /// #1267), and that must not read as an old server. A probe that fails is+ /// not this function's business — the verify window that follows asks the+ /// same thing and fails loud — so it returns `None` without a warning.+ /// It never aborts the run: an old server is slow, not wrong.+ fn probe_server_generation(+ pr: &crate::progress::Progress,+ windows: u64,+ threshold: std::time::Duration,+ mut window_query: impl FnMut() -> Result<std::time::Duration>,+ server_version: impl FnOnce() -> Option<String>,+ ) -> Option<std::time::Duration> {+ let cold = window_query().ok()?;+ if cold <= threshold {+ return Some(cold);+ }+ let warm = window_query().ok()?;+ if warm <= threshold {+ return Some(warm);+ }+ let client = env!("CARGO_PKG_VERSION");+ let version = match server_version() {+ Some(server) if server != client => {+ format!(" The server reports XERJ {server}; this client is {client}.")+ }+ Some(_) => format!(+ " The server reports the same version as this client ({client}), so it is \+ likely a branch or development build cut before the fix."+ ),+ None => String::new(),+ };+ let projected_s = warm.as_secs_f64() * windows as f64;+ pr.warn(&format!(+ "autoindex: warning: one finalize-verify window took {:.1}s (twice; under {}ms is \+ expected). The server is likely older than this client and lacks the columnar \+ term+exists count lane (#1260/#1270), so every verify window brute-scans its \+ index: {windows} windows at this rate is ~{:.1}h.{version} Continuing — the \+ result will be correct, only slow. To avoid the wait, stop this run, upgrade the \+ server to this client's version, and re-run the same command: the journal resumes \+ where it stopped.",+ warm.as_secs_f64(),+ threshold.as_millis(),+ projected_s / 3600.0,+ ));+ Some(warm)+ }++ /// The XERJ version the server reports on `/v1/health` (`data.version`),+ /// or `None` when the endpoint is absent or unreadable.+ fn server_xerj_version(&self) -> Option<String> {+ self.es+ .get_json("/v1/health")+ .ok()?+ .pointer("/data/version")+ .and_then(Value::as_str)+ .map(str::to_owned)+ }+fn exact_dataset_catalog_stats(&self,desired: &GenerationManifest,@@ -1622,6 +1706,43 @@ impl SyncOperationBackend for EsSyncBackend<'_> {}self.pr.phase("finalize-verify", readbacks, 0);+ // #1267 operational: one representative window, timed, before the+ // walk. The semantic leg is preferred because its `term`+`exists`+ // shape is exactly the one a server without the #1260 exists arm+ // brute-scans; a generation with no semantic field probes a record+ // window instead. One digest is enough — the brute path's cost is the+ // index size, not the window width.+ let probe = semantic_wanted+ .iter()+ .find_map(|((index, field), values)| {+ let exists = serde_json::json!({"exists": {"field": field}});+ values.first().map(|value| (index, value, vec![exists]))+ })+ .or_else(|| {+ record_wanted+ .iter()+ .find_map(|(index, values)| values.first().map(|value| (index, value, vec![])))+ });+ if let Some((index, value, extra)) = probe {+ let _probing = self.pr.file(index, 0);+ Self::probe_server_generation(+ self.pr,+ readbacks,+ Self::SERVER_PROBE_SLOW,+ || {+ let started = std::time::Instant::now();+ self.windowed_value_counts(+ index,+ "ax_file",+ std::slice::from_ref(value),+ &extra,+ )?;+ Ok(started.elapsed())+ },+ || self.server_xerj_version(),+ );+ }+let mut record_counts: HashMap<String, HashMap<String, u64>> = HashMap::new();for (index, values) in &record_wanted {let _reading = self.pr.file(index, 0);@@ -5310,6 +5431,140 @@ mod tests {drop(ticker);}+ // #1267 operational: the server-generation latency probe that runs before+ // `finalize-verify`. Timings are injected, so these pin the decision+ // without a node or a clock.++ fn run_probe(+ surface: crate::progress::Surface,+ timings: &[std::time::Duration],+ version: Option<&str>,+ ) -> (Option<std::time::Duration>, usize, String) {+ let (pr, buffer) =+ crate::progress::Progress::capture(surface, std::time::Duration::from_secs(3600));+ let queue = std::cell::RefCell::new(timings.to_vec());+ let calls = std::cell::Cell::new(0usize);+ let verdict = EsSyncBackend::probe_server_generation(+ &pr,+ 400,+ EsSyncBackend::SERVER_PROBE_SLOW,+ || {+ calls.set(calls.get() + 1);+ let mut queue = queue.borrow_mut();+ anyhow::ensure!(!queue.is_empty(), "transport down");+ Ok(queue.remove(0))+ },+ || version.map(str::to_owned),+ );+ let out = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();+ (verdict, calls.get(), out)+ }++ #[test]+ fn a_fast_verify_window_probes_once_and_stays_silent() {+ let ms = std::time::Duration::from_millis;+ let (verdict, calls, out) =+ run_probe(crate::progress::Surface::Plain, &[ms(3)], Some("0.0.0-old"));+ assert_eq!(verdict, Some(ms(3)));+ assert_eq!(calls, 1, "a fast first answer is not re-asked");+ assert!(+ !out.contains("warning"),+ "a fixed server is not warned about: {out}"+ );+ }++ #[test]+ fn a_cold_first_window_is_reasked_and_not_mistaken_for_an_old_server() {+ // #1267's own numbers: 8.8 s cold page-in on the FIXED binary, then 2.7 ms.+ let ms = std::time::Duration::from_millis;+ let (verdict, calls, out) =+ run_probe(crate::progress::Surface::Plain, &[ms(8800), ms(3)], None);+ assert_eq!(verdict, Some(ms(3)));+ assert_eq!(calls, 2);+ assert!(+ !out.contains("warning"),+ "cold page-in is not an old server: {out}"+ );+ }++ #[test]+ fn a_slow_warm_window_warns_on_the_plain_surface_and_does_not_abort() {+ let ms = std::time::Duration::from_millis;+ let (verdict, calls, out) = run_probe(+ crate::progress::Surface::Plain,+ &[ms(8800), ms(3400)],+ Some("1.0.0-rc.80"),+ );+ assert_eq!(+ verdict,+ Some(ms(3400)),+ "the probe reports and the run continues"+ );+ assert_eq!(calls, 2);+ for needle in [+ "warning",+ "3.4s",+ "older than this client",+ "#1260",+ "400 windows",+ "~0.4h",+ "XERJ 1.0.0-rc.80",+ env!("CARGO_PKG_VERSION"),+ "upgrade the server",+ "Continuing",+ ] {+ assert!(out.contains(needle), "warning lacks {needle:?}: {out}");+ }+ }++ #[test]+ fn a_slow_server_with_the_same_version_is_named_a_development_build() {+ let ms = std::time::Duration::from_millis;+ let (_, _, out) = run_probe(+ crate::progress::Surface::Plain,+ &[ms(3400), ms(3400)],+ Some(env!("CARGO_PKG_VERSION")),+ );+ assert!(out.contains("same version as this client"), "{out}");+ assert!(out.contains("development build"), "{out}");+ }++ #[test]+ fn the_slow_server_warning_is_one_json_event_on_the_machine_surface() {+ let ms = std::time::Duration::from_millis;+ let (_, _, out) = run_probe(crate::progress::Surface::Json, &[ms(3400), ms(3400)], None);+ let events: Vec<Value> = out+ .lines()+ .map(|line| serde_json::from_str(line).expect("one JSON object per line"))+ .collect();+ assert_eq!(events.len(), 1, "{out}");+ assert_eq!(events[0]["event"], "warning");+ assert!(events[0]["message"]+ .as_str()+ .unwrap()+ .contains("older than this client"));+ }++ #[test]+ fn the_slow_server_warning_survives_the_silent_surface() {+ // `--quiet` drops routine notes; a four-hour wait is not routine.+ let ms = std::time::Duration::from_millis;+ let (_, _, out) = run_probe(+ crate::progress::Surface::Silent,+ &[ms(3400), ms(3400)],+ None,+ );+ assert!(out.contains("older than this client"), "{out}");+ }++ #[test]+ fn a_failing_probe_neither_warns_nor_propagates() {+ let (verdict, calls, out) = run_probe(crate::progress::Surface::Plain, &[], None);+ assert_eq!(verdict, None);+ assert_eq!(calls, 1);+ assert!(out.is_empty(), "{out}");+ }+#[test]fn a_duplicate_id_across_pages_fails_the_walk_loudly() {let pr = crate::progress::Progress::silent();