AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): surface the catalog read-back walk as its own progress phase (#1267)
xerj-org2e6ee469b47d
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 14ce592a0..7eaab7c20 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -4929,12 +4929,37 @@ fn the_generated_path_reports_indexing_as_indexing_not_as_a_stalled_scan() {"snapshot","index","finalize-catalog",+ // #1267: the `run_id`-scoped catalog read-back is its own phase.+ // On a full-rewrite resume it runs for ~35 min, and used to do so+ // behind the finished `finalize-catalog` ticker — a line that only+ // climbed `since_progress_s`, i.e. looked exactly like a hang.+ "finalize-catalog-readback","finalize-refresh","finalize-verify"],"each long step of the generated route is its own phase, in order:\n{stream}");+ // #1267: that read-back phase opens with the engine's own match total as+ // its denominator — a moving bar with an ETA from the first page, not a+ // count with no total. The two-dataset generation writes at least its run+ // document, so the denominator is non-zero.+ let readback_opened = lines+ .iter()+ .find(|line| line["phase"] == "finalize-catalog-readback")+ .expect("the catalog read-back walk opens a phase of its own");+ assert_eq!(readback_opened["basis"], "items", "{readback_opened:?}");+ let (done, total) = readback_opened["items"].split_once('/').unwrap();+ assert_eq!(+ done, "0",+ "the read-back phase opens at zero: {readback_opened:?}"+ );+ assert!(+ total.parse::<u64>().unwrap() > 0,+ "the read-back phase opens with the engine's match total, not an unknown \+ denominator: {readback_opened:?}"+ );+// `scan` never reappears once the run has moved on — the exact symptom.let left_scan = lines.iter()diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex e58d1c6a6..576fe848e 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -945,26 +945,85 @@ impl<'a> EsSyncBackend<'a> {Ok(observation)}+ /// Page size of the `run_id`-scoped read-back walk. The engine reports the+ /// full match count on every page, so a page shorter than this is the walk+ /// reaching the end of the generation's documents.+ const CATALOG_WALK_PAGE: usize = 1000;++ /// The phase [`Self::catalog_generation`] surfaces itself under. A resume+ /// that rewrote the whole catalog gives this walk a `run_id` matching every+ /// document in the index (402,814 on the cve-records node), so the+ /// "O(changed) by construction" invariant degenerates to O(corpus): 403+ /// sorted pages at ~5.2 s each ≈ 35 min, during which the surface used to+ /// show the FINISHED `finalize-catalog` ticker (every dataset already+ /// counted done) with a `since_progress_s` that only climbed — which reads+ /// exactly like a hang, and cost an hour of "is it deadlocked?" diagnosis+ /// on the #1267 resume. The walk is its own phase now, with the engine's+ /// own match total as a real denominator, so the long read-back shows a+ /// moving bar and an ETA from its first page.+ const CATALOG_READBACK_PHASE: &'static str = "finalize-catalog-readback";+fn catalog_generation(&self, run_id: &str) -> Result<BTreeMap<String, Value>> {+ Self::walk_run_id_pages(+ self.pr,+ Self::CATALOG_READBACK_PHASE,+ Self::CATALOG_WALK_PAGE,+ |search_after| {+ let mut body = serde_json::json!({+ "size": Self::CATALOG_WALK_PAGE,+ "sort": [{"_id": "asc"}],+ "query": {"term": {"run_id": run_id}}+ });+ if let Some(after) = search_after {+ body["search_after"] = after.clone();+ }+ // #1212: a `search_page`, not a `search` — the engine answers a+ // missed 30 s default deadline with a partial 200 (`timed_out`),+ // and this walk's short-page break read one partial page as the+ // end of the catalog: 56,441 documents observed against 58,568+ // that were all there, on a node whose sorted pages took 11-40 s+ // while its breaker drained. The page variant retries a timed-out+ // page at the same continuation key instead of returning it.+ self.es.search_page(crate::catalog::CATALOG_INDEX, &body)+ },+ )+ }++ /// Drive a `run_id`-scoped, `_id asc` + `search_after` generation walk to+ /// completion, collecting `_id -> _source` for every matched document and+ /// surfacing the walk as its own progress phase (#1267). `fetch_page` is+ /// handed the previous page's continuation sort key (`None` for the first+ /// page) and returns the raw search response; isolating it this way lets+ /// the paging contract — the duplicate-id guard, the short-page terminator,+ /// the continuation-key check, and the per-page progress ticks — be pinned+ /// without a node, the way [`Self::rewalk_while_short`] is. `page` is the+ /// request `size`: a page shorter than it terminates the walk, so a test+ /// drives real multi-page paging with a handful of documents.+ ///+ /// The phase opens on the first page using the engine's reported match+ /// total as the item denominator, then ticks once per collected document,+ /// so the surface moves and carries an ETA across a walk that can run for+ /// over half an hour. A total the engine does not report leaves the phase+ /// without a denominator (`pct=unknown`) rather than inventing one.+ fn walk_run_id_pages(+ pr: &crate::progress::Progress,+ phase: &'static str,+ page: usize,+ mut fetch_page: impl FnMut(Option<&Value>) -> Result<Value>,+ ) -> Result<BTreeMap<String, Value>> {let mut documents = BTreeMap::new();let mut search_after: Option<Value> = None;+ let mut phase_opened = false;loop {- let mut body = serde_json::json!({- "size": 1000,- "sort": [{"_id": "asc"}],- "query": {"term": {"run_id": run_id}}- });- if let Some(after) = &search_after {- body["search_after"] = after.clone();+ let response = fetch_page(search_after.as_ref())?;+ if !phase_opened {+ let total = response+ .pointer("/hits/total/value")+ .and_then(Value::as_u64)+ .unwrap_or(0);+ pr.phase(phase, total, 0);+ phase_opened = true;}- // #1212: a `search_page`, not a `search` — the engine answers a- // missed 30 s default deadline with a partial 200 (`timed_out`),- // and this walk's short-page break read one partial page as the- // end of the catalog: 56,441 documents observed against 58,568- // that were all there, on a node whose sorted pages took 11-40 s- // while its breaker drained. The page variant retries a timed-out- // page at the same continuation key instead of returning it.- let response = self.es.search_page(crate::catalog::CATALOG_INDEX, &body)?;let hits = response.pointer("/hits/hits").and_then(Value::as_array)@@ -983,8 +1042,9 @@ impl<'a> EsSyncBackend<'a> {documents.insert(id.clone(), source).is_none(),"catalog generation query returned duplicate ID {id}");+ pr.item_done(0);}- if hits.len() < 1000 {+ if hits.len() < page {break;}search_after = hits.last().and_then(|hit| hit.get("sort")).cloned();@@ -5135,6 +5195,154 @@ mod tests {assert!(verdict.is_err(), "an error is not a short walk");assert_eq!(refreshes.get(), 0, "no refresh over a dead transport");}++ // #1267: the `run_id`-scoped catalog read-back walk. On a resume that+ // rewrote the whole catalog its `run_id` matches every document, so it+ // runs for ~35 min — and used to do so invisibly, behind a finished+ // `finalize-catalog` ticker that read as a hang. These pin the walk as its+ // own phase with the engine's match total as the denominator, plus the+ // paging contract, without a node.++ /// One page of a `run_id` walk: every hit carries a `sort` key (the+ /// continuation the next page is fetched from) and the engine reports the+ /// full match `total` on every page, not just the first.+ fn walk_page(total: u64, ids: &[&str]) -> Value {+ let hits: Vec<Value> = ids+ .iter()+ .map(|id| serde_json::json!({"_id": id, "_source": {"run": id}, "sort": [id]}))+ .collect();+ serde_json::json!({"hits": {"total": {"value": total}, "hits": hits}})+ }++ #[test]+ fn a_run_id_walk_collects_every_page_and_opens_an_honest_phase() {+ let (pr, buffer) = crate::progress::Progress::capture(+ crate::progress::Surface::Plain,+ std::time::Duration::from_secs(3600),+ );+ // Page size 2: two full pages, then a short page ends the walk. The+ // engine reports total=5 on every page.+ let pages = std::cell::RefCell::new(vec![+ walk_page(5, &["a", "b"]),+ walk_page(5, &["c", "d"]),+ walk_page(5, &["e"]),+ ]);+ let calls = std::cell::Cell::new(0usize);+ let observed = EsSyncBackend::walk_run_id_pages(+ &pr,+ "finalize-catalog-readback",+ 2,+ |_search_after| {+ calls.set(calls.get() + 1);+ Ok(pages.borrow_mut().remove(0))+ },+ )+ .unwrap();+ assert_eq!(+ calls.get(),+ 3,+ "two full pages then one short page ends the walk"+ );+ assert_eq!(+ observed.len(),+ 5,+ "every document across every page is collected"+ );+ assert!(+ observed.contains_key("e"),+ "the short final page is not dropped"+ );++ let out = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();+ assert!(+ out.contains("phase=finalize-catalog-readback"),+ "the walk is its own phase, not a frozen finalize-catalog ticker: {out}"+ );+ // The denominator is the engine's own match total from page one — not+ // the page size, not the running count — so the bar is honest from the+ // first page instead of after 35 min of silence (#1267).+ assert!(+ out.contains("items=0/5"),+ "the phase opens with the full match total as its denominator: {out}"+ );+ }++ #[test]+ fn the_read_back_walk_advances_its_surface_per_document() {+ // The whole point of #1267: the walk must TICK, so a long read-back is+ // never a stalled line. A ticker on a short cadence proves the surface+ // reaches `items=3/3` without any item needing to "complete" a file.+ let (pr, buffer) = crate::progress::Progress::capture(+ crate::progress::Surface::Plain,+ std::time::Duration::from_millis(40),+ );+ let ticker = pr.spawn_ticker();+ // One short page (size 1000 default; 3 < 1000 ends the walk).+ let observed = EsSyncBackend::walk_run_id_pages(+ &pr,+ "finalize-catalog-readback",+ 1000,+ |_search_after| Ok(walk_page(3, &["a", "b", "c"])),+ )+ .unwrap();+ assert_eq!(observed.len(), 3);++ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);+ loop {+ let out = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();+ if out.contains("phase=finalize-catalog-readback") && out.contains("items=3/3") {+ break;+ }+ assert!(+ std::time::Instant::now() < deadline,+ "the ticker never showed the walk advance to items=3/3: {out}"+ );+ std::thread::yield_now();+ }+ drop(ticker);+ }++ #[test]+ fn a_duplicate_id_across_pages_fails_the_walk_loudly() {+ let pr = crate::progress::Progress::silent();+ let pages =+ std::cell::RefCell::new(vec![walk_page(4, &["a", "b"]), walk_page(4, &["b", "c"])]);+ let verdict = EsSyncBackend::walk_run_id_pages(+ &pr,+ "finalize-catalog-readback",+ 2,+ |_search_after| Ok(pages.borrow_mut().remove(0)),+ );+ let err = verdict.unwrap_err().to_string();+ assert!(+ err.contains("duplicate ID b"),+ "a document seen twice is a loud failure, never a silent overwrite: {err}"+ );+ }++ #[test]+ fn a_full_page_without_a_continuation_key_cannot_advance() {+ let pr = crate::progress::Progress::silent();+ // A full page (size 2) whose hits carry no `sort`: the walk cannot form+ // the next `search_after` and must refuse rather than loop or truncate.+ let page = serde_json::json!({+ "hits": {"total": {"value": 9}, "hits": [+ {"_id": "a", "_source": {}},+ {"_id": "b", "_source": {}}+ ]}+ });+ let verdict = EsSyncBackend::walk_run_id_pages(+ &pr,+ "finalize-catalog-readback",+ 2,+ |_search_after| Ok(page.clone()),+ );+ let err = verdict.unwrap_err().to_string();+ assert!(+ err.contains("continuation sort key"),+ "a full page with no sort key is a loud failure: {err}"+ );+ }}/// Issue #482 regression guard.