AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): fail a graph-path run whose catalog rows are not visible (#2224)
xerj-orgfcd72567a72c
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rsindex ce826326e..a392d55e2 100644--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs@@ -354,6 +354,26 @@ fn search(index: &str, body: &Value, docs: &Arc<Mutex<Docs>>) -> Value {.collect();return json!({"hits": {"total": {"value": hits.len()}, "hits": hits}});}+ // #1977: the graph path reads the catalog back with `query.ids` after+ // the write. This mock used to fall through to the size:0 stub below+ // and answer `hits: []`, so every stored row looked missing and a+ // healthy run failed the visibility gate. Return the requested ids that+ // are actually in the store; an id that was never written stays absent,+ // which is what the gate is there to catch.+ if let Some(values) = body.pointer("/query/ids/values").and_then(Value::as_array) {+ let wanted: Vec<&str> = values.iter().filter_map(Value::as_str).collect();+ let size = body.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;+ let matched: Vec<(&String, &Value)> = store+ .iter()+ .filter(|(id, _)| wanted.contains(&id.as_str()))+ .collect();+ let hits: Vec<Value> = matched+ .iter()+ .take(size)+ .map(|(id, source)| json!({"_id": id, "_source": source}))+ .collect();+ return json!({"hits": {"total": {"value": matched.len()}, "hits": hits}});+ }// size:0 totals (dataset counts, map summary): total = matching docs.let live_only = body.pointer("/query/bool/must_not").is_some();let total = storediff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex bf498f5cc..7d47d2363 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -173,6 +173,14 @@ struct MockState {max_actions_per_bulk: Option<usize>,/// Line count of every bulk `max_actions_per_bulk` refused, in order.oversize_bulks_refused: Vec<usize>,+ /// Opt-in (#1977): answer the catalog bulk `errors: false` and persist+ /// nothing. The data path's zero-live gate already refuses this shape;+ /// the catalog write used to exit 0 with an empty `autoindex-catalog`.+ swallow_catalog_bulks: bool,+ /// Opt-in (#1977): refuse every catalog action with a per-item 400 and+ /// persist nothing. Status 400 is not a `server_error`, so it used to be+ /// a warning on an exit-0 run.+ reject_catalog_items: bool,}struct MockEndpoint {@@ -472,6 +480,48 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {"hits": {"total": {"value": 0, "relation": "gte"}, "hits": []}}),)+ } else if method == "POST"+ && path.starts_with("/autoindex-catalog/")+ && path.ends_with("/_search")+ && serde_json::from_slice::<Value>(&body)+ .ok()+ .is_some_and(|query| {+ query+ .pointer("/query/ids/values")+ .and_then(Value::as_array)+ .is_some()+ })+ {+ // #1977: the graph path reads catalog rows back by `_id` after the+ // write. Answer with the documents actually stored — an empty `hits`+ // array here would fail every successful run.+ let query: Value = serde_json::from_slice(&body).unwrap();+ let wanted = query+ .pointer("/query/ids/values")+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default();+ let locked = state.lock().unwrap();+ let mut hits: Vec<Value> = wanted+ .iter()+ .filter_map(Value::as_str)+ .filter_map(|id| {+ locked+ .catalog_docs+ .get(id)+ .map(|doc| json!({"_id": id, "_source": doc}))+ })+ .collect();+ hits.sort_by(|left, right| left["_id"].as_str().cmp(&right["_id"].as_str()));+ (+ 200,+ json!({+ "hits": {+ "total": {"value": hits.len(), "relation": "eq"},+ "hits": hits+ }+ }),+ )} else if method == "POST" && path.ends_with("/_search") {// Report the REAL number of stored docs: `run_index` verifies live// counts against the journal (#195), so a mock that always answered 0@@ -678,6 +728,29 @@ fn bulk_response(body: &[u8], state: &Arc<Mutex<MockState>>) -> Value {if locked.catalog_preexists && !locked.catalog_mapping_upgraded {locked.catalog_bulk_before_upgrade = true;}+ if locked.swallow_catalog_bulks {+ return json!({"errors": false, "items": []});+ }+ if locked.reject_catalog_items {+ let mut items = Vec::new();+ let mut line = 0;+ while line < lines.len() {+ let Ok(action) = serde_json::from_slice::<Value>(lines[line]) else {+ line += 1;+ continue;+ };+ let is_delete = action.get("delete").is_some();+ items.push(json!({"index": {+ "status": 400,+ "error": {+ "type": "document_parsing_exception",+ "reason": "failed to parse field [time_min] of type [date]"+ }+ }}));+ line += if is_delete { 1 } else { 2 };+ }+ return json!({"errors": true, "items": items});+ }let mut line = 0;while line < lines.len() {let Ok(action) = serde_json::from_slice::<Value>(lines[line]) else {@@ -1147,6 +1220,162 @@ fn a_catalog_bulk_the_server_refuses_as_too_large_still_lands_whole() {);}+/// Locks the three fixtures this module's HTTP tests share.+fn http_test_guards() -> (+ std::sync::MutexGuard<'static, ()>,+ std::sync::MutexGuard<'static, ()>,+ std::sync::MutexGuard<'static, ()>,+) {+ (+ FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner()),+ state::FILE_DONE_IO_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner()),+ crate::progress::SINK_TEST_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner()),+ )+}++/// #1977: one repository per directory, the shape of the ~210-index report.+/// Every dataset the run publishes has a catalog row before the run exits 0.+///+/// 201 is one past the read-back chunk of 200, so the gate's second ids+/// query is part of the passing run and not only the failure path.+#[test]+fn many_datasets_in_one_run_each_get_a_catalog_row() {+ let _guards = http_test_guards();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ const REPOS: usize = 201;+ for n in 0..REPOS {+ let repo = corpus.path().join(format!("repo-{n:02}"));+ fs::create_dir_all(repo.join(".git")).unwrap();+ fs::write(repo.join(".git").join("HEAD"), "ref: refs/heads/main\n").unwrap();+ fs::write(+ repo.join("notes.md"),+ format!("# repo {n}\n\nnotes for repository {n}.\n"),+ )+ .unwrap();+ }+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert!(+ !config.no_graph,+ "the empty-catalog report is the graph path"+ );+ let (code, _) = run_index_report(config).unwrap();+ assert_eq!(code, 0);+ let datasets = dataset_catalog_docs(&endpoint);+ assert_eq!(+ datasets.len(),+ REPOS,+ "each repository is its own dataset and must have a catalog row: {datasets:?}"+ );+ let runs = endpoint+ .state+ .lock()+ .unwrap()+ .catalog_docs+ .values()+ .filter(|doc| doc.get("doc_kind").and_then(Value::as_str) == Some("run"))+ .count();+ assert_eq!(runs, 1, "the run document is part of the catalog gate");+}++/// #1977: the server answers the catalog bulk as a clean success and writes+/// nothing. That is the data path's swallowed-bulk shape (#195). On the graph+/// path it used to exit 0 with `autoindex-catalog` still empty, so `xerj_map`+/// reported no datasets. The run must fail, and the same command must recover+/// once the server accepts the write — the journal is not finished until the+/// rows are visible.+#[test]+fn a_catalog_bulk_answered_success_but_not_stored_does_not_exit_zero() {+ let _guards = http_test_guards();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("rows.csv"),+ "id,value\n1,first\n2,second\n",+ )+ .unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ endpoint.state.lock().unwrap().swallow_catalog_bulks = true;+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ let error = run_index_report(config.clone()).unwrap_err();+ let message = format!("{error:#}");+ assert!(+ message.contains("not visible"),+ "the failure must say the catalog rows are missing: {message}"+ );+ assert!(+ message.contains("rerun the same command"),+ "the failure must name the recovery: {message}"+ );+ {+ let locked = endpoint.state.lock().unwrap();+ assert!(+ locked.catalog_docs.is_empty(),+ "the swallowed bulk must not have stored a catalog row"+ );+ assert_eq!(+ data_rows(&locked).len(),+ 2,+ "the data indices are searchable; only the catalog write was lost"+ );+ }+ assert_eq!(+ event_count(state_dir.path(), "finish"),+ 0,+ "a catalog the map cannot see must not finish the resume journal"+ );+ endpoint.state.lock().unwrap().swallow_catalog_bulks = false;+ let (code, _) = run_index_report(config).unwrap();+ assert_eq!(code, 0, "the same command rewrites the catalog");+ assert_eq!(+ dataset_catalog_docs(&endpoint).len(),+ 1,+ "the recovered run publishes the dataset row"+ );+}++/// #1977: a per-item 400 on every catalog action is not a `server_error`, so+/// it used to be a warning on exit 0 while the catalog stayed empty.+#[test]+fn a_catalog_bulk_refused_item_by_item_does_not_exit_zero() {+ let _guards = http_test_guards();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("rows.csv"), "id,value\n1,first\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ endpoint.state.lock().unwrap().reject_catalog_items = true;+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ let error = run_index_report(config.clone()).unwrap_err();+ let message = format!("{error:#}");+ assert!(+ message.contains("refused") && message.contains("catalog"),+ "the failure must name the refused catalog documents: {message}"+ );+ assert!(+ endpoint.state.lock().unwrap().catalog_docs.is_empty(),+ "a refused catalog bulk stores nothing"+ );+ assert_eq!(+ event_count(state_dir.path(), "finish"),+ 0,+ "a refused catalog must not finish the resume journal"+ );+ endpoint.state.lock().unwrap().reject_catalog_items = false;+ let (code, _) = run_index_report(config).unwrap();+ assert_eq!(+ code, 0,+ "clearing the refusal lets the same command recover"+ );+ assert_eq!(dataset_catalog_docs(&endpoint).len(), 1);+}+/// #944: the shape that aborted a 48,533-file run at 60% — one data bulk/// answered HTTP 200 with every item `status: 429` by the engine's memory/// circuit breaker, which engages and releases within a second. The legacydiff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 26bae1a0b..dd56df071 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -854,6 +854,22 @@ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Valuelet must_not_exists = query.pointer("/query/bool/must_not/0/exists/field").and_then(Value::as_str);+ // #1977: the catalog gate reads rows back with `query.ids`. Dropping that+ // clause made the search a match-all, and `size` (the published id count)+ // then kept the `ds:` rows plus the `file:` rows that sort between `ds:`+ // and `run:`. The run document was in the store and still looked missing.+ // An id that was never written stays out of the hits, which is the case+ // the gate exists to fail.+ let ids: Option<Vec<String>> = query+ .pointer("/query/ids/values")+ .and_then(Value::as_array)+ .map(|values| {+ values+ .iter()+ .filter_map(Value::as_str)+ .map(str::to_owned)+ .collect()+ });let locked = state.lock().unwrap();// #890: a catalog field an older build left as `text` is ANALYZED, so a// `term` on it matches tokens, not the raw value. `legacy` is empty for@@ -869,8 +885,9 @@ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Valuelet mut matching: Vec<_> = locked.docs.iter()- .filter(|((index, _), doc)| {+ .filter(|((index, id), doc)| {indices.contains(&index.as_str())+ && ids.as_ref().is_none_or(|wanted| wanted.contains(id))&& term.map(|(field, value)| term_matches(doc, field, value, legacy_for(index))).unwrap_or(true)diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex fb041cd9f..804dfa39a 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -6087,6 +6087,94 @@ fn delete_catalog_hits_where())}+/// The graph path's catalog gate (#1977).+///+/// `autoindex-catalog` is created empty during prepare, and the data indices+/// are refreshed — and therefore searchable — before this write. The bulk that+/// fills the catalog used to treat a per-item refusal as a warning (#955) and+/// never read the rows back, so a response the client classified as success+/// could leave the index at zero documents and the run at exit 0. `xerj_map`+/// then reports no datasets. The data path already refuses that shape (the+/// zero-live gate, #195). This is the same check for the rows `xerj_map` reads:+/// every published dataset document and the run document, visible by `_id`+/// with the `doc_kind` the map filters on.+///+/// Called only after the catalog refresh, and before the resume journal is+/// finished, so a failure leaves the next invocation of the same command to+/// rewrite these ids.+fn require_visible_catalog_rows(es: &Es, ids: &[String]) -> Result<()> {+ // One ids query stays under the default result window. 200 is also the+ // point at which a ~210-dataset batch (the #1977 report) takes two+ // requests instead of one oversized one.+ const CHUNK: usize = 200;+ let mut missing_at: Vec<usize> = Vec::new();+ for (chunk_index, chunk) in ids.chunks(CHUNK).enumerate() {+ let response = es+ .search(+ catalog::CATALOG_INDEX,+ &json!({+ "size": chunk.len(),+ "query": {"ids": {"values": chunk}},+ "_source": ["doc_kind"]+ }),+ )+ .context(+ "write catalog: reading the catalog back failed, so this run cannot claim the \+ data map was published",+ )?;+ let hits = response+ .pointer("/hits/hits")+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default();+ let mut seen: HashSet<String> = HashSet::new();+ for hit in &hits {+ let Some(id) = hit.get("_id").and_then(Value::as_str) else {+ continue;+ };+ let kind = hit.pointer("/_source/doc_kind").and_then(Value::as_str);+ let expected = if id.starts_with("run:") {+ "run"+ } else if id.starts_with("ds:") {+ "dataset"+ } else {+ continue;+ };+ if kind == Some(expected) {+ seen.insert(id.to_string());+ }+ }+ let base = chunk_index * CHUNK;+ for (offset, id) in chunk.iter().enumerate() {+ if !seen.contains(id) {+ missing_at.push(base + offset);+ }+ }+ }+ if missing_at.is_empty() {+ return Ok(());+ }+ let shown: Vec<&str> = missing_at+ .iter()+ .take(8)+ .map(|&index| ids[index].as_str())+ .collect();+ let extra = missing_at.len().saturating_sub(shown.len());+ let extra = if extra > 0 {+ format!(" (+{extra} more)")+ } else {+ String::new()+ };+ anyhow::bail!(+ "write catalog: {} of {} catalog document(s) are not visible after the write ({}{extra}). \+ The data indices are already indexed; rerun the same command to rewrite the catalog. \+ `xerj autoindex map` stays empty until those rows exist",+ missing_at.len(),+ ids.len(),+ shown.join(", "),+ )+}+/// #589: sweep the documents an exclusion left behind. A file still on disk/// that a widened ignore/hidden/`.xerjignore` rule now skips must have its/// already-published documents removed — an exclusion that cannot remove data@@ -10718,19 +10806,33 @@ fn run_index_report_inner(.unwrap_or("unknown server error"));}- // A catalog document the server refused is a file or dataset that- // `xerj autoindex map` will not show. It has never ended a run and- // does not now, but it is no longer silent either (#955).+ // A refused catalog document is a dataset or file `xerj autoindex map`+ // will not show. Warning and exiting 0 left a ~210-index batch with+ // an empty catalog and no signal to rerun (#1977). Same rule as the+ // generation path's `check_bulk_outcome`: a catalog item the server+ // refused fails the run, before the journal is finished, so the same+ // command rewrites it.if outcome.item_errors > 0 {- pr.note(&format!(- "autoindex: WARNING — the server refused {} catalog document(s): {}. The \- records are indexed; `xerj autoindex map` will be missing those entries",+ anyhow::bail!(+ "write catalog: the server refused {} catalog document(s): {}. The data indices \+ are already indexed, but `xerj autoindex map` would be missing those entries. \+ Rerun the same command to rewrite the catalog",outcome.item_errors,outcome.first_error.as_deref().unwrap_or("no reason given")- ));+ );}+ // Refresh is what makes the rows searchable. Swallowing a failure+ // (`.ok()`) reported success over a catalog `xerj_map` could not see.+ es.refresh(catalog::CATALOG_INDEX)+ .context("write catalog: refresh autoindex-catalog so the new rows are searchable")?;+ let mut published_ids: Vec<String> = plan+ .datasets+ .iter()+ .map(|dataset| format!("ds:{}:{}", cfg.prefix, dataset.slug))+ .collect();+ published_ids.push(format!("run:{run_id}"));+ require_visible_catalog_rows(&es, &published_ids)?;}- es.refresh(catalog::CATALOG_INDEX).ok();pr.item_done(0);// ── durable junk record, written last (#238) ─────────────────────────//