Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): fail a graph-path run whose catalog rows are not visible (#2224)

xerj-orgfcd72567a72c
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rs
index ce826326e..a392d55e2 100644
--- a/engine/crates/xerj-autoindex/src/detect/e2e.rs
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@ -354,6 +354,26 @@ fn search(index: &str, body: &Value, docs: &Arc<Mutex<Docs>>) -> Value {
.collect();
return json!({"hits": {"total": {"value": hits.len()}, "hits": hits}});
}
+ // #1977: the graph path reads the catalog back with `query.ids` after
+ // the write. This mock used to fall through to the size:0 stub below
+ // and answer `hits: []`, so every stored row looked missing and a
+ // healthy run failed the visibility gate. Return the requested ids that
+ // are actually in the store; an id that was never written stays absent,
+ // which is what the gate is there to catch.
+ if let Some(values) = body.pointer("/query/ids/values").and_then(Value::as_array) {
+ let wanted: Vec<&str> = values.iter().filter_map(Value::as_str).collect();
+ let size = body.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;
+ let matched: Vec<(&String, &Value)> = store
+ .iter()
+ .filter(|(id, _)| wanted.contains(&id.as_str()))
+ .collect();
+ let hits: Vec<Value> = matched
+ .iter()
+ .take(size)
+ .map(|(id, source)| json!({"_id": id, "_source": source}))
+ .collect();
+ return json!({"hits": {"total": {"value": matched.len()}, "hits": hits}});
+ }
// size:0 totals (dataset counts, map summary): total = matching docs.
let live_only = body.pointer("/query/bool/must_not").is_some();
let total = store
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index bf498f5cc..7d47d2363 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -173,6 +173,14 @@ struct MockState {
max_actions_per_bulk: Option<usize>,
/// Line count of every bulk `max_actions_per_bulk` refused, in order.
oversize_bulks_refused: Vec<usize>,
+ /// Opt-in (#1977): answer the catalog bulk `errors: false` and persist
+ /// nothing. The data path's zero-live gate already refuses this shape;
+ /// the catalog write used to exit 0 with an empty `autoindex-catalog`.
+ swallow_catalog_bulks: bool,
+ /// Opt-in (#1977): refuse every catalog action with a per-item 400 and
+ /// persist nothing. Status 400 is not a `server_error`, so it used to be
+ /// a warning on an exit-0 run.
+ reject_catalog_items: bool,
}
struct MockEndpoint {
@@ -472,6 +480,48 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {
"hits": {"total": {"value": 0, "relation": "gte"}, "hits": []}
}),
)
+ } else if method == "POST"
+ && path.starts_with("/autoindex-catalog/")
+ && path.ends_with("/_search")
+ && serde_json::from_slice::<Value>(&body)
+ .ok()
+ .is_some_and(|query| {
+ query
+ .pointer("/query/ids/values")
+ .and_then(Value::as_array)
+ .is_some()
+ })
+ {
+ // #1977: the graph path reads catalog rows back by `_id` after the
+ // write. Answer with the documents actually stored — an empty `hits`
+ // array here would fail every successful run.
+ let query: Value = serde_json::from_slice(&body).unwrap();
+ let wanted = query
+ .pointer("/query/ids/values")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ let locked = state.lock().unwrap();
+ let mut hits: Vec<Value> = wanted
+ .iter()
+ .filter_map(Value::as_str)
+ .filter_map(|id| {
+ locked
+ .catalog_docs
+ .get(id)
+ .map(|doc| json!({"_id": id, "_source": doc}))
+ })
+ .collect();
+ hits.sort_by(|left, right| left["_id"].as_str().cmp(&right["_id"].as_str()));
+ (
+ 200,
+ json!({
+ "hits": {
+ "total": {"value": hits.len(), "relation": "eq"},
+ "hits": hits
+ }
+ }),
+ )
} else if method == "POST" && path.ends_with("/_search") {
// Report the REAL number of stored docs: `run_index` verifies live
// counts against the journal (#195), so a mock that always answered 0
@@ -678,6 +728,29 @@ fn bulk_response(body: &[u8], state: &Arc<Mutex<MockState>>) -> Value {
if locked.catalog_preexists && !locked.catalog_mapping_upgraded {
locked.catalog_bulk_before_upgrade = true;
}
+ if locked.swallow_catalog_bulks {
+ return json!({"errors": false, "items": []});
+ }
+ if locked.reject_catalog_items {
+ let mut items = Vec::new();
+ let mut line = 0;
+ while line < lines.len() {
+ let Ok(action) = serde_json::from_slice::<Value>(lines[line]) else {
+ line += 1;
+ continue;
+ };
+ let is_delete = action.get("delete").is_some();
+ items.push(json!({"index": {
+ "status": 400,
+ "error": {
+ "type": "document_parsing_exception",
+ "reason": "failed to parse field [time_min] of type [date]"
+ }
+ }}));
+ line += if is_delete { 1 } else { 2 };
+ }
+ return json!({"errors": true, "items": items});
+ }
let mut line = 0;
while line < lines.len() {
let Ok(action) = serde_json::from_slice::<Value>(lines[line]) else {
@@ -1147,6 +1220,162 @@ fn a_catalog_bulk_the_server_refuses_as_too_large_still_lands_whole() {
);
}
+/// Locks the three fixtures this module's HTTP tests share.
+fn http_test_guards() -> (
+ std::sync::MutexGuard<'static, ()>,
+ std::sync::MutexGuard<'static, ()>,
+ std::sync::MutexGuard<'static, ()>,
+) {
+ (
+ FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner()),
+ state::FILE_DONE_IO_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner()),
+ crate::progress::SINK_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner()),
+ )
+}
+
+/// #1977: one repository per directory, the shape of the ~210-index report.
+/// Every dataset the run publishes has a catalog row before the run exits 0.
+///
+/// 201 is one past the read-back chunk of 200, so the gate's second ids
+/// query is part of the passing run and not only the failure path.
+#[test]
+fn many_datasets_in_one_run_each_get_a_catalog_row() {
+ let _guards = http_test_guards();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ const REPOS: usize = 201;
+ for n in 0..REPOS {
+ let repo = corpus.path().join(format!("repo-{n:02}"));
+ fs::create_dir_all(repo.join(".git")).unwrap();
+ fs::write(repo.join(".git").join("HEAD"), "ref: refs/heads/main\n").unwrap();
+ fs::write(
+ repo.join("notes.md"),
+ format!("# repo {n}\n\nnotes for repository {n}.\n"),
+ )
+ .unwrap();
+ }
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert!(
+ !config.no_graph,
+ "the empty-catalog report is the graph path"
+ );
+ let (code, _) = run_index_report(config).unwrap();
+ assert_eq!(code, 0);
+ let datasets = dataset_catalog_docs(&endpoint);
+ assert_eq!(
+ datasets.len(),
+ REPOS,
+ "each repository is its own dataset and must have a catalog row: {datasets:?}"
+ );
+ let runs = endpoint
+ .state
+ .lock()
+ .unwrap()
+ .catalog_docs
+ .values()
+ .filter(|doc| doc.get("doc_kind").and_then(Value::as_str) == Some("run"))
+ .count();
+ assert_eq!(runs, 1, "the run document is part of the catalog gate");
+}
+
+/// #1977: the server answers the catalog bulk as a clean success and writes
+/// nothing. That is the data path's swallowed-bulk shape (#195). On the graph
+/// path it used to exit 0 with `autoindex-catalog` still empty, so `xerj_map`
+/// reported no datasets. The run must fail, and the same command must recover
+/// once the server accepts the write — the journal is not finished until the
+/// rows are visible.
+#[test]
+fn a_catalog_bulk_answered_success_but_not_stored_does_not_exit_zero() {
+ let _guards = http_test_guards();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("rows.csv"),
+ "id,value\n1,first\n2,second\n",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ endpoint.state.lock().unwrap().swallow_catalog_bulks = true;
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ let error = run_index_report(config.clone()).unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("not visible"),
+ "the failure must say the catalog rows are missing: {message}"
+ );
+ assert!(
+ message.contains("rerun the same command"),
+ "the failure must name the recovery: {message}"
+ );
+ {
+ let locked = endpoint.state.lock().unwrap();
+ assert!(
+ locked.catalog_docs.is_empty(),
+ "the swallowed bulk must not have stored a catalog row"
+ );
+ assert_eq!(
+ data_rows(&locked).len(),
+ 2,
+ "the data indices are searchable; only the catalog write was lost"
+ );
+ }
+ assert_eq!(
+ event_count(state_dir.path(), "finish"),
+ 0,
+ "a catalog the map cannot see must not finish the resume journal"
+ );
+ endpoint.state.lock().unwrap().swallow_catalog_bulks = false;
+ let (code, _) = run_index_report(config).unwrap();
+ assert_eq!(code, 0, "the same command rewrites the catalog");
+ assert_eq!(
+ dataset_catalog_docs(&endpoint).len(),
+ 1,
+ "the recovered run publishes the dataset row"
+ );
+}
+
+/// #1977: a per-item 400 on every catalog action is not a `server_error`, so
+/// it used to be a warning on exit 0 while the catalog stayed empty.
+#[test]
+fn a_catalog_bulk_refused_item_by_item_does_not_exit_zero() {
+ let _guards = http_test_guards();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("rows.csv"), "id,value\n1,first\n").unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ endpoint.state.lock().unwrap().reject_catalog_items = true;
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ let error = run_index_report(config.clone()).unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("refused") && message.contains("catalog"),
+ "the failure must name the refused catalog documents: {message}"
+ );
+ assert!(
+ endpoint.state.lock().unwrap().catalog_docs.is_empty(),
+ "a refused catalog bulk stores nothing"
+ );
+ assert_eq!(
+ event_count(state_dir.path(), "finish"),
+ 0,
+ "a refused catalog must not finish the resume journal"
+ );
+ endpoint.state.lock().unwrap().reject_catalog_items = false;
+ let (code, _) = run_index_report(config).unwrap();
+ assert_eq!(
+ code, 0,
+ "clearing the refusal lets the same command recover"
+ );
+ assert_eq!(dataset_catalog_docs(&endpoint).len(), 1);
+}
+
/// #944: the shape that aborted a 48,533-file run at 60% — one data bulk
/// answered HTTP 200 with every item `status: 429` by the engine's memory
/// circuit breaker, which engages and releases within a second. The legacy
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 26bae1a0b..dd56df071 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -854,6 +854,22 @@ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value
let must_not_exists = query
.pointer("/query/bool/must_not/0/exists/field")
.and_then(Value::as_str);
+ // #1977: the catalog gate reads rows back with `query.ids`. Dropping that
+ // clause made the search a match-all, and `size` (the published id count)
+ // then kept the `ds:` rows plus the `file:` rows that sort between `ds:`
+ // and `run:`. The run document was in the store and still looked missing.
+ // An id that was never written stays out of the hits, which is the case
+ // the gate exists to fail.
+ let ids: Option<Vec<String>> = query
+ .pointer("/query/ids/values")
+ .and_then(Value::as_array)
+ .map(|values| {
+ values
+ .iter()
+ .filter_map(Value::as_str)
+ .map(str::to_owned)
+ .collect()
+ });
let locked = state.lock().unwrap();
// #890: a catalog field an older build left as `text` is ANALYZED, so a
// `term` on it matches tokens, not the raw value. `legacy` is empty for
@@ -869,8 +885,9 @@ fn search_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> Value
let mut matching: Vec<_> = locked
.docs
.iter()
- .filter(|((index, _), doc)| {
+ .filter(|((index, id), doc)| {
indices.contains(&index.as_str())
+ && ids.as_ref().is_none_or(|wanted| wanted.contains(id))
&& term
.map(|(field, value)| term_matches(doc, field, value, legacy_for(index)))
.unwrap_or(true)
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index fb041cd9f..804dfa39a 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -6087,6 +6087,94 @@ fn delete_catalog_hits_where(
))
}
+/// The graph path's catalog gate (#1977).
+///
+/// `autoindex-catalog` is created empty during prepare, and the data indices
+/// are refreshed — and therefore searchable — before this write. The bulk that
+/// fills the catalog used to treat a per-item refusal as a warning (#955) and
+/// never read the rows back, so a response the client classified as success
+/// could leave the index at zero documents and the run at exit 0. `xerj_map`
+/// then reports no datasets. The data path already refuses that shape (the
+/// zero-live gate, #195). This is the same check for the rows `xerj_map` reads:
+/// every published dataset document and the run document, visible by `_id`
+/// with the `doc_kind` the map filters on.
+///
+/// Called only after the catalog refresh, and before the resume journal is
+/// finished, so a failure leaves the next invocation of the same command to
+/// rewrite these ids.
+fn require_visible_catalog_rows(es: &Es, ids: &[String]) -> Result<()> {
+ // One ids query stays under the default result window. 200 is also the
+ // point at which a ~210-dataset batch (the #1977 report) takes two
+ // requests instead of one oversized one.
+ const CHUNK: usize = 200;
+ let mut missing_at: Vec<usize> = Vec::new();
+ for (chunk_index, chunk) in ids.chunks(CHUNK).enumerate() {
+ let response = es
+ .search(
+ catalog::CATALOG_INDEX,
+ &json!({
+ "size": chunk.len(),
+ "query": {"ids": {"values": chunk}},
+ "_source": ["doc_kind"]
+ }),
+ )
+ .context(
+ "write catalog: reading the catalog back failed, so this run cannot claim the \
+ data map was published",
+ )?;
+ let hits = response
+ .pointer("/hits/hits")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ let mut seen: HashSet<String> = HashSet::new();
+ for hit in &hits {
+ let Some(id) = hit.get("_id").and_then(Value::as_str) else {
+ continue;
+ };
+ let kind = hit.pointer("/_source/doc_kind").and_then(Value::as_str);
+ let expected = if id.starts_with("run:") {
+ "run"
+ } else if id.starts_with("ds:") {
+ "dataset"
+ } else {
+ continue;
+ };
+ if kind == Some(expected) {
+ seen.insert(id.to_string());
+ }
+ }
+ let base = chunk_index * CHUNK;
+ for (offset, id) in chunk.iter().enumerate() {
+ if !seen.contains(id) {
+ missing_at.push(base + offset);
+ }
+ }
+ }
+ if missing_at.is_empty() {
+ return Ok(());
+ }
+ let shown: Vec<&str> = missing_at
+ .iter()
+ .take(8)
+ .map(|&index| ids[index].as_str())
+ .collect();
+ let extra = missing_at.len().saturating_sub(shown.len());
+ let extra = if extra > 0 {
+ format!(" (+{extra} more)")
+ } else {
+ String::new()
+ };
+ anyhow::bail!(
+ "write catalog: {} of {} catalog document(s) are not visible after the write ({}{extra}). \
+ The data indices are already indexed; rerun the same command to rewrite the catalog. \
+ `xerj autoindex map` stays empty until those rows exist",
+ missing_at.len(),
+ ids.len(),
+ shown.join(", "),
+ )
+}
+
/// #589: sweep the documents an exclusion left behind. A file still on disk
/// that a widened ignore/hidden/`.xerjignore` rule now skips must have its
/// already-published documents removed — an exclusion that cannot remove data
@@ -10718,19 +10806,33 @@ fn run_index_report_inner(
.unwrap_or("unknown server error")
);
}
- // A catalog document the server refused is a file or dataset that
- // `xerj autoindex map` will not show. It has never ended a run and
- // does not now, but it is no longer silent either (#955).
+ // A refused catalog document is a dataset or file `xerj autoindex map`
+ // will not show. Warning and exiting 0 left a ~210-index batch with
+ // an empty catalog and no signal to rerun (#1977). Same rule as the
+ // generation path's `check_bulk_outcome`: a catalog item the server
+ // refused fails the run, before the journal is finished, so the same
+ // command rewrites it.
if outcome.item_errors > 0 {
- pr.note(&format!(
- "autoindex: WARNING — the server refused {} catalog document(s): {}. The \
- records are indexed; `xerj autoindex map` will be missing those entries",
+ anyhow::bail!(
+ "write catalog: the server refused {} catalog document(s): {}. The data indices \
+ are already indexed, but `xerj autoindex map` would be missing those entries. \
+ Rerun the same command to rewrite the catalog",
outcome.item_errors,
outcome.first_error.as_deref().unwrap_or("no reason given")
- ));
+ );
}
+ // Refresh is what makes the rows searchable. Swallowing a failure
+ // (`.ok()`) reported success over a catalog `xerj_map` could not see.
+ es.refresh(catalog::CATALOG_INDEX)
+ .context("write catalog: refresh autoindex-catalog so the new rows are searchable")?;
+ let mut published_ids: Vec<String> = plan
+ .datasets
+ .iter()
+ .map(|dataset| format!("ds:{}:{}", cfg.prefix, dataset.slug))
+ .collect();
+ published_ids.push(format!("run:{run_id}"));
+ require_visible_catalog_rows(&es, &published_ids)?;
}
- es.refresh(catalog::CATALOG_INDEX).ok();
pr.item_done(0);
// ── durable junk record, written last (#238) ─────────────────────────
//