AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): fail a run when one dataset index is empty beside live ones (#2407)
Term563221f0e75fbe
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 031595f2d..88dd16ef1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -21,6 +21,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- `xerj autoindex` fails a run when one dataset index holds no documents+ although the journal says records were written to it, even when the other+ datasets are live (#2407, defect 2). The check summed live documents over+ every dataset, so a populated `-json` index let a re-run print `ok: indexed+ and searchable` over an empty `-docs` index. The error names each empty+ index and the records and files the journal sent it.- SQL source extraction no longer slices statement heads inside a UTF-8character, so unrecognized CJK statement/comment heads do not panic (#2547).- Short Markdown files with YAML frontmatter retain their prose body evendiff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex a7a5cd7ba..627378b47 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -137,6 +137,10 @@ struct MockState {/// `timed_out`. Models a node that accepted the writes and then could/// not finish a count (merge, deadline). Must not look like zero docs.fail_counts: bool,+ /// Opt-in (#2407): `_count` on these indices answers 0. The fixture keeps+ /// one document map for every index, so this is how a test models one+ /// dataset index holding nothing while its siblings are live.+ empty_count_indices: Vec<String>,/// Answer the catalog's duplicate-alias sweep — and ONLY that request —/// with a 500 whose body names the condition. Scoped to the one call so/// the fixture proves the sweep is what wedged the run (#345), not that a@@ -466,10 +470,17 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {}(200, json!({"deleted": removed, "failures": []}))} else if method == "GET" && path.ends_with("/_count") {- if state.lock().unwrap().fail_counts {+ let locked = state.lock().unwrap();+ if locked.fail_counts {(200, json!({"error": "count unavailable"}))+ } else if locked+ .empty_count_indices+ .iter()+ .any(|index| path == format!("/{index}/_count"))+ {+ (200, json!({"count": 0}))} else {- (200, json!({"count": state.lock().unwrap().docs.len()}))+ (200, json!({"count": locked.docs.len()}))}} else if method == "POST"&& path.ends_with("/_search")@@ -3995,6 +4006,65 @@ fn zero_live_documents_after_journaled_records_fails_the_run() {);}+/// #2407: the zero-live gate summed live documents over every dataset, so a+/// populated sibling hid an index that held nothing. The reporter's re-run+/// printed `ok: indexed and searchable` over a 0-document `-docs` index+/// beside a live `-json` one. The empty dataset must fail the run by name,+/// and it must not be the all-empty error `xerj brain` reads as a wiped+/// destination.+#[test]+fn an_empty_dataset_beside_a_live_one_fails_the_run() {+ let _guard = FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let write_corpus = |root: &Path| {+ fs::write(root.join("rows.csv"), "id,value\n1,first\n2,second\n").unwrap();+ fs::write(+ root.join("notes.md"),+ "# Notes\n\nTiered merge policy picks segments of similar size.\n",+ )+ .unwrap();+ };+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ write_corpus(corpus.path());+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ let prefix = config.prefix.clone();+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ // Learn the prose dataset's index rather than hard-coding the slug scheme.+ let prose = {+ let locked = endpoint.state.lock().unwrap();+ let slug = data_rows(&locked)+ .into_iter()+ .find(|doc| doc["ax_path"] == "notes.md")+ .and_then(|doc| doc["ax_dataset"].as_str())+ .expect("notes.md is indexed")+ .to_owned();+ format!("{prefix}-{slug}")+ };++ // The re-run resumes over the completed journal while the prose index+ // answers 0, the shape of writes the server refused.+ endpoint.state.lock().unwrap().empty_count_indices = vec![prose.clone()];+ let error = run_index(config).unwrap_err();+ assert!(+ !error.is::<crate::ZeroLiveVerificationError>(),+ "a live sibling means the destination was not wiped: {error:#}"+ );+ let error = format!("{error:#}");+ assert!(error.contains("verification failed"), "{error}");+ assert!(+ error.contains(&format!("{prose} (")) && error.contains("record(s) from 1 file(s))"),+ "the empty index is named with what the journal sent it: {error}"+ );+ assert!(error.contains("1 of the 2 dataset index(es)"), "{error}");+ assert!(error.contains("--fresh"), "{error}");+}+/// A node that accepted the bulks and then could not count (the size:0/// fallback comes back `timed_out` with total 0) must not abort finalize./// Treating that 0 as "nothing is live" is the count-mismatch abort: thediff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 4d1395292..008c2b5a8 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -6516,6 +6516,66 @@ impl std::fmt::Display for ZeroLiveVerificationError {impl std::error::Error for ZeroLiveVerificationError {}+/// A dataset index the resume journal says received records, and that answers+/// with zero live documents.+#[derive(Debug, Clone, PartialEq, Eq)]+struct EmptyDataset {+ index: String,+ /// Records journaled by completed files that write only to empty datasets.+ records: u64,+ files: usize,+}++/// The per-dataset half of the #195 gate (#2407). Summing live documents over+/// every dataset let a populated sibling hide an index that held nothing, so a+/// re-run reported `ok: indexed and searchable` over an empty `-docs` index.+///+/// The journal keeps one record count per file, and a file's groups can feed+/// several datasets, so records are attributed to a file's datasets only as a+/// set: a completed file with records is missing when EVERY dataset it feeds+/// answers zero. A dataset with no count (refused, or the count did not+/// answer) is unknown, not empty, and never makes a file missing.+fn datasets_left_empty(+ plan: &Plan,+ done: &HashMap<String, FileDone>,+ ds_counts: &HashMap<String, u64>,+) -> Vec<EmptyDataset> {+ let mut empty: BTreeMap<String, (u64, usize)> = BTreeMap::new();+ for (key, file) in done {+ if file.records == 0 {+ continue;+ }+ let Some(assignment) = plan.files.get(key) else {+ continue;+ };+ let slugs: BTreeSet<&str> = assignment+ .assignments+ .iter()+ .map(|(_, slug)| slug.as_str())+ .collect();+ if slugs.is_empty() || !slugs.iter().all(|slug| ds_counts.get(*slug) == Some(&0)) {+ continue;+ }+ for slug in slugs {+ let entry = empty.entry(slug.to_string()).or_default();+ entry.0 += file.records;+ entry.1 += 1;+ }+ }+ plan.datasets+ .iter()+ .filter_map(|dataset| {+ empty+ .get(&dataset.slug)+ .map(|(records, files)| EmptyDataset {+ index: dataset.index.clone(),+ records: *records,+ files: *files,+ })+ })+ .collect()+}+#[derive(Debug)]struct UnsupportedInventoryDeltaError {delta: UnsupportedInventoryDelta,@@ -11797,8 +11857,9 @@ fn run_index_report_inner(// "the corpus has no match" from "nothing was ever written". This is// the last-resort catch for any rejection path the per-bulk// classification does not recognise.- let (files_done_journaled, journal_records) = {+ let (files_done_journaled, journal_records, left_empty) = {let journal = journal_mx.lock().unwrap();+ let left_empty = datasets_left_empty(&plan, &journal.done, &ds_counts);// #2314: a file swept because it left the folder is still in// `journal.done` until the next plan write forgets to republish it.// Those records are supposed to be gone. Counting them made a folder@@ -11811,7 +11872,7 @@ fn run_index_report_inner(journal_records += done.records;}}- (files_done_journaled, journal_records)+ (files_done_journaled, journal_records, left_empty)};let live_records: u64 = ds_counts.values().sum();if !count_unknown.is_empty() {@@ -11839,6 +11900,31 @@ fn run_index_report_inner(state_dir: state_dir.display().to_string(),}.into());+ } else if !left_empty.is_empty() {+ // #2407: other datasets are live, so this is not the wiped+ // destination `ZeroLiveVerificationError` stands for, and `xerj+ // brain` must not read it as one.+ let named: Vec<String> = left_empty+ .iter()+ .map(|d| {+ format!(+ "{} ({} record(s) from {} file(s))",+ d.index, d.records, d.files+ )+ })+ .collect();+ anyhow::bail!(+ "autoindex verification failed: {} of the {} dataset index(es) hold 0 live \+ documents although the resume journal records records written to them: {}. \+ The other datasets are live, so the run is not reported as indexed. A \+ server-side rejection of that dataset's writes (for example an embedding \+ width its mapping does not fit) or a deleted index leaves this. Journal \+ state dir: {}. Fix the cause, then rerun with --fresh to re-index from scratch",+ left_empty.len(),+ plan.datasets.len(),+ named.join(", "),+ state_dir.display()+ );}// correlations@@ -14616,6 +14702,99 @@ mod map_metadata_tests {}}+ fn dataset(slug: &str) -> PlanDataset {+ PlanDataset {+ slug: slug.into(),+ index: format!("ax-{slug}"),+ family: "text".into(),+ group: None,+ specs: vec![],+ time_field: None,+ semantic_field: None,+ text_analyzer: None,+ scope: None,+ sampled_records: 1,+ file_count: 1,+ }+ }++ fn done(key: &str, records: u64) -> (String, FileDone) {+ (+ key.into(),+ FileDone {+ file_key: key.into(),+ path: format!("{key}.dat"),+ records,+ junk: 0,+ bytes: 1,+ dropped_by_dataset: HashMap::new(),+ generation: None,+ },+ )+ }++ /// #2407: a live sibling must not hide a dataset that holds nothing. The+ /// prose file wrote only to `docs`, which answers 0 while `json` is live.+ /// A file that also feeds a live dataset cannot be shown missing (the+ /// journal has one count per file), and a dataset with no count is+ /// unknown, not empty.+ #[test]+ fn a_dataset_left_empty_beside_a_live_sibling_is_named() {+ let plan = Plan {+ datasets: vec![dataset("docs"), dataset("json"), dataset("mixed")],+ files: HashMap::from([+ ("prose-1".into(), assignment(&["docs"])),+ ("prose-2".into(), assignment(&["docs"])),+ ("rows".into(), assignment(&["json"])),+ ("both".into(), assignment(&["mixed", "json"])),+ ("unknown".into(), assignment(&["refused"])),+ ]),+ ..Plan::default()+ };+ let journal = HashMap::from([+ done("prose-1", 3),+ done("prose-2", 2),+ done("rows", 120),+ done("both", 4),+ done("unknown", 9),+ // Swept from the folder: in the journal, not in the plan (#2314).+ done("gone", 7),+ ]);+ let counts = HashMap::from([+ ("docs".to_string(), 0),+ ("json".to_string(), 120),+ ("mixed".to_string(), 0),+ ]);+ assert_eq!(+ datasets_left_empty(&plan, &journal, &counts),+ [EmptyDataset {+ index: "ax-docs".into(),+ records: 5,+ files: 2,+ }]+ );++ // Every dataset live: nothing to name.+ let live = HashMap::from([+ ("docs".to_string(), 5),+ ("json".to_string(), 120),+ ("mixed".to_string(), 0),+ ]);+ assert!(datasets_left_empty(&plan, &journal, &live).is_empty());++ // A file whose every dataset is empty names each of them.+ let both_empty = HashMap::from([+ ("docs".to_string(), 5),+ ("json".to_string(), 0),+ ("mixed".to_string(), 0),+ ]);+ let named: Vec<String> = datasets_left_empty(&plan, &journal, &both_empty)+ .into_iter()+ .map(|d| d.index)+ .collect();+ assert_eq!(named, ["ax-json", "ax-mixed"]);+ }+#[test]fn unchanged_resume_keeps_durable_assignment_aware_dataset_bytes() {let _guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK