Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): fail a run when one dataset index is empty beside live ones (#2407)

Term563221f0e75fbe
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 031595f2d..88dd16ef1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- `xerj autoindex` fails a run when one dataset index holds no documents
+ although the journal says records were written to it, even when the other
+ datasets are live (#2407, defect 2). The check summed live documents over
+ every dataset, so a populated `-json` index let a re-run print `ok: indexed
+ and searchable` over an empty `-docs` index. The error names each empty
+ index and the records and files the journal sent it.
- SQL source extraction no longer slices statement heads inside a UTF-8
character, so unrecognized CJK statement/comment heads do not panic (#2547).
- Short Markdown files with YAML frontmatter retain their prose body even
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index a7a5cd7ba..627378b47 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -137,6 +137,10 @@ struct MockState {
/// `timed_out`. Models a node that accepted the writes and then could
/// not finish a count (merge, deadline). Must not look like zero docs.
fail_counts: bool,
+ /// Opt-in (#2407): `_count` on these indices answers 0. The fixture keeps
+ /// one document map for every index, so this is how a test models one
+ /// dataset index holding nothing while its siblings are live.
+ empty_count_indices: Vec<String>,
/// Answer the catalog's duplicate-alias sweep — and ONLY that request —
/// with a 500 whose body names the condition. Scoped to the one call so
/// the fixture proves the sweep is what wedged the run (#345), not that a
@@ -466,10 +470,17 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {
}
(200, json!({"deleted": removed, "failures": []}))
} else if method == "GET" && path.ends_with("/_count") {
- if state.lock().unwrap().fail_counts {
+ let locked = state.lock().unwrap();
+ if locked.fail_counts {
(200, json!({"error": "count unavailable"}))
+ } else if locked
+ .empty_count_indices
+ .iter()
+ .any(|index| path == format!("/{index}/_count"))
+ {
+ (200, json!({"count": 0}))
} else {
- (200, json!({"count": state.lock().unwrap().docs.len()}))
+ (200, json!({"count": locked.docs.len()}))
}
} else if method == "POST"
&& path.ends_with("/_search")
@@ -3995,6 +4006,65 @@ fn zero_live_documents_after_journaled_records_fails_the_run() {
);
}
+/// #2407: the zero-live gate summed live documents over every dataset, so a
+/// populated sibling hid an index that held nothing. The reporter's re-run
+/// printed `ok: indexed and searchable` over a 0-document `-docs` index
+/// beside a live `-json` one. The empty dataset must fail the run by name,
+/// and it must not be the all-empty error `xerj brain` reads as a wiped
+/// destination.
+#[test]
+fn an_empty_dataset_beside_a_live_one_fails_the_run() {
+ let _guard = FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let write_corpus = |root: &Path| {
+ fs::write(root.join("rows.csv"), "id,value\n1,first\n2,second\n").unwrap();
+ fs::write(
+ root.join("notes.md"),
+ "# Notes\n\nTiered merge policy picks segments of similar size.\n",
+ )
+ .unwrap();
+ };
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ write_corpus(corpus.path());
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ let prefix = config.prefix.clone();
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ // Learn the prose dataset's index rather than hard-coding the slug scheme.
+ let prose = {
+ let locked = endpoint.state.lock().unwrap();
+ let slug = data_rows(&locked)
+ .into_iter()
+ .find(|doc| doc["ax_path"] == "notes.md")
+ .and_then(|doc| doc["ax_dataset"].as_str())
+ .expect("notes.md is indexed")
+ .to_owned();
+ format!("{prefix}-{slug}")
+ };
+
+ // The re-run resumes over the completed journal while the prose index
+ // answers 0, the shape of writes the server refused.
+ endpoint.state.lock().unwrap().empty_count_indices = vec![prose.clone()];
+ let error = run_index(config).unwrap_err();
+ assert!(
+ !error.is::<crate::ZeroLiveVerificationError>(),
+ "a live sibling means the destination was not wiped: {error:#}"
+ );
+ let error = format!("{error:#}");
+ assert!(error.contains("verification failed"), "{error}");
+ assert!(
+ error.contains(&format!("{prose} (")) && error.contains("record(s) from 1 file(s))"),
+ "the empty index is named with what the journal sent it: {error}"
+ );
+ assert!(error.contains("1 of the 2 dataset index(es)"), "{error}");
+ assert!(error.contains("--fresh"), "{error}");
+}
+
/// A node that accepted the bulks and then could not count (the size:0
/// fallback comes back `timed_out` with total 0) must not abort finalize.
/// Treating that 0 as "nothing is live" is the count-mismatch abort: the
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 4d1395292..008c2b5a8 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -6516,6 +6516,66 @@ impl std::fmt::Display for ZeroLiveVerificationError {
impl std::error::Error for ZeroLiveVerificationError {}
+/// A dataset index the resume journal says received records, and that answers
+/// with zero live documents.
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct EmptyDataset {
+ index: String,
+ /// Records journaled by completed files that write only to empty datasets.
+ records: u64,
+ files: usize,
+}
+
+/// The per-dataset half of the #195 gate (#2407). Summing live documents over
+/// every dataset let a populated sibling hide an index that held nothing, so a
+/// re-run reported `ok: indexed and searchable` over an empty `-docs` index.
+///
+/// The journal keeps one record count per file, and a file's groups can feed
+/// several datasets, so records are attributed to a file's datasets only as a
+/// set: a completed file with records is missing when EVERY dataset it feeds
+/// answers zero. A dataset with no count (refused, or the count did not
+/// answer) is unknown, not empty, and never makes a file missing.
+fn datasets_left_empty(
+ plan: &Plan,
+ done: &HashMap<String, FileDone>,
+ ds_counts: &HashMap<String, u64>,
+) -> Vec<EmptyDataset> {
+ let mut empty: BTreeMap<String, (u64, usize)> = BTreeMap::new();
+ for (key, file) in done {
+ if file.records == 0 {
+ continue;
+ }
+ let Some(assignment) = plan.files.get(key) else {
+ continue;
+ };
+ let slugs: BTreeSet<&str> = assignment
+ .assignments
+ .iter()
+ .map(|(_, slug)| slug.as_str())
+ .collect();
+ if slugs.is_empty() || !slugs.iter().all(|slug| ds_counts.get(*slug) == Some(&0)) {
+ continue;
+ }
+ for slug in slugs {
+ let entry = empty.entry(slug.to_string()).or_default();
+ entry.0 += file.records;
+ entry.1 += 1;
+ }
+ }
+ plan.datasets
+ .iter()
+ .filter_map(|dataset| {
+ empty
+ .get(&dataset.slug)
+ .map(|(records, files)| EmptyDataset {
+ index: dataset.index.clone(),
+ records: *records,
+ files: *files,
+ })
+ })
+ .collect()
+}
+
#[derive(Debug)]
struct UnsupportedInventoryDeltaError {
delta: UnsupportedInventoryDelta,
@@ -11797,8 +11857,9 @@ fn run_index_report_inner(
// "the corpus has no match" from "nothing was ever written". This is
// the last-resort catch for any rejection path the per-bulk
// classification does not recognise.
- let (files_done_journaled, journal_records) = {
+ let (files_done_journaled, journal_records, left_empty) = {
let journal = journal_mx.lock().unwrap();
+ let left_empty = datasets_left_empty(&plan, &journal.done, &ds_counts);
// #2314: a file swept because it left the folder is still in
// `journal.done` until the next plan write forgets to republish it.
// Those records are supposed to be gone. Counting them made a folder
@@ -11811,7 +11872,7 @@ fn run_index_report_inner(
journal_records += done.records;
}
}
- (files_done_journaled, journal_records)
+ (files_done_journaled, journal_records, left_empty)
};
let live_records: u64 = ds_counts.values().sum();
if !count_unknown.is_empty() {
@@ -11839,6 +11900,31 @@ fn run_index_report_inner(
state_dir: state_dir.display().to_string(),
}
.into());
+ } else if !left_empty.is_empty() {
+ // #2407: other datasets are live, so this is not the wiped
+ // destination `ZeroLiveVerificationError` stands for, and `xerj
+ // brain` must not read it as one.
+ let named: Vec<String> = left_empty
+ .iter()
+ .map(|d| {
+ format!(
+ "{} ({} record(s) from {} file(s))",
+ d.index, d.records, d.files
+ )
+ })
+ .collect();
+ anyhow::bail!(
+ "autoindex verification failed: {} of the {} dataset index(es) hold 0 live \
+ documents although the resume journal records records written to them: {}. \
+ The other datasets are live, so the run is not reported as indexed. A \
+ server-side rejection of that dataset's writes (for example an embedding \
+ width its mapping does not fit) or a deleted index leaves this. Journal \
+ state dir: {}. Fix the cause, then rerun with --fresh to re-index from scratch",
+ left_empty.len(),
+ plan.datasets.len(),
+ named.join(", "),
+ state_dir.display()
+ );
}
// correlations
@@ -14616,6 +14702,99 @@ mod map_metadata_tests {
}
}
+ fn dataset(slug: &str) -> PlanDataset {
+ PlanDataset {
+ slug: slug.into(),
+ index: format!("ax-{slug}"),
+ family: "text".into(),
+ group: None,
+ specs: vec![],
+ time_field: None,
+ semantic_field: None,
+ text_analyzer: None,
+ scope: None,
+ sampled_records: 1,
+ file_count: 1,
+ }
+ }
+
+ fn done(key: &str, records: u64) -> (String, FileDone) {
+ (
+ key.into(),
+ FileDone {
+ file_key: key.into(),
+ path: format!("{key}.dat"),
+ records,
+ junk: 0,
+ bytes: 1,
+ dropped_by_dataset: HashMap::new(),
+ generation: None,
+ },
+ )
+ }
+
+ /// #2407: a live sibling must not hide a dataset that holds nothing. The
+ /// prose file wrote only to `docs`, which answers 0 while `json` is live.
+ /// A file that also feeds a live dataset cannot be shown missing (the
+ /// journal has one count per file), and a dataset with no count is
+ /// unknown, not empty.
+ #[test]
+ fn a_dataset_left_empty_beside_a_live_sibling_is_named() {
+ let plan = Plan {
+ datasets: vec![dataset("docs"), dataset("json"), dataset("mixed")],
+ files: HashMap::from([
+ ("prose-1".into(), assignment(&["docs"])),
+ ("prose-2".into(), assignment(&["docs"])),
+ ("rows".into(), assignment(&["json"])),
+ ("both".into(), assignment(&["mixed", "json"])),
+ ("unknown".into(), assignment(&["refused"])),
+ ]),
+ ..Plan::default()
+ };
+ let journal = HashMap::from([
+ done("prose-1", 3),
+ done("prose-2", 2),
+ done("rows", 120),
+ done("both", 4),
+ done("unknown", 9),
+ // Swept from the folder: in the journal, not in the plan (#2314).
+ done("gone", 7),
+ ]);
+ let counts = HashMap::from([
+ ("docs".to_string(), 0),
+ ("json".to_string(), 120),
+ ("mixed".to_string(), 0),
+ ]);
+ assert_eq!(
+ datasets_left_empty(&plan, &journal, &counts),
+ [EmptyDataset {
+ index: "ax-docs".into(),
+ records: 5,
+ files: 2,
+ }]
+ );
+
+ // Every dataset live: nothing to name.
+ let live = HashMap::from([
+ ("docs".to_string(), 5),
+ ("json".to_string(), 120),
+ ("mixed".to_string(), 0),
+ ]);
+ assert!(datasets_left_empty(&plan, &journal, &live).is_empty());
+
+ // A file whose every dataset is empty names each of them.
+ let both_empty = HashMap::from([
+ ("docs".to_string(), 5),
+ ("json".to_string(), 0),
+ ("mixed".to_string(), 0),
+ ]);
+ let named: Vec<String> = datasets_left_empty(&plan, &journal, &both_empty)
+ .into_iter()
+ .map(|d| d.index)
+ .collect();
+ assert_eq!(named, ["ax-json", "ax-mixed"]);
+ }
+
#[test]
fn unchanged_resume_keeps_durable_assignment_aware_dataset_bytes() {
let _guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK