AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): sweep catalog entries for skipped files that leave the corpus
xerj-orgdea30360c82a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 83d7c03fa..00ff4fc62 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- **`autoindex` no longer leaves immortal catalog entries for skipped files**+ ([#238](https://github.com/xerj-org/xerj/issues/238)). A file added after the+ resume plan was frozen is skipped and reported in the catalog — but that+ report was written from a per-run `Vec` that nothing durable remembered, so+ once the file left the corpus no run could ever remove its `file:{key}`+ document. The catalog is the data map every `map`, `status` and agent query+ reads, and it kept advertising files that were gone. Skipped files are now+ recorded in the durable plan and swept from the catalog when they leave the+ corpus. The sweep is safe to do completely because a skipped file is never+ indexed and never enters the graph corpus — that one catalog document is its+ entire live footprint. Ordering is deliberate: the plan entry is added only+ after the document is written and dropped only after the delete has landed,+ so a failed catalog bulk is retried by the next run instead of stranding the+ document. This does **not** implement add/change/delete reconciliation for+ *indexed* files, which remains open.+- **The installer is now fail-closed on checksum *verification*, not just onchecksum *download*.** `landing/get` printed`warning: sha256sum/shasum not found — skipping checksum verification` anddiff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rsindex b49e9d823..0e0c72f6a 100644--- a/engine/crates/xerj-autoindex/src/catalog.rs+++ b/engine/crates/xerj-autoindex/src/catalog.rs@@ -103,6 +103,13 @@ pub fn dataset_doc(inp: &DatasetDocInput) -> (String, Value) {(id, doc)}+/// Catalog document id for a file. One definition, because the sweep that+/// removes a junk/skipped file's document (`lib.rs`, "stale junk-catalog+/// sweep") has to name the id without holding the document.+pub fn file_id(file_key: &str) -> String {+ format!("file:{file_key}")+}+#[allow(clippy::too_many_arguments)] // 1:1 with the file-status doc's fieldspub fn file_doc(file_key: &str,@@ -116,7 +123,7 @@ pub fn file_doc(run_id: &str,) -> (String, Value) {(- format!("file:{file_key}"),+ file_id(file_key),json!({"doc_kind": "file","file_key": file_key,diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex a4f8618ce..c1c1e3df1 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -19,6 +19,10 @@ static FAILPOINT_TEST_LOCK: Mutex<()> = Mutex::new(());#[derive(Default)]struct MockState {docs: HashMap<String, Value>,+ /// Catalog documents, kept apart from `docs` on purpose: `docs.len()` is+ /// what `_count`/`_search` answer with, and the run verifies live data+ /// counts against the journal (#195). Catalog rows are not data rows.+ catalog_docs: HashMap<String, Value>,data_bulk_number: usize,fail_data_bulk: usize,failed_once: bool,@@ -168,6 +172,35 @@ fn bulk_response(body: &[u8], state: &Arc<Mutex<MockState>>) -> Value {.and_then(|index| index.as_str().map(str::to_owned)).is_some_and(|index| index != catalog::CATALOG_INDEX);if !is_data {+ // Catalog bulks mix `index` and `delete` actions (stale aliases, and+ // the junk sweep of #238), so walk the NDJSON rather than assuming+ // action/document pairs — a delete carries no document line.+ let mut locked = state.lock().unwrap();+ let mut line = 0;+ while line < lines.len() {+ let Ok(action) = serde_json::from_slice::<Value>(lines[line]) else {+ line += 1;+ continue;+ };+ if action.pointer("/delete/_index").and_then(Value::as_str)+ == Some(catalog::CATALOG_INDEX)+ {+ if let Some(id) = action.pointer("/delete/_id").and_then(Value::as_str) {+ locked.catalog_docs.remove(id);+ }+ line += 1;+ } else if action.pointer("/index/_index").and_then(Value::as_str)+ == Some(catalog::CATALOG_INDEX)+ && line + 1 < lines.len()+ {+ let id = action.pointer("/index/_id").unwrap().as_str().unwrap();+ let doc: Value = serde_json::from_slice(lines[line + 1]).unwrap();+ locked.catalog_docs.insert(id.to_owned(), doc);+ line += 2;+ } else {+ line += 1;+ }+ }return json!({"errors": false, "items": []});}@@ -822,6 +855,98 @@ fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {.all(|doc| doc["value"].as_str() == Some("rewritten")));}+/// #238: a file added after the plan was frozen is skipped and reported in the+/// catalog. That report used to be immortal — `new_unplanned` was a per-run+/// `Vec`, so once the file left the corpus nothing remembered the document had+/// ever been written and no run could remove it. The catalog is the data map+/// every `map`/`status`/agent query reads; it must not keep advertising a file+/// that is gone.+#[test]+fn an_added_then_deleted_file_leaves_no_immortal_catalog_entry() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("planned.csv"),+ "id,value\n0,planned\n1,planned\n",+ )+ .unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ // Appears after the plan froze: not indexed, but reported as skipped.+ fs::write(+ corpus.path().join("added.csv"),+ "id,value\n2,added\n3,added\n",+ )+ .unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 3);+ let skipped_id = {+ let locked = endpoint.state.lock().unwrap();+ let (id, doc) = locked+ .catalog_docs+ .iter()+ .find(|(_, doc)| doc["path"].as_str() == Some("added.csv"))+ .expect("a file skipped for being post-freeze is reported in the catalog");+ assert_eq!(doc["status"].as_str(), Some("skipped"));+ assert_eq!(doc["doc_kind"].as_str(), Some("file"));+ id.clone()+ };++ // The durable plan is the only record that the document exists. Without+ // it no later run can ever delete it.+ let replay = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert!(+ replay+ .plan+ .as_ref()+ .unwrap()+ .junk_files+ .iter()+ .any(|junk| junk.rel == "added.csv"),+ "the skipped file must be recorded in the durable plan"+ );+ drop(replay);++ // Delete it: the catalog entry goes with it, and the run is clean again.+ fs::remove_file(corpus.path().join("added.csv")).unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ {+ let locked = endpoint.state.lock().unwrap();+ assert!(+ !locked.catalog_docs.contains_key(&skipped_id),+ "the catalog entry for a deleted skipped file must not survive"+ );+ assert!(+ locked+ .catalog_docs+ .values()+ .all(|doc| doc["path"].as_str() != Some("added.csv")),+ "no catalog document may still name the deleted file"+ );+ // The file that is still there keeps its entry.+ assert!(locked+ .catalog_docs+ .values()+ .any(|doc| doc["path"].as_str() == Some("planned.csv")));+ }++ // Converged: nothing left to add or sweep, so no further plan is appended.+ let plans_before = event_count(state_dir.path(), "plan");+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(event_count(state_dir.path(), "plan"), plans_before);+}+#[test]fn deleting_an_entire_duplicate_group_strands_no_pending_replacement() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 0adfee682..1cb92710e 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -1109,7 +1109,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {// ── Phase A: inference (skipped when a frozen plan exists) ──────────let mut clusters_rt: Option<Vec<dataset::Cluster>> = None;- let plan: Plan = if let Some(p) = journal.plan.clone() {+ let mut plan: Plan = if let Some(p) = journal.plan.clone() {p} else {if !cfg.quiet {@@ -1134,6 +1134,37 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {plan};+ // ── stale junk-catalog sweep (#238) ──────────────────────────────────+ //+ // A junk/skipped file is never indexed and never enters the graph corpus+ // — both walk `plan.files`, which by construction does not contain it. Its+ // ENTIRE live footprint is one `file:{key}` catalog document, so removing+ // that document is a complete removal, unlike an indexed file where+ // dropping the catalog entry while its records stay live would be a lie.+ //+ // The durable plan is the only thing that remembers the document exists:+ // nothing else in a run remembers a file it deliberately did not read. So+ // the sweep is driven from the plan, and the plan entry may be dropped+ // only after the delete has actually landed — the same order quickwit's+ // GC keeps, deleting split files first and removing metastore records+ // only for the deletes that succeeded (quickwit,+ // quickwit/quickwit-index-management/src/garbage_collection.rs:484-534,+ // Apache-2.0; approach only, no code taken). Dropping the record first+ // would strand the document exactly as before.+ //+ // A key that is somehow BOTH planned and junk is left alone: deleting+ // `file:{key}` would take out a live indexed file's catalog entry, and an+ // immortal junk row is the cheaper of those two failures.+ let live_keys: std::collections::HashSet<&str> = keys.iter().map(String::as_str).collect();+ let stale_junk_keys: std::collections::HashSet<String> = plan+ .junk_files+ .iter()+ .filter(|junk| {+ !live_keys.contains(junk.file_key.as_str()) && !plan.files.contains_key(&junk.file_key)+ })+ .map(|junk| junk.file_key.clone())+ .collect();+if cfg.dry_run {println!("{}", serde_json::to_string_pretty(&plan)?);eprintln!("(dry run — nothing indexed)");@@ -2129,6 +2160,20 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {cat_buf.extend_from_slice(action.to_string().as_bytes());cat_buf.push(b'\n');}+ // Junk/skipped files that left the corpus (#238). Deletes lead the buffer+ // so a key that is re-reported in the same run — a junk file whose bytes+ // changed, a legacy plan key rewritten under the current scheme — is+ // deleted before its replacement document is indexed, never after.+ let mut swept_junk_keys: Vec<&str> = stale_junk_keys.iter().map(String::as_str).collect();+ swept_junk_keys.sort_unstable();+ for key in &swept_junk_keys {+ let action = json!({"delete": {+ "_index": catalog::CATALOG_INDEX,+ "_id": catalog::file_id(key),+ }});+ cat_buf.extend_from_slice(action.to_string().as_bytes());+ cat_buf.push(b'\n');+ }// dataset docslet mut junk_records_by_run: u64 = junk_records.load(Ordering::Relaxed);@@ -2217,9 +2262,17 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {}}let extra = extra_junk.into_inner().unwrap();- let mut all_junk: Vec<&JunkFile> = plan.junk_files.iter().collect();+ let mut all_junk: Vec<&JunkFile> = plan+ .junk_files+ .iter()+ .filter(|junk| !stale_junk_keys.contains(&junk.file_key))+ .collect();all_junk.extend(extra.iter());all_junk.extend(new_unplanned.iter());+ // Counted now: every later reader of this number outlives the borrows+ // `all_junk` holds on `plan` and `new_unplanned`, which the durable+ // junk-plan update below mutates.+ let junk_file_count = all_junk.len();for jf in &all_junk {let (id, doc) = catalog::file_doc(&jf.file_key,@@ -2309,7 +2362,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {"unique_content_files": files.len(),"files_indexed": journal_mx.lock().unwrap().done.len(),"duplicate_files": plan.duplicate_files.len(),- "files_junk": all_junk.len(),+ "files_junk": junk_file_count,"records_total": total_records,"junk_records_total": junk_records_by_run,// This-run submission accounting (#195): what THIS invocation sent@@ -2345,6 +2398,27 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {}}es.refresh(catalog::CATALOG_INDEX).ok();+ // ── durable junk record, written last (#238) ─────────────────────────+ //+ // Only now is the plan allowed to claim what the catalog holds: the+ // documents above are written and the swept ones deleted. A failed+ // catalog bulk bailed out before this point with the plan untouched, so+ // the next run recomputes the identical additions and removals and+ // retries them — losing a record here is what makes a document immortal.+ //+ // Post-freeze skipped files are re-derived on every rerun, so this+ // converges: once they are in the plan they are `planned_junk`, the+ // additions are empty, and no further plan record is appended.+ if !new_unplanned.is_empty() || !stale_junk_keys.is_empty() {+ plan.junk_files+ .retain(|junk| !stale_junk_keys.contains(&junk.file_key));+ plan.junk_files.append(&mut new_unplanned);+ journal_mx+ .lock()+ .unwrap()+ .write_plan(&plan)+ .context("record junk/skipped catalog entries in the resume plan")?;+ }journal_mx.lock().unwrap().finish(&run_doc)?;// ── summary ──────────────────────────────────────────────────────────@@ -2353,7 +2427,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {println!("{run_doc}");} else if !cfg.quiet {println!("\ndone in {wall:.1}s — {} datasets, {} records live, {} duplicate aliases, {} junk records, {} junk/skipped files",- plan.datasets.len(), total_records, plan.duplicate_files.len(), junk_total_records, all_junk.len());+ plan.datasets.len(), total_records, plan.duplicate_files.len(), junk_total_records, junk_file_count);// Indexed-vs-submitted honesty line (#195): the live count against// what this run actually submitted, so a silent-rejection mismatch// is visible in the client output alone. Units differ on purpose: a@@ -2397,7 +2471,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {cfg.url, cfg.prefix);}- let code = if junk_total_records > 0 || !all_junk.is_empty() {+ let code = if junk_total_records > 0 || junk_file_count > 0 {3} else {0