Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): sweep catalog entries for skipped files that leave the corpus

xerj-orgdea30360c82a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83d7c03fa..00ff4fc62 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- **`autoindex` no longer leaves immortal catalog entries for skipped files**
+ ([#238](https://github.com/xerj-org/xerj/issues/238)). A file added after the
+ resume plan was frozen is skipped and reported in the catalog — but that
+ report was written from a per-run `Vec` that nothing durable remembered, so
+ once the file left the corpus no run could ever remove its `file:{key}`
+ document. The catalog is the data map every `map`, `status` and agent query
+ reads, and it kept advertising files that were gone. Skipped files are now
+ recorded in the durable plan and swept from the catalog when they leave the
+ corpus. The sweep is safe to do completely because a skipped file is never
+ indexed and never enters the graph corpus — that one catalog document is its
+ entire live footprint. Ordering is deliberate: the plan entry is added only
+ after the document is written and dropped only after the delete has landed,
+ so a failed catalog bulk is retried by the next run instead of stranding the
+ document. This does **not** implement add/change/delete reconciliation for
+ *indexed* files, which remains open.
+
- **The installer is now fail-closed on checksum *verification*, not just on
checksum *download*.** `landing/get` printed
`warning: sha256sum/shasum not found — skipping checksum verification` and
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rs
index b49e9d823..0e0c72f6a 100644
--- a/engine/crates/xerj-autoindex/src/catalog.rs
+++ b/engine/crates/xerj-autoindex/src/catalog.rs
@@ -103,6 +103,13 @@ pub fn dataset_doc(inp: &DatasetDocInput) -> (String, Value) {
(id, doc)
}
+/// Catalog document id for a file. One definition, because the sweep that
+/// removes a junk/skipped file's document (`lib.rs`, "stale junk-catalog
+/// sweep") has to name the id without holding the document.
+pub fn file_id(file_key: &str) -> String {
+ format!("file:{file_key}")
+}
+
#[allow(clippy::too_many_arguments)] // 1:1 with the file-status doc's fields
pub fn file_doc(
file_key: &str,
@@ -116,7 +123,7 @@ pub fn file_doc(
run_id: &str,
) -> (String, Value) {
(
- format!("file:{file_key}"),
+ file_id(file_key),
json!({
"doc_kind": "file",
"file_key": file_key,
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index a4f8618ce..c1c1e3df1 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -19,6 +19,10 @@ static FAILPOINT_TEST_LOCK: Mutex<()> = Mutex::new(());
#[derive(Default)]
struct MockState {
docs: HashMap<String, Value>,
+ /// Catalog documents, kept apart from `docs` on purpose: `docs.len()` is
+ /// what `_count`/`_search` answer with, and the run verifies live data
+ /// counts against the journal (#195). Catalog rows are not data rows.
+ catalog_docs: HashMap<String, Value>,
data_bulk_number: usize,
fail_data_bulk: usize,
failed_once: bool,
@@ -168,6 +172,35 @@ fn bulk_response(body: &[u8], state: &Arc<Mutex<MockState>>) -> Value {
.and_then(|index| index.as_str().map(str::to_owned))
.is_some_and(|index| index != catalog::CATALOG_INDEX);
if !is_data {
+ // Catalog bulks mix `index` and `delete` actions (stale aliases, and
+ // the junk sweep of #238), so walk the NDJSON rather than assuming
+ // action/document pairs — a delete carries no document line.
+ let mut locked = state.lock().unwrap();
+ let mut line = 0;
+ while line < lines.len() {
+ let Ok(action) = serde_json::from_slice::<Value>(lines[line]) else {
+ line += 1;
+ continue;
+ };
+ if action.pointer("/delete/_index").and_then(Value::as_str)
+ == Some(catalog::CATALOG_INDEX)
+ {
+ if let Some(id) = action.pointer("/delete/_id").and_then(Value::as_str) {
+ locked.catalog_docs.remove(id);
+ }
+ line += 1;
+ } else if action.pointer("/index/_index").and_then(Value::as_str)
+ == Some(catalog::CATALOG_INDEX)
+ && line + 1 < lines.len()
+ {
+ let id = action.pointer("/index/_id").unwrap().as_str().unwrap();
+ let doc: Value = serde_json::from_slice(lines[line + 1]).unwrap();
+ locked.catalog_docs.insert(id.to_owned(), doc);
+ line += 2;
+ } else {
+ line += 1;
+ }
+ }
return json!({"errors": false, "items": []});
}
@@ -822,6 +855,98 @@ fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {
.all(|doc| doc["value"].as_str() == Some("rewritten")));
}
+/// #238: a file added after the plan was frozen is skipped and reported in the
+/// catalog. That report used to be immortal — `new_unplanned` was a per-run
+/// `Vec`, so once the file left the corpus nothing remembered the document had
+/// ever been written and no run could remove it. The catalog is the data map
+/// every `map`/`status`/agent query reads; it must not keep advertising a file
+/// that is gone.
+#[test]
+fn an_added_then_deleted_file_leaves_no_immortal_catalog_entry() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("planned.csv"),
+ "id,value\n0,planned\n1,planned\n",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ // Appears after the plan froze: not indexed, but reported as skipped.
+ fs::write(
+ corpus.path().join("added.csv"),
+ "id,value\n2,added\n3,added\n",
+ )
+ .unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 3);
+ let skipped_id = {
+ let locked = endpoint.state.lock().unwrap();
+ let (id, doc) = locked
+ .catalog_docs
+ .iter()
+ .find(|(_, doc)| doc["path"].as_str() == Some("added.csv"))
+ .expect("a file skipped for being post-freeze is reported in the catalog");
+ assert_eq!(doc["status"].as_str(), Some("skipped"));
+ assert_eq!(doc["doc_kind"].as_str(), Some("file"));
+ id.clone()
+ };
+
+ // The durable plan is the only record that the document exists. Without
+ // it no later run can ever delete it.
+ let replay = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert!(
+ replay
+ .plan
+ .as_ref()
+ .unwrap()
+ .junk_files
+ .iter()
+ .any(|junk| junk.rel == "added.csv"),
+ "the skipped file must be recorded in the durable plan"
+ );
+ drop(replay);
+
+ // Delete it: the catalog entry goes with it, and the run is clean again.
+ fs::remove_file(corpus.path().join("added.csv")).unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ {
+ let locked = endpoint.state.lock().unwrap();
+ assert!(
+ !locked.catalog_docs.contains_key(&skipped_id),
+ "the catalog entry for a deleted skipped file must not survive"
+ );
+ assert!(
+ locked
+ .catalog_docs
+ .values()
+ .all(|doc| doc["path"].as_str() != Some("added.csv")),
+ "no catalog document may still name the deleted file"
+ );
+ // The file that is still there keeps its entry.
+ assert!(locked
+ .catalog_docs
+ .values()
+ .any(|doc| doc["path"].as_str() == Some("planned.csv")));
+ }
+
+ // Converged: nothing left to add or sweep, so no further plan is appended.
+ let plans_before = event_count(state_dir.path(), "plan");
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(event_count(state_dir.path(), "plan"), plans_before);
+}
+
#[test]
fn deleting_an_entire_duplicate_group_strands_no_pending_replacement() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 0adfee682..1cb92710e 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -1109,7 +1109,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
// ── Phase A: inference (skipped when a frozen plan exists) ──────────
let mut clusters_rt: Option<Vec<dataset::Cluster>> = None;
- let plan: Plan = if let Some(p) = journal.plan.clone() {
+ let mut plan: Plan = if let Some(p) = journal.plan.clone() {
p
} else {
if !cfg.quiet {
@@ -1134,6 +1134,37 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
plan
};
+ // ── stale junk-catalog sweep (#238) ──────────────────────────────────
+ //
+ // A junk/skipped file is never indexed and never enters the graph corpus
+ // — both walk `plan.files`, which by construction does not contain it. Its
+ // ENTIRE live footprint is one `file:{key}` catalog document, so removing
+ // that document is a complete removal, unlike an indexed file where
+ // dropping the catalog entry while its records stay live would be a lie.
+ //
+ // The durable plan is the only thing that remembers the document exists:
+ // nothing else in a run remembers a file it deliberately did not read. So
+ // the sweep is driven from the plan, and the plan entry may be dropped
+ // only after the delete has actually landed — the same order quickwit's
+ // GC keeps, deleting split files first and removing metastore records
+ // only for the deletes that succeeded (quickwit,
+ // quickwit/quickwit-index-management/src/garbage_collection.rs:484-534,
+ // Apache-2.0; approach only, no code taken). Dropping the record first
+ // would strand the document exactly as before.
+ //
+ // A key that is somehow BOTH planned and junk is left alone: deleting
+ // `file:{key}` would take out a live indexed file's catalog entry, and an
+ // immortal junk row is the cheaper of those two failures.
+ let live_keys: std::collections::HashSet<&str> = keys.iter().map(String::as_str).collect();
+ let stale_junk_keys: std::collections::HashSet<String> = plan
+ .junk_files
+ .iter()
+ .filter(|junk| {
+ !live_keys.contains(junk.file_key.as_str()) && !plan.files.contains_key(&junk.file_key)
+ })
+ .map(|junk| junk.file_key.clone())
+ .collect();
+
if cfg.dry_run {
println!("{}", serde_json::to_string_pretty(&plan)?);
eprintln!("(dry run — nothing indexed)");
@@ -2129,6 +2160,20 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
cat_buf.extend_from_slice(action.to_string().as_bytes());
cat_buf.push(b'\n');
}
+ // Junk/skipped files that left the corpus (#238). Deletes lead the buffer
+ // so a key that is re-reported in the same run — a junk file whose bytes
+ // changed, a legacy plan key rewritten under the current scheme — is
+ // deleted before its replacement document is indexed, never after.
+ let mut swept_junk_keys: Vec<&str> = stale_junk_keys.iter().map(String::as_str).collect();
+ swept_junk_keys.sort_unstable();
+ for key in &swept_junk_keys {
+ let action = json!({"delete": {
+ "_index": catalog::CATALOG_INDEX,
+ "_id": catalog::file_id(key),
+ }});
+ cat_buf.extend_from_slice(action.to_string().as_bytes());
+ cat_buf.push(b'\n');
+ }
// dataset docs
let mut junk_records_by_run: u64 = junk_records.load(Ordering::Relaxed);
@@ -2217,9 +2262,17 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
}
}
let extra = extra_junk.into_inner().unwrap();
- let mut all_junk: Vec<&JunkFile> = plan.junk_files.iter().collect();
+ let mut all_junk: Vec<&JunkFile> = plan
+ .junk_files
+ .iter()
+ .filter(|junk| !stale_junk_keys.contains(&junk.file_key))
+ .collect();
all_junk.extend(extra.iter());
all_junk.extend(new_unplanned.iter());
+ // Counted now: every later reader of this number outlives the borrows
+ // `all_junk` holds on `plan` and `new_unplanned`, which the durable
+ // junk-plan update below mutates.
+ let junk_file_count = all_junk.len();
for jf in &all_junk {
let (id, doc) = catalog::file_doc(
&jf.file_key,
@@ -2309,7 +2362,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
"unique_content_files": files.len(),
"files_indexed": journal_mx.lock().unwrap().done.len(),
"duplicate_files": plan.duplicate_files.len(),
- "files_junk": all_junk.len(),
+ "files_junk": junk_file_count,
"records_total": total_records,
"junk_records_total": junk_records_by_run,
// This-run submission accounting (#195): what THIS invocation sent
@@ -2345,6 +2398,27 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
}
}
es.refresh(catalog::CATALOG_INDEX).ok();
+ // ── durable junk record, written last (#238) ─────────────────────────
+ //
+ // Only now is the plan allowed to claim what the catalog holds: the
+ // documents above are written and the swept ones deleted. A failed
+ // catalog bulk bailed out before this point with the plan untouched, so
+ // the next run recomputes the identical additions and removals and
+ // retries them — losing a record here is what makes a document immortal.
+ //
+ // Post-freeze skipped files are re-derived on every rerun, so this
+ // converges: once they are in the plan they are `planned_junk`, the
+ // additions are empty, and no further plan record is appended.
+ if !new_unplanned.is_empty() || !stale_junk_keys.is_empty() {
+ plan.junk_files
+ .retain(|junk| !stale_junk_keys.contains(&junk.file_key));
+ plan.junk_files.append(&mut new_unplanned);
+ journal_mx
+ .lock()
+ .unwrap()
+ .write_plan(&plan)
+ .context("record junk/skipped catalog entries in the resume plan")?;
+ }
journal_mx.lock().unwrap().finish(&run_doc)?;
// ── summary ──────────────────────────────────────────────────────────
@@ -2353,7 +2427,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
println!("{run_doc}");
} else if !cfg.quiet {
println!("\ndone in {wall:.1}s — {} datasets, {} records live, {} duplicate aliases, {} junk records, {} junk/skipped files",
- plan.datasets.len(), total_records, plan.duplicate_files.len(), junk_total_records, all_junk.len());
+ plan.datasets.len(), total_records, plan.duplicate_files.len(), junk_total_records, junk_file_count);
// Indexed-vs-submitted honesty line (#195): the live count against
// what this run actually submitted, so a silent-rejection mismatch
// is visible in the client output alone. Units differ on purpose: a
@@ -2397,7 +2471,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
cfg.url, cfg.prefix
);
}
- let code = if junk_total_records > 0 || !all_junk.is_empty() {
+ let code = if junk_total_records > 0 || junk_file_count > 0 {
3
} else {
0