Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): let the #238 junk sweep run instead of refusing it

xerj-org45e2634d63cb
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c455cc26..cd45b230f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -543,7 +543,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
indices and the state directory, or rebuild under a new state directory,
prefix and brain. `--fresh` is refused for the same case, because it does
not delete those documents either. `--json` emits the same facts as
- `xerj.autoindex.unsupported_sync_delta.v1` on stdout with exit 1.
+ `xerj.autoindex.unsupported_sync_delta.v1` on stdout with exit 1. Deleting a
+ file that was only ever *skipped* is not refused: a junk file publishes no
+ documents, and the stale junk-catalog sweep removes its one catalog row, so
+ nothing is stranded.
- **Files added after a plan was frozen are now called out on stderr.** They
are still not indexed by a rerun that resumes an existing plan — the plan is
a crash-resume boundary, not a folder-sync generation — but the run now
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 98c34becd..a5775994d 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -748,7 +748,7 @@ fn unchanged_planned_junk_is_not_reported_as_added_content() {
}
#[test]
-fn deleted_planned_junk_fails_closed_before_catalog_can_stay_stale() {
+fn deleted_planned_junk_is_swept_rather_than_refused() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
@@ -763,8 +763,33 @@ fn deleted_planned_junk_fails_closed_before_catalog_can_stay_stale() {
let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
assert_eq!(run_index(config.clone()).unwrap(), 3);
+ // A junk file publishes exactly one catalog row and nothing else, and the
+ // #238 sweep deletes that row. Nothing is stranded, so the removal gate
+ // must not fire — refusing here would block a case the pipeline handles
+ // completely. Only a file that published DOCUMENTS refuses a rerun.
fs::remove_file(corpus.path().join("opaque.bin")).unwrap();
- assert_unsupported_delta_without_remote_mutation(&endpoint, config, &[], &["opaque.bin"]);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ {
+ let locked = endpoint.state.lock().unwrap();
+ assert!(
+ locked
+ .catalog_docs
+ .values()
+ .all(|doc| doc["path"].as_str() != Some("opaque.bin")),
+ "the deleted junk file's catalog row must be swept, not left immortal"
+ );
+ assert!(
+ locked
+ .catalog_docs
+ .values()
+ .any(|doc| doc["path"].as_str() == Some("rows.csv")),
+ "the surviving indexed file keeps its entry"
+ );
+ }
+
+ // Deleting the INDEXED file is still refused: its documents are live.
+ fs::remove_file(corpus.path().join("rows.csv")).unwrap();
+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &[], &["rows.csv"]);
}
#[test]
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 923f84b95..afae28b85 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -1301,15 +1301,14 @@ impl UnsupportedInventoryDelta {
path: assignment.rel.clone(),
})
.collect();
- vanished_content_groups.extend(
- plan.junk_files
- .iter()
- .filter(|junk| !current_keys.contains(junk.file_key.as_str()))
- .map(|junk| InventoryDeltaEntry {
- file_key: junk.file_key.clone(),
- path: junk.rel.clone(),
- }),
- );
+ // Deliberately NOT extended with `plan.junk_files`. A junk/skipped file
+ // published no documents, no aliases and no graph edges — its entire
+ // live footprint is one `file:{key}` catalog row, and the stale
+ // junk-catalog sweep below (#238) deletes that row before dropping the
+ // plan entry. Removing it therefore strands nothing, so refusing the
+ // rerun would block a case the pipeline now handles completely. Junk
+ // keys stay in `durable_keys` above: an unchanged skipped file is still
+ // not an addition.
let stable_order = |left: &InventoryDeltaEntry, right: &InventoryDeltaEntry| {
left.path