AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): an edited file is a replacement, not a removal
xerj-org2308a8517dbe
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex a5775994d..62082ab0a 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -678,6 +678,125 @@ fn completed_plan_rejects_mixed_membership_delta_in_stable_order() {assert_eq!(vanished, ["a-old.csv", "b-old.csv"]);}+/// An in-place EDIT is a replacement, not a removal. `--fresh` is the route+/// the refusal itself recommends for picking up added and changed files, so+/// the gate must not classify the edited file's superseded content key as a+/// vanished group — the path is still there, and the pipeline republishes it.+#[test]+fn fresh_absorbs_an_edited_file_instead_of_calling_it_a_removal() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("notes.csv"), "id,value\n1,before\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(corpus.path().join("notes.csv"), "id,value\n1,after\n").unwrap();+ config.fresh = true;+ assert_eq!(+ run_index(config).unwrap(),+ 0,+ "an edited file is a same-path replacement, not a vanished content group"+ );+ let locked = endpoint.state.lock().unwrap();+ assert!(+ locked+ .docs+ .values()+ .any(|doc| doc["value"].as_str() == Some("after")),+ "the new content is live: {:?}",+ locked.docs+ );+ // Documented, and NOT what this gate is for: `--fresh` rebuilds the plan,+ // it does not reconcile the destination. Document ids derive from the+ // content key (`ids::doc_id`), so the pre-edit record stays live beside+ // the new one — the same outcome `--fresh` has always had. An ordinary+ // rerun is the clean route for an edit: it keeps the planned key and runs+ // a delete-before-replace transaction on it. What must never happen is+ // this run being REFUSED by a message claiming a file that is sitting in+ // the folder has vanished.+ assert!(+ locked+ .docs+ .values()+ .any(|doc| doc["value"].as_str() == Some("before")),+ "known --fresh gap: superseded records are not deleted: {:?}",+ locked.docs+ );+}++/// The clean route for the same edit: an ordinary rerun keeps the planned key+/// and replaces its records, so nothing is stranded. This is the contrast that+/// makes the `--fresh` gap above a documented trade-off rather than a silent+/// one.+#[test]+fn an_ordinary_rerun_replaces_an_edited_file_without_stranding_records() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("notes.csv"), "id,value\n1,before\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(corpus.path().join("notes.csv"), "id,value\n1,after\n").unwrap();+ assert_eq!(run_index(config).unwrap(), 0);+ let locked = endpoint.state.lock().unwrap();+ let values: Vec<&str> = locked+ .docs+ .values()+ .filter_map(|doc| doc["value"].as_str())+ .collect();+ assert_eq!(values, ["after"], "the superseded record is replaced");+}++/// The composite workflow the run's own stderr recommends: index, then add one+/// file and edit another, rerun (added file reported and skipped by the frozen+/// plan), then `--fresh` to absorb both.+#[test]+fn fresh_absorbs_an_addition_and_an_edit_after_a_reported_rerun() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("first.csv"), "id,value\n1,before\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ fs::write(corpus.path().join("first.csv"), "id,value\n1,after\n").unwrap();+ fs::write(corpus.path().join("second.csv"), "id,value\n2,second\n").unwrap();+ // 3 = completed-with-junk: the frozen plan cannot absorb the added file,+ // so the rerun reports it as skipped rather than refusing.+ assert_eq!(run_index(config.clone()).unwrap(), 3);++ config.fresh = true;+ assert_eq!(+ run_index(config).unwrap(),+ 0,+ "--fresh is the documented way out of exactly this state"+ );+ let locked = endpoint.state.lock().unwrap();+ let live: std::collections::HashSet<&str> = locked+ .docs+ .values()+ .filter_map(|doc| doc["ax_path"].as_str())+ .collect();+ assert!(live.contains("first.csv"), "{live:?}");+ assert!(live.contains("second.csv"), "{live:?}");+ assert!(+ locked+ .docs+ .values()+ .any(|doc| doc["value"].as_str() == Some("after")),+ "the edited file's new content is live: {:?}",+ locked.docs+ );+}+#[test]fn fresh_cannot_erase_the_plan_and_bypass_the_removal_gate() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex afae28b85..421d0682d 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -1282,6 +1282,27 @@ impl UnsupportedInventoryDelta {.map(String::as_str).chain(plan.junk_files.iter().map(|junk| junk.file_key.as_str())).collect();+ // Path identity, not just content identity. An in-place EDIT gives the+ // same file a new content key, so a key-only comparison reads the+ // superseded key as vanished and the new one as added — the SAME path+ // listed as both removed and added. That is a replacement, and the file+ // is still there to be republished, so it must never be refused.+ // Ordinary resume already reaches this conclusion by mapping each file+ // onto its planned key (`select_resume_plan_keys`); doing it here makes+ // `--fresh`, which has no plan to map through, agree — without making+ // the gate more fatal than the open it precedes.+ let current_rels: std::collections::HashSet<&str> =+ files.iter().map(|file| file.rel.as_str()).collect();+ let current_path_ids: std::collections::HashSet<&str> = files+ .iter()+ .map(|file| file.rel_id.as_str())+ .filter(|id| !id.is_empty())+ .collect();+ let path_survives = |assignment: &FileAssignment| {+ current_rels.contains(assignment.rel.as_str())+ || (!assignment.path_id.is_empty()+ && current_path_ids.contains(assignment.path_id.as_str()))+ };let mut added_content_groups: Vec<InventoryDeltaEntry> = files.iter()@@ -1295,7 +1316,9 @@ impl UnsupportedInventoryDelta {let mut vanished_content_groups: Vec<InventoryDeltaEntry> = plan.files.iter()- .filter(|(key, _)| !current_keys.contains(key.as_str()))+ .filter(|(key, assignment)| {+ !current_keys.contains(key.as_str()) && !path_survives(assignment)+ }).map(|(key, assignment)| InventoryDeltaEntry {file_key: key.clone(),path: assignment.rel.clone(),