AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): exclusive plan-key ownership, complete delete_by_query, honest recovery guidance
xerj-orgfd7e453299a4
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 6e45d0ab0..f90b9c03e 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -212,8 +212,24 @@ impl Es {/// Remove every document matching `query` before a file-level replacement./// Refresh is requested so a retry cannot observe or retain an older- /// locator set alongside the replacement.+ /// locator set alongside the replacement. The server executes ONE bounded+ /// search-and-delete pass per call (size-capped at 10k docs), so a single+ /// response is not complete removal: repeat until a pass deletes nothing.pub fn delete_by_query(&self, index: &str, query: &Value) -> Result<()> {+ const MAX_PASSES: usize = 1_000;+ for _ in 0..MAX_PASSES {+ if self.delete_by_query_pass(index, query)? == 0 {+ return Ok(());+ }+ }+ Err(anyhow!(+ "POST /{index}/_delete_by_query still reported deletions after {MAX_PASSES} passes; \+ refusing to treat the previous generation as fully removed"+ ))+ }++ /// One server-side delete pass; returns the reported `deleted` count.+ fn delete_by_query_pass(&self, index: &str, query: &Value) -> Result<u64> {self.with_retry("delete_by_query",|| {@@ -234,7 +250,7 @@ impl Es {.and_then(Value::as_array).is_none_or(Vec::is_empty){- Ok(())+ Ok(body.get("deleted").and_then(Value::as_u64).unwrap_or(0))} else {Err(anyhow!("POST /{index}/_delete_by_query failed: HTTP {status}: {body}"@@ -415,7 +431,10 @@ mod tests {}fn success(stream: &mut std::net::TcpStream) {- let body = br#"{"errors":false,"items":[]}"#;+ respond_json(stream, br#"{"errors":false,"items":[]}"#);+ }++ fn respond_json(stream: &mut std::net::TcpStream, body: &[u8]) {write!(stream,"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",@@ -496,6 +515,84 @@ mod tests {server.join().unwrap();}+ #[test]+ fn delete_by_query_repeats_until_a_pass_deletes_nothing() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let requests = Arc::new(Mutex::new(Vec::new()));+ let requests_server = requests.clone();+ let server = std::thread::spawn(move || {+ // A previous generation larger than one server pass: the first+ // pass removes its 10k cap and only the second reports done.+ for body in [+ br#"{"deleted":10000,"failures":[]}"#.as_slice(),+ br#"{"deleted":0,"failures":[]}"#.as_slice(),+ ] {+ let (mut stream, _) = listener.accept().unwrap();+ requests_server+ .lock()+ .unwrap()+ .push(read_request(&mut stream));+ respond_json(&mut stream, body);+ }+ });+ let es = Es::with_bulk_policy(+ &format!("http://{address}"),+ None,+ Duration::from_millis(100),+ Duration::from_millis(10),+ Duration::from_millis(20),+ )+ .unwrap();+ es.delete_by_query("data", &serde_json::json!({"term": {"ax_file": "key"}}))+ .unwrap();+ server.join().unwrap();+ let requests = requests.lock().unwrap();+ assert_eq!(requests.len(), 2);+ for request in requests.iter() {+ let text = String::from_utf8_lossy(request);+ assert!(text.contains("POST /data/_delete_by_query"), "{text}");+ }+ }++ #[test]+ fn delete_by_query_that_never_reaches_zero_fails_at_the_pass_cap() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let server = std::thread::spawn(move || {+ let mut served = 0usize;+ loop {+ let (mut stream, _) = listener.accept().unwrap();+ let request = read_request(&mut stream);+ if request.starts_with(b"STOP") {+ break;+ }+ respond_json(&mut stream, br#"{"deleted":10000,"failures":[]}"#);+ served += 1;+ }+ served+ });+ let es = Es::with_bulk_policy(+ &format!("http://{address}"),+ None,+ Duration::from_millis(100),+ Duration::from_millis(10),+ Duration::from_millis(20),+ )+ .unwrap();+ let error = es+ .delete_by_query("data", &serde_json::json!({"term": {"ax_file": "key"}}))+ .unwrap_err();+ assert!(+ format!("{error:#}").contains("still reported deletions"),+ "{error:#}"+ );+ let mut stop = std::net::TcpStream::connect(address).unwrap();+ stop.write_all(b"STOP\r\n\r\n").unwrap();+ drop(stop);+ assert_eq!(server.join().unwrap(), 1_000);+ }+#[test]fn ambiguous_dropped_response_retries_byte_identical_bulk_body() {let listener = TcpListener::bind("127.0.0.1:0").unwrap();diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 9e3d0c612..765ff8d41 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -320,7 +320,7 @@ fn legacy_plan_without_completion_cleans_possible_partial_visibility() {}#[test]-fn legacy_key_collision_fails_before_visibility_and_points_to_fresh() {+fn legacy_key_collision_fails_before_visibility_with_scoped_guidance() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();let state_dir = tempfile::tempdir().unwrap();@@ -371,7 +371,14 @@ fn legacy_key_collision_fails_before_visibility_and_points_to_fresh() {let error = run_index(config).unwrap_err();let message = format!("{error:#}");assert!(message.contains("collides with legacy resume key"));- assert!(message.contains("rerun with --fresh"));+ assert!(message.contains("remove or move one of these two files"));+ assert!(message.contains(+ &state_dir+ .path()+ .join("journal.ndjson")+ .display()+ .to_string()+ ));let state = endpoint.state.lock().unwrap();assert_eq!(state.data_bulk_number, bulks_before);assert_eq!(state.delete_calls, deletes_before);@@ -642,6 +649,105 @@ fn pending_generation_b_is_superseded_when_source_changes_to_c() {.all(|doc| doc["value"].as_str() == Some("generation-c")));}+#[test]+fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let original = "id,value\n0,original\n1,original\n";+ fs::write(corpus.path().join("a.csv"), original).unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);++ // a.csv keeps its planned key by path while its OLD bytes reappear as+ // b.csv — whose content key is exactly the planned key a.csv claims. Both+ // files owning one ax_file key would let each replacement delete the+ // other's freshly published documents.+ fs::write(+ corpus.path().join("a.csv"),+ "id,value\n0,rewritten\n1,rewritten\n",+ )+ .unwrap();+ fs::write(corpus.path().join("b.csv"), original).unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 3);+ {+ let locked = endpoint.state.lock().unwrap();+ assert_eq!(locked.docs.len(), 2);+ assert!(locked.docs.values().all(|doc| {+ doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("rewritten")+ }));+ }+ let replay = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert!(replay.pending_replacements.is_empty());+ assert_eq!(replay.done.len(), 1);+ let plan = replay.plan.clone().unwrap();+ assert_eq!(plan.junk_files.len(), 1);+ assert_eq!(plan.junk_files[0].rel, "b.csv");+ assert!(plan.junk_files[0]+ .reason+ .contains("key ownership exclusive"));+ drop(replay);++ // The divergence is durable and deterministic: an identical rerun keeps+ // the same owner, appends no new plan, and changes no documents.+ let plans_before = event_count(state_dir.path(), "plan");+ assert_eq!(run_index(config).unwrap(), 3);+ assert_eq!(event_count(state_dir.path(), "plan"), plans_before);+ let locked = endpoint.state.lock().unwrap();+ assert_eq!(locked.docs.len(), 2);+ assert!(locked+ .docs+ .values()+ .all(|doc| doc["value"].as_str() == Some("rewritten")));+}++#[test]+fn deleting_an_entire_duplicate_group_strands_no_pending_replacement() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("dup-a.csv"), "id,value\n0,dup\n1,dup\n").unwrap();+ fs::write(corpus.path().join("dup-b.csv"), "id,value\n0,dup\n1,dup\n").unwrap();+ fs::write(corpus.path().join("keep.csv"), "id,value\n2,keep\n3,keep\n").unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ let starts = event_count(state_dir.path(), "file_replace_start");++ // The whole duplicate group disappears. There is no current file left to+ // republish its key, so no replacement intent may be journaled — a+ // stranded intent would be re-appended forever without ever committing.+ fs::remove_file(corpus.path().join("dup-a.csv")).unwrap();+ fs::remove_file(corpus.path().join("dup-b.csv")).unwrap();+ for _ in 0..2 {+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ assert_eq!(+ event_count(state_dir.path(), "file_replace_start"),+ starts,+ "an orphaned duplicate-group key must not schedule a replacement"+ );+ let replay = state::Journal::open(+ state_dir.path(),+ &config.root.to_string_lossy(),+ &config.url,+ &config.prefix,+ config.bulk_timeout_secs,+ false,+ )+ .unwrap();+ assert!(replay.pending_replacements.is_empty());+ }+}+#[test]fn partial_file_done_is_rolled_back_before_another_worker_commits() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex a346d6124..0aa3f3380 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -254,6 +254,7 @@ fn select_resume_plan_keys(files: &[walk::FileEntry],content_keys: &[String],plan: &Plan,+ journal_path: &Path,) -> Result<Vec<Option<String>>> {let mut planned_by_rel: HashMap<&str, &str> = HashMap::new();let mut planned_by_path_id: HashMap<&str, &str> = HashMap::new();@@ -308,13 +309,28 @@ fn select_resume_plan_keys(if claimed.contains(legacy_key.as_str()) || has_exact_current_owner {anyhow::bail!("{} collides with legacy resume key {} already owned by {}. No documents \- were changed; rerun with --fresh to build distinct full-content keys",+ were changed; remove or move one of these two files out of the corpus \+ and rerun — every other file keeps its resume state. Deleting the \+ journal at {} (or rerunning with --fresh) also clears the collision, \+ but re-extracts and re-embeds the entire corpus",file.rel,legacy_key,- assignment.rel+ assignment.rel,+ journal_path.display());}Some(legacy_key)+ } else if claimed.contains(content_key.as_str()) {+ // Another current file already owns this planned key. Ownership+ // must stay exclusive — two owners would each run the+ // replacement transaction on one ax_file key and delete each+ // other's freshly published documents. Divert this file to a+ // deterministic path-derived key, the same discriminator scheme+ // content::resolve uses for byte-proven digest collisions.+ Some(format!(+ "{content_key}-claimed-{:032x}",+ xxhash_rust::xxh3::xxh3_128(file.rel_id.as_bytes())+ ))} else {None}@@ -416,6 +432,7 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {// filesystems. A metadata-only shortcut could leave stale live documents// forever after a same-size rewrite with restored or stale timestamps.let mut inventory = content::resolve(discovered_files)?;+ let journal_path = journal.path().to_path_buf();let mut content_changed = std::collections::HashSet::new();let mut stale_alias_ids = Vec::new();let mut alias_paths_to_replace = std::collections::HashSet::new();@@ -431,10 +448,44 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {.iter().map(|old| catalog::duplicate_file_id(&old.file_key, &old.rel, &old.path_id)),);- let selected_plan_keys = select_resume_plan_keys(&inventory.files, &inventory.keys, plan)?;+ let selected_plan_keys =+ select_resume_plan_keys(&inventory.files, &inventory.keys, plan, &journal_path)?;for (index, planned_key) in selected_plan_keys.into_iter().enumerate() {let file = &inventory.files[index];if let Some(planned_key) = planned_key {+ if !plan.files.contains_key(&planned_key) {+ // The file was diverted off a planned key exclusively+ // owned by another current file. Record the divergence in+ // the durable plan so every resume deterministically skips+ // this path instead of racing one ax_file key.+ if !plan+ .junk_files+ .iter()+ .any(|junk| junk.file_key == planned_key)+ {+ let owner = plan+ .files+ .get(&inventory.keys[index])+ .map(|assignment| assignment.rel.as_str())+ .unwrap_or("another file");+ plan.junk_files.push(JunkFile {+ file_key: planned_key.clone(),+ rel: file.rel.clone(),+ format: "unknown".into(),+ status: "skipped".into(),+ reason: format!(+ "content resolves to planned key {} owned by {owner}; skipped to \+ keep key ownership exclusive (remove one of the two files and \+ rerun to index the survivor)",+ inventory.keys[index]+ ),+ bytes: file.size,+ });+ plan_changed = true;+ }+ inventory.keys[index] = planned_key;+ continue;+ }let assignment = plan.files.get_mut(&planned_key).expect("planned key");if assignment.rel != file.rel {content_changed.insert(planned_key.clone());@@ -480,11 +531,20 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {// The historical global flag cannot identify which live documents// already carry ax_paths. Its one-time migration must rewrite every// canonical key; ordinary alias changes remain scoped per key.- content_changed.extend(alias_keys_to_reindex(- &previous_aliases,- &inventory.duplicates,- needs_alias_path_migration.then_some(inventory.keys.as_slice()),- ));+ // A key whose entire duplicate group was deleted has no current file+ // to republish; scheduling it would strand a pending replacement that+ // every later run re-journals without ever committing.+ let current_keys: std::collections::HashSet<&str> =+ inventory.keys.iter().map(String::as_str).collect();+ content_changed.extend(+ alias_keys_to_reindex(+ &previous_aliases,+ &inventory.duplicates,+ needs_alias_path_migration.then_some(inventory.keys.as_slice()),+ )+ .into_iter()+ .filter(|key| current_keys.contains(key.as_str())),+ );if needs_alias_path_migration {plan.alias_paths_indexed = true;plan_changed = true;@@ -1632,10 +1692,57 @@ mod duplicate_integration_tests {let mut plan = Plan::default();plan.files.insert(legacy.clone(), legacy_assignment("b.txt"));- let error = select_resume_plan_keys(&inventory.files, &inventory.keys, &plan).unwrap_err();+ let error = select_resume_plan_keys(+ &inventory.files,+ &inventory.keys,+ &plan,+ Path::new("/state/journal.ndjson"),+ )+ .unwrap_err();let message = format!("{error:#}");assert!(message.contains("collides with legacy resume key"));- assert!(message.contains("rerun with --fresh"));+ // Recovery advice must stay scoped to the two colliding files and be+ // honest that discarding the journal re-embeds the whole corpus.+ assert!(message.contains("remove or move one of these two files"));+ assert!(message.contains("/state/journal.ndjson"));+ assert!(message.contains("re-extracts and re-embeds the entire corpus"));+ }++ #[test]+ fn planned_key_claimed_by_path_diverts_the_content_claimant_deterministically() {+ let corpus = tempfile::tempdir().unwrap();+ // a.txt was planned under its old digest; its content has since+ // changed, while b.txt now holds exactly the bytes a.txt was planned+ // with — so b.txt's content key IS the planned key a.txt claims by rel.+ fs::write(corpus.path().join("a.txt"), b"rewritten content\n").unwrap();+ fs::write(corpus.path().join("b.txt"), b"original planned content\n").unwrap();+ let inventory = content::resolve(walk::walk(corpus.path(), false).unwrap()).unwrap();+ let planned_key = inventory.keys[1].clone();+ let mut plan = Plan::default();+ plan.files+ .insert(planned_key.clone(), legacy_assignment("a.txt"));++ let selected = select_resume_plan_keys(+ &inventory.files,+ &inventory.keys,+ &plan,+ Path::new("/state/journal.ndjson"),+ )+ .unwrap();+ assert_eq!(selected[0].as_deref(), Some(planned_key.as_str()));+ let diverted = selected[1].as_deref().expect("diverted key");+ assert_ne!(diverted, planned_key);+ assert!(diverted.starts_with(&format!("{planned_key}-claimed-")));+ // The divergence is a pure function of (digest, path identity):+ // resumes select the same exclusive owner and the same diverted key.+ let again = select_resume_plan_keys(+ &inventory.files,+ &inventory.keys,+ &plan,+ Path::new("/state/journal.ndjson"),+ )+ .unwrap();+ assert_eq!(selected, again);}#[test]diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex bee9c444f..363fceb70 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -312,7 +312,12 @@ impl Journal {anyhow::bail!("journal corruption at byte {record_start} in {}: malformed \newline-terminated record ({error}). Refusing to discard records \- after corruption; restore the journal or use --fresh",+ after corruption. Restore the journal from a backup, or truncate it \+ to exactly {record_start} bytes to keep every completion recorded \+ before the corruption (files journaled after that point are \+ re-verified, re-indexed and re-embedded on the next run). Deleting \+ the whole journal (or rerunning with --fresh) also recovers, but \+ re-extracts and re-embeds the entire corpus",jpath.display());}@@ -674,7 +679,10 @@ mod compatibility_tests {.expect("corruption must fail closed");let message = format!("{error:#}");assert!(message.contains("journal corruption"));- assert!(message.contains("use --fresh"));+ // Recovery guidance must stay scoped and honest: byte-exact truncation+ // keeps prior completions; discarding the journal re-embeds everything.+ assert!(message.contains("truncate it to exactly"));+ assert!(message.contains("re-extracts and re-embeds the entire corpus"));}#[test]