Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): exclusive plan-key ownership, complete delete_by_query, honest recovery guidance

xerj-orgfd7e453299a4
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 6e45d0ab0..f90b9c03e 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -212,8 +212,24 @@ impl Es {
/// Remove every document matching `query` before a file-level replacement.
/// Refresh is requested so a retry cannot observe or retain an older
- /// locator set alongside the replacement.
+ /// locator set alongside the replacement. The server executes ONE bounded
+ /// search-and-delete pass per call (size-capped at 10k docs), so a single
+ /// response is not complete removal: repeat until a pass deletes nothing.
pub fn delete_by_query(&self, index: &str, query: &Value) -> Result<()> {
+ const MAX_PASSES: usize = 1_000;
+ for _ in 0..MAX_PASSES {
+ if self.delete_by_query_pass(index, query)? == 0 {
+ return Ok(());
+ }
+ }
+ Err(anyhow!(
+ "POST /{index}/_delete_by_query still reported deletions after {MAX_PASSES} passes; \
+ refusing to treat the previous generation as fully removed"
+ ))
+ }
+
+ /// One server-side delete pass; returns the reported `deleted` count.
+ fn delete_by_query_pass(&self, index: &str, query: &Value) -> Result<u64> {
self.with_retry(
"delete_by_query",
|| {
@@ -234,7 +250,7 @@ impl Es {
.and_then(Value::as_array)
.is_none_or(Vec::is_empty)
{
- Ok(())
+ Ok(body.get("deleted").and_then(Value::as_u64).unwrap_or(0))
} else {
Err(anyhow!(
"POST /{index}/_delete_by_query failed: HTTP {status}: {body}"
@@ -415,7 +431,10 @@ mod tests {
}
fn success(stream: &mut std::net::TcpStream) {
- let body = br#"{"errors":false,"items":[]}"#;
+ respond_json(stream, br#"{"errors":false,"items":[]}"#);
+ }
+
+ fn respond_json(stream: &mut std::net::TcpStream, body: &[u8]) {
write!(
stream,
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
@@ -496,6 +515,84 @@ mod tests {
server.join().unwrap();
}
+ #[test]
+ fn delete_by_query_repeats_until_a_pass_deletes_nothing() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let requests = Arc::new(Mutex::new(Vec::new()));
+ let requests_server = requests.clone();
+ let server = std::thread::spawn(move || {
+ // A previous generation larger than one server pass: the first
+ // pass removes its 10k cap and only the second reports done.
+ for body in [
+ br#"{"deleted":10000,"failures":[]}"#.as_slice(),
+ br#"{"deleted":0,"failures":[]}"#.as_slice(),
+ ] {
+ let (mut stream, _) = listener.accept().unwrap();
+ requests_server
+ .lock()
+ .unwrap()
+ .push(read_request(&mut stream));
+ respond_json(&mut stream, body);
+ }
+ });
+ let es = Es::with_bulk_policy(
+ &format!("http://{address}"),
+ None,
+ Duration::from_millis(100),
+ Duration::from_millis(10),
+ Duration::from_millis(20),
+ )
+ .unwrap();
+ es.delete_by_query("data", &serde_json::json!({"term": {"ax_file": "key"}}))
+ .unwrap();
+ server.join().unwrap();
+ let requests = requests.lock().unwrap();
+ assert_eq!(requests.len(), 2);
+ for request in requests.iter() {
+ let text = String::from_utf8_lossy(request);
+ assert!(text.contains("POST /data/_delete_by_query"), "{text}");
+ }
+ }
+
+ #[test]
+ fn delete_by_query_that_never_reaches_zero_fails_at_the_pass_cap() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = std::thread::spawn(move || {
+ let mut served = 0usize;
+ loop {
+ let (mut stream, _) = listener.accept().unwrap();
+ let request = read_request(&mut stream);
+ if request.starts_with(b"STOP") {
+ break;
+ }
+ respond_json(&mut stream, br#"{"deleted":10000,"failures":[]}"#);
+ served += 1;
+ }
+ served
+ });
+ let es = Es::with_bulk_policy(
+ &format!("http://{address}"),
+ None,
+ Duration::from_millis(100),
+ Duration::from_millis(10),
+ Duration::from_millis(20),
+ )
+ .unwrap();
+ let error = es
+ .delete_by_query("data", &serde_json::json!({"term": {"ax_file": "key"}}))
+ .unwrap_err();
+ assert!(
+ format!("{error:#}").contains("still reported deletions"),
+ "{error:#}"
+ );
+ let mut stop = std::net::TcpStream::connect(address).unwrap();
+ stop.write_all(b"STOP\r\n\r\n").unwrap();
+ drop(stop);
+ assert_eq!(server.join().unwrap(), 1_000);
+ }
+
#[test]
fn ambiguous_dropped_response_retries_byte_identical_bulk_body() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 9e3d0c612..765ff8d41 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -320,7 +320,7 @@ fn legacy_plan_without_completion_cleans_possible_partial_visibility() {
}
#[test]
-fn legacy_key_collision_fails_before_visibility_and_points_to_fresh() {
+fn legacy_key_collision_fails_before_visibility_with_scoped_guidance() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
let state_dir = tempfile::tempdir().unwrap();
@@ -371,7 +371,14 @@ fn legacy_key_collision_fails_before_visibility_and_points_to_fresh() {
let error = run_index(config).unwrap_err();
let message = format!("{error:#}");
assert!(message.contains("collides with legacy resume key"));
- assert!(message.contains("rerun with --fresh"));
+ assert!(message.contains("remove or move one of these two files"));
+ assert!(message.contains(
+ &state_dir
+ .path()
+ .join("journal.ndjson")
+ .display()
+ .to_string()
+ ));
let state = endpoint.state.lock().unwrap();
assert_eq!(state.data_bulk_number, bulks_before);
assert_eq!(state.delete_calls, deletes_before);
@@ -642,6 +649,105 @@ fn pending_generation_b_is_superseded_when_source_changes_to_c() {
.all(|doc| doc["value"].as_str() == Some("generation-c")));
}
+#[test]
+fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let original = "id,value\n0,original\n1,original\n";
+ fs::write(corpus.path().join("a.csv"), original).unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ // a.csv keeps its planned key by path while its OLD bytes reappear as
+ // b.csv — whose content key is exactly the planned key a.csv claims. Both
+ // files owning one ax_file key would let each replacement delete the
+ // other's freshly published documents.
+ fs::write(
+ corpus.path().join("a.csv"),
+ "id,value\n0,rewritten\n1,rewritten\n",
+ )
+ .unwrap();
+ fs::write(corpus.path().join("b.csv"), original).unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 3);
+ {
+ let locked = endpoint.state.lock().unwrap();
+ assert_eq!(locked.docs.len(), 2);
+ assert!(locked.docs.values().all(|doc| {
+ doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("rewritten")
+ }));
+ }
+ let replay = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert!(replay.pending_replacements.is_empty());
+ assert_eq!(replay.done.len(), 1);
+ let plan = replay.plan.clone().unwrap();
+ assert_eq!(plan.junk_files.len(), 1);
+ assert_eq!(plan.junk_files[0].rel, "b.csv");
+ assert!(plan.junk_files[0]
+ .reason
+ .contains("key ownership exclusive"));
+ drop(replay);
+
+ // The divergence is durable and deterministic: an identical rerun keeps
+ // the same owner, appends no new plan, and changes no documents.
+ let plans_before = event_count(state_dir.path(), "plan");
+ assert_eq!(run_index(config).unwrap(), 3);
+ assert_eq!(event_count(state_dir.path(), "plan"), plans_before);
+ let locked = endpoint.state.lock().unwrap();
+ assert_eq!(locked.docs.len(), 2);
+ assert!(locked
+ .docs
+ .values()
+ .all(|doc| doc["value"].as_str() == Some("rewritten")));
+}
+
+#[test]
+fn deleting_an_entire_duplicate_group_strands_no_pending_replacement() {
+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("dup-a.csv"), "id,value\n0,dup\n1,dup\n").unwrap();
+ fs::write(corpus.path().join("dup-b.csv"), "id,value\n0,dup\n1,dup\n").unwrap();
+ fs::write(corpus.path().join("keep.csv"), "id,value\n2,keep\n3,keep\n").unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let starts = event_count(state_dir.path(), "file_replace_start");
+
+ // The whole duplicate group disappears. There is no current file left to
+ // republish its key, so no replacement intent may be journaled — a
+ // stranded intent would be re-appended forever without ever committing.
+ fs::remove_file(corpus.path().join("dup-a.csv")).unwrap();
+ fs::remove_file(corpus.path().join("dup-b.csv")).unwrap();
+ for _ in 0..2 {
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ assert_eq!(
+ event_count(state_dir.path(), "file_replace_start"),
+ starts,
+ "an orphaned duplicate-group key must not schedule a replacement"
+ );
+ let replay = state::Journal::open(
+ state_dir.path(),
+ &config.root.to_string_lossy(),
+ &config.url,
+ &config.prefix,
+ config.bulk_timeout_secs,
+ false,
+ )
+ .unwrap();
+ assert!(replay.pending_replacements.is_empty());
+ }
+}
+
#[test]
fn partial_file_done_is_rolled_back_before_another_worker_commits() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index a346d6124..0aa3f3380 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -254,6 +254,7 @@ fn select_resume_plan_keys(
files: &[walk::FileEntry],
content_keys: &[String],
plan: &Plan,
+ journal_path: &Path,
) -> Result<Vec<Option<String>>> {
let mut planned_by_rel: HashMap<&str, &str> = HashMap::new();
let mut planned_by_path_id: HashMap<&str, &str> = HashMap::new();
@@ -308,13 +309,28 @@ fn select_resume_plan_keys(
if claimed.contains(legacy_key.as_str()) || has_exact_current_owner {
anyhow::bail!(
"{} collides with legacy resume key {} already owned by {}. No documents \
- were changed; rerun with --fresh to build distinct full-content keys",
+ were changed; remove or move one of these two files out of the corpus \
+ and rerun — every other file keeps its resume state. Deleting the \
+ journal at {} (or rerunning with --fresh) also clears the collision, \
+ but re-extracts and re-embeds the entire corpus",
file.rel,
legacy_key,
- assignment.rel
+ assignment.rel,
+ journal_path.display()
);
}
Some(legacy_key)
+ } else if claimed.contains(content_key.as_str()) {
+ // Another current file already owns this planned key. Ownership
+ // must stay exclusive — two owners would each run the
+ // replacement transaction on one ax_file key and delete each
+ // other's freshly published documents. Divert this file to a
+ // deterministic path-derived key, the same discriminator scheme
+ // content::resolve uses for byte-proven digest collisions.
+ Some(format!(
+ "{content_key}-claimed-{:032x}",
+ xxhash_rust::xxh3::xxh3_128(file.rel_id.as_bytes())
+ ))
} else {
None
}
@@ -416,6 +432,7 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
// filesystems. A metadata-only shortcut could leave stale live documents
// forever after a same-size rewrite with restored or stale timestamps.
let mut inventory = content::resolve(discovered_files)?;
+ let journal_path = journal.path().to_path_buf();
let mut content_changed = std::collections::HashSet::new();
let mut stale_alias_ids = Vec::new();
let mut alias_paths_to_replace = std::collections::HashSet::new();
@@ -431,10 +448,44 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
.iter()
.map(|old| catalog::duplicate_file_id(&old.file_key, &old.rel, &old.path_id)),
);
- let selected_plan_keys = select_resume_plan_keys(&inventory.files, &inventory.keys, plan)?;
+ let selected_plan_keys =
+ select_resume_plan_keys(&inventory.files, &inventory.keys, plan, &journal_path)?;
for (index, planned_key) in selected_plan_keys.into_iter().enumerate() {
let file = &inventory.files[index];
if let Some(planned_key) = planned_key {
+ if !plan.files.contains_key(&planned_key) {
+ // The file was diverted off a planned key exclusively
+ // owned by another current file. Record the divergence in
+ // the durable plan so every resume deterministically skips
+ // this path instead of racing one ax_file key.
+ if !plan
+ .junk_files
+ .iter()
+ .any(|junk| junk.file_key == planned_key)
+ {
+ let owner = plan
+ .files
+ .get(&inventory.keys[index])
+ .map(|assignment| assignment.rel.as_str())
+ .unwrap_or("another file");
+ plan.junk_files.push(JunkFile {
+ file_key: planned_key.clone(),
+ rel: file.rel.clone(),
+ format: "unknown".into(),
+ status: "skipped".into(),
+ reason: format!(
+ "content resolves to planned key {} owned by {owner}; skipped to \
+ keep key ownership exclusive (remove one of the two files and \
+ rerun to index the survivor)",
+ inventory.keys[index]
+ ),
+ bytes: file.size,
+ });
+ plan_changed = true;
+ }
+ inventory.keys[index] = planned_key;
+ continue;
+ }
let assignment = plan.files.get_mut(&planned_key).expect("planned key");
if assignment.rel != file.rel {
content_changed.insert(planned_key.clone());
@@ -480,11 +531,20 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
// The historical global flag cannot identify which live documents
// already carry ax_paths. Its one-time migration must rewrite every
// canonical key; ordinary alias changes remain scoped per key.
- content_changed.extend(alias_keys_to_reindex(
- &previous_aliases,
- &inventory.duplicates,
- needs_alias_path_migration.then_some(inventory.keys.as_slice()),
- ));
+ // A key whose entire duplicate group was deleted has no current file
+ // to republish; scheduling it would strand a pending replacement that
+ // every later run re-journals without ever committing.
+ let current_keys: std::collections::HashSet<&str> =
+ inventory.keys.iter().map(String::as_str).collect();
+ content_changed.extend(
+ alias_keys_to_reindex(
+ &previous_aliases,
+ &inventory.duplicates,
+ needs_alias_path_migration.then_some(inventory.keys.as_slice()),
+ )
+ .into_iter()
+ .filter(|key| current_keys.contains(key.as_str())),
+ );
if needs_alias_path_migration {
plan.alias_paths_indexed = true;
plan_changed = true;
@@ -1632,10 +1692,57 @@ mod duplicate_integration_tests {
let mut plan = Plan::default();
plan.files
.insert(legacy.clone(), legacy_assignment("b.txt"));
- let error = select_resume_plan_keys(&inventory.files, &inventory.keys, &plan).unwrap_err();
+ let error = select_resume_plan_keys(
+ &inventory.files,
+ &inventory.keys,
+ &plan,
+ Path::new("/state/journal.ndjson"),
+ )
+ .unwrap_err();
let message = format!("{error:#}");
assert!(message.contains("collides with legacy resume key"));
- assert!(message.contains("rerun with --fresh"));
+ // Recovery advice must stay scoped to the two colliding files and be
+ // honest that discarding the journal re-embeds the whole corpus.
+ assert!(message.contains("remove or move one of these two files"));
+ assert!(message.contains("/state/journal.ndjson"));
+ assert!(message.contains("re-extracts and re-embeds the entire corpus"));
+ }
+
+ #[test]
+ fn planned_key_claimed_by_path_diverts_the_content_claimant_deterministically() {
+ let corpus = tempfile::tempdir().unwrap();
+ // a.txt was planned under its old digest; its content has since
+ // changed, while b.txt now holds exactly the bytes a.txt was planned
+ // with — so b.txt's content key IS the planned key a.txt claims by rel.
+ fs::write(corpus.path().join("a.txt"), b"rewritten content\n").unwrap();
+ fs::write(corpus.path().join("b.txt"), b"original planned content\n").unwrap();
+ let inventory = content::resolve(walk::walk(corpus.path(), false).unwrap()).unwrap();
+ let planned_key = inventory.keys[1].clone();
+ let mut plan = Plan::default();
+ plan.files
+ .insert(planned_key.clone(), legacy_assignment("a.txt"));
+
+ let selected = select_resume_plan_keys(
+ &inventory.files,
+ &inventory.keys,
+ &plan,
+ Path::new("/state/journal.ndjson"),
+ )
+ .unwrap();
+ assert_eq!(selected[0].as_deref(), Some(planned_key.as_str()));
+ let diverted = selected[1].as_deref().expect("diverted key");
+ assert_ne!(diverted, planned_key);
+ assert!(diverted.starts_with(&format!("{planned_key}-claimed-")));
+ // The divergence is a pure function of (digest, path identity):
+ // resumes select the same exclusive owner and the same diverted key.
+ let again = select_resume_plan_keys(
+ &inventory.files,
+ &inventory.keys,
+ &plan,
+ Path::new("/state/journal.ndjson"),
+ )
+ .unwrap();
+ assert_eq!(selected, again);
}
#[test]
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index bee9c444f..363fceb70 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -312,7 +312,12 @@ impl Journal {
anyhow::bail!(
"journal corruption at byte {record_start} in {}: malformed \
newline-terminated record ({error}). Refusing to discard records \
- after corruption; restore the journal or use --fresh",
+ after corruption. Restore the journal from a backup, or truncate it \
+ to exactly {record_start} bytes to keep every completion recorded \
+ before the corruption (files journaled after that point are \
+ re-verified, re-indexed and re-embedded on the next run). Deleting \
+ the whole journal (or rerunning with --fresh) also recovers, but \
+ re-extracts and re-embeds the entire corpus",
jpath.display()
);
}
@@ -674,7 +679,10 @@ mod compatibility_tests {
.expect("corruption must fail closed");
let message = format!("{error:#}");
assert!(message.contains("journal corruption"));
- assert!(message.contains("use --fresh"));
+ // Recovery guidance must stay scoped and honest: byte-exact truncation
+ // keeps prior completions; discarding the journal re-embeds everything.
+ assert!(message.contains("truncate it to exactly"));
+ assert!(message.contains("re-extracts and re-embeds the entire corpus"));
}
#[test]