AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): prefix-scope the exclusion catalog sweeps so a sibling corpus is not over-deleted (#737)
xerj-org0ea096a83017
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rsindex 9ab7b8aec..1790c1464 100644--- a/engine/crates/xerj-autoindex/src/catalog.rs+++ b/engine/crates/xerj-autoindex/src/catalog.rs@@ -108,6 +108,12 @@ pub fn catalog_mapping() -> Value {json!({"type": "keyword"}),);properties.insert("junk_records_this_run".into(), json!({"type": "long"}));+ // #737: corpus scope, so a delete_by_query can constrain a sweep to THIS+ // corpus's docs. The doc `_id`s are already prefix-scoped (`file:{prefix}:…`)+ // but `_id` is not term-queryable; a byte-identical file shared with a live+ // sibling corpus would otherwise be caught by an unscoped `file_key`/`path`+ // sweep. Written by `file_doc`/`duplicate_file_doc` (the sweep's targets).+ properties.insert("prefix".into(), json!({"type": "keyword"}));mapping}@@ -193,6 +199,7 @@ pub fn file_doc(file_id(prefix, file_key),json!({"doc_kind": "file",+ "prefix": prefix, // #737: corpus scope for a scoped exclusion sweep"file_key": file_key,"path": path,"format": format,@@ -220,6 +227,7 @@ pub fn duplicate_file_doc(alias_id,json!({"doc_kind": "file",+ "prefix": prefix, // #737: corpus scope for a scoped exclusion sweep"file_key": file_key,"path": path,"format": "duplicate",diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 8e4a86ec2..70839788d 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -306,21 +306,45 @@ fn delete_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> usize.next().unwrap();let query: Value = serde_json::from_slice(body).unwrap();- let term = query+ // Collect every term constraint: the single `/query/term` form AND each+ // `/query/bool/filter[].term` (#737 scopes catalog deletes with a+ // `prefix` + `path`/`file_key` bool filter). A doc is deleted iff it+ // matches EVERY collected term.+ let mut terms: Vec<(String, Value)> = Vec::new();+ if let Some((field, value)) = query.pointer("/query/term").and_then(Value::as_object).and_then(|term| term.iter().next())- .map(|(field, value)| (field.clone(), value.clone()));+ {+ terms.push((field.clone(), value.clone()));+ }+ if let Some(filters) = query+ .pointer("/query/bool/filter")+ .and_then(Value::as_array)+ {+ for filter in filters {+ if let Some((field, value)) = filter+ .pointer("/term")+ .and_then(Value::as_object)+ .and_then(|term| term.iter().next())+ {+ terms.push((field.clone(), value.clone()));+ }+ }+ }let mut locked = state.lock().unwrap();let before = locked.docs.len();locked.docs.retain(|(candidate, _), doc| {if candidate != index {return true;}- let Some((field, expected)) = &term else {- return false;- };- doc.get(field) != Some(expected)+ if terms.is_empty() {+ return false; // no constraint: a match-all delete+ }+ // Retain (do NOT delete) unless the doc matches every term.+ !terms+ .iter()+ .all(|(field, expected)| doc.get(field) == Some(expected))});before - locked.docs.len()}@@ -2048,11 +2072,11 @@ fn sweep_excluded_groups_deletes_only_the_excluded_files_documents() {);st.docs.insert((catalog::CATALOG_INDEX.to_string(), "file:excluded".into()),- json!({"doc_kind": "file", "path": "secret.csv"}),+ json!({"doc_kind": "file", "prefix": "ax", "path": "secret.csv"}),);st.docs.insert((catalog::CATALOG_INDEX.to_string(), "file:kept".into()),- json!({"doc_kind": "file", "path": "kept.csv"}),+ json!({"doc_kind": "file", "prefix": "ax", "path": "kept.csv"}),);}@@ -2076,7 +2100,7 @@ fn sweep_excluded_groups_deletes_only_the_excluded_files_documents() {path: "secret.csv".into(),}];- sweep_excluded_groups(&es, &plan, &excluded, None, 0).expect("sweep");+ sweep_excluded_groups(&es, "ax", &plan, &excluded, None, 0).expect("sweep");let st = ep.state.lock().unwrap();// The excluded file's dataset record is gone.@@ -2159,7 +2183,7 @@ fn sweep_excluded_groups_invalidates_the_excluded_files_edges() {}];let now_ms = 1_724_500_000_000i64;- sweep_excluded_groups(&es, &plan, &excluded, Some(edges), now_ms).expect("sweep");+ sweep_excluded_groups(&es, "ax", &plan, &excluded, Some(edges), now_ms).expect("sweep");let st = ep.state.lock().unwrap();// The excluded file's edge is soft-invalidated (present, stamped invalid_at).@@ -2201,16 +2225,16 @@ fn sweep_excluded_groups_purges_the_excluded_files_alias_catalog_docs() {// `file_key`.st.docs.insert((catalog::CATALOG_INDEX.to_string(), "file:excluded".into()),- json!({"doc_kind": "file", "file_key": "key-excluded", "path": "canonical.csv"}),+ json!({"doc_kind": "file", "prefix": "ax", "file_key": "key-excluded", "path": "canonical.csv"}),);st.docs.insert((catalog::CATALOG_INDEX.to_string(), "file-alias:ax:excluded".into()),- json!({"doc_kind": "file", "file_key": "key-excluded", "path": "dup.csv", "status": "duplicate"}),+ json!({"doc_kind": "file", "prefix": "ax", "file_key": "key-excluded", "path": "dup.csv", "status": "duplicate"}),);// A different file's alias (distinct file_key) must survive.st.docs.insert((catalog::CATALOG_INDEX.to_string(), "file-alias:ax:kept".into()),- json!({"doc_kind": "file", "file_key": "key-kept", "path": "kept-dup.csv", "status": "duplicate"}),+ json!({"doc_kind": "file", "prefix": "ax", "file_key": "key-kept", "path": "kept-dup.csv", "status": "duplicate"}),);}@@ -2233,9 +2257,9 @@ fn sweep_excluded_groups_purges_the_excluded_files_alias_catalog_docs() {path: "canonical.csv".into(),}];- // 5-arg signature (post-#694): this test does not exercise edges, so pass- // `None`/`0` — only the catalog file_key sweep is under test here.- sweep_excluded_groups(&es, &plan, &excluded, None, 0).expect("sweep");+ // 6-arg signature (post-#694 edges, post-#737 prefix): this test does not+ // exercise edges, so pass `None`/`0`; the corpus prefix is "ax".+ sweep_excluded_groups(&es, "ax", &plan, &excluded, None, 0).expect("sweep");let st = ep.state.lock().unwrap();// The excluded file's alias catalog doc is gone (the #693 fix).@@ -2264,6 +2288,69 @@ fn sweep_excluded_groups_purges_the_excluded_files_alias_catalog_docs() {);}+/// #737: the catalog sweep is scoped to THIS corpus's `prefix`. The+/// `autoindex-catalog` index is shared across corpora; a byte-identical common+/// file (LICENSE, a lockfile) indexed by a still-live SIBLING corpus shares the+/// same `file_key`/`path`, so an unscoped sweep would delete the sibling's live+/// catalog docs too. Fail-before: dropping the `prefix` filter (unscoped+/// `{term:{file_key}}`) deletes the sibling-corpus doc as well.+#[test]+fn sweep_excluded_groups_does_not_delete_a_sibling_corpus_catalog_doc() {+ let ep = HttpEndpoint::start();++ {+ let mut st = ep.state.lock().unwrap();+ // Corpus "ax" excludes a file; corpus "bx" is a live sibling holding a+ // byte-identical copy (same file_key) under its own prefix-scoped id.+ st.docs.insert(+ (catalog::CATALOG_INDEX.to_string(), "file:ax:excluded".into()),+ json!({"doc_kind": "file", "prefix": "ax", "file_key": "shared-key", "path": "LICENSE"}),+ );+ st.docs.insert(+ (catalog::CATALOG_INDEX.to_string(), "file:bx:sibling".into()),+ json!({"doc_kind": "file", "prefix": "bx", "file_key": "shared-key", "path": "LICENSE"}),+ );+ }++ let es = Es::with_bulk_timeout(&ep.url, None, 30).expect("es client");+ let mut plan = Plan::default();+ plan.datasets.push(crate::state::PlanDataset {+ slug: "ds".into(),+ index: "incremental-http-ds".into(),+ family: "csv".into(),+ group: None,+ specs: Vec::new(),+ time_field: None,+ semantic_field: None,+ sampled_records: 1,+ file_count: 1,+ });+ let excluded = vec![InventoryDeltaEntry {+ file_key: "shared-key".into(),+ path: "LICENSE".into(),+ }];++ sweep_excluded_groups(&es, "ax", &plan, &excluded, None, 0).expect("sweep");++ let st = ep.state.lock().unwrap();+ // Corpus ax's own doc is swept (same file_key/path, matching prefix).+ assert!(+ !st.docs.contains_key(&(+ catalog::CATALOG_INDEX.to_string(),+ "file:ax:excluded".to_string()+ )),+ "#737: the excluding corpus's own catalog doc must still be swept"+ );+ // The live sibling corpus's byte-identical doc SURVIVES (prefix-scoped).+ assert!(+ st.docs.contains_key(&(+ catalog::CATALOG_INDEX.to_string(),+ "file:bx:sibling".to_string()+ )),+ "#737: a sibling corpus's byte-identical catalog doc must NOT be swept"+ );+}+/// #585 (case 1): a pending (uncommitted) `--no-graph` generation must NOT be/// resumed and committed on the default graph path — doing so silently commits/// a no-graph generation under a graph authority (the #584 hazard, but for adiff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex ba5bec272..612235a01 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -3172,6 +3172,7 @@ fn finish_generated_run(es: &Es, journal: &mut state::Journal, cfg: &IndexCfg) -/// `None`) there is no edges index to touch.fn sweep_excluded_groups(es: &Es,+ prefix: &str,plan: &Plan,excluded: &[InventoryDeltaEntry],edges_index: Option<&str>,@@ -3193,9 +3194,18 @@ fn sweep_excluded_groups()})?;}+ // #737: BOTH catalog deletes are scoped to THIS corpus's `prefix`. The+ // `autoindex-catalog` index is shared across corpora; an unscoped `path`+ // or `file_key` term would also delete a byte-identical file's catalog+ // docs owned by a still-live SIBLING corpus (`current_keys` only guards+ // the same corpus). The `prefix` keyword field (written by `file_doc`/+ // `duplicate_file_doc`) makes the scope term-queryable.es.delete_by_query(catalog::CATALOG_INDEX,- &json!({"term": {"path": entry.path}}),+ &json!({"bool": {"filter": [+ {"term": {"prefix": prefix}},+ {"term": {"path": entry.path}},+ ]}}),).with_context(|| format!("sweep catalog entry for newly-excluded {}", entry.path))?;// #693: also purge the file's `file-alias:` duplicate catalog docs@@ -3205,10 +3215,14 @@ fn sweep_excluded_groups(// and every `file-alias:` doc carry `file_key`, and the delta classifies// a content group as excluded ONLY when no surviving file bears its key// (`UnsupportedInventoryDelta::between`: the `current_keys` guard), so a- // `file_key` term cannot strand a still-live byte-identical duplicate.+ // `file_key` term cannot strand a still-live byte-identical duplicate in+ // the same corpus; the `prefix` filter (#737) guards the cross-corpus axis.es.delete_by_query(catalog::CATALOG_INDEX,- &json!({"term": {"file_key": entry.file_key}}),+ &json!({"bool": {"filter": [+ {"term": {"prefix": prefix}},+ {"term": {"file_key": entry.file_key}},+ ]}}),).with_context(|| format!("sweep catalog aliases for newly-excluded {}", entry.path))?;// #694: soft-invalidate the edges this file taught. `invalidate_prior_edges`@@ -3827,6 +3841,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let edges_index = graph_edges_index(&cfg);sweep_excluded_groups(&es,+ &cfg.prefix,prior_plan,&delta.excluded_content_groups,edges_index.as_deref(),@@ -4094,6 +4109,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {let edges_index = graph_edges_index(&cfg);sweep_excluded_groups(&es,+ &cfg.prefix,&plan,&delta.excluded_content_groups,edges_index.as_deref(),