Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): prefix-scope the exclusion catalog sweeps so a sibling corpus is not over-deleted (#737)

xerj-org0ea096a83017
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rs
index 9ab7b8aec..1790c1464 100644
--- a/engine/crates/xerj-autoindex/src/catalog.rs
+++ b/engine/crates/xerj-autoindex/src/catalog.rs
@@ -108,6 +108,12 @@ pub fn catalog_mapping() -> Value {
json!({"type": "keyword"}),
);
properties.insert("junk_records_this_run".into(), json!({"type": "long"}));
+ // #737: corpus scope, so a delete_by_query can constrain a sweep to THIS
+ // corpus's docs. The doc `_id`s are already prefix-scoped (`file:{prefix}:…`)
+ // but `_id` is not term-queryable; a byte-identical file shared with a live
+ // sibling corpus would otherwise be caught by an unscoped `file_key`/`path`
+ // sweep. Written by `file_doc`/`duplicate_file_doc` (the sweep's targets).
+ properties.insert("prefix".into(), json!({"type": "keyword"}));
mapping
}
@@ -193,6 +199,7 @@ pub fn file_doc(
file_id(prefix, file_key),
json!({
"doc_kind": "file",
+ "prefix": prefix, // #737: corpus scope for a scoped exclusion sweep
"file_key": file_key,
"path": path,
"format": format,
@@ -220,6 +227,7 @@ pub fn duplicate_file_doc(
alias_id,
json!({
"doc_kind": "file",
+ "prefix": prefix, // #737: corpus scope for a scoped exclusion sweep
"file_key": file_key,
"path": path,
"format": "duplicate",
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 8e4a86ec2..70839788d 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -306,21 +306,45 @@ fn delete_http(path: &str, body: &[u8], state: &Arc<Mutex<HttpState>>) -> usize
.next()
.unwrap();
let query: Value = serde_json::from_slice(body).unwrap();
- let term = query
+ // Collect every term constraint: the single `/query/term` form AND each
+ // `/query/bool/filter[].term` (#737 scopes catalog deletes with a
+ // `prefix` + `path`/`file_key` bool filter). A doc is deleted iff it
+ // matches EVERY collected term.
+ let mut terms: Vec<(String, Value)> = Vec::new();
+ if let Some((field, value)) = query
.pointer("/query/term")
.and_then(Value::as_object)
.and_then(|term| term.iter().next())
- .map(|(field, value)| (field.clone(), value.clone()));
+ {
+ terms.push((field.clone(), value.clone()));
+ }
+ if let Some(filters) = query
+ .pointer("/query/bool/filter")
+ .and_then(Value::as_array)
+ {
+ for filter in filters {
+ if let Some((field, value)) = filter
+ .pointer("/term")
+ .and_then(Value::as_object)
+ .and_then(|term| term.iter().next())
+ {
+ terms.push((field.clone(), value.clone()));
+ }
+ }
+ }
let mut locked = state.lock().unwrap();
let before = locked.docs.len();
locked.docs.retain(|(candidate, _), doc| {
if candidate != index {
return true;
}
- let Some((field, expected)) = &term else {
- return false;
- };
- doc.get(field) != Some(expected)
+ if terms.is_empty() {
+ return false; // no constraint: a match-all delete
+ }
+ // Retain (do NOT delete) unless the doc matches every term.
+ !terms
+ .iter()
+ .all(|(field, expected)| doc.get(field) == Some(expected))
});
before - locked.docs.len()
}
@@ -2048,11 +2072,11 @@ fn sweep_excluded_groups_deletes_only_the_excluded_files_documents() {
);
st.docs.insert(
(catalog::CATALOG_INDEX.to_string(), "file:excluded".into()),
- json!({"doc_kind": "file", "path": "secret.csv"}),
+ json!({"doc_kind": "file", "prefix": "ax", "path": "secret.csv"}),
);
st.docs.insert(
(catalog::CATALOG_INDEX.to_string(), "file:kept".into()),
- json!({"doc_kind": "file", "path": "kept.csv"}),
+ json!({"doc_kind": "file", "prefix": "ax", "path": "kept.csv"}),
);
}
@@ -2076,7 +2100,7 @@ fn sweep_excluded_groups_deletes_only_the_excluded_files_documents() {
path: "secret.csv".into(),
}];
- sweep_excluded_groups(&es, &plan, &excluded, None, 0).expect("sweep");
+ sweep_excluded_groups(&es, "ax", &plan, &excluded, None, 0).expect("sweep");
let st = ep.state.lock().unwrap();
// The excluded file's dataset record is gone.
@@ -2159,7 +2183,7 @@ fn sweep_excluded_groups_invalidates_the_excluded_files_edges() {
}];
let now_ms = 1_724_500_000_000i64;
- sweep_excluded_groups(&es, &plan, &excluded, Some(edges), now_ms).expect("sweep");
+ sweep_excluded_groups(&es, "ax", &plan, &excluded, Some(edges), now_ms).expect("sweep");
let st = ep.state.lock().unwrap();
// The excluded file's edge is soft-invalidated (present, stamped invalid_at).
@@ -2201,16 +2225,16 @@ fn sweep_excluded_groups_purges_the_excluded_files_alias_catalog_docs() {
// `file_key`.
st.docs.insert(
(catalog::CATALOG_INDEX.to_string(), "file:excluded".into()),
- json!({"doc_kind": "file", "file_key": "key-excluded", "path": "canonical.csv"}),
+ json!({"doc_kind": "file", "prefix": "ax", "file_key": "key-excluded", "path": "canonical.csv"}),
);
st.docs.insert(
(catalog::CATALOG_INDEX.to_string(), "file-alias:ax:excluded".into()),
- json!({"doc_kind": "file", "file_key": "key-excluded", "path": "dup.csv", "status": "duplicate"}),
+ json!({"doc_kind": "file", "prefix": "ax", "file_key": "key-excluded", "path": "dup.csv", "status": "duplicate"}),
);
// A different file's alias (distinct file_key) must survive.
st.docs.insert(
(catalog::CATALOG_INDEX.to_string(), "file-alias:ax:kept".into()),
- json!({"doc_kind": "file", "file_key": "key-kept", "path": "kept-dup.csv", "status": "duplicate"}),
+ json!({"doc_kind": "file", "prefix": "ax", "file_key": "key-kept", "path": "kept-dup.csv", "status": "duplicate"}),
);
}
@@ -2233,9 +2257,9 @@ fn sweep_excluded_groups_purges_the_excluded_files_alias_catalog_docs() {
path: "canonical.csv".into(),
}];
- // 5-arg signature (post-#694): this test does not exercise edges, so pass
- // `None`/`0` — only the catalog file_key sweep is under test here.
- sweep_excluded_groups(&es, &plan, &excluded, None, 0).expect("sweep");
+ // 6-arg signature (post-#694 edges, post-#737 prefix): this test does not
+ // exercise edges, so pass `None`/`0`; the corpus prefix is "ax".
+ sweep_excluded_groups(&es, "ax", &plan, &excluded, None, 0).expect("sweep");
let st = ep.state.lock().unwrap();
// The excluded file's alias catalog doc is gone (the #693 fix).
@@ -2264,6 +2288,69 @@ fn sweep_excluded_groups_purges_the_excluded_files_alias_catalog_docs() {
);
}
+/// #737: the catalog sweep is scoped to THIS corpus's `prefix`. The
+/// `autoindex-catalog` index is shared across corpora; a byte-identical common
+/// file (LICENSE, a lockfile) indexed by a still-live SIBLING corpus shares the
+/// same `file_key`/`path`, so an unscoped sweep would delete the sibling's live
+/// catalog docs too. Fail-before: dropping the `prefix` filter (unscoped
+/// `{term:{file_key}}`) deletes the sibling-corpus doc as well.
+#[test]
+fn sweep_excluded_groups_does_not_delete_a_sibling_corpus_catalog_doc() {
+ let ep = HttpEndpoint::start();
+
+ {
+ let mut st = ep.state.lock().unwrap();
+ // Corpus "ax" excludes a file; corpus "bx" is a live sibling holding a
+ // byte-identical copy (same file_key) under its own prefix-scoped id.
+ st.docs.insert(
+ (catalog::CATALOG_INDEX.to_string(), "file:ax:excluded".into()),
+ json!({"doc_kind": "file", "prefix": "ax", "file_key": "shared-key", "path": "LICENSE"}),
+ );
+ st.docs.insert(
+ (catalog::CATALOG_INDEX.to_string(), "file:bx:sibling".into()),
+ json!({"doc_kind": "file", "prefix": "bx", "file_key": "shared-key", "path": "LICENSE"}),
+ );
+ }
+
+ let es = Es::with_bulk_timeout(&ep.url, None, 30).expect("es client");
+ let mut plan = Plan::default();
+ plan.datasets.push(crate::state::PlanDataset {
+ slug: "ds".into(),
+ index: "incremental-http-ds".into(),
+ family: "csv".into(),
+ group: None,
+ specs: Vec::new(),
+ time_field: None,
+ semantic_field: None,
+ sampled_records: 1,
+ file_count: 1,
+ });
+ let excluded = vec![InventoryDeltaEntry {
+ file_key: "shared-key".into(),
+ path: "LICENSE".into(),
+ }];
+
+ sweep_excluded_groups(&es, "ax", &plan, &excluded, None, 0).expect("sweep");
+
+ let st = ep.state.lock().unwrap();
+ // Corpus ax's own doc is swept (same file_key/path, matching prefix).
+ assert!(
+ !st.docs.contains_key(&(
+ catalog::CATALOG_INDEX.to_string(),
+ "file:ax:excluded".to_string()
+ )),
+ "#737: the excluding corpus's own catalog doc must still be swept"
+ );
+ // The live sibling corpus's byte-identical doc SURVIVES (prefix-scoped).
+ assert!(
+ st.docs.contains_key(&(
+ catalog::CATALOG_INDEX.to_string(),
+ "file:bx:sibling".to_string()
+ )),
+ "#737: a sibling corpus's byte-identical catalog doc must NOT be swept"
+ );
+}
+
/// #585 (case 1): a pending (uncommitted) `--no-graph` generation must NOT be
/// resumed and committed on the default graph path — doing so silently commits
/// a no-graph generation under a graph authority (the #584 hazard, but for a
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index ba5bec272..612235a01 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3172,6 +3172,7 @@ fn finish_generated_run(es: &Es, journal: &mut state::Journal, cfg: &IndexCfg) -
/// `None`) there is no edges index to touch.
fn sweep_excluded_groups(
es: &Es,
+ prefix: &str,
plan: &Plan,
excluded: &[InventoryDeltaEntry],
edges_index: Option<&str>,
@@ -3193,9 +3194,18 @@ fn sweep_excluded_groups(
)
})?;
}
+ // #737: BOTH catalog deletes are scoped to THIS corpus's `prefix`. The
+ // `autoindex-catalog` index is shared across corpora; an unscoped `path`
+ // or `file_key` term would also delete a byte-identical file's catalog
+ // docs owned by a still-live SIBLING corpus (`current_keys` only guards
+ // the same corpus). The `prefix` keyword field (written by `file_doc`/
+ // `duplicate_file_doc`) makes the scope term-queryable.
es.delete_by_query(
catalog::CATALOG_INDEX,
- &json!({"term": {"path": entry.path}}),
+ &json!({"bool": {"filter": [
+ {"term": {"prefix": prefix}},
+ {"term": {"path": entry.path}},
+ ]}}),
)
.with_context(|| format!("sweep catalog entry for newly-excluded {}", entry.path))?;
// #693: also purge the file's `file-alias:` duplicate catalog docs
@@ -3205,10 +3215,14 @@ fn sweep_excluded_groups(
// and every `file-alias:` doc carry `file_key`, and the delta classifies
// a content group as excluded ONLY when no surviving file bears its key
// (`UnsupportedInventoryDelta::between`: the `current_keys` guard), so a
- // `file_key` term cannot strand a still-live byte-identical duplicate.
+ // `file_key` term cannot strand a still-live byte-identical duplicate in
+ // the same corpus; the `prefix` filter (#737) guards the cross-corpus axis.
es.delete_by_query(
catalog::CATALOG_INDEX,
- &json!({"term": {"file_key": entry.file_key}}),
+ &json!({"bool": {"filter": [
+ {"term": {"prefix": prefix}},
+ {"term": {"file_key": entry.file_key}},
+ ]}}),
)
.with_context(|| format!("sweep catalog aliases for newly-excluded {}", entry.path))?;
// #694: soft-invalidate the edges this file taught. `invalidate_prior_edges`
@@ -3827,6 +3841,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let edges_index = graph_edges_index(&cfg);
sweep_excluded_groups(
&es,
+ &cfg.prefix,
prior_plan,
&delta.excluded_content_groups,
edges_index.as_deref(),
@@ -4094,6 +4109,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
let edges_index = graph_edges_index(&cfg);
sweep_excluded_groups(
&es,
+ &cfg.prefix,
&plan,
&delta.excluded_content_groups,
edges_index.as_deref(),