AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): invalidate the edges of a superseded anchor node (#736) (#885)
xerj-org6d727bd445e8
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 5a7c97900..3180cddbb 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -2703,6 +2703,185 @@ fn sweep_excluded_groups_invalidates_inbound_edges() {);}+/// #736 (replacement half): a run that SUPERSEDES a file's anchor node must+/// soft-invalidate the edges pointing AT the old anchor, not only the edges the+/// old generation taught.+///+/// The reachable trigger is `--fresh`. An ordinary resume maps every file back+/// onto its planned key (`select_resume_plan_keys`), so an in-place edit keeps+/// its `file_key` and its anchor id — nothing is superseded and the inbound+/// edges keep pointing at a live node. `--fresh` discards the plan and rebuilds+/// every key from current content, so an edited file gets a NEW key and a NEW+/// anchor, while `cleanup_required` (the journal's record of what is live) has+/// just been wiped with the journal — so before this fix the run invalidated+/// nothing at all, and every edge a surviving file taught into the old anchor+/// stayed live and searchable, pointing at a superseded node.+///+/// Fail-before: with the superseded-anchor invalidation reverted, the two+/// `a.md → old b anchor` edges come back with no `invalid_at` and the first+/// assertion fires.+#[test]+fn fresh_run_invalidates_inbound_edges_to_a_superseded_anchor() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("a.md"),+ "# A\n\nSee [[b]] for the details of the target document.\n",+ )+ .unwrap();+ fs::write(+ corpus.path().join("b.md"),+ "# B\n\nThe target document says one thing.\n",+ )+ .unwrap();+ let endpoint = HttpEndpoint::start();+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ config.no_graph = false;+ config.brain = Some("testbrain".into());+ assert_eq!(run_index(config.clone()).unwrap(), 0);+ let edges_index = detect::edges_index_name("testbrain");++ // Anchor ids are read off the published file cards (`ax_locator: "file"`),+ // so the test never has to recompute `ids::doc_id` itself.+ let anchors_of = |path: &str| -> Vec<String> {+ let st = endpoint.state.lock().unwrap();+ let mut ids: Vec<String> = st+ .docs+ .iter()+ .filter(|((index, _), doc)| {+ index != catalog::CATALOG_INDEX+ && doc.get("ax_locator").and_then(Value::as_str) == Some("file")+ && doc.get("ax_path").and_then(Value::as_str) == Some(path)+ })+ .map(|((_, id), _)| id.clone())+ .collect();+ ids.sort();+ ids+ };+ let edges_of = |predicate: &dyn Fn(&Value) -> bool| -> Vec<(String, Value)> {+ let st = endpoint.state.lock().unwrap();+ let mut rows: Vec<(String, Value)> = st+ .docs+ .iter()+ .filter(|((index, _), doc)| *index == edges_index && predicate(doc))+ .map(|((_, id), doc)| (id.clone(), doc.clone()))+ .collect();+ rows.sort_by(|left, right| left.0.cmp(&right.0));+ rows+ };++ let old_b_anchor = {+ let mut found = anchors_of("b.md");+ assert_eq!(found.len(), 1, "run 1 publishes exactly one b.md file card");+ found.pop().unwrap()+ };+ // The inbound edges the fix has to reach: taught by the SURVIVING file a.md,+ // pointing at b.md's anchor. Asserted before the change so a corpus that+ // stopped producing them could never make this test vacuously pass.+ let inbound_before = edges_of(&|doc| {+ doc.get("src_file").and_then(Value::as_str) == Some("a.md")+ && doc.get("dst").and_then(Value::as_str) == Some(old_b_anchor.as_str())+ });+ assert!(+ !inbound_before.is_empty(),+ "#736: the fixture must actually teach a.md → b.md edges"+ );+ // a.md's other edges (its own card → section chain) must survive untouched:+ // a.md is byte-identical across the two runs, and #868's skip means an+ // unchanged file must neither re-teach nor lose its edges.+ let a_internal_before: Vec<String> = edges_of(&|doc| {+ doc.get("src_file").and_then(Value::as_str) == Some("a.md")+ && doc.get("dst").and_then(Value::as_str) != Some(old_b_anchor.as_str())+ })+ .into_iter()+ .map(|(id, _)| id)+ .collect();+ assert!(+ !a_internal_before.is_empty(),+ "#736: the fixture must have edges that the sweep MUST NOT touch"+ );++ // Edit b.md, then rebuild in place with --fresh: b.md's content key, and so+ // its anchor node id, is superseded.+ fs::write(+ corpus.path().join("b.md"),+ "# B\n\nThe target document now says something completely different today.\n",+ )+ .unwrap();+ let mut fresh = config.clone();+ fresh.fresh = true;+ assert_eq!(run_index(fresh).unwrap(), 0);++ let new_b_anchor = {+ let found = anchors_of("b.md");+ let fresh_ids: Vec<&String> = found.iter().filter(|id| **id != old_b_anchor).collect();+ assert_eq!(+ fresh_ids.len(),+ 1,+ "--fresh must publish a new b.md file card after the edit, got {found:?}"+ );+ fresh_ids[0].clone()+ };+ assert_ne!(+ new_b_anchor, old_b_anchor,+ "the fixture must actually supersede b.md's anchor"+ );++ // The bug: every live edge pointing at the superseded anchor.+ let inbound_after =+ edges_of(&|doc| doc.get("dst").and_then(Value::as_str) == Some(old_b_anchor.as_str()));+ assert_eq!(+ inbound_after.len(),+ inbound_before.len(),+ "#736: prior edges are soft-invalidated, never deleted — the bi-temporal \+ record must still be there for `as_of`"+ );+ for (id, doc) in &inbound_after {+ assert!(+ doc.get("invalid_at").is_some(),+ "#736: edge {id} still points at b.md's superseded anchor {old_b_anchor} \+ and was left live: {doc}"+ );+ }+ // The old generation's own outbound edges go too — same root cause, same+ // pass: `src_file == b.md` was never invalidated under --fresh either.+ for (id, doc) in+ edges_of(&|doc| doc.get("src").and_then(Value::as_str) == Some(old_b_anchor.as_str()))+ {+ assert!(+ doc.get("invalid_at").is_some(),+ "#736: edge {id} is taught BY b.md's superseded anchor and was left live: {doc}"+ );+ }++ // …and the graph is still a graph: a.md links to the NEW anchor, live.+ let inbound_new = edges_of(&|doc| {+ doc.get("src_file").and_then(Value::as_str) == Some("a.md")+ && doc.get("dst").and_then(Value::as_str) == Some(new_b_anchor.as_str())+ && doc.get("invalid_at").is_none()+ });+ assert!(+ !inbound_new.is_empty(),+ "#736: the re-run must re-teach a.md → b.md against the new anchor"+ );+ // #868: the unchanged file's unrelated edges are untouched.+ for id in &a_internal_before {+ let st = endpoint.state.lock().unwrap();+ let doc = st+ .docs+ .get(&(edges_index.clone(), id.clone()))+ .unwrap_or_else(|| panic!("#868: a.md's edge {id} must survive the re-run"));+ assert!(+ doc.get("invalid_at").is_none(),+ "#868: byte-identical a.md's own edge {id} must stay live: {doc}"+ );+ }+}+/// #585 (case 1): a pending (uncommitted) `--no-graph` generation must NOT be/// resumed and committed on the default graph path — doing so silently commits/// a no-graph generation under a graph authority (the #584 hazard, but for adiff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 7e4f70176..6e23b7548 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -3447,6 +3447,40 @@ fn graph_edges_index(cfg: &IndexCfg) -> Option<String> {.map(|()| detect::edges_index_name(&brain))}+/// #736: every graph ANCHOR node a plan owns — `ids::doc_id(slug, file_key,+/// FILE_CARD_LOCATOR)` for each dataset the file is assigned to — paired with+/// that file's `rel`. Sorted and deduped so two calls are directly comparable.+///+/// The anchor is the node every cross-file edge points AT (`EdgeDraft::dst` is+/// `CorpusFile::anchor_doc_id` in every detector that links two files), so the+/// anchor set is exactly the set of edge endpoints a plan can own.+///+/// The graph phase compares the PRIOR plan's anchors against this run's. An+/// anchor the prior plan owned and this one does not is SUPERSEDED — its node+/// belongs to a generation this run replaced — so the edges pointing at it+/// (`dst`) and the edges that generation taught (`src_file`) must both be+/// soft-invalidated. A file whose key and dataset assignments are unchanged+/// appears in both sets and is therefore never touched, which is what keeps the+/// #868 byte-identical skip honest: an unchanged file neither re-teaches nor+/// loses its edges.+fn plan_anchor_nodes(plan: &Plan) -> Vec<(String, String)> {+ let mut anchors: Vec<(String, String)> = plan+ .files+ .iter()+ .flat_map(|(key, assignment)| {+ assignment.assignments.iter().map(move |(_, slug)| {+ (+ crate::ids::doc_id(slug, key, detect::FILE_CARD_LOCATOR),+ assignment.rel.clone(),+ )+ })+ })+ .collect();+ anchors.sort();+ anchors.dedup();+ anchors+}+/// Catalog doc id for a time correlation, corpus-prefix-scoped (#673): the/// `autoindex-catalog` index is shared across corpora, so a time correlation/// keyed only by dataset slugs would overwrite the same-slug correlation from@@ -4121,6 +4155,18 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {.context("sweep newly-excluded content groups")?;}}+ // #736: snapshot the PRIOR plan's anchor nodes HERE, before the journal is+ // opened. `--fresh` deletes `journal.ndjson` inside `open_after_preflight`,+ // taking the prior plan with it — and `--fresh` is precisely the path that+ // supersedes a changed file's anchor, because it rebuilds every key from+ // current content instead of mapping each file onto its planned key the way+ // `select_resume_plan_keys` does on an ordinary resume. Only a run that can+ // write edges has any to invalidate, so `--no-graph` (and an unusable brain+ // name) pays nothing here.+ let prior_anchor_nodes: Vec<(String, String)> = match preflight.plan.as_ref() {+ Some(prior) if graph_edges_index(&cfg).is_some() => plan_anchor_nodes(prior),+ _ => Vec::new(),+ };let mut journal = state::Journal::open_after_preflight(preflight,&root_str,@@ -4969,6 +5015,25 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {.filter(|&&i| cleanup_required.contains(&keys[i])).map(|&i| files[i].rel.as_str()).collect();+ // #736: `cleanup_required` is the journal's own record of what is+ // already live, so it is EMPTY after `--fresh` wiped the journal —+ // and `--fresh` is the one path that gives a content-changed file a+ // new `file_key`, hence a new anchor node. Nothing above notices, so+ // the prior generation's edges stayed live: the ones the old anchor+ // taught (`src_file`) AND the ones a surviving file taught pointing+ // AT it (`dst`), which is the inbound half #736 was filed for. An+ // ordinary resume supersedes no anchor (`select_resume_plan_keys`+ // preserves the planned key across an in-place edit), so this set is+ // empty there and the resume behaviour is unchanged.+ let live_anchors: std::collections::HashSet<String> = plan_anchor_nodes(&plan)+ .into_iter()+ .map(|(anchor, _)| anchor)+ .collect();+ let superseded: Vec<&(String, String)> = prior_anchor_nodes+ .iter()+ .filter(|(anchor, _)| !live_anchors.contains(anchor))+ .collect();+ replaced_rels.extend(superseded.iter().map(|(_, rel)| rel.as_str()));replaced_rels.sort_unstable();replaced_rels.dedup();for rel in replaced_rels {@@ -4976,6 +5041,24 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {detect::invalidate_prior_edges(&es, &edges_index, rel, created_at_ms).with_context(|| format!("invalidate prior edges taught by {rel}"))?;}+ // The inbound half. Keyed on the SUPERSEDED anchor only — never on a+ // surviving file — so an unchanged file's edges are left exactly as+ // they are (#868). This runs before any edge this run writes, and+ // this run's edges name the NEW anchor, so it can only ever match+ // prior generations. Same soft-invalidate as the src side: the+ // bi-temporal record survives for `as_of` time travel.+ for (anchor, rel) in superseded {+ invalidated += detect::invalidate_edges_by_field(+ &es,+ &edges_index,+ "dst",+ anchor,+ created_at_ms,+ )+ .with_context(|| {+ format!("invalidate inbound edges to the superseded anchor node of {rel}")+ })?;+ }}// Structural detection (samedir chains) + bulk write, cut at the same