Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): invalidate the edges of a superseded anchor node (#736) (#885)

xerj-org6d727bd445e8
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 5a7c97900..3180cddbb 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -2703,6 +2703,185 @@ fn sweep_excluded_groups_invalidates_inbound_edges() {
);
}
+/// #736 (replacement half): a run that SUPERSEDES a file's anchor node must
+/// soft-invalidate the edges pointing AT the old anchor, not only the edges the
+/// old generation taught.
+///
+/// The reachable trigger is `--fresh`. An ordinary resume maps every file back
+/// onto its planned key (`select_resume_plan_keys`), so an in-place edit keeps
+/// its `file_key` and its anchor id — nothing is superseded and the inbound
+/// edges keep pointing at a live node. `--fresh` discards the plan and rebuilds
+/// every key from current content, so an edited file gets a NEW key and a NEW
+/// anchor, while `cleanup_required` (the journal's record of what is live) has
+/// just been wiped with the journal — so before this fix the run invalidated
+/// nothing at all, and every edge a surviving file taught into the old anchor
+/// stayed live and searchable, pointing at a superseded node.
+///
+/// Fail-before: with the superseded-anchor invalidation reverted, the two
+/// `a.md → old b anchor` edges come back with no `invalid_at` and the first
+/// assertion fires.
+#[test]
+fn fresh_run_invalidates_inbound_edges_to_a_superseded_anchor() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("a.md"),
+ "# A\n\nSee [[b]] for the details of the target document.\n",
+ )
+ .unwrap();
+ fs::write(
+ corpus.path().join("b.md"),
+ "# B\n\nThe target document says one thing.\n",
+ )
+ .unwrap();
+ let endpoint = HttpEndpoint::start();
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ config.no_graph = false;
+ config.brain = Some("testbrain".into());
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let edges_index = detect::edges_index_name("testbrain");
+
+ // Anchor ids are read off the published file cards (`ax_locator: "file"`),
+ // so the test never has to recompute `ids::doc_id` itself.
+ let anchors_of = |path: &str| -> Vec<String> {
+ let st = endpoint.state.lock().unwrap();
+ let mut ids: Vec<String> = st
+ .docs
+ .iter()
+ .filter(|((index, _), doc)| {
+ index != catalog::CATALOG_INDEX
+ && doc.get("ax_locator").and_then(Value::as_str) == Some("file")
+ && doc.get("ax_path").and_then(Value::as_str) == Some(path)
+ })
+ .map(|((_, id), _)| id.clone())
+ .collect();
+ ids.sort();
+ ids
+ };
+ let edges_of = |predicate: &dyn Fn(&Value) -> bool| -> Vec<(String, Value)> {
+ let st = endpoint.state.lock().unwrap();
+ let mut rows: Vec<(String, Value)> = st
+ .docs
+ .iter()
+ .filter(|((index, _), doc)| *index == edges_index && predicate(doc))
+ .map(|((_, id), doc)| (id.clone(), doc.clone()))
+ .collect();
+ rows.sort_by(|left, right| left.0.cmp(&right.0));
+ rows
+ };
+
+ let old_b_anchor = {
+ let mut found = anchors_of("b.md");
+ assert_eq!(found.len(), 1, "run 1 publishes exactly one b.md file card");
+ found.pop().unwrap()
+ };
+ // The inbound edges the fix has to reach: taught by the SURVIVING file a.md,
+ // pointing at b.md's anchor. Asserted before the change so a corpus that
+ // stopped producing them could never make this test vacuously pass.
+ let inbound_before = edges_of(&|doc| {
+ doc.get("src_file").and_then(Value::as_str) == Some("a.md")
+ && doc.get("dst").and_then(Value::as_str) == Some(old_b_anchor.as_str())
+ });
+ assert!(
+ !inbound_before.is_empty(),
+ "#736: the fixture must actually teach a.md → b.md edges"
+ );
+ // a.md's other edges (its own card → section chain) must survive untouched:
+ // a.md is byte-identical across the two runs, and #868's skip means an
+ // unchanged file must neither re-teach nor lose its edges.
+ let a_internal_before: Vec<String> = edges_of(&|doc| {
+ doc.get("src_file").and_then(Value::as_str) == Some("a.md")
+ && doc.get("dst").and_then(Value::as_str) != Some(old_b_anchor.as_str())
+ })
+ .into_iter()
+ .map(|(id, _)| id)
+ .collect();
+ assert!(
+ !a_internal_before.is_empty(),
+ "#736: the fixture must have edges that the sweep MUST NOT touch"
+ );
+
+ // Edit b.md, then rebuild in place with --fresh: b.md's content key, and so
+ // its anchor node id, is superseded.
+ fs::write(
+ corpus.path().join("b.md"),
+ "# B\n\nThe target document now says something completely different today.\n",
+ )
+ .unwrap();
+ let mut fresh = config.clone();
+ fresh.fresh = true;
+ assert_eq!(run_index(fresh).unwrap(), 0);
+
+ let new_b_anchor = {
+ let found = anchors_of("b.md");
+ let fresh_ids: Vec<&String> = found.iter().filter(|id| **id != old_b_anchor).collect();
+ assert_eq!(
+ fresh_ids.len(),
+ 1,
+ "--fresh must publish a new b.md file card after the edit, got {found:?}"
+ );
+ fresh_ids[0].clone()
+ };
+ assert_ne!(
+ new_b_anchor, old_b_anchor,
+ "the fixture must actually supersede b.md's anchor"
+ );
+
+ // The bug: every live edge pointing at the superseded anchor.
+ let inbound_after =
+ edges_of(&|doc| doc.get("dst").and_then(Value::as_str) == Some(old_b_anchor.as_str()));
+ assert_eq!(
+ inbound_after.len(),
+ inbound_before.len(),
+ "#736: prior edges are soft-invalidated, never deleted — the bi-temporal \
+ record must still be there for `as_of`"
+ );
+ for (id, doc) in &inbound_after {
+ assert!(
+ doc.get("invalid_at").is_some(),
+ "#736: edge {id} still points at b.md's superseded anchor {old_b_anchor} \
+ and was left live: {doc}"
+ );
+ }
+ // The old generation's own outbound edges go too — same root cause, same
+ // pass: `src_file == b.md` was never invalidated under --fresh either.
+ for (id, doc) in
+ edges_of(&|doc| doc.get("src").and_then(Value::as_str) == Some(old_b_anchor.as_str()))
+ {
+ assert!(
+ doc.get("invalid_at").is_some(),
+ "#736: edge {id} is taught BY b.md's superseded anchor and was left live: {doc}"
+ );
+ }
+
+ // …and the graph is still a graph: a.md links to the NEW anchor, live.
+ let inbound_new = edges_of(&|doc| {
+ doc.get("src_file").and_then(Value::as_str) == Some("a.md")
+ && doc.get("dst").and_then(Value::as_str) == Some(new_b_anchor.as_str())
+ && doc.get("invalid_at").is_none()
+ });
+ assert!(
+ !inbound_new.is_empty(),
+ "#736: the re-run must re-teach a.md → b.md against the new anchor"
+ );
+ // #868: the unchanged file's unrelated edges are untouched.
+ for id in &a_internal_before {
+ let st = endpoint.state.lock().unwrap();
+ let doc = st
+ .docs
+ .get(&(edges_index.clone(), id.clone()))
+ .unwrap_or_else(|| panic!("#868: a.md's edge {id} must survive the re-run"));
+ assert!(
+ doc.get("invalid_at").is_none(),
+ "#868: byte-identical a.md's own edge {id} must stay live: {doc}"
+ );
+ }
+}
+
/// #585 (case 1): a pending (uncommitted) `--no-graph` generation must NOT be
/// resumed and committed on the default graph path — doing so silently commits
/// a no-graph generation under a graph authority (the #584 hazard, but for a
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 7e4f70176..6e23b7548 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3447,6 +3447,40 @@ fn graph_edges_index(cfg: &IndexCfg) -> Option<String> {
.map(|()| detect::edges_index_name(&brain))
}
+/// #736: every graph ANCHOR node a plan owns — `ids::doc_id(slug, file_key,
+/// FILE_CARD_LOCATOR)` for each dataset the file is assigned to — paired with
+/// that file's `rel`. Sorted and deduped so two calls are directly comparable.
+///
+/// The anchor is the node every cross-file edge points AT (`EdgeDraft::dst` is
+/// `CorpusFile::anchor_doc_id` in every detector that links two files), so the
+/// anchor set is exactly the set of edge endpoints a plan can own.
+///
+/// The graph phase compares the PRIOR plan's anchors against this run's. An
+/// anchor the prior plan owned and this one does not is SUPERSEDED — its node
+/// belongs to a generation this run replaced — so the edges pointing at it
+/// (`dst`) and the edges that generation taught (`src_file`) must both be
+/// soft-invalidated. A file whose key and dataset assignments are unchanged
+/// appears in both sets and is therefore never touched, which is what keeps the
+/// #868 byte-identical skip honest: an unchanged file neither re-teaches nor
+/// loses its edges.
+fn plan_anchor_nodes(plan: &Plan) -> Vec<(String, String)> {
+ let mut anchors: Vec<(String, String)> = plan
+ .files
+ .iter()
+ .flat_map(|(key, assignment)| {
+ assignment.assignments.iter().map(move |(_, slug)| {
+ (
+ crate::ids::doc_id(slug, key, detect::FILE_CARD_LOCATOR),
+ assignment.rel.clone(),
+ )
+ })
+ })
+ .collect();
+ anchors.sort();
+ anchors.dedup();
+ anchors
+}
+
/// Catalog doc id for a time correlation, corpus-prefix-scoped (#673): the
/// `autoindex-catalog` index is shared across corpora, so a time correlation
/// keyed only by dataset slugs would overwrite the same-slug correlation from
@@ -4121,6 +4155,18 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
.context("sweep newly-excluded content groups")?;
}
}
+ // #736: snapshot the PRIOR plan's anchor nodes HERE, before the journal is
+ // opened. `--fresh` deletes `journal.ndjson` inside `open_after_preflight`,
+ // taking the prior plan with it — and `--fresh` is precisely the path that
+ // supersedes a changed file's anchor, because it rebuilds every key from
+ // current content instead of mapping each file onto its planned key the way
+ // `select_resume_plan_keys` does on an ordinary resume. Only a run that can
+ // write edges has any to invalidate, so `--no-graph` (and an unusable brain
+ // name) pays nothing here.
+ let prior_anchor_nodes: Vec<(String, String)> = match preflight.plan.as_ref() {
+ Some(prior) if graph_edges_index(&cfg).is_some() => plan_anchor_nodes(prior),
+ _ => Vec::new(),
+ };
let mut journal = state::Journal::open_after_preflight(
preflight,
&root_str,
@@ -4969,6 +5015,25 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
.filter(|&&i| cleanup_required.contains(&keys[i]))
.map(|&i| files[i].rel.as_str())
.collect();
+ // #736: `cleanup_required` is the journal's own record of what is
+ // already live, so it is EMPTY after `--fresh` wiped the journal —
+ // and `--fresh` is the one path that gives a content-changed file a
+ // new `file_key`, hence a new anchor node. Nothing above notices, so
+ // the prior generation's edges stayed live: the ones the old anchor
+ // taught (`src_file`) AND the ones a surviving file taught pointing
+ // AT it (`dst`), which is the inbound half #736 was filed for. An
+ // ordinary resume supersedes no anchor (`select_resume_plan_keys`
+ // preserves the planned key across an in-place edit), so this set is
+ // empty there and the resume behaviour is unchanged.
+ let live_anchors: std::collections::HashSet<String> = plan_anchor_nodes(&plan)
+ .into_iter()
+ .map(|(anchor, _)| anchor)
+ .collect();
+ let superseded: Vec<&(String, String)> = prior_anchor_nodes
+ .iter()
+ .filter(|(anchor, _)| !live_anchors.contains(anchor))
+ .collect();
+ replaced_rels.extend(superseded.iter().map(|(_, rel)| rel.as_str()));
replaced_rels.sort_unstable();
replaced_rels.dedup();
for rel in replaced_rels {
@@ -4976,6 +5041,24 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
detect::invalidate_prior_edges(&es, &edges_index, rel, created_at_ms)
.with_context(|| format!("invalidate prior edges taught by {rel}"))?;
}
+ // The inbound half. Keyed on the SUPERSEDED anchor only — never on a
+ // surviving file — so an unchanged file's edges are left exactly as
+ // they are (#868). This runs before any edge this run writes, and
+ // this run's edges name the NEW anchor, so it can only ever match
+ // prior generations. Same soft-invalidate as the src side: the
+ // bi-temporal record survives for `as_of` time travel.
+ for (anchor, rel) in superseded {
+ invalidated += detect::invalidate_edges_by_field(
+ &es,
+ &edges_index,
+ "dst",
+ anchor,
+ created_at_ms,
+ )
+ .with_context(|| {
+ format!("invalidate inbound edges to the superseded anchor node of {rel}")
+ })?;
+ }
}
// Structural detection (samedir chains) + bulk write, cut at the same