Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): keep repository scope so sibling forks route added files correctly (#1230)

xerj-orgb2464e5e2e44
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rs
index f684a4fb3..b94719282 100644
--- a/engine/crates/xerj-autoindex/src/catalog.rs
+++ b/engine/crates/xerj-autoindex/src/catalog.rs
@@ -837,6 +837,7 @@ mod sample_query_tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
}
diff --git a/engine/crates/xerj-autoindex/src/generation_catalog.rs b/engine/crates/xerj-autoindex/src/generation_catalog.rs
index 721c0b24e..8f84b096a 100644
--- a/engine/crates/xerj-autoindex/src/generation_catalog.rs
+++ b/engine/crates/xerj-autoindex/src/generation_catalog.rs
@@ -522,6 +522,7 @@ mod tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 2,
file_count: 1,
}
diff --git a/engine/crates/xerj-autoindex/src/generation_catalog_http_tests.rs b/engine/crates/xerj-autoindex/src/generation_catalog_http_tests.rs
index 8a9488836..7b8a4badf 100644
--- a/engine/crates/xerj-autoindex/src/generation_catalog_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/generation_catalog_http_tests.rs
@@ -74,6 +74,7 @@ fn dataset() -> PlanDataset {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 2,
file_count: 0,
}
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 44b9889b9..14ce592a0 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -3155,6 +3155,7 @@ fn sweep_excluded_groups_deletes_only_the_excluded_files_documents() {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -3239,6 +3240,7 @@ fn sweep_excluded_groups_invalidates_the_excluded_files_edges() {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -3314,6 +3316,7 @@ fn sweep_excluded_groups_purges_the_excluded_files_alias_catalog_docs() {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -3389,6 +3392,7 @@ fn sweep_excluded_groups_does_not_delete_a_sibling_corpus_catalog_doc() {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -3478,6 +3482,7 @@ fn sweep_excluded_groups_scopes_on_the_keyword_corpus_scope_field() {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -3588,6 +3593,7 @@ fn sweep_excluded_groups_cannot_reach_a_sibling_corpus_on_a_text_mapped_prefix()
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -4079,6 +4085,7 @@ fn sweep_excluded_groups_deletes_a_legacy_main_doc_by_id() {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -4146,6 +4153,7 @@ fn sweep_excluded_groups_deletes_a_legacy_alias_doc_by_id() {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -4219,6 +4227,7 @@ fn sweep_excluded_groups_invalidates_inbound_edges() {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
@@ -6085,6 +6094,7 @@ mod delete_gate {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
}],
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 5ff701dd7..e89b1fea2 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3642,6 +3642,12 @@ fn build_phase_a(
time_field,
semantic_field,
text_analyzer,
+ // #1230: freeze the cluster's repository scope so the incremental
+ // reconciler can route a newly added file to the right sibling
+ // `docs` dataset when forks share a schema. `Some("")` is the root
+ // scope; it is a real recorded value, distinct from a legacy
+ // plan's `None`.
+ scope: Some(c.scope.clone()),
sampled_records: c.records,
file_count: c.members.len(),
});
@@ -10596,6 +10602,7 @@ mod generation_contract_identity_tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 0,
file_count: 0,
}
@@ -10841,6 +10848,7 @@ mod stem_default_tests {
time_field: None,
semantic_field: Some("body".into()),
text_analyzer: text_analyzer.map(str::to_string),
+ scope: None,
sampled_records: 1,
file_count: 1,
}
diff --git a/engine/crates/xerj-autoindex/src/reconcile_plan.rs b/engine/crates/xerj-autoindex/src/reconcile_plan.rs
index 6036462a9..db7577d55 100644
--- a/engine/crates/xerj-autoindex/src/reconcile_plan.rs
+++ b/engine/crates/xerj-autoindex/src/reconcile_plan.rs
@@ -324,7 +324,7 @@ pub(crate) fn reconcile_plan(
} else {
sniffed.family.as_str()
};
- classify_new(group, family, fields, &classification_candidates)
+ classify_new(group, family, fields, &file.rel, &classification_candidates)
.with_context(|| format!("project new file {}", file.rel))?
};
assignments.push((group.clone(), slug));
@@ -479,6 +479,7 @@ fn classify_new(
group: &Option<String>,
family: &str,
fields: &HashMap<String, FieldAcc>,
+ rel: &str,
datasets: &[PlanDataset],
) -> Result<String> {
let mut candidates = Vec::new();
@@ -489,6 +490,7 @@ fn classify_new(
if ensure_compatible(fields, dataset).is_err() {
continue;
}
+ let affinity = scope_affinity(dataset.scope.as_deref(), rel);
let (intersection, union) = field_overlap(fields, &dataset.specs);
let threshold_met = if dataset.family == "docs" {
// A `docs` dataset's specs are the UNION of every document/config
@@ -525,13 +527,24 @@ fn classify_new(
intersection * 10 >= union * 7 // 0.7
};
if threshold_met {
- candidates.push((dataset, intersection, union));
+ candidates.push((dataset, intersection, union, affinity));
}
}
- candidates.sort_by(|(left, li, lu), (right, ri, ru)| {
- ratio_cmp(*ri, *ru, *li, *lu).then_with(|| left.slug.cmp(&right.slug))
+ // #1230: field overlap is the primary key, then repository scope. Two
+ // sibling forks produce byte-identical `docs` schemas, so a newly added
+ // file overlaps both equally and the schema ratio alone is a genuine tie —
+ // the refusal this used to raise. The file's own path resolves it: a
+ // dataset whose frozen scope *contains* the file (nearest `.git` ancestor
+ // wins, hence longest match) is the one a fresh plan would have folded it
+ // into. Scope only ever breaks a ratio tie; it can never promote a
+ // worse-overlapping dataset over a better one.
+ candidates.sort_by(|(left, li, lu, la), (right, ri, ru, ra)| {
+ ratio_cmp(*ri, *ru, *li, *lu)
+ .then_with(|| ra.cmp(la))
+ .then_with(|| left.slug.cmp(&right.slug))
});
- let Some((winner, winner_intersection, winner_union)) = candidates.first() else {
+ let Some((winner, winner_intersection, winner_union, winner_affinity)) = candidates.first()
+ else {
anyhow::bail!(
"no frozen dataset accepts family {family}, group {:?}, and fields {:?}; \
this requires unsupported dataset/schema evolution (use a new prefix)",
@@ -539,14 +552,21 @@ fn classify_new(
sorted_field_names(fields)
);
};
- if let Some((runner_up, runner_up_intersection, runner_up_union)) = candidates.get(1) {
+ if let Some((runner_up, runner_up_intersection, runner_up_union, runner_up_affinity)) =
+ candidates.get(1)
+ {
+ // Ambiguous only when neither field overlap NOR repository scope can
+ // separate the top two. A pre-#1230 plan carries no scope (`None` for
+ // every dataset), so both affinities are `None`, they compare equal,
+ // and this fails closed exactly as before.
anyhow::ensure!(
ratio_cmp(
*winner_intersection,
*winner_union,
*runner_up_intersection,
*runner_up_union
- ) != Ordering::Equal,
+ ) != Ordering::Equal
+ || winner_affinity != runner_up_affinity,
"new file matches frozen datasets {} and {} equally for family {family}, group {:?}, \
and fields {:?}; refusing ambiguous assignment (use a new prefix)",
winner.slug,
@@ -558,6 +578,29 @@ fn classify_new(
Ok(winner.slug.clone())
}
+/// How strongly a frozen dataset's repository scope lays claim to a file at
+/// `rel` (#1230). `None` means the scope does not contain the file (or the
+/// plan predates scope recording and carries no scope at all); a `Some` value
+/// is the matched scope's byte length, so the *nearest* enclosing repository —
+/// the longest owning scope, which is what `compute_scopes` picks at genesis —
+/// ranks highest. The autoindex root (`Some("")`) owns every file but at the
+/// lowest precedence, below any nested repository. `Option<usize>` already
+/// orders `None < Some(0) < Some(len)`, which is exactly that precedence.
+fn scope_affinity(scope: Option<&str>, rel: &str) -> Option<usize> {
+ let scope = scope?;
+ if scope.is_empty() {
+ Some(0)
+ } else if rel == scope
+ || rel
+ .strip_prefix(scope)
+ .is_some_and(|rest| rest.starts_with('/'))
+ {
+ Some(scope.len())
+ } else {
+ None
+ }
+}
+
fn field_overlap(fields: &HashMap<String, FieldAcc>, specs: &[FieldSpec]) -> (usize, usize) {
// Keep null-only and object-shaped fields in the schema fingerprint too.
// FieldAcc deliberately does not count those as scalar observations, but
@@ -686,6 +729,7 @@ mod tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
}
@@ -1039,6 +1083,152 @@ mod tests {
);
}
+ /// Two frozen scope-keyed `docs` datasets (one per nested repository) whose
+ /// genesis schemas are byte-identical. A code document is a group-less,
+ /// empty-key-field shape, so it routes to the `docs` family (see
+ /// `new_document_file_joins_the_docs_dataset_on_incremental`) and classifies
+ /// as a document.
+ fn two_fork_docs_scopes() -> Plan {
+ let docs = |slug: &str, scope: &str| PlanDataset {
+ family: "docs".into(),
+ scope: Some(scope.into()),
+ ..dataset(slug, vec![spec("title", "text"), spec("body", "text")])
+ };
+ Plan {
+ datasets: vec![
+ docs("bitcoin-core-docs", "bitcoin-core"),
+ docs("bitcoin-knots-docs", "bitcoin-knots"),
+ ],
+ ..Plan::default()
+ }
+ }
+
+ fn fork_doc_scan() -> FileScan {
+ // `title`/`body`: the docs shape, present in both forks' identical
+ // specs, so field overlap is a genuine tie. Empty key_fields + no group
+ // + family code -> a NEW document that routes to the `docs` family.
+ let fields = HashMap::from([
+ ("title".into(), acc(&[json!("adjacent.ts")])),
+ (
+ "body".into(),
+ acc(&[json!("export const adjacent = true;")]),
+ ),
+ ]);
+ let mut scan = scan(fields);
+ scan.sniffed.as_mut().unwrap().family = Family::Code;
+ scan
+ }
+
+ /// #1230: adding a file inside one of several sibling repositories with
+ /// matching schemas must route to THAT repository's frozen `docs` dataset,
+ /// not abort the whole run as an ambiguous assignment. The file's path
+ /// lands it under exactly one frozen scope, which breaks the field-overlap
+ /// tie the identical fork schemas create.
+ #[test]
+ fn new_file_in_a_fork_joins_its_own_repository_scope_not_abort() {
+ let previous = two_fork_docs_scopes();
+ let plan = reconcile_plan(
+ &inventory("bitcoin-core/adjacent.ts", "unix:a1", "core-add"),
+ &previous,
+ vec![fork_doc_scan()],
+ 50,
+ )
+ .expect("an adjacent file inside bitcoin-core must not abort the run");
+ assert_eq!(
+ plan.files["core-add"].assignments,
+ vec![(None, "bitcoin-core-docs".into())],
+ "a file under bitcoin-core/ must join bitcoin-core-docs, not its sibling fork"
+ );
+
+ // The sibling repository resolves symmetrically.
+ let plan = reconcile_plan(
+ &inventory("bitcoin-knots/adjacent.ts", "unix:a2", "knots-add"),
+ &previous,
+ vec![fork_doc_scan()],
+ 50,
+ )
+ .expect("an adjacent file inside bitcoin-knots must not abort the run");
+ assert_eq!(
+ plan.files["knots-add"].assignments,
+ vec![(None, "bitcoin-knots-docs".into())],
+ );
+ }
+
+ /// The scope tiebreak only ever breaks a field-overlap TIE; it never
+ /// overrides a dataset that overlaps the file better. A file under
+ /// bitcoin-knots whose fields overlap bitcoin-core-docs strictly more still
+ /// joins bitcoin-core-docs.
+ #[test]
+ fn scope_never_overrides_a_strictly_better_field_overlap() {
+ let core = PlanDataset {
+ family: "jsonl".into(),
+ scope: Some("bitcoin-core".into()),
+ ..dataset(
+ "bitcoin-core-rows",
+ vec![spec("id", "long"), spec("body", "text")],
+ )
+ };
+ let knots = PlanDataset {
+ family: "jsonl".into(),
+ scope: Some("bitcoin-knots".into()),
+ ..dataset("bitcoin-knots-rows", vec![spec("id", "long")])
+ };
+ let previous = Plan {
+ datasets: vec![core, knots],
+ ..Plan::default()
+ };
+ // {id, body}: a perfect overlap with bitcoin-core-rows, a partial one
+ // with bitcoin-knots-rows — even though the file sits under knots.
+ let fields = HashMap::from([
+ ("id".into(), acc(&[json!(7)])),
+ ("body".into(), acc(&[json!("hello world")])),
+ ]);
+ let plan = reconcile_plan(
+ &inventory("bitcoin-knots/rows.jsonl", "unix:a3", "row-add"),
+ &previous,
+ vec![scan(fields)],
+ 50,
+ )
+ .unwrap();
+ assert_eq!(
+ plan.files["row-add"].assignments,
+ vec![(None, "bitcoin-core-rows".into())],
+ "field overlap is primary; scope only breaks a genuine tie"
+ );
+ }
+
+ /// A plan frozen before #1230 carries no scope (`None` for every dataset),
+ /// so equal field overlap stays genuinely ambiguous and still fails closed —
+ /// the fix is additive, never a silent change for legacy state dirs.
+ #[test]
+ fn legacy_plan_without_scope_still_fails_closed_on_an_equal_tie() {
+ let previous = Plan {
+ datasets: vec![
+ PlanDataset {
+ family: "docs".into(),
+ ..dataset("z-docs", vec![spec("title", "text"), spec("body", "text")])
+ },
+ PlanDataset {
+ family: "docs".into(),
+ ..dataset("a-docs", vec![spec("title", "text"), spec("body", "text")])
+ },
+ ],
+ ..Plan::default()
+ };
+ let error = reconcile_plan(
+ &inventory("bitcoin-core/adjacent.ts", "unix:a4", "legacy-add"),
+ &previous,
+ vec![fork_doc_scan()],
+ 50,
+ )
+ .unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("refusing ambiguous assignment"),
+ "{message}"
+ );
+ }
+
#[test]
fn rename_by_content_retains_assignment_and_rebuilds_aliases_and_counts() {
let mut previous = Plan {
diff --git a/engine/crates/xerj-autoindex/src/refused_dataset_tests.rs b/engine/crates/xerj-autoindex/src/refused_dataset_tests.rs
index cb60b1f9c..029b7ff51 100644
--- a/engine/crates/xerj-autoindex/src/refused_dataset_tests.rs
+++ b/engine/crates/xerj-autoindex/src/refused_dataset_tests.rs
@@ -20,6 +20,7 @@ fn dataset(slug: &str) -> PlanDataset {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 0,
}
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index 5cd35e7db..8e4988fa5 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -66,6 +66,30 @@ pub struct PlanDataset {
/// (`install_dataset_mappings`).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text_analyzer: Option<String>,
+ /// Repository scope this dataset was clustered under (#1230): the
+ /// root-relative, "/"-separated path of the nearest enclosing `.git`
+ /// ancestor at genesis, or `Some("")` for the autoindex root. A `docs`
+ /// dataset is one-per-scope (`dataset::cluster`), so this is what tells two
+ /// sibling forks with byte-identical schemas apart — without it the
+ /// `--no-graph` incremental reconciler cannot say which frozen `docs`
+ /// dataset a newly added file belongs to and refuses the whole run as an
+ /// ambiguous assignment.
+ ///
+ /// Like [`Self::text_analyzer`] it rides the PLAN and NOT
+ /// [`crate::infer::FieldSpec`]: `generation_contract_identities` serializes
+ /// the schema contract through named keys (`slug`/`family`/`group`/`specs`/
+ /// `time_field`/`semantic_field`) only, so a new field here is invisible to
+ /// the identity digests and does not move any committed generation's
+ /// `schema_identity`/`index_identity` — adding it never aborts an existing
+ /// state dir on the no-change arm.
+ ///
+ /// `#[serde(default)]` so a plan frozen by a pre-#1230 build deserializes
+ /// with `None`. `None` means "scope was never recorded"; the reconciler
+ /// then falls back to its prior schema-only classification (an older plan
+ /// that hit the ambiguity simply keeps hitting it), so the fix is additive
+ /// and changes nothing for single-scope trees.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub scope: Option<String>,
pub sampled_records: u64,
pub file_count: usize,
}
@@ -1615,6 +1639,7 @@ mod sync_journal_tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 0,
});
@@ -1635,6 +1660,7 @@ mod sync_journal_tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
});
diff --git a/engine/crates/xerj-autoindex/src/sync.rs b/engine/crates/xerj-autoindex/src/sync.rs
index dd89e25f0..30571a5fe 100644
--- a/engine/crates/xerj-autoindex/src/sync.rs
+++ b/engine/crates/xerj-autoindex/src/sync.rs
@@ -1989,6 +1989,7 @@ mod tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 0,
file_count: 0,
});
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index f47eba57f..e58d1c6a6 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -3238,6 +3238,7 @@ mod tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: 1,
}],
@@ -3292,6 +3293,7 @@ mod tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 1,
file_count: files.len(),
}],
@@ -3647,6 +3649,7 @@ mod tests {
time_field: None,
semantic_field: None,
text_analyzer: None,
+ scope: None,
sampled_records: 2,
file_count: 1,
};