AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): keep repository scope so sibling forks route added files correctly (#1230)
xerj-orgb2464e5e2e44
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rsindex f684a4fb3..b94719282 100644--- a/engine/crates/xerj-autoindex/src/catalog.rs+++ b/engine/crates/xerj-autoindex/src/catalog.rs@@ -837,6 +837,7 @@ mod sample_query_tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,}diff --git a/engine/crates/xerj-autoindex/src/generation_catalog.rs b/engine/crates/xerj-autoindex/src/generation_catalog.rsindex 721c0b24e..8f84b096a 100644--- a/engine/crates/xerj-autoindex/src/generation_catalog.rs+++ b/engine/crates/xerj-autoindex/src/generation_catalog.rs@@ -522,6 +522,7 @@ mod tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 2,file_count: 1,}diff --git a/engine/crates/xerj-autoindex/src/generation_catalog_http_tests.rs b/engine/crates/xerj-autoindex/src/generation_catalog_http_tests.rsindex 8a9488836..7b8a4badf 100644--- a/engine/crates/xerj-autoindex/src/generation_catalog_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/generation_catalog_http_tests.rs@@ -74,6 +74,7 @@ fn dataset() -> PlanDataset {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 2,file_count: 0,}diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex 44b9889b9..14ce592a0 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -3155,6 +3155,7 @@ fn sweep_excluded_groups_deletes_only_the_excluded_files_documents() {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -3239,6 +3240,7 @@ fn sweep_excluded_groups_invalidates_the_excluded_files_edges() {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -3314,6 +3316,7 @@ fn sweep_excluded_groups_purges_the_excluded_files_alias_catalog_docs() {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -3389,6 +3392,7 @@ fn sweep_excluded_groups_does_not_delete_a_sibling_corpus_catalog_doc() {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -3478,6 +3482,7 @@ fn sweep_excluded_groups_scopes_on_the_keyword_corpus_scope_field() {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -3588,6 +3593,7 @@ fn sweep_excluded_groups_cannot_reach_a_sibling_corpus_on_a_text_mapped_prefix()time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -4079,6 +4085,7 @@ fn sweep_excluded_groups_deletes_a_legacy_main_doc_by_id() {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -4146,6 +4153,7 @@ fn sweep_excluded_groups_deletes_a_legacy_alias_doc_by_id() {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -4219,6 +4227,7 @@ fn sweep_excluded_groups_invalidates_inbound_edges() {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});@@ -6085,6 +6094,7 @@ mod delete_gate {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,}],diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 5ff701dd7..e89b1fea2 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -3642,6 +3642,12 @@ fn build_phase_a(time_field,semantic_field,text_analyzer,+ // #1230: freeze the cluster's repository scope so the incremental+ // reconciler can route a newly added file to the right sibling+ // `docs` dataset when forks share a schema. `Some("")` is the root+ // scope; it is a real recorded value, distinct from a legacy+ // plan's `None`.+ scope: Some(c.scope.clone()),sampled_records: c.records,file_count: c.members.len(),});@@ -10596,6 +10602,7 @@ mod generation_contract_identity_tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 0,file_count: 0,}@@ -10841,6 +10848,7 @@ mod stem_default_tests {time_field: None,semantic_field: Some("body".into()),text_analyzer: text_analyzer.map(str::to_string),+ scope: None,sampled_records: 1,file_count: 1,}diff --git a/engine/crates/xerj-autoindex/src/reconcile_plan.rs b/engine/crates/xerj-autoindex/src/reconcile_plan.rsindex 6036462a9..db7577d55 100644--- a/engine/crates/xerj-autoindex/src/reconcile_plan.rs+++ b/engine/crates/xerj-autoindex/src/reconcile_plan.rs@@ -324,7 +324,7 @@ pub(crate) fn reconcile_plan(} else {sniffed.family.as_str()};- classify_new(group, family, fields, &classification_candidates)+ classify_new(group, family, fields, &file.rel, &classification_candidates).with_context(|| format!("project new file {}", file.rel))?};assignments.push((group.clone(), slug));@@ -479,6 +479,7 @@ fn classify_new(group: &Option<String>,family: &str,fields: &HashMap<String, FieldAcc>,+ rel: &str,datasets: &[PlanDataset],) -> Result<String> {let mut candidates = Vec::new();@@ -489,6 +490,7 @@ fn classify_new(if ensure_compatible(fields, dataset).is_err() {continue;}+ let affinity = scope_affinity(dataset.scope.as_deref(), rel);let (intersection, union) = field_overlap(fields, &dataset.specs);let threshold_met = if dataset.family == "docs" {// A `docs` dataset's specs are the UNION of every document/config@@ -525,13 +527,24 @@ fn classify_new(intersection * 10 >= union * 7 // 0.7};if threshold_met {- candidates.push((dataset, intersection, union));+ candidates.push((dataset, intersection, union, affinity));}}- candidates.sort_by(|(left, li, lu), (right, ri, ru)| {- ratio_cmp(*ri, *ru, *li, *lu).then_with(|| left.slug.cmp(&right.slug))+ // #1230: field overlap is the primary key, then repository scope. Two+ // sibling forks produce byte-identical `docs` schemas, so a newly added+ // file overlaps both equally and the schema ratio alone is a genuine tie —+ // the refusal this used to raise. The file's own path resolves it: a+ // dataset whose frozen scope *contains* the file (nearest `.git` ancestor+ // wins, hence longest match) is the one a fresh plan would have folded it+ // into. Scope only ever breaks a ratio tie; it can never promote a+ // worse-overlapping dataset over a better one.+ candidates.sort_by(|(left, li, lu, la), (right, ri, ru, ra)| {+ ratio_cmp(*ri, *ru, *li, *lu)+ .then_with(|| ra.cmp(la))+ .then_with(|| left.slug.cmp(&right.slug))});- let Some((winner, winner_intersection, winner_union)) = candidates.first() else {+ let Some((winner, winner_intersection, winner_union, winner_affinity)) = candidates.first()+ else {anyhow::bail!("no frozen dataset accepts family {family}, group {:?}, and fields {:?}; \this requires unsupported dataset/schema evolution (use a new prefix)",@@ -539,14 +552,21 @@ fn classify_new(sorted_field_names(fields));};- if let Some((runner_up, runner_up_intersection, runner_up_union)) = candidates.get(1) {+ if let Some((runner_up, runner_up_intersection, runner_up_union, runner_up_affinity)) =+ candidates.get(1)+ {+ // Ambiguous only when neither field overlap NOR repository scope can+ // separate the top two. A pre-#1230 plan carries no scope (`None` for+ // every dataset), so both affinities are `None`, they compare equal,+ // and this fails closed exactly as before.anyhow::ensure!(ratio_cmp(*winner_intersection,*winner_union,*runner_up_intersection,*runner_up_union- ) != Ordering::Equal,+ ) != Ordering::Equal+ || winner_affinity != runner_up_affinity,"new file matches frozen datasets {} and {} equally for family {family}, group {:?}, \and fields {:?}; refusing ambiguous assignment (use a new prefix)",winner.slug,@@ -558,6 +578,29 @@ fn classify_new(Ok(winner.slug.clone())}+/// How strongly a frozen dataset's repository scope lays claim to a file at+/// `rel` (#1230). `None` means the scope does not contain the file (or the+/// plan predates scope recording and carries no scope at all); a `Some` value+/// is the matched scope's byte length, so the *nearest* enclosing repository —+/// the longest owning scope, which is what `compute_scopes` picks at genesis —+/// ranks highest. The autoindex root (`Some("")`) owns every file but at the+/// lowest precedence, below any nested repository. `Option<usize>` already+/// orders `None < Some(0) < Some(len)`, which is exactly that precedence.+fn scope_affinity(scope: Option<&str>, rel: &str) -> Option<usize> {+ let scope = scope?;+ if scope.is_empty() {+ Some(0)+ } else if rel == scope+ || rel+ .strip_prefix(scope)+ .is_some_and(|rest| rest.starts_with('/'))+ {+ Some(scope.len())+ } else {+ None+ }+}+fn field_overlap(fields: &HashMap<String, FieldAcc>, specs: &[FieldSpec]) -> (usize, usize) {// Keep null-only and object-shaped fields in the schema fingerprint too.// FieldAcc deliberately does not count those as scalar observations, but@@ -686,6 +729,7 @@ mod tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,}@@ -1039,6 +1083,152 @@ mod tests {);}+ /// Two frozen scope-keyed `docs` datasets (one per nested repository) whose+ /// genesis schemas are byte-identical. A code document is a group-less,+ /// empty-key-field shape, so it routes to the `docs` family (see+ /// `new_document_file_joins_the_docs_dataset_on_incremental`) and classifies+ /// as a document.+ fn two_fork_docs_scopes() -> Plan {+ let docs = |slug: &str, scope: &str| PlanDataset {+ family: "docs".into(),+ scope: Some(scope.into()),+ ..dataset(slug, vec![spec("title", "text"), spec("body", "text")])+ };+ Plan {+ datasets: vec![+ docs("bitcoin-core-docs", "bitcoin-core"),+ docs("bitcoin-knots-docs", "bitcoin-knots"),+ ],+ ..Plan::default()+ }+ }++ fn fork_doc_scan() -> FileScan {+ // `title`/`body`: the docs shape, present in both forks' identical+ // specs, so field overlap is a genuine tie. Empty key_fields + no group+ // + family code -> a NEW document that routes to the `docs` family.+ let fields = HashMap::from([+ ("title".into(), acc(&[json!("adjacent.ts")])),+ (+ "body".into(),+ acc(&[json!("export const adjacent = true;")]),+ ),+ ]);+ let mut scan = scan(fields);+ scan.sniffed.as_mut().unwrap().family = Family::Code;+ scan+ }++ /// #1230: adding a file inside one of several sibling repositories with+ /// matching schemas must route to THAT repository's frozen `docs` dataset,+ /// not abort the whole run as an ambiguous assignment. The file's path+ /// lands it under exactly one frozen scope, which breaks the field-overlap+ /// tie the identical fork schemas create.+ #[test]+ fn new_file_in_a_fork_joins_its_own_repository_scope_not_abort() {+ let previous = two_fork_docs_scopes();+ let plan = reconcile_plan(+ &inventory("bitcoin-core/adjacent.ts", "unix:a1", "core-add"),+ &previous,+ vec![fork_doc_scan()],+ 50,+ )+ .expect("an adjacent file inside bitcoin-core must not abort the run");+ assert_eq!(+ plan.files["core-add"].assignments,+ vec![(None, "bitcoin-core-docs".into())],+ "a file under bitcoin-core/ must join bitcoin-core-docs, not its sibling fork"+ );++ // The sibling repository resolves symmetrically.+ let plan = reconcile_plan(+ &inventory("bitcoin-knots/adjacent.ts", "unix:a2", "knots-add"),+ &previous,+ vec![fork_doc_scan()],+ 50,+ )+ .expect("an adjacent file inside bitcoin-knots must not abort the run");+ assert_eq!(+ plan.files["knots-add"].assignments,+ vec![(None, "bitcoin-knots-docs".into())],+ );+ }++ /// The scope tiebreak only ever breaks a field-overlap TIE; it never+ /// overrides a dataset that overlaps the file better. A file under+ /// bitcoin-knots whose fields overlap bitcoin-core-docs strictly more still+ /// joins bitcoin-core-docs.+ #[test]+ fn scope_never_overrides_a_strictly_better_field_overlap() {+ let core = PlanDataset {+ family: "jsonl".into(),+ scope: Some("bitcoin-core".into()),+ ..dataset(+ "bitcoin-core-rows",+ vec![spec("id", "long"), spec("body", "text")],+ )+ };+ let knots = PlanDataset {+ family: "jsonl".into(),+ scope: Some("bitcoin-knots".into()),+ ..dataset("bitcoin-knots-rows", vec![spec("id", "long")])+ };+ let previous = Plan {+ datasets: vec![core, knots],+ ..Plan::default()+ };+ // {id, body}: a perfect overlap with bitcoin-core-rows, a partial one+ // with bitcoin-knots-rows — even though the file sits under knots.+ let fields = HashMap::from([+ ("id".into(), acc(&[json!(7)])),+ ("body".into(), acc(&[json!("hello world")])),+ ]);+ let plan = reconcile_plan(+ &inventory("bitcoin-knots/rows.jsonl", "unix:a3", "row-add"),+ &previous,+ vec![scan(fields)],+ 50,+ )+ .unwrap();+ assert_eq!(+ plan.files["row-add"].assignments,+ vec![(None, "bitcoin-core-rows".into())],+ "field overlap is primary; scope only breaks a genuine tie"+ );+ }++ /// A plan frozen before #1230 carries no scope (`None` for every dataset),+ /// so equal field overlap stays genuinely ambiguous and still fails closed —+ /// the fix is additive, never a silent change for legacy state dirs.+ #[test]+ fn legacy_plan_without_scope_still_fails_closed_on_an_equal_tie() {+ let previous = Plan {+ datasets: vec![+ PlanDataset {+ family: "docs".into(),+ ..dataset("z-docs", vec![spec("title", "text"), spec("body", "text")])+ },+ PlanDataset {+ family: "docs".into(),+ ..dataset("a-docs", vec![spec("title", "text"), spec("body", "text")])+ },+ ],+ ..Plan::default()+ };+ let error = reconcile_plan(+ &inventory("bitcoin-core/adjacent.ts", "unix:a4", "legacy-add"),+ &previous,+ vec![fork_doc_scan()],+ 50,+ )+ .unwrap_err();+ let message = format!("{error:#}");+ assert!(+ message.contains("refusing ambiguous assignment"),+ "{message}"+ );+ }+#[test]fn rename_by_content_retains_assignment_and_rebuilds_aliases_and_counts() {let mut previous = Plan {diff --git a/engine/crates/xerj-autoindex/src/refused_dataset_tests.rs b/engine/crates/xerj-autoindex/src/refused_dataset_tests.rsindex cb60b1f9c..029b7ff51 100644--- a/engine/crates/xerj-autoindex/src/refused_dataset_tests.rs+++ b/engine/crates/xerj-autoindex/src/refused_dataset_tests.rs@@ -20,6 +20,7 @@ fn dataset(slug: &str) -> PlanDataset {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 0,}diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex 5cd35e7db..8e4988fa5 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -66,6 +66,30 @@ pub struct PlanDataset {/// (`install_dataset_mappings`).#[serde(default, skip_serializing_if = "Option::is_none")]pub text_analyzer: Option<String>,+ /// Repository scope this dataset was clustered under (#1230): the+ /// root-relative, "/"-separated path of the nearest enclosing `.git`+ /// ancestor at genesis, or `Some("")` for the autoindex root. A `docs`+ /// dataset is one-per-scope (`dataset::cluster`), so this is what tells two+ /// sibling forks with byte-identical schemas apart — without it the+ /// `--no-graph` incremental reconciler cannot say which frozen `docs`+ /// dataset a newly added file belongs to and refuses the whole run as an+ /// ambiguous assignment.+ ///+ /// Like [`Self::text_analyzer`] it rides the PLAN and NOT+ /// [`crate::infer::FieldSpec`]: `generation_contract_identities` serializes+ /// the schema contract through named keys (`slug`/`family`/`group`/`specs`/+ /// `time_field`/`semantic_field`) only, so a new field here is invisible to+ /// the identity digests and does not move any committed generation's+ /// `schema_identity`/`index_identity` — adding it never aborts an existing+ /// state dir on the no-change arm.+ ///+ /// `#[serde(default)]` so a plan frozen by a pre-#1230 build deserializes+ /// with `None`. `None` means "scope was never recorded"; the reconciler+ /// then falls back to its prior schema-only classification (an older plan+ /// that hit the ambiguity simply keeps hitting it), so the fix is additive+ /// and changes nothing for single-scope trees.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub scope: Option<String>,pub sampled_records: u64,pub file_count: usize,}@@ -1615,6 +1639,7 @@ mod sync_journal_tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 0,});@@ -1635,6 +1660,7 @@ mod sync_journal_tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,});diff --git a/engine/crates/xerj-autoindex/src/sync.rs b/engine/crates/xerj-autoindex/src/sync.rsindex dd89e25f0..30571a5fe 100644--- a/engine/crates/xerj-autoindex/src/sync.rs+++ b/engine/crates/xerj-autoindex/src/sync.rs@@ -1989,6 +1989,7 @@ mod tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 0,file_count: 0,});diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex f47eba57f..e58d1c6a6 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -3238,6 +3238,7 @@ mod tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: 1,}],@@ -3292,6 +3293,7 @@ mod tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 1,file_count: files.len(),}],@@ -3647,6 +3649,7 @@ mod tests {time_field: None,semantic_field: None,text_analyzer: None,+ scope: None,sampled_records: 2,file_count: 1,};