AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): keep a changed file in a field-capped dataset (#1815) (#2215)
xerj-orgf07bdde19708
diff --git a/engine/crates/xerj-autoindex/src/reconcile_plan.rs b/engine/crates/xerj-autoindex/src/reconcile_plan.rsindex 874a29868..f0c452ecd 100644--- a/engine/crates/xerj-autoindex/src/reconcile_plan.rs+++ b/engine/crates/xerj-autoindex/src/reconcile_plan.rs@@ -811,6 +811,11 @@ fn ensure_compatible(if widenable_field_name(name) {continue;}+ // #1815: genesis already published this name as an unmapped+ // overflow field of a dataset that hit the field cap.+ if cap_overflow_fields(dataset).is_some_and(|overflow| overflow.contains(name)) {+ continue;+ }return Err(AssignFailure::File(format!("field {name} is absent from frozen dataset {} and cannot be added to its mapping",dataset.slug@@ -847,6 +852,20 @@ fn widenable_field_name(name: &str) -> bool {chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')}+/// The overflow field names genesis recorded when `infer` truncated the+/// dataset at [`crate::infer::MAX_FIELDS_PER_DATASET`], or `None` when the+/// dataset never hit the cap.+fn cap_overflow_fields(dataset: &PlanDataset) -> Option<HashSet<&str>> {+ dataset.specs.iter().find_map(|spec| {+ spec.notes.iter().find_map(|note| {+ note.strip_prefix(CAP_OVERFLOW_NOTE)+ .map(|list| list.split(", ").filter(|name| !name.is_empty()).collect())+ })+ })+}++const CAP_OVERFLOW_NOTE: &str = "dataset field cap hit; unmapped overflow fields: ";+fn widen_dataset(dataset: &PlanDataset,added: &mut Vec<FieldSpec>,@@ -859,6 +878,11 @@ fn widen_dataset(.map(|spec| spec.name.as_str()).collect();known.extend(added.iter().map(|spec| spec.name.as_str()));+ // #1815: a dataset that hit the field cap at genesis has an explicitly+ // partial schema. Its overflow fields were published unmapped and pass+ // through coercion (`coerce.rs`). A changed file that re-observes them, or+ // grows another field, is the same shape genesis accepted, not drift.+ let overflow = cap_overflow_fields(dataset);let mut pending: HashMap<String, FieldAcc> = HashMap::new();let mut names: Vec<&str> = fields.keys().map(String::as_str).collect();names.sort_unstable();@@ -867,6 +891,13 @@ fn widen_dataset(if known.contains(name) || absent_field_is_tolerated(name, acc, dataset) || acc.n == 0 {continue;}+ if let Some(overflow) = &overflow {+ if overflow.contains(name) || widenable_field_name(name) {+ // Unmapped, exactly like the genesis overflow: the cap leaves+ // no room to map it, and refusing it would drop the file.+ continue;+ }+ }if !widenable_field_name(name) {return Err(AssignFailure::File(format!("field {name} is absent from frozen dataset {} and cannot be added to its mapping",@@ -1286,6 +1317,62 @@ mod tests {);}+ /// #1815: a dataset genesis truncated at the field cap keeps its overflow+ /// fields unmapped. A retained file that changes a value and grows a new+ /// field must stay assigned, not be refused for "passing the field cap".+ #[test]+ fn capped_dataset_retains_a_changed_file_with_overflow_fields() {+ let mut genesis: HashMap<String, FieldAcc> = (0..crate::infer::MAX_FIELDS_PER_DATASET + 3)+ .map(|i| (format!("f_{i:04}"), acc(&[json!(i)])))+ .collect();+ genesis.insert("id".into(), acc(&[json!(1)]));+ let specs = crate::infer::infer_fields(&genesis, 1, true);+ assert_eq!(specs.len(), crate::infer::MAX_FIELDS_PER_DATASET);+ let mut ds = dataset("wide", specs);+ let overflow: Vec<String> = cap_overflow_fields(&ds)+ .expect("genesis records the cap overflow")+ .into_iter()+ .map(str::to_string)+ .collect();+ assert_eq!(overflow.len(), 4);+ ds.file_count = 1;+ let mut previous = Plan {+ datasets: vec![ds],+ ..Plan::default()+ };+ previous+ .files+ .insert("old".into(), assignment("wide.json", "unix:77", "wide"));++ let mut changed = genesis.clone();+ changed.insert("id".into(), acc(&[json!(2)]));+ changed.insert(+ "paths_v1_new_route_get".into(),+ acc(&[json!("get /v1/new")]),+ );+ let plan = reconcile_plan(+ &inventory("wide.json", "unix:77", "new"),+ &previous,+ vec![scan(changed)],+ 50,+ REBUILD,+ )+ .unwrap();+ assert!(plan.junk_files.is_empty(), "{:?}", plan.junk_files);+ assert_eq!(plan.files["new"].assignments, vec![(None, "wide".into())]);+ assert!(+ plan.incremental_fields+ .get("wide")+ .is_none_or(|added| added.is_empty()),+ "a capped dataset cannot map more fields; overflow passes through"+ );+ }++ #[test]+ fn an_uncapped_dataset_has_no_overflow_fields() {+ assert!(cap_overflow_fields(&dataset("rows", vec![spec("id", "long")])).is_none());+ }+/// A NEW document file (empty `key_fields`, no group — the shape a fresh/// plan folds into the scope's docs dataset, dataset.rs:119) added to an/// already-indexed folder must join the frozen `docs` dataset on the