AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): let a watched file grow a field without failing the pass (#2193)
xerj-org922921aec0d6
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex e7e7241a8..5eddd164d 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -70,6 +70,14 @@ struct HttpState {unavailable_dataset_mappings: Vec<String>,/// Every dataset index a `_mapping` PUT was attempted for, in order.dataset_mapping_puts: Vec<String>,+ /// Properties installed for a dataset index: the create body the first+ /// time the index is seen, then additive `_mapping` merges. A later+ /// `_mapping` whose `type` disagrees with what is already stored is the+ /// engine's type-change refusal (`field [X] already exists as [old],+ /// cannot add [new]`). A create of an index that already exists is+ /// ignored, the way `Es::ensure_index` swallows "exists" without+ /// re-sending the body.+ installed_dataset_properties: HashMap<String, serde_json::Map<String, Value>>,/// Opt-in (#931): fail the Nth data bulk (1-based) the way/// `fail_next_data_bulk` fails the next one. Lets a replay be interrupted/// AFTER some operations committed, which is the only state in which@@ -188,6 +196,16 @@ impl HttpEndpoint {self.state.lock().unwrap().data_bulk_requests}+ /// Properties the fake has installed for one dataset index (#2176).+ fn dataset_properties(&self, index: &str) -> Option<Value> {+ self.state+ .lock()+ .unwrap()+ .installed_dataset_properties+ .get(index)+ .map(|properties| Value::Object(properties.clone()))+ }+/// #755: the catalog mapping the endpoint finally acknowledged.fn installed_catalog_properties(&self) -> Value {self.state@@ -308,6 +326,7 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {let mut conflict: Option<String> = None;let mut unavailable = false;let mut refused_dataset: Option<String> = None;+ let mut type_conflict: Option<String> = None;if path.ends_with("/_mapping") {assert!(value.get("properties").is_some());assert!(value.get("mappings").is_none());@@ -337,11 +356,19 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {if selected(&locked.refused_dataset_mappings) {refused_dataset = Some(index.clone());}+ if refused_dataset.is_none() && !unavailable {+ let incoming = value["properties"].as_object().cloned().unwrap_or_default();+ type_conflict = apply_dataset_mapping(&mut locked, &index, &incoming);+ }locked.dataset_mapping_puts.push(index);}} else {assert!(value.pointer("/mappings/properties").is_some());assert!(value.get("properties").is_none());+ let index = path.trim_start_matches('/');+ if !index.is_empty() && index != catalog::CATALOG_INDEX {+ remember_dataset_create(&mut locked, index, &value);+ }}if let Some(index) = refused_dataset {// Byte for byte the shape of the refusal in #929.@@ -378,6 +405,23 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {"status": 400}),)+ } else if let Some(reason) = type_conflict {+ // Same shape as the catalog conflict above, and the same words+ // `xerj-api` uses when `PUT /_mapping` changes a field's type+ // (`es_compat.rs`). `update_mapping` does not swallow a body that+ // contains "exists"; `ensure_index` does, which is why a create+ // of an existing index never reaches this branch.+ (+ 400,+ json!({+ "error": {+ "root_cause": [{"type": "mapper_parsing_exception", "reason": reason}],+ "type": "mapper_parsing_exception",+ "reason": reason+ },+ "status": 400+ }),+ )} else {(200, json!({"acknowledged": true}))}@@ -402,6 +446,60 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {.unwrap();}+/// Store the create-body properties the first time an index is seen.+/// A later create is the "index exists" case: the body is not applied.+fn remember_dataset_create(state: &mut HttpState, index: &str, body: &Value) {+ if state.installed_dataset_properties.contains_key(index) {+ return;+ }+ let Some(properties) = body+ .pointer("/mappings/properties")+ .and_then(Value::as_object)+ .cloned()+ else {+ return;+ };+ state+ .installed_dataset_properties+ .insert(index.to_owned(), properties);+}++/// Merge a dataset `_mapping` into the stored properties, or refuse a type+/// change. Returns the engine's reason when a stored field's `type` would+/// change; the stored map is left untouched in that case.+fn apply_dataset_mapping(+ state: &mut HttpState,+ index: &str,+ incoming: &serde_json::Map<String, Value>,+) -> Option<String> {+ if let Some(existing) = state.installed_dataset_properties.get(index) {+ for (name, spec) in incoming {+ let (Some(old), Some(new)) = (+ existing+ .get(name)+ .and_then(|value| value.get("type"))+ .and_then(Value::as_str),+ spec.get("type").and_then(Value::as_str),+ ) else {+ continue;+ };+ if old != new {+ return Some(format!(+ "field [{name}] already exists as [{old}], cannot add [{new}]"+ ));+ }+ }+ }+ let slot = state+ .installed_dataset_properties+ .entry(index.to_owned())+ .or_default();+ for (name, spec) in incoming {+ slot.entry(name.clone()).or_insert_with(|| spec.clone());+ }+ None+}+fn bulk_http(body: &[u8], state: &Arc<Mutex<HttpState>>) -> (u16, Value) {let lines: Vec<&[u8]> = body.split(|byte| *byte == b'\n')@@ -5655,6 +5753,189 @@ fn a_watch_pass_rehashes_only_what_changed_and_publishes_it() {);}+fn joined_docs(endpoint: &HttpEndpoint) -> String {+ endpoint+ .data_docs()+ .iter()+ .map(|doc| doc.to_string())+ .collect::<Vec<_>>()+ .join("\n")+}++/// #2176: a Python file indexed as a comment and an import freezes `docs`+/// without a `code` field. Adding a function while `--watch` is running must+/// publish the new source, install `code` on the dataset, and still index a+/// sibling file that changed in the same pass. On the failing build this+/// pass returns the frozen-schema error and neither edit lands.+#[test]+fn adding_a_python_function_under_watch_indexes_the_new_field_and_sibling_edits() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let root = corpus.path().canonicalize().unwrap();+ fs::write(+ root.join("module.py"),+ "# The birchbefore marker identifies the cancellation module.\nimport math\n",+ )+ .unwrap();+ fs::write(root.join("notes.md"), "birchdoc-before\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(&root, state_dir.path(), &endpoint.url, false);+ let mut carry = crate::watch::Carry::default();+ let (outcome, _) =+ crate::watch::one_pass(&config, &root, &mut carry, &crate::watch::ChangeSet::full());+ assert_eq!(outcome.unwrap().0, 0, "the symbol-less file must index");+ let first = joined_docs(&endpoint);+ assert!(+ first.contains("birchbefore"),+ "the first pass must publish the original source: {first}"+ );+ assert!(first.contains("birchdoc-before"), "{first}");++ fs::write(+ root.join("module.py"),+ "# The birchafter marker identifies the cancellation module.\n\+ import math\n\+ \n\+ def cancel_request():\n\+ return \"birchfunction\"\n",+ )+ .unwrap();+ fs::write(root.join("notes.md"), "birchdoc-after\n").unwrap();+ let burst =+ crate::watch::ChangeSet::from_paths(&[root.join("module.py"), root.join("notes.md")]);+ let (outcome, _) = crate::watch::one_pass(&config, &root, &mut carry, &burst);+ let (code, _) = outcome+ .expect("adding `code` to a frozen docs dataset must not fail the watch pass (#2176)");+ assert_eq!(code, 0, "both files are compatible once the mapping grows");+ let indexed = joined_docs(&endpoint);+ assert!(+ indexed.contains("birchafter"),+ "the rewritten file must be live: {indexed}"+ );+ assert!(+ indexed.contains("birchfunction"),+ "the function body must be searchable: {indexed}"+ );+ assert!(+ indexed.contains("birchdoc-after"),+ "a sibling edit in the same pass must still be indexed: {indexed}"+ );+ assert!(+ !indexed.contains("birchbefore"),+ "the old source must not stay searchable: {indexed}"+ );+ assert!(+ !indexed.contains("birchdoc-before"),+ "the old note must not stay searchable: {indexed}"+ );+ assert!(+ endpoint.data_docs().iter().any(|doc| {+ doc.get("code")+ .and_then(Value::as_str)+ .is_some_and(|code| code.contains("cancel_request"))+ }),+ "a symbol document must carry the new `code` field: {indexed}"+ );+ let properties = endpoint+ .dataset_properties("incremental-http-docs")+ .expect("the docs index mapping must have been installed");+ let code_type = properties["code"]["type"]+ .as_str()+ .expect("code must be a mapped field");+ assert!(+ code_type == "text" || code_type == "keyword",+ "code must be a searchable scalar, got {code_type}"+ );+ a_rerun_changes_nothing(&config, &endpoint, "function added under watch");+}++/// A type change cannot be applied to a frozen field. Only that file is+/// skipped; the other file in the pass is published, and the printed rebuild+/// command is a real `autoindex` of a new state dir and prefix.+#[test]+fn a_type_change_skips_one_file_and_the_printed_rebuild_indexes_it() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let root = corpus.path().canonicalize().unwrap();+ fs::write(root.join("good.csv"), "id,value\n2,beta\n").unwrap();+ fs::write(root.join("bad.csv"), "id,value\n1,alpha\n").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(&root, state_dir.path(), &endpoint.url, false);+ let mut carry = crate::watch::Carry::default();+ assert_eq!(+ crate::watch::one_pass(&config, &root, &mut carry, &crate::watch::ChangeSet::full())+ .0+ .unwrap()+ .0,+ 0+ );++ // The data row keeps a number so the sniffer still treats the first line+ // as a header. `id` stays the frozen `long` field and the string is a+ // type conflict, not a new `col_1` schema.+ fs::write(root.join("good.csv"), "id,value\n2,good-new\n").unwrap();+ fs::write(root.join("bad.csv"), "id,value\nnot-a-number,2\n").unwrap();+ let burst = crate::watch::ChangeSet::from_paths(&[root.join("good.csv"), root.join("bad.csv")]);+ let (outcome, _) = crate::watch::one_pass(&config, &root, &mut carry, &burst);+ let (code, summary) = outcome.expect("a type change must not fail the rest of the pass");+ assert_eq!(code, 3, "the unmappable file is junk, and junk exits 3");+ let skipped = summary.expect("the pass must report the skipped file")["skipped"].clone();+ let rendered = skipped.to_string();+ let command = incremental_schema_rebuild_command(&config, state_dir.path());+ assert!(+ rendered.contains("incompatible"),+ "the reason must name the type conflict: {rendered}"+ );+ assert!(+ rendered.contains("bad.csv"),+ "the reason must name the file: {rendered}"+ );+ assert!(+ rendered.contains(&command),+ "the reason must print the rebuild command\nreason: {rendered}\ncommand: {command}"+ );+ assert!(+ rendered.contains("Documents already indexed for this file are dropped"),+ "{rendered}"+ );+ let indexed = joined_docs(&endpoint);+ assert!(+ indexed.contains("good-new"),+ "the sibling file must be published: {indexed}"+ );+ assert!(+ !indexed.contains("not-a-number"),+ "the type-changed file must not be published: {indexed}"+ );+ assert!(+ !indexed.contains("alpha"),+ "documents already indexed for the skipped file are dropped: {indexed}"+ );++ let journal_before = fs::read(state_dir.path().join("journal.ndjson")).unwrap();+ let rebuild_state = PathBuf::from(format!("{}-reindex", state_dir.path().display()));+ fs::create_dir_all(&rebuild_state).unwrap();+ let mut rebuild = config.clone();+ rebuild.prefix = format!("{}-reindex", config.prefix);+ rebuild.state_dir = Some(rebuild_state);+ assert!(+ matches!(run_index(rebuild).unwrap(), 0 | 3),+ "the printed destination must accept a fresh index of the same folder"+ );+ let rebuilt = joined_docs(&endpoint);+ assert!(+ rebuilt.contains("not-a-number"),+ "the rebuild must index the file the watcher skipped: {rebuilt}"+ );+ assert_eq!(+ fs::read(state_dir.path().join("journal.ndjson")).unwrap(),+ journal_before,+ "the rebuild uses its own state dir and must not rewrite the watcher journal"+ );+}+#[test]fn a_deleted_files_records_stop_appearing() {let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex d6ff5c608..9925b32a6 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -422,7 +422,7 @@ fn install_dataset_mappings() -> Result<Vec<MappingRefusal>> {let mut refused = Vec::new();for dataset in &plan.datasets {- let create_body = dataset_create_body(dataset);+ let create_body = dataset_mapping_body(dataset, extra_fields(plan, &dataset.slug));let update_body = json!({"properties": create_body["mappings"]["properties"].clone()});@@ -475,7 +475,39 @@ fn install_dataset_mappings(/// marker therefore rides the plan frozen in the journal: a pending/// generation replayed after a restart still declares the analyzer.fn dataset_create_body(dataset: &PlanDataset) -> Value {+ dataset_mapping_body(dataset, &[])+}++fn extra_fields<'a>(plan: &'a Plan, slug: &str) -> &'a [infer::FieldSpec] {+ plan.incremental_fields+ .get(slug)+ .map(Vec::as_slice)+ .unwrap_or(&[])+}++/// The index-create body, plus fields an incremental run added after the+/// schema was frozen (#2176). An extra name that the frozen mapping already+/// declares is left alone, so this cannot change `impl` from `text` to a+/// re-inferred `keyword`.+fn dataset_mapping_body(dataset: &PlanDataset, extra: &[infer::FieldSpec]) -> Value {let mut create_body = build_mapping(&dataset.specs);+ if let Some(props) = create_body+ .pointer_mut("/mappings/properties")+ .and_then(Value::as_object_mut)+ {+ for spec in extra {+ if props.contains_key(&spec.name) {+ continue;+ }+ let property = match spec.es_type.as_str() {+ "date" => {+ json!({"type": "date", "format": "strict_date_optional_time||epoch_millis"})+ }+ es_type => json!({"type": es_type}),+ };+ props.insert(spec.name.clone(), property);+ }+ }create_body["mappings"]["properties"]["ax_paths"] = json!({"type": "keyword"});if dataset.family == "docs" || dataset.family == "code" {if let Some(props) = create_body@@ -1032,7 +1064,41 @@ fn project_reconcile_plan(}).collect()});- reconcile_plan::reconcile_plan(inventory, base_plan, scans, cfg.sample)+ let rebuild = incremental_schema_rebuild_command(cfg, state_dir);+ reconcile_plan::reconcile_plan(inventory, base_plan, scans, cfg.sample, &rebuild)+}++/// The command a per-file schema refusal tells the operator to run.+///+/// A new `--state-dir` and a new `--prefix` are both required: the watcher's+/// journal owns the current pair, and `preflight` refuses a different prefix+/// in that directory. The destination sits beside the live state dir so the+/// watcher can keep running until the operator stops it. Every flag that+/// changes what gets indexed is carried by [`legacy_rebuild_argv`].+fn incremental_schema_rebuild_command(cfg: &IndexCfg, state_dir: &Path) -> String {+ let rebuild_dir = format!("{}-reindex", state_dir.display());+ let rebuild_prefix = format!("{}-reindex", cfg.prefix);+ legacy_rebuild_argv(cfg, Path::new(&rebuild_dir), &rebuild_prefix)+ .iter()+ .map(|arg| quote_shell(arg))+ .collect::<Vec<_>>()+ .join(" ")+}++/// Shell-quote one argv element. Safe characters are the same set+/// `catalog::shell_quote` uses; that helper is private to its module.+fn quote_shell(s: &str) -> String {+ if s.is_empty() {+ return "''".to_string();+ }+ let safe = s+ .chars()+ .all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '.' | '_' | '-' | ':' | '\\'));+ if safe {+ s.to_string()+ } else {+ format!("'{}'", s.replace('\'', "'\\''"))+ }}/// Code/AST coverage for one corpus: how much of what a person would call@@ -4435,6 +4501,7 @@ fn build_phase_a(duplicate_files,alias_paths_indexed: true,refused_datasets: Vec::new(),+ incremental_fields: Default::default(),};PhaseA {plan,diff --git a/engine/crates/xerj-autoindex/src/reconcile_plan.rs b/engine/crates/xerj-autoindex/src/reconcile_plan.rsindex 689f2f1d4..874a29868 100644--- a/engine/crates/xerj-autoindex/src/reconcile_plan.rs+++ b/engine/crates/xerj-autoindex/src/reconcile_plan.rs@@ -2,8 +2,13 @@//!//! Fresh-run clustering is intentionally not reused here: cluster formation//! and slug election depend on corpus order and membership. Incremental runs-//! must preserve the committed dataset/schema identity and fail closed when a-//! file would require a new dataset or mapping.+//! keep the committed dataset identity. A new scalar field on a file that+//! already belongs to one of those datasets is added to that dataset's+//! mapping — the server accepts an additive `PUT _mapping` — so a code file+//! that grows a function (`code`, `defs`, `name`, …) does not wedge the+//! watcher (#2176). A change that cannot be applied in place (a type+//! conflict, a brand-new dataset, an ambiguous assignment) drops only that+//! file and names the rebuild command; the rest of the pass still commits.use crate::content::Inventory;use crate::infer::{FieldAcc, FieldSpec};@@ -11,7 +16,7 @@ use crate::state::{FileAssignment, JunkFile, Plan, PlanDataset, RefusedDataset};use crate::FileScan;use anyhow::{Context, Result};use std::cmp::Ordering;-use std::collections::{HashMap, HashSet};+use std::collections::{BTreeMap, HashMap, HashSet};use std::path::Path;/// Field shape of a file as the *document* renderer sees it.@@ -41,13 +46,16 @@ fn document_sample(path: &Path, gzip: bool, sample: usize) -> HashMap<String, Fi/// Project the current, byte-verified inventory onto committed dataset schemas.////// Assignment precedence is stable content identity, then stable native path-/// identity, then deterministic schema matching. Dataset definitions are-/// copied byte-for-byte except for their derived `file_count`.+/// identity, then deterministic schema matching. A dataset keeps its committed+/// specs — those are the generation identity — and `file_count` is recomputed.+/// A scalar field the specs do not name is recorded beside them and installed+/// by the mapping update (#2176).pub(crate) fn reconcile_plan(inventory: &Inventory,previous: &Plan,scans: Vec<FileScan>,sample: usize,+ rebuild_command: &str,) -> Result<Plan> {anyhow::ensure!(inventory.files.len() == inventory.keys.len()@@ -136,16 +144,16 @@ pub(crate) fn reconcile_plan(.iter().map(|junk| (junk.file_key.as_str(), junk)).collect();- let classification_candidates: Vec<PlanDataset> = previous- .datasets+ // Specs stay frozen. A new scalar is recorded in `incremental_fields` and+ // installed by the mapping update; writing it into `specs` would move the+ // generation identity and abort the pass. Refused definitions are not+ // widened — growing a mapping the server already rejected cannot help.+ let mut owned_datasets = previous.datasets.clone();+ let mut incremental_fields = previous.incremental_fields.clone();+ let refused_datasets_frozen: Vec<&PlanDataset> = previous+ .refused_datasets.iter()- .chain(- previous- .refused_datasets- .iter()- .map(|refusal| &refusal.dataset),- )- .cloned()+ .map(|refusal| &refusal.dataset).collect();let mut refused_keys: Vec<Vec<String>> = vec![Vec::new(); previous.refused_datasets.len()];@@ -310,12 +318,19 @@ pub(crate) fn reconcile_plan(};let mut assignments = Vec::with_capacity(scan.sketches.len());+ let mut file_reject: Option<String> = None;for sketch in &scan.sketches {let group = &sketch.group;let fields = document_fields.as_ref().unwrap_or(&sketch.fields);- let slug = if let Some(owner) = retained {- retained_slug(owner, group, sniffed.family.as_str(), fields, &datasets)- .with_context(|| format!("project retained file {}", file.rel))?+ let assigned = if let Some(owner) = retained {+ retained_slug(+ owner,+ group,+ sniffed.family.as_str(),+ fields,+ &owned_datasets,+ &incremental_fields,+ )} else {// A routed document/config classifies as "docs" — its fresh-plan// fold target — instead of aborting on its raw family.@@ -324,10 +339,37 @@ pub(crate) fn reconcile_plan(} else {sniffed.family.as_str()};- classify_new(group, family, fields, &file.rel, &classification_candidates)- .with_context(|| format!("project new file {}", file.rel))?+ classify_new(+ group,+ family,+ fields,+ &file.rel,+ &owned_datasets,+ &refused_datasets_frozen,+ &incremental_fields,+ )};- assignments.push((group.clone(), slug));+ match assigned {+ Ok(slug) => assignments.push((group.clone(), slug, fields, sketch.records)),+ Err(AssignFailure::File(detail)) => {+ file_reject = Some(detail);+ break;+ }+ Err(AssignFailure::Fatal(error)) => {+ return Err(error.context(format!("project file {}", file.rel)));+ }+ }+ }+ if let Some(detail) = file_reject {+ junk_files.push(JunkFile {+ file_key: content_id.clone(),+ rel: file.rel.clone(),+ format: crate::format_str(scan.sniffed.as_ref()),+ status: "junk".into(),+ reason: file_schema_reason(&detail, retained.is_some(), rebuild_command),+ bytes: file.size,+ });+ continue;}// A new or changed file whose shape belongs to a refused dataset is// refused with it — whole, for the reason `refuse_datasets` gives: a@@ -335,7 +377,7 @@ pub(crate) fn reconcile_plan(// refused slug so the choice does not depend on sketch order.if let Some(owner) = assignments.iter()- .filter_map(|(_, slug)| refused_by_slug.get(slug.as_str()).copied())+ .filter_map(|(_, slug, _, _)| refused_by_slug.get(slug.as_str()).copied()).min_by_key(|owner| previous.refused_datasets[*owner].dataset.slug.as_str()){junk_files.push(JunkFile {@@ -349,13 +391,74 @@ pub(crate) fn reconcile_plan(refused_keys[owner].push(content_id.clone());continue;}+ // Widen only after every sketch has a published dataset, and roll the+ // appends back if a later sketch cannot grow the mapping — a file is+ // assigned whole or not at all. The frozen specs are not touched.+ let mut widened_to: HashMap<String, usize> = HashMap::new();+ let mut widen_reject: Option<String> = None;+ for (_, slug, fields, records) in &assignments {+ let Some(index) = owned_datasets+ .iter()+ .position(|dataset| dataset.slug == *slug)+ else {+ continue;+ };+ let before = incremental_fields.get(slug.as_str()).map_or(0, Vec::len);+ match widen_dataset(+ &owned_datasets[index],+ incremental_fields.entry(slug.clone()).or_default(),+ fields,+ *records,+ ) {+ Ok(()) => {+ if incremental_fields.get(slug.as_str()).map_or(0, Vec::len) > before {+ widened_to.entry(slug.clone()).or_insert(before);+ }+ }+ Err(AssignFailure::File(detail)) => {+ rollback_widens(&mut incremental_fields, &widened_to);+ widen_reject = Some(detail);+ break;+ }+ Err(AssignFailure::Fatal(error)) => {+ return Err(error.context(format!("project file {}", file.rel)));+ }+ }+ }+ if let Some(detail) = widen_reject {+ junk_files.push(JunkFile {+ file_key: content_id.clone(),+ rel: file.rel.clone(),+ format: crate::format_str(scan.sniffed.as_ref()),+ status: "junk".into(),+ reason: file_schema_reason(&detail, retained.is_some(), rebuild_command),+ bytes: file.size,+ });+ continue;+ }+ let mut assignments: Vec<(Option<String>, String)> = assignments+ .into_iter()+ .map(|(group, slug, _, _)| (group, slug))+ .collect();assignments.sort();assignments.dedup();- anyhow::ensure!(- assignments.len() == scan.sketches.len(),- "{} has duplicate dataset groups",- file.rel- );+ if assignments.len() != scan.sketches.len() {+ junk_files.push(JunkFile {+ file_key: content_id.clone(),+ rel: file.rel.clone(),+ format: crate::format_str(scan.sniffed.as_ref()),+ status: "junk".into(),+ reason: file_schema_reason(+ &format!("{} has duplicate dataset groups", file.rel),+ retained.is_some(),+ rebuild_command,+ ),+ bytes: file.size,+ });+ // The widens above belong to a file that is not indexed.+ rollback_widens(&mut incremental_fields, &widened_to);+ continue;+ }let replaced = files.insert(content_id.clone(),FileAssignment {@@ -394,8 +497,7 @@ pub(crate) fn reconcile_plan(.then_with(|| left.rel.cmp(&right.rel))});- let mut frozen_datasets = previous.datasets.clone();- for dataset in &mut frozen_datasets {+ for dataset in &mut owned_datasets {dataset.file_count = files.values().filter(|assignment| {@@ -424,44 +526,70 @@ pub(crate) fn reconcile_plan(}).collect();+ incremental_fields.retain(|_, specs| !specs.is_empty());+Ok(Plan {- datasets: frozen_datasets,+ datasets: owned_datasets,files,junk_files,duplicate_files,alias_paths_indexed: previous.alias_paths_indexed,refused_datasets,+ incremental_fields,})}+/// A per-file projection failure, or a frozen plan that is internally inconsistent.+enum AssignFailure {+ /// This file cannot join the frozen mapping. The rest of the pass can.+ File(String),+ /// The committed plan itself is contradictory. Abort; do not publish a+ /// partial projection of a corrupt journal.+ Fatal(anyhow::Error),+}++impl From<anyhow::Error> for AssignFailure {+ fn from(error: anyhow::Error) -> Self {+ AssignFailure::Fatal(error)+ }+}+fn retained_slug(owner: &FileAssignment,group: &Option<String>,family: &str,fields: &HashMap<String, FieldAcc>,- datasets: &HashMap<&str, &PlanDataset>,-) -> Result<String> {- anyhow::ensure!(- owner.family == family,- "detected family changed from {} to {family}",- owner.family- );+ datasets: &[PlanDataset],+ incremental: &BTreeMap<String, Vec<FieldSpec>>,+) -> Result<String, AssignFailure> {+ if owner.family != family {+ return Err(AssignFailure::File(format!(+ "detected family changed from {} to {family}",+ owner.family+ )));+ }let matches = owner.assignments.iter().filter(|(assigned_group, _)| assigned_group == group).collect::<Vec<_>>();- anyhow::ensure!(- matches.len() == 1,- "frozen assignment has {} matches for group {:?}",- matches.len(),- group- );+ if matches.len() != 1 {+ return Err(AssignFailure::Fatal(anyhow::anyhow!(+ "frozen assignment has {} matches for group {:?}",+ matches.len(),+ group+ )));+ }let slug = &matches[0].1;let dataset = datasets- .get(slug.as_str())+ .iter()+ .find(|dataset| dataset.slug == *slug).with_context(|| format!("frozen assignment references absent dataset {slug}"))?;- ensure_compatible(fields, dataset)?;+ let extra = incremental+ .get(slug.as_str())+ .map(Vec::as_slice)+ .unwrap_or(&[]);+ ensure_compatible(fields, dataset, extra)?;Ok(slug.clone())}@@ -481,30 +609,44 @@ fn classify_new(fields: &HashMap<String, FieldAcc>,rel: &str,datasets: &[PlanDataset],-) -> Result<String> {+ refused: &[&PlanDataset],+ incremental: &BTreeMap<String, Vec<FieldSpec>>,+) -> Result<String, AssignFailure> {let mut candidates = Vec::new();+ let mut rejected: Option<String> = None;for dataset in datasets.iter()+ .chain(refused.iter().copied()).filter(|dataset| dataset.family == family && &dataset.group == group){- if ensure_compatible(fields, dataset).is_err() {- continue;+ let extra = incremental+ .get(dataset.slug.as_str())+ .map(Vec::as_slice)+ .unwrap_or(&[]);+ match ensure_compatible(fields, dataset, extra) {+ Ok(()) => {}+ Err(AssignFailure::File(detail)) => {+ rejected = Some(detail);+ continue;+ }+ Err(failure @ AssignFailure::Fatal(_)) => return Err(failure),}let affinity = scope_affinity(dataset.scope.as_deref(), rel);let (intersection, union) = field_overlap(fields, &dataset.specs);let threshold_met = if dataset.family == "docs" {// A `docs` dataset's specs are the UNION of every document/config- // the fold produced. `ensure_compatible` (run just above) already- // requires every observed SCALAR field to be present in the frozen- // specs — a field absent from the specs is tolerated ONLY when- // `acc.n == 0` (pruned: e.g. a code file's object-valued `symbols`- // sidecar, #580). So a document/config that reached here belongs to- // the docs dataset regardless of how many other fields the union- // carries; accept it. (The 0.7 Jaccard would wrongly reject a- // document with fewer fields than the union, and `== fields.len()`- // wrongly counts the pruned `symbols` key and aborted a new code- // file — the #729 central case.) `let _` keeps the shared- // `field_overlap` result in scope for the data-family branches.+ // the fold produced. `ensure_compatible` (run just above) accepts a+ // scalar field the frozen specs do not name yet — `widen_dataset`+ // records it beside the specs, and the mapping update installs it —+ // and still tolerates `acc.n == 0` (pruned: e.g. a+ // code file's object-valued `symbols` sidecar, #580). So a+ // document/config that reached here belongs to the docs dataset+ // regardless of how many other fields the union carries; accept+ // it. (The 0.7 Jaccard would wrongly reject a document with fewer+ // fields than the union, and `== fields.len()` wrongly counts the+ // pruned `symbols` key and aborted a new code file — the #729+ // central case.) `let _` keeps the shared `field_overlap` result+ // in scope for the data-family branches.//// #731 (3): this also silently admits a routed file with ZERO// rendered fields (e.g. an unreadable/empty document) — harmless,@@ -545,12 +687,16 @@ fn classify_new(});let Some((winner, winner_intersection, winner_union, winner_affinity)) = candidates.first()else {- anyhow::bail!(+ let because = rejected+ .as_deref()+ .map(|detail| format!(" Last candidate refused: {detail}"))+ .unwrap_or_default();+ return Err(AssignFailure::File(format!("no frozen dataset accepts family {family}, group {:?}, and fields {:?}; \- this requires unsupported dataset/schema evolution (use a new prefix)",+ this requires unsupported dataset/schema evolution (use a new prefix).{because}",group,sorted_field_names(fields)- );+ )));};if let Some((runner_up, runner_up_intersection, runner_up_union, runner_up_affinity)) =candidates.get(1)@@ -559,21 +705,23 @@ fn classify_new(// separate the top two. A pre-#1230 plan carries no scope (`None` for// every dataset), so both affinities are `None`, they compare equal,// and this fails closed exactly as before.- anyhow::ensure!(- ratio_cmp(- *winner_intersection,- *winner_union,- *runner_up_intersection,- *runner_up_union- ) != Ordering::Equal- || winner_affinity != runner_up_affinity,- "new file matches frozen datasets {} and {} equally for family {family}, group {:?}, \- and fields {:?}; refusing ambiguous assignment (use a new prefix)",- winner.slug,- runner_up.slug,- group,- sorted_field_names(fields)- );+ if ratio_cmp(+ *winner_intersection,+ *winner_union,+ *runner_up_intersection,+ *runner_up_union,+ ) == Ordering::Equal+ && winner_affinity == runner_up_affinity+ {+ return Err(AssignFailure::File(format!(+ "new file matches frozen datasets {} and {} equally for family {family}, group {:?}, \+ and fields {:?}; refusing ambiguous assignment (use a new prefix)",+ winner.slug,+ runner_up.slug,+ group,+ sorted_field_names(fields)+ )));+ }}Ok(winner.slug.clone())}@@ -605,7 +753,12 @@ fn field_overlap(fields: &HashMap<String, FieldAcc>, specs: &[FieldSpec]) -> (us// Keep null-only and object-shaped fields in the schema fingerprint too.// FieldAcc deliberately does not count those as scalar observations, but// allowing an unknown one through would let the server create a dynamic- // mapping outside the frozen plan.+ // mapping outside the frozen plan. A scalar the specs do not name yet is+ // also a mismatch here: `widen_dataset` adds it only after a file has+ // already been assigned (a retained file, or a docs file the fold accepts).+ // Dropping it from this ratio would make a dataset that lacks the field+ // look identical to one that has it, and scope would then pick the worse+ // one.let observed: HashSet<&str> = fields.keys().map(String::as_str).collect();let frozen: HashSet<&str> = specs.iter().map(|spec| spec.name.as_str()).collect();(@@ -618,48 +771,163 @@ fn ratio_cmp(left_n: usize, left_d: usize, right_n: usize, right_d: usize) -> Or(left_n * right_d).cmp(&(right_n * left_d))}-fn ensure_compatible(fields: &HashMap<String, FieldAcc>, dataset: &PlanDataset) -> Result<()> {- let specs: HashMap<&str, &FieldSpec> = dataset+fn ensure_compatible(+ fields: &HashMap<String, FieldAcc>,+ dataset: &PlanDataset,+ extra: &[FieldSpec],+) -> Result<(), AssignFailure> {+ let mut specs: HashMap<&str, &FieldSpec> = dataset.specs.iter().map(|spec| (spec.name.as_str(), spec)).collect();- for (name, acc) in fields {- let Some(spec) = specs.get(name.as_str()) else {+ for spec in extra {+ specs.entry(spec.name.as_str()).or_insert(spec);+ }+ // Sorted so the reported field does not depend on hash-map order.+ let mut names: Vec<&str> = fields.keys().map(String::as_str).collect();+ names.sort_unstable();+ for name in names {+ let acc = &fields[name];+ let Some(spec) = specs.get(name) else {// A field observed with NO scalar values (`acc.n == 0`) — e.g. the// code file's `symbols` sidecar (an array of objects, which// `FieldAcc` deliberately does not scalar-type) — legitimately never// received a frozen `FieldSpec` at genesis: it has nothing to type.- // Its absence is therefore not drift; there is nothing to validate,- // so mirror `compatible_with_spec`'s `acc.n == 0 => true` and skip- // it. Without this, re-indexing ANY code file aborted the whole run- // (#580), because `symbols` is observed on every generation but was- // pruned from the frozen schema. A field that DOES carry scalar- // values yet is absent is still a real incompatibility.+ // Its absence is therefore not drift (#580).//- // #1237: `impl` and `end_line` on symbol records were added after- // #579 froze the five symbol scalar fields; tolerate their absence- // on pre-#1237 frozen `docs`/`code` datasets so incremental runs- // over existing state dirs do not abort.- let pre_1237_symbol_field = (name == "impl" || name == "end_line")- && (dataset.family == "docs" || dataset.family == "code");- anyhow::ensure!(- acc.n == 0 || pre_1237_symbol_field,- "field {name} is absent from frozen dataset {}",+ // #1237: `impl` and `end_line` are installed on every docs/code+ // mapping by `dataset_create_body`, outside the frozen specs.+ // Re-inferring them here can elect `keyword` for a short `impl`+ // and the server then refuses to change the existing `text` field.+ //+ // Any other absent scalar with a name the mapping can grow is+ // added by `widen_dataset` (#2176). A name that cannot be added+ // (empty, dotted, leading `_`) is a per-file refusal: letting it+ // through would dynamic-map it.+ if absent_field_is_tolerated(name, acc, dataset) {+ continue;+ }+ if widenable_field_name(name) {+ continue;+ }+ return Err(AssignFailure::File(format!(+ "field {name} is absent from frozen dataset {} and cannot be added to its mapping",dataset.slug- );- continue;+ )));};- anyhow::ensure!(- compatible_with_spec(acc, spec),- "field {name} is incompatible with frozen {} mapping in dataset {}",- spec.es_type,- dataset.slug- );+ if !compatible_with_spec(acc, spec) {+ return Err(AssignFailure::File(format!(+ "field {name} is incompatible with frozen {} mapping in dataset {}",+ spec.es_type, dataset.slug+ )));+ }}Ok(())}+/// `impl` / `end_line` and object-valued sidecars are already accounted for+/// and must not be re-inferred onto the frozen specs.+fn absent_field_is_tolerated(name: &str, acc: &FieldAcc, dataset: &PlanDataset) -> bool {+ if acc.n == 0 {+ return true;+ }+ (name == "impl" || name == "end_line") && (dataset.family == "docs" || dataset.family == "code")+}++/// A top-level field name `PUT /_mapping` can add without nesting or colliding+/// with metadata. Dotted names become nested properties; a leading `_` is+/// reserved. Anything else is refused per file instead of dynamic-mapped.+fn widenable_field_name(name: &str) -> bool {+ let mut chars = name.chars();+ match chars.next() {+ Some(first) if first.is_ascii_alphabetic() => {}+ _ => return false,+ }+ chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')+}++fn widen_dataset(+ dataset: &PlanDataset,+ added: &mut Vec<FieldSpec>,+ fields: &HashMap<String, FieldAcc>,+ records: u64,+) -> Result<(), AssignFailure> {+ let mut known: HashSet<&str> = dataset+ .specs+ .iter()+ .map(|spec| spec.name.as_str())+ .collect();+ known.extend(added.iter().map(|spec| spec.name.as_str()));+ let mut pending: HashMap<String, FieldAcc> = HashMap::new();+ let mut names: Vec<&str> = fields.keys().map(String::as_str).collect();+ names.sort_unstable();+ for name in names {+ let acc = &fields[name];+ if known.contains(name) || absent_field_is_tolerated(name, acc, dataset) || acc.n == 0 {+ continue;+ }+ if !widenable_field_name(name) {+ return Err(AssignFailure::File(format!(+ "field {name} is absent from frozen dataset {} and cannot be added to its mapping",+ dataset.slug+ )));+ }+ pending.insert(name.to_string(), acc.clone());+ }+ if pending.is_empty() {+ return Ok(());+ }+ if dataset.specs.len() + added.len() + pending.len() > crate::infer::MAX_FIELDS_PER_DATASET {+ let pending_len = pending.len();+ return Err(AssignFailure::File(format!(+ "dataset {} already maps {} fields; adding {pending_len} more would pass the {} field cap",+ dataset.slug,+ dataset.specs.len() + added.len(),+ crate::infer::MAX_FIELDS_PER_DATASET+ )));+ }+ // `no_semantic`: a new field must not elect `semantic_text`. That would+ // change the dataset's embedding contract without updating `semantic_field`,+ // and a fresh mapping install would then refuse or embed a field this+ // generation does not declare as the body. Lexical text/keyword/long still+ // search.+ let mut inferred = crate::infer::infer_fields(&pending, records.max(1), true);+ for spec in &mut inferred {+ if spec.es_type == "semantic_text" {+ spec.es_type = "text".into();+ }+ spec.notes+ .push("added on incremental reconcile when the field first appeared".into());+ }+ added.extend(inferred);+ Ok(())+}++fn rollback_widens(+ fields: &mut BTreeMap<String, Vec<FieldSpec>>,+ widened: &HashMap<String, usize>,+) {+ for (slug, len) in widened {+ if let Some(specs) = fields.get_mut(slug) {+ specs.truncate(*len);+ }+ }+ fields.retain(|_, specs| !specs.is_empty());+}++fn file_schema_reason(detail: &str, retained: bool, rebuild_command: &str) -> String {+ let fate = if retained {+ "Documents already indexed for this file are dropped with this generation."+ } else {+ "Nothing was indexed for this file."+ };+ format!(+ "{detail} This file was not indexed; every other file in this pass still is. {fate} \+ Stop the watcher and rebuild the folder so the schema is inferred again: {rebuild_command}"+ )+}+fn compatible_with_spec(acc: &FieldAcc, spec: &FieldSpec) -> bool {if acc.n == 0 {return true;@@ -697,6 +965,11 @@ mod tests {use serde_json::json;use std::path::PathBuf;+ /// Printed verbatim by a per-file schema refusal. A reworded error that+ /// drops this command is a failed recovery contract.+ const REBUILD: &str =+ "xerj autoindex /src --prefix p-reindex --state-dir /state-reindex --no-graph";+fn acc(values: &[serde_json::Value]) -> FieldAcc {let mut acc = FieldAcc::default();for value in values {@@ -821,6 +1094,7 @@ mod tests {&previous,vec![scan(fields)],50,+ REBUILD,).unwrap();assert_eq!(@@ -854,6 +1128,7 @@ mod tests {&previous,vec![scan(fields)],50,+ REBUILD,).unwrap();assert_eq!(plan.files["new"].assignments, vec![(None, "winner".into())]);@@ -874,66 +1149,141 @@ mod tests {("id".into(), acc(&[json!(7)])),("body".into(), acc(&[json!("hello")])),]);- let error = reconcile_plan(+ let plan = reconcile_plan(&inventory("new.jsonl", "unix:6e", "new"),&previous,vec![scan(fields)],50,+ REBUILD,)- .unwrap_err();- let message = format!("{error:#}");+ .unwrap();+ assert!(+ plan.files.is_empty(),+ "an equal tie must not assign a dataset"+ );+ assert_eq!(plan.junk_files.len(), 1);+ let message = &plan.junk_files[0].reason;assert!(message.contains("refusing ambiguous assignment"),"{message}");assert!(message.contains("a-tie"), "{message}");assert!(message.contains("z-tie"), "{message}");+ assert!(message.contains(REBUILD), "{message}");+ assert!(+ message.contains("Nothing was indexed for this file."),+ "{message}"+ );}#[test]- fn new_field_and_incompatible_type_fail_before_plan_changes() {+ fn a_new_scalar_field_widens_and_an_unmappable_change_is_one_file() {let previous = Plan {datasets: vec![dataset("rows", vec![spec("id", "long")])],..Plan::default()};+ // A file already assigned to this dataset grows a scalar the frozen+ // specs do not name. The mapping is extended. It used to abort the plan.+ let mut retained = previous.clone();+ retained+ .files+ .insert("old".into(), assignment("rows.jsonl", "unix:6e", "rows"));let unknown = HashMap::from([("id".into(), acc(&[json!(7)])),("surprise".into(), acc(&[json!("new mapping")])),]);- let error = reconcile_plan(- &inventory("new.jsonl", "unix:6e", "new"),- &previous,+ let plan = reconcile_plan(+ &inventory("rows.jsonl", "unix:6e", "new"),+ &retained,vec![scan(unknown)],50,+ REBUILD,)- .unwrap_err();- assert!(format!("{error:#}").contains("unsupported dataset/schema evolution"));+ .unwrap();+ assert_eq!(plan.files["new"].assignments, vec![(None, "rows".into())]);+ assert!(plan.junk_files.is_empty());+ assert!(+ plan.datasets[0].specs.iter().all(|spec| spec.name == "id"),+ "frozen specs stay the generation identity"+ );+ assert!(+ plan.incremental_fields["rows"]+ .iter()+ .any(|spec| spec.name == "surprise"),+ "the new scalar is installed beside the frozen specs"+ );- // Object-shaped values are not counted as scalar FieldAcc evidence,- // but their field name must still be rejected rather than dynamically- // mapped by the server.+ // The same extra field on a file with no committed assignment stays+ // below the group-less overlap threshold, so it does not join — and+ // it does not take the rest of the plan down with it.+ let plan = reconcile_plan(+ &inventory("new.jsonl", "unix:6f", "brand-new"),+ &previous,+ vec![scan(HashMap::from([+ ("id".into(), acc(&[json!(7)])),+ ("surprise".into(), acc(&[json!("new mapping")])),+ ]))],+ 50,+ REBUILD,+ )+ .unwrap();+ assert!(plan.files.is_empty());+ assert!(+ plan.junk_files[0]+ .reason+ .contains("unsupported dataset/schema evolution"),+ "{}",+ plan.junk_files[0].reason+ );+ assert!(plan.junk_files[0].reason.contains(REBUILD));+ assert!(plan.datasets[0].specs.iter().all(|spec| spec.name == "id"));+ assert!(plan.incremental_fields.is_empty());++ // Object-shaped values are not scalar FieldAcc evidence. Their name+ // stays in the overlap fingerprint, so a data file that introduces one+ // still fails to join — and only that file is junked.let object_field = HashMap::from([("id".into(), acc(&[json!(7)])),("object".into(), acc(&[json!({"nested": true})])),]);- let error = reconcile_plan(+ let plan = reconcile_plan(&inventory("new.jsonl", "unix:6e", "new"),&previous,vec![scan(object_field)],50,+ REBUILD,)- .unwrap_err();- assert!(format!("{error:#}").contains("unsupported dataset/schema evolution"));+ .unwrap();+ assert!(plan.files.is_empty());+ let message = &plan.junk_files[0].reason;+ assert!(+ message.contains("unsupported dataset/schema evolution"),+ "{message}"+ );+ assert!(message.contains(REBUILD), "{message}");+ assert!(+ plan.datasets[0].specs.iter().all(|spec| spec.name == "id"),+ "a refused file must not widen the frozen specs"+ );+ assert!(plan.incremental_fields.is_empty());let wrong_type = HashMap::from([("id".into(), acc(&[json!("not-a-number")]))]);- let error = reconcile_plan(+ let plan = reconcile_plan(&inventory("new.jsonl", "unix:6e", "new"),&previous,vec![scan(wrong_type)],50,+ REBUILD,)- .unwrap_err();- assert!(format!("{error:#}").contains("unsupported dataset/schema evolution"));+ .unwrap();+ assert!(plan.files.is_empty());+ let message = &plan.junk_files[0].reason;+ assert!(message.contains("incompatible"), "{message}");+ assert!(message.contains(REBUILD), "{message}");+ assert!(+ message.contains("Nothing was indexed for this file."),+ "{message}"+ );}/// A NEW document file (empty `key_fields`, no group — the shape a fresh@@ -971,6 +1321,7 @@ mod tests {&previous,vec![file_scan],50,+ REBUILD,).unwrap();assert_eq!(@@ -1031,7 +1382,7 @@ mod tests {digests: vec!["digest-cfg".into()],duplicates: vec![],};- let plan = reconcile_plan(&inv, &previous, vec![file_scan], 50).unwrap();+ let plan = reconcile_plan(&inv, &previous, vec![file_scan], 50, REBUILD).unwrap();assert_eq!(plan.files["cfg"].assignments, vec![(None, "docs".into())]);assert!(plan.files["cfg"].as_document,@@ -1082,6 +1433,7 @@ mod tests {&previous,vec![file_scan],50,+ REBUILD,).unwrap();assert_eq!(plan.files["code"].assignments, vec![(None, "docs".into())]);@@ -1091,6 +1443,93 @@ mod tests {);}+ /// #2176: a code file indexed while it had no symbols freezes `docs` as+ /// title/language/body. Adding a function emits scalar fields (`code`,+ /// `name`, `defs`, …) the frozen specs do not name. The retained file must+ /// stay on `docs` and those fields must be recorded beside the specs —+ /// not written into them, which would move the generation identity.+ /// `impl` and `end_line` stay out of that list too: the create body+ /// already maps them, and re-inferring `impl` can elect `keyword`.+ #[test]+ fn retained_code_file_grows_a_function_and_docs_gains_code() {+ let mut previous = Plan {+ datasets: vec![PlanDataset {+ family: "docs".into(),+ ..dataset(+ "docs",+ vec![+ spec("title", "text"),+ spec("language", "keyword"),+ spec("body", "text"),+ ],+ )+ }],+ ..Plan::default()+ };+ let mut prior = assignment("module.py", "unix:70", "docs");+ prior.family = "code".into();+ previous.files.insert("old-module".into(), prior);+ let fields = HashMap::from([+ ("title".to_string(), acc(&[json!("module.py")])),+ ("language".to_string(), acc(&[json!("python")])),+ (+ "body".to_string(),+ acc(&[json!("def cancel_request():\n return \"birchfunction\"")]),+ ),+ ("defs".to_string(), acc(&[json!("function cancel_request")])),+ ("symbol_count".to_string(), acc(&[json!(1)])),+ ("code".to_string(), acc(&[json!("def cancel_request():")])),+ ("name".to_string(), acc(&[json!("cancel_request")])),+ ("kind".to_string(), acc(&[json!("function")])),+ ("line".to_string(), acc(&[json!(3)])),+ (+ "symbols".to_string(),+ acc(&[json!({"name": "cancel_request"})]),+ ),+ ]);+ let mut file_scan = scan(fields);+ file_scan.sniffed.as_mut().unwrap().family = Family::Code;++ let plan = reconcile_plan(+ &inventory("module.py", "unix:70", "new-module"),+ &previous,+ vec![file_scan],+ 50,+ REBUILD,+ )+ .unwrap();+ assert!(+ plan.junk_files.is_empty(),+ "adding a function must not junk the file: {:?}",+ plan.junk_files+ );+ assert_eq!(+ plan.files["new-module"].assignments,+ vec![(None, "docs".into())]+ );+ let frozen: Vec<&str> = plan.datasets[0]+ .specs+ .iter()+ .map(|spec| spec.name.as_str())+ .collect();+ assert_eq!(frozen, ["title", "language", "body"]);+ let names: Vec<&str> = plan.incremental_fields["docs"]+ .iter()+ .map(|spec| spec.name.as_str())+ .collect();+ for field in ["code", "name", "kind", "line", "defs", "symbol_count"] {+ assert!(names.contains(&field), "missing {field} in {names:?}");+ }+ assert!(!names.contains(&"impl"));+ assert!(!names.contains(&"end_line"));+ assert!(!names.contains(&"symbols"));+ let reason = plan.files["new-module"].rel.as_str();+ assert_eq!(reason, "module.py");+ let message = file_schema_reason("field id is incompatible", true, REBUILD);+ assert!(message.contains("Documents already indexed for this file are dropped"));+ assert!(message.contains(REBUILD));+ }+/// Two frozen scope-keyed `docs` datasets (one per nested repository) whose/// genesis schemas are byte-identical. A code document is a group-less,/// empty-key-field shape, so it routes to the `docs` family (see@@ -1140,6 +1579,7 @@ mod tests {&previous,vec![fork_doc_scan()],50,+ REBUILD,).expect("an adjacent file inside bitcoin-core must not abort the run");assert_eq!(@@ -1154,6 +1594,7 @@ mod tests {&previous,vec![fork_doc_scan()],50,+ REBUILD,).expect("an adjacent file inside bitcoin-knots must not abort the run");assert_eq!(@@ -1196,6 +1637,7 @@ mod tests {&previous,vec![scan(fields)],50,+ REBUILD,).unwrap();assert_eq!(@@ -1223,18 +1665,24 @@ mod tests {],..Plan::default()};- let error = reconcile_plan(+ let plan = reconcile_plan(&inventory("bitcoin-core/adjacent.ts", "unix:a4", "legacy-add"),&previous,vec![fork_doc_scan()],50,+ REBUILD,)- .unwrap_err();- let message = format!("{error:#}");+ .unwrap();+ assert!(+ plan.files.is_empty(),+ "a legacy plan with no scope must not silently assign the tie"+ );+ let message = &plan.junk_files[0].reason;assert!(message.contains("refusing ambiguous assignment"),"{message}");+ assert!(message.contains(REBUILD), "{message}");}#[test]@@ -1256,7 +1704,7 @@ mod tests {bytes: 10,});let fields = HashMap::from([("id".into(), acc(&[json!(7)]))]);- let plan = reconcile_plan(¤t, &previous, vec![scan(fields)], 50).unwrap();+ let plan = reconcile_plan(¤t, &previous, vec![scan(fields)], 50, REBUILD).unwrap();assert_eq!(plan.files["same"].rel, "renamed.jsonl");assert_eq!(plan.datasets[0].file_count, 1);assert_eq!(plan.duplicate_files.len(), 1);@@ -1277,7 +1725,8 @@ mod tests {let fields = || HashMap::from([("id".into(), acc(&[json!(7)]))]);let conflict = inventory("a.jsonl", "unix:62", "content-a");- let error = reconcile_plan(&conflict, &previous, vec![scan(fields())], 50).unwrap_err();+ let error =+ reconcile_plan(&conflict, &previous, vec![scan(fields())], 50, REBUILD).unwrap_err();assert!(format!("{error:#}").contains("conflicting committed content and path owners"));let duplicate = Inventory {@@ -1291,6 +1740,7 @@ mod tests {&previous,vec![scan(fields()), scan(fields())],50,+ REBUILD,).unwrap_err();assert!(format!("{error:#}").contains("duplicate content identity"));diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex 0debae8c8..b9f308834 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -7,7 +7,7 @@ use anyhow::{Context, Result};use fs2::FileExt;use serde::{Deserialize, Serialize};use serde_json::Value;-use std::collections::{HashMap, HashSet};+use std::collections::{BTreeMap, HashMap, HashSet};use std::io::{BufRead, Write};use std::path::{Path, PathBuf};@@ -566,6 +566,18 @@ pub struct Plan {/// a spurious "plan changed" generation.#[serde(default, skip_serializing_if = "Vec::is_empty")]pub refused_datasets: Vec<RefusedDataset>,+ /// Scalar fields installed on a frozen dataset after the first pass (#2176).+ ///+ /// Keyed by dataset slug. Empty — and omitted from the serialized plan —+ /// until a watched file grows a field the frozen `specs` do not name.+ /// Those specs stay frozen: `schema_identity`, `index_identity`, and the+ /// generation-to-generation dataset equality check all read them, and+ /// changing them aborts the next generation. The mapping update sends+ /// this list on top of the frozen specs. The server accepts the additive+ /// `PUT /_mapping`. A no-op re-run copies the list forward, so the plan+ /// compares equal and commits nothing.+ #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]+ pub incremental_fields: BTreeMap<String, Vec<crate::infer::FieldSpec>>,}/// A dataset the plan inferred and the server refused to map (#929).