Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): let a watched file grow a field without failing the pass (#2193)

xerj-org922921aec0d6
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index e7e7241a8..5eddd164d 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -70,6 +70,14 @@ struct HttpState {
unavailable_dataset_mappings: Vec<String>,
/// Every dataset index a `_mapping` PUT was attempted for, in order.
dataset_mapping_puts: Vec<String>,
+ /// Properties installed for a dataset index: the create body the first
+ /// time the index is seen, then additive `_mapping` merges. A later
+ /// `_mapping` whose `type` disagrees with what is already stored is the
+ /// engine's type-change refusal (`field [X] already exists as [old],
+ /// cannot add [new]`). A create of an index that already exists is
+ /// ignored, the way `Es::ensure_index` swallows "exists" without
+ /// re-sending the body.
+ installed_dataset_properties: HashMap<String, serde_json::Map<String, Value>>,
/// Opt-in (#931): fail the Nth data bulk (1-based) the way
/// `fail_next_data_bulk` fails the next one. Lets a replay be interrupted
/// AFTER some operations committed, which is the only state in which
@@ -188,6 +196,16 @@ impl HttpEndpoint {
self.state.lock().unwrap().data_bulk_requests
}
+ /// Properties the fake has installed for one dataset index (#2176).
+ fn dataset_properties(&self, index: &str) -> Option<Value> {
+ self.state
+ .lock()
+ .unwrap()
+ .installed_dataset_properties
+ .get(index)
+ .map(|properties| Value::Object(properties.clone()))
+ }
+
/// #755: the catalog mapping the endpoint finally acknowledged.
fn installed_catalog_properties(&self) -> Value {
self.state
@@ -308,6 +326,7 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {
let mut conflict: Option<String> = None;
let mut unavailable = false;
let mut refused_dataset: Option<String> = None;
+ let mut type_conflict: Option<String> = None;
if path.ends_with("/_mapping") {
assert!(value.get("properties").is_some());
assert!(value.get("mappings").is_none());
@@ -337,11 +356,19 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {
if selected(&locked.refused_dataset_mappings) {
refused_dataset = Some(index.clone());
}
+ if refused_dataset.is_none() && !unavailable {
+ let incoming = value["properties"].as_object().cloned().unwrap_or_default();
+ type_conflict = apply_dataset_mapping(&mut locked, &index, &incoming);
+ }
locked.dataset_mapping_puts.push(index);
}
} else {
assert!(value.pointer("/mappings/properties").is_some());
assert!(value.get("properties").is_none());
+ let index = path.trim_start_matches('/');
+ if !index.is_empty() && index != catalog::CATALOG_INDEX {
+ remember_dataset_create(&mut locked, index, &value);
+ }
}
if let Some(index) = refused_dataset {
// Byte for byte the shape of the refusal in #929.
@@ -378,6 +405,23 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {
"status": 400
}),
)
+ } else if let Some(reason) = type_conflict {
+ // Same shape as the catalog conflict above, and the same words
+ // `xerj-api` uses when `PUT /_mapping` changes a field's type
+ // (`es_compat.rs`). `update_mapping` does not swallow a body that
+ // contains "exists"; `ensure_index` does, which is why a create
+ // of an existing index never reaches this branch.
+ (
+ 400,
+ json!({
+ "error": {
+ "root_cause": [{"type": "mapper_parsing_exception", "reason": reason}],
+ "type": "mapper_parsing_exception",
+ "reason": reason
+ },
+ "status": 400
+ }),
+ )
} else {
(200, json!({"acknowledged": true}))
}
@@ -402,6 +446,60 @@ fn handle_http(mut stream: TcpStream, state: &Arc<Mutex<HttpState>>) {
.unwrap();
}
+/// Store the create-body properties the first time an index is seen.
+/// A later create is the "index exists" case: the body is not applied.
+fn remember_dataset_create(state: &mut HttpState, index: &str, body: &Value) {
+ if state.installed_dataset_properties.contains_key(index) {
+ return;
+ }
+ let Some(properties) = body
+ .pointer("/mappings/properties")
+ .and_then(Value::as_object)
+ .cloned()
+ else {
+ return;
+ };
+ state
+ .installed_dataset_properties
+ .insert(index.to_owned(), properties);
+}
+
+/// Merge a dataset `_mapping` into the stored properties, or refuse a type
+/// change. Returns the engine's reason when a stored field's `type` would
+/// change; the stored map is left untouched in that case.
+fn apply_dataset_mapping(
+ state: &mut HttpState,
+ index: &str,
+ incoming: &serde_json::Map<String, Value>,
+) -> Option<String> {
+ if let Some(existing) = state.installed_dataset_properties.get(index) {
+ for (name, spec) in incoming {
+ let (Some(old), Some(new)) = (
+ existing
+ .get(name)
+ .and_then(|value| value.get("type"))
+ .and_then(Value::as_str),
+ spec.get("type").and_then(Value::as_str),
+ ) else {
+ continue;
+ };
+ if old != new {
+ return Some(format!(
+ "field [{name}] already exists as [{old}], cannot add [{new}]"
+ ));
+ }
+ }
+ }
+ let slot = state
+ .installed_dataset_properties
+ .entry(index.to_owned())
+ .or_default();
+ for (name, spec) in incoming {
+ slot.entry(name.clone()).or_insert_with(|| spec.clone());
+ }
+ None
+}
+
fn bulk_http(body: &[u8], state: &Arc<Mutex<HttpState>>) -> (u16, Value) {
let lines: Vec<&[u8]> = body
.split(|byte| *byte == b'\n')
@@ -5655,6 +5753,189 @@ fn a_watch_pass_rehashes_only_what_changed_and_publishes_it() {
);
}
+fn joined_docs(endpoint: &HttpEndpoint) -> String {
+ endpoint
+ .data_docs()
+ .iter()
+ .map(|doc| doc.to_string())
+ .collect::<Vec<_>>()
+ .join("\n")
+}
+
+/// #2176: a Python file indexed as a comment and an import freezes `docs`
+/// without a `code` field. Adding a function while `--watch` is running must
+/// publish the new source, install `code` on the dataset, and still index a
+/// sibling file that changed in the same pass. On the failing build this
+/// pass returns the frozen-schema error and neither edit lands.
+#[test]
+fn adding_a_python_function_under_watch_indexes_the_new_field_and_sibling_edits() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let root = corpus.path().canonicalize().unwrap();
+ fs::write(
+ root.join("module.py"),
+ "# The birchbefore marker identifies the cancellation module.\nimport math\n",
+ )
+ .unwrap();
+ fs::write(root.join("notes.md"), "birchdoc-before\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(&root, state_dir.path(), &endpoint.url, false);
+ let mut carry = crate::watch::Carry::default();
+ let (outcome, _) =
+ crate::watch::one_pass(&config, &root, &mut carry, &crate::watch::ChangeSet::full());
+ assert_eq!(outcome.unwrap().0, 0, "the symbol-less file must index");
+ let first = joined_docs(&endpoint);
+ assert!(
+ first.contains("birchbefore"),
+ "the first pass must publish the original source: {first}"
+ );
+ assert!(first.contains("birchdoc-before"), "{first}");
+
+ fs::write(
+ root.join("module.py"),
+ "# The birchafter marker identifies the cancellation module.\n\
+ import math\n\
+ \n\
+ def cancel_request():\n\
+ return \"birchfunction\"\n",
+ )
+ .unwrap();
+ fs::write(root.join("notes.md"), "birchdoc-after\n").unwrap();
+ let burst =
+ crate::watch::ChangeSet::from_paths(&[root.join("module.py"), root.join("notes.md")]);
+ let (outcome, _) = crate::watch::one_pass(&config, &root, &mut carry, &burst);
+ let (code, _) = outcome
+ .expect("adding `code` to a frozen docs dataset must not fail the watch pass (#2176)");
+ assert_eq!(code, 0, "both files are compatible once the mapping grows");
+ let indexed = joined_docs(&endpoint);
+ assert!(
+ indexed.contains("birchafter"),
+ "the rewritten file must be live: {indexed}"
+ );
+ assert!(
+ indexed.contains("birchfunction"),
+ "the function body must be searchable: {indexed}"
+ );
+ assert!(
+ indexed.contains("birchdoc-after"),
+ "a sibling edit in the same pass must still be indexed: {indexed}"
+ );
+ assert!(
+ !indexed.contains("birchbefore"),
+ "the old source must not stay searchable: {indexed}"
+ );
+ assert!(
+ !indexed.contains("birchdoc-before"),
+ "the old note must not stay searchable: {indexed}"
+ );
+ assert!(
+ endpoint.data_docs().iter().any(|doc| {
+ doc.get("code")
+ .and_then(Value::as_str)
+ .is_some_and(|code| code.contains("cancel_request"))
+ }),
+ "a symbol document must carry the new `code` field: {indexed}"
+ );
+ let properties = endpoint
+ .dataset_properties("incremental-http-docs")
+ .expect("the docs index mapping must have been installed");
+ let code_type = properties["code"]["type"]
+ .as_str()
+ .expect("code must be a mapped field");
+ assert!(
+ code_type == "text" || code_type == "keyword",
+ "code must be a searchable scalar, got {code_type}"
+ );
+ a_rerun_changes_nothing(&config, &endpoint, "function added under watch");
+}
+
+/// A type change cannot be applied to a frozen field. Only that file is
+/// skipped; the other file in the pass is published, and the printed rebuild
+/// command is a real `autoindex` of a new state dir and prefix.
+#[test]
+fn a_type_change_skips_one_file_and_the_printed_rebuild_indexes_it() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let root = corpus.path().canonicalize().unwrap();
+ fs::write(root.join("good.csv"), "id,value\n2,beta\n").unwrap();
+ fs::write(root.join("bad.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(&root, state_dir.path(), &endpoint.url, false);
+ let mut carry = crate::watch::Carry::default();
+ assert_eq!(
+ crate::watch::one_pass(&config, &root, &mut carry, &crate::watch::ChangeSet::full())
+ .0
+ .unwrap()
+ .0,
+ 0
+ );
+
+ // The data row keeps a number so the sniffer still treats the first line
+ // as a header. `id` stays the frozen `long` field and the string is a
+ // type conflict, not a new `col_1` schema.
+ fs::write(root.join("good.csv"), "id,value\n2,good-new\n").unwrap();
+ fs::write(root.join("bad.csv"), "id,value\nnot-a-number,2\n").unwrap();
+ let burst = crate::watch::ChangeSet::from_paths(&[root.join("good.csv"), root.join("bad.csv")]);
+ let (outcome, _) = crate::watch::one_pass(&config, &root, &mut carry, &burst);
+ let (code, summary) = outcome.expect("a type change must not fail the rest of the pass");
+ assert_eq!(code, 3, "the unmappable file is junk, and junk exits 3");
+ let skipped = summary.expect("the pass must report the skipped file")["skipped"].clone();
+ let rendered = skipped.to_string();
+ let command = incremental_schema_rebuild_command(&config, state_dir.path());
+ assert!(
+ rendered.contains("incompatible"),
+ "the reason must name the type conflict: {rendered}"
+ );
+ assert!(
+ rendered.contains("bad.csv"),
+ "the reason must name the file: {rendered}"
+ );
+ assert!(
+ rendered.contains(&command),
+ "the reason must print the rebuild command\nreason: {rendered}\ncommand: {command}"
+ );
+ assert!(
+ rendered.contains("Documents already indexed for this file are dropped"),
+ "{rendered}"
+ );
+ let indexed = joined_docs(&endpoint);
+ assert!(
+ indexed.contains("good-new"),
+ "the sibling file must be published: {indexed}"
+ );
+ assert!(
+ !indexed.contains("not-a-number"),
+ "the type-changed file must not be published: {indexed}"
+ );
+ assert!(
+ !indexed.contains("alpha"),
+ "documents already indexed for the skipped file are dropped: {indexed}"
+ );
+
+ let journal_before = fs::read(state_dir.path().join("journal.ndjson")).unwrap();
+ let rebuild_state = PathBuf::from(format!("{}-reindex", state_dir.path().display()));
+ fs::create_dir_all(&rebuild_state).unwrap();
+ let mut rebuild = config.clone();
+ rebuild.prefix = format!("{}-reindex", config.prefix);
+ rebuild.state_dir = Some(rebuild_state);
+ assert!(
+ matches!(run_index(rebuild).unwrap(), 0 | 3),
+ "the printed destination must accept a fresh index of the same folder"
+ );
+ let rebuilt = joined_docs(&endpoint);
+ assert!(
+ rebuilt.contains("not-a-number"),
+ "the rebuild must index the file the watcher skipped: {rebuilt}"
+ );
+ assert_eq!(
+ fs::read(state_dir.path().join("journal.ndjson")).unwrap(),
+ journal_before,
+ "the rebuild uses its own state dir and must not rewrite the watcher journal"
+ );
+}
+
#[test]
fn a_deleted_files_records_stop_appearing() {
let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index d6ff5c608..9925b32a6 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -422,7 +422,7 @@ fn install_dataset_mappings(
) -> Result<Vec<MappingRefusal>> {
let mut refused = Vec::new();
for dataset in &plan.datasets {
- let create_body = dataset_create_body(dataset);
+ let create_body = dataset_mapping_body(dataset, extra_fields(plan, &dataset.slug));
let update_body = json!({
"properties": create_body["mappings"]["properties"].clone()
});
@@ -475,7 +475,39 @@ fn install_dataset_mappings(
/// marker therefore rides the plan frozen in the journal: a pending
/// generation replayed after a restart still declares the analyzer.
fn dataset_create_body(dataset: &PlanDataset) -> Value {
+ dataset_mapping_body(dataset, &[])
+}
+
+fn extra_fields<'a>(plan: &'a Plan, slug: &str) -> &'a [infer::FieldSpec] {
+ plan.incremental_fields
+ .get(slug)
+ .map(Vec::as_slice)
+ .unwrap_or(&[])
+}
+
+/// The index-create body, plus fields an incremental run added after the
+/// schema was frozen (#2176). An extra name that the frozen mapping already
+/// declares is left alone, so this cannot change `impl` from `text` to a
+/// re-inferred `keyword`.
+fn dataset_mapping_body(dataset: &PlanDataset, extra: &[infer::FieldSpec]) -> Value {
let mut create_body = build_mapping(&dataset.specs);
+ if let Some(props) = create_body
+ .pointer_mut("/mappings/properties")
+ .and_then(Value::as_object_mut)
+ {
+ for spec in extra {
+ if props.contains_key(&spec.name) {
+ continue;
+ }
+ let property = match spec.es_type.as_str() {
+ "date" => {
+ json!({"type": "date", "format": "strict_date_optional_time||epoch_millis"})
+ }
+ es_type => json!({"type": es_type}),
+ };
+ props.insert(spec.name.clone(), property);
+ }
+ }
create_body["mappings"]["properties"]["ax_paths"] = json!({"type": "keyword"});
if dataset.family == "docs" || dataset.family == "code" {
if let Some(props) = create_body
@@ -1032,7 +1064,41 @@ fn project_reconcile_plan(
})
.collect()
});
- reconcile_plan::reconcile_plan(inventory, base_plan, scans, cfg.sample)
+ let rebuild = incremental_schema_rebuild_command(cfg, state_dir);
+ reconcile_plan::reconcile_plan(inventory, base_plan, scans, cfg.sample, &rebuild)
+}
+
+/// The command a per-file schema refusal tells the operator to run.
+///
+/// A new `--state-dir` and a new `--prefix` are both required: the watcher's
+/// journal owns the current pair, and `preflight` refuses a different prefix
+/// in that directory. The destination sits beside the live state dir so the
+/// watcher can keep running until the operator stops it. Every flag that
+/// changes what gets indexed is carried by [`legacy_rebuild_argv`].
+fn incremental_schema_rebuild_command(cfg: &IndexCfg, state_dir: &Path) -> String {
+ let rebuild_dir = format!("{}-reindex", state_dir.display());
+ let rebuild_prefix = format!("{}-reindex", cfg.prefix);
+ legacy_rebuild_argv(cfg, Path::new(&rebuild_dir), &rebuild_prefix)
+ .iter()
+ .map(|arg| quote_shell(arg))
+ .collect::<Vec<_>>()
+ .join(" ")
+}
+
+/// Shell-quote one argv element. Safe characters are the same set
+/// `catalog::shell_quote` uses; that helper is private to its module.
+fn quote_shell(s: &str) -> String {
+ if s.is_empty() {
+ return "''".to_string();
+ }
+ let safe = s
+ .chars()
+ .all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '.' | '_' | '-' | ':' | '\\'));
+ if safe {
+ s.to_string()
+ } else {
+ format!("'{}'", s.replace('\'', "'\\''"))
+ }
}
/// Code/AST coverage for one corpus: how much of what a person would call
@@ -4435,6 +4501,7 @@ fn build_phase_a(
duplicate_files,
alias_paths_indexed: true,
refused_datasets: Vec::new(),
+ incremental_fields: Default::default(),
};
PhaseA {
plan,
diff --git a/engine/crates/xerj-autoindex/src/reconcile_plan.rs b/engine/crates/xerj-autoindex/src/reconcile_plan.rs
index 689f2f1d4..874a29868 100644
--- a/engine/crates/xerj-autoindex/src/reconcile_plan.rs
+++ b/engine/crates/xerj-autoindex/src/reconcile_plan.rs
@@ -2,8 +2,13 @@
//!
//! Fresh-run clustering is intentionally not reused here: cluster formation
//! and slug election depend on corpus order and membership. Incremental runs
-//! must preserve the committed dataset/schema identity and fail closed when a
-//! file would require a new dataset or mapping.
+//! keep the committed dataset identity. A new scalar field on a file that
+//! already belongs to one of those datasets is added to that dataset's
+//! mapping — the server accepts an additive `PUT _mapping` — so a code file
+//! that grows a function (`code`, `defs`, `name`, …) does not wedge the
+//! watcher (#2176). A change that cannot be applied in place (a type
+//! conflict, a brand-new dataset, an ambiguous assignment) drops only that
+//! file and names the rebuild command; the rest of the pass still commits.
use crate::content::Inventory;
use crate::infer::{FieldAcc, FieldSpec};
@@ -11,7 +16,7 @@ use crate::state::{FileAssignment, JunkFile, Plan, PlanDataset, RefusedDataset};
use crate::FileScan;
use anyhow::{Context, Result};
use std::cmp::Ordering;
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::Path;
/// Field shape of a file as the *document* renderer sees it.
@@ -41,13 +46,16 @@ fn document_sample(path: &Path, gzip: bool, sample: usize) -> HashMap<String, Fi
/// Project the current, byte-verified inventory onto committed dataset schemas.
///
/// Assignment precedence is stable content identity, then stable native path
-/// identity, then deterministic schema matching. Dataset definitions are
-/// copied byte-for-byte except for their derived `file_count`.
+/// identity, then deterministic schema matching. A dataset keeps its committed
+/// specs — those are the generation identity — and `file_count` is recomputed.
+/// A scalar field the specs do not name is recorded beside them and installed
+/// by the mapping update (#2176).
pub(crate) fn reconcile_plan(
inventory: &Inventory,
previous: &Plan,
scans: Vec<FileScan>,
sample: usize,
+ rebuild_command: &str,
) -> Result<Plan> {
anyhow::ensure!(
inventory.files.len() == inventory.keys.len()
@@ -136,16 +144,16 @@ pub(crate) fn reconcile_plan(
.iter()
.map(|junk| (junk.file_key.as_str(), junk))
.collect();
- let classification_candidates: Vec<PlanDataset> = previous
- .datasets
+ // Specs stay frozen. A new scalar is recorded in `incremental_fields` and
+ // installed by the mapping update; writing it into `specs` would move the
+ // generation identity and abort the pass. Refused definitions are not
+ // widened — growing a mapping the server already rejected cannot help.
+ let mut owned_datasets = previous.datasets.clone();
+ let mut incremental_fields = previous.incremental_fields.clone();
+ let refused_datasets_frozen: Vec<&PlanDataset> = previous
+ .refused_datasets
.iter()
- .chain(
- previous
- .refused_datasets
- .iter()
- .map(|refusal| &refusal.dataset),
- )
- .cloned()
+ .map(|refusal| &refusal.dataset)
.collect();
let mut refused_keys: Vec<Vec<String>> = vec![Vec::new(); previous.refused_datasets.len()];
@@ -310,12 +318,19 @@ pub(crate) fn reconcile_plan(
};
let mut assignments = Vec::with_capacity(scan.sketches.len());
+ let mut file_reject: Option<String> = None;
for sketch in &scan.sketches {
let group = &sketch.group;
let fields = document_fields.as_ref().unwrap_or(&sketch.fields);
- let slug = if let Some(owner) = retained {
- retained_slug(owner, group, sniffed.family.as_str(), fields, &datasets)
- .with_context(|| format!("project retained file {}", file.rel))?
+ let assigned = if let Some(owner) = retained {
+ retained_slug(
+ owner,
+ group,
+ sniffed.family.as_str(),
+ fields,
+ &owned_datasets,
+ &incremental_fields,
+ )
} else {
// A routed document/config classifies as "docs" — its fresh-plan
// fold target — instead of aborting on its raw family.
@@ -324,10 +339,37 @@ pub(crate) fn reconcile_plan(
} else {
sniffed.family.as_str()
};
- classify_new(group, family, fields, &file.rel, &classification_candidates)
- .with_context(|| format!("project new file {}", file.rel))?
+ classify_new(
+ group,
+ family,
+ fields,
+ &file.rel,
+ &owned_datasets,
+ &refused_datasets_frozen,
+ &incremental_fields,
+ )
};
- assignments.push((group.clone(), slug));
+ match assigned {
+ Ok(slug) => assignments.push((group.clone(), slug, fields, sketch.records)),
+ Err(AssignFailure::File(detail)) => {
+ file_reject = Some(detail);
+ break;
+ }
+ Err(AssignFailure::Fatal(error)) => {
+ return Err(error.context(format!("project file {}", file.rel)));
+ }
+ }
+ }
+ if let Some(detail) = file_reject {
+ junk_files.push(JunkFile {
+ file_key: content_id.clone(),
+ rel: file.rel.clone(),
+ format: crate::format_str(scan.sniffed.as_ref()),
+ status: "junk".into(),
+ reason: file_schema_reason(&detail, retained.is_some(), rebuild_command),
+ bytes: file.size,
+ });
+ continue;
}
// A new or changed file whose shape belongs to a refused dataset is
// refused with it — whole, for the reason `refuse_datasets` gives: a
@@ -335,7 +377,7 @@ pub(crate) fn reconcile_plan(
// refused slug so the choice does not depend on sketch order.
if let Some(owner) = assignments
.iter()
- .filter_map(|(_, slug)| refused_by_slug.get(slug.as_str()).copied())
+ .filter_map(|(_, slug, _, _)| refused_by_slug.get(slug.as_str()).copied())
.min_by_key(|owner| previous.refused_datasets[*owner].dataset.slug.as_str())
{
junk_files.push(JunkFile {
@@ -349,13 +391,74 @@ pub(crate) fn reconcile_plan(
refused_keys[owner].push(content_id.clone());
continue;
}
+ // Widen only after every sketch has a published dataset, and roll the
+ // appends back if a later sketch cannot grow the mapping — a file is
+ // assigned whole or not at all. The frozen specs are not touched.
+ let mut widened_to: HashMap<String, usize> = HashMap::new();
+ let mut widen_reject: Option<String> = None;
+ for (_, slug, fields, records) in &assignments {
+ let Some(index) = owned_datasets
+ .iter()
+ .position(|dataset| dataset.slug == *slug)
+ else {
+ continue;
+ };
+ let before = incremental_fields.get(slug.as_str()).map_or(0, Vec::len);
+ match widen_dataset(
+ &owned_datasets[index],
+ incremental_fields.entry(slug.clone()).or_default(),
+ fields,
+ *records,
+ ) {
+ Ok(()) => {
+ if incremental_fields.get(slug.as_str()).map_or(0, Vec::len) > before {
+ widened_to.entry(slug.clone()).or_insert(before);
+ }
+ }
+ Err(AssignFailure::File(detail)) => {
+ rollback_widens(&mut incremental_fields, &widened_to);
+ widen_reject = Some(detail);
+ break;
+ }
+ Err(AssignFailure::Fatal(error)) => {
+ return Err(error.context(format!("project file {}", file.rel)));
+ }
+ }
+ }
+ if let Some(detail) = widen_reject {
+ junk_files.push(JunkFile {
+ file_key: content_id.clone(),
+ rel: file.rel.clone(),
+ format: crate::format_str(scan.sniffed.as_ref()),
+ status: "junk".into(),
+ reason: file_schema_reason(&detail, retained.is_some(), rebuild_command),
+ bytes: file.size,
+ });
+ continue;
+ }
+ let mut assignments: Vec<(Option<String>, String)> = assignments
+ .into_iter()
+ .map(|(group, slug, _, _)| (group, slug))
+ .collect();
assignments.sort();
assignments.dedup();
- anyhow::ensure!(
- assignments.len() == scan.sketches.len(),
- "{} has duplicate dataset groups",
- file.rel
- );
+ if assignments.len() != scan.sketches.len() {
+ junk_files.push(JunkFile {
+ file_key: content_id.clone(),
+ rel: file.rel.clone(),
+ format: crate::format_str(scan.sniffed.as_ref()),
+ status: "junk".into(),
+ reason: file_schema_reason(
+ &format!("{} has duplicate dataset groups", file.rel),
+ retained.is_some(),
+ rebuild_command,
+ ),
+ bytes: file.size,
+ });
+ // The widens above belong to a file that is not indexed.
+ rollback_widens(&mut incremental_fields, &widened_to);
+ continue;
+ }
let replaced = files.insert(
content_id.clone(),
FileAssignment {
@@ -394,8 +497,7 @@ pub(crate) fn reconcile_plan(
.then_with(|| left.rel.cmp(&right.rel))
});
- let mut frozen_datasets = previous.datasets.clone();
- for dataset in &mut frozen_datasets {
+ for dataset in &mut owned_datasets {
dataset.file_count = files
.values()
.filter(|assignment| {
@@ -424,44 +526,70 @@ pub(crate) fn reconcile_plan(
})
.collect();
+ incremental_fields.retain(|_, specs| !specs.is_empty());
+
Ok(Plan {
- datasets: frozen_datasets,
+ datasets: owned_datasets,
files,
junk_files,
duplicate_files,
alias_paths_indexed: previous.alias_paths_indexed,
refused_datasets,
+ incremental_fields,
})
}
+/// A per-file projection failure, or a frozen plan that is internally inconsistent.
+enum AssignFailure {
+ /// This file cannot join the frozen mapping. The rest of the pass can.
+ File(String),
+ /// The committed plan itself is contradictory. Abort; do not publish a
+ /// partial projection of a corrupt journal.
+ Fatal(anyhow::Error),
+}
+
+impl From<anyhow::Error> for AssignFailure {
+ fn from(error: anyhow::Error) -> Self {
+ AssignFailure::Fatal(error)
+ }
+}
+
fn retained_slug(
owner: &FileAssignment,
group: &Option<String>,
family: &str,
fields: &HashMap<String, FieldAcc>,
- datasets: &HashMap<&str, &PlanDataset>,
-) -> Result<String> {
- anyhow::ensure!(
- owner.family == family,
- "detected family changed from {} to {family}",
- owner.family
- );
+ datasets: &[PlanDataset],
+ incremental: &BTreeMap<String, Vec<FieldSpec>>,
+) -> Result<String, AssignFailure> {
+ if owner.family != family {
+ return Err(AssignFailure::File(format!(
+ "detected family changed from {} to {family}",
+ owner.family
+ )));
+ }
let matches = owner
.assignments
.iter()
.filter(|(assigned_group, _)| assigned_group == group)
.collect::<Vec<_>>();
- anyhow::ensure!(
- matches.len() == 1,
- "frozen assignment has {} matches for group {:?}",
- matches.len(),
- group
- );
+ if matches.len() != 1 {
+ return Err(AssignFailure::Fatal(anyhow::anyhow!(
+ "frozen assignment has {} matches for group {:?}",
+ matches.len(),
+ group
+ )));
+ }
let slug = &matches[0].1;
let dataset = datasets
- .get(slug.as_str())
+ .iter()
+ .find(|dataset| dataset.slug == *slug)
.with_context(|| format!("frozen assignment references absent dataset {slug}"))?;
- ensure_compatible(fields, dataset)?;
+ let extra = incremental
+ .get(slug.as_str())
+ .map(Vec::as_slice)
+ .unwrap_or(&[]);
+ ensure_compatible(fields, dataset, extra)?;
Ok(slug.clone())
}
@@ -481,30 +609,44 @@ fn classify_new(
fields: &HashMap<String, FieldAcc>,
rel: &str,
datasets: &[PlanDataset],
-) -> Result<String> {
+ refused: &[&PlanDataset],
+ incremental: &BTreeMap<String, Vec<FieldSpec>>,
+) -> Result<String, AssignFailure> {
let mut candidates = Vec::new();
+ let mut rejected: Option<String> = None;
for dataset in datasets
.iter()
+ .chain(refused.iter().copied())
.filter(|dataset| dataset.family == family && &dataset.group == group)
{
- if ensure_compatible(fields, dataset).is_err() {
- continue;
+ let extra = incremental
+ .get(dataset.slug.as_str())
+ .map(Vec::as_slice)
+ .unwrap_or(&[]);
+ match ensure_compatible(fields, dataset, extra) {
+ Ok(()) => {}
+ Err(AssignFailure::File(detail)) => {
+ rejected = Some(detail);
+ continue;
+ }
+ Err(failure @ AssignFailure::Fatal(_)) => return Err(failure),
}
let affinity = scope_affinity(dataset.scope.as_deref(), rel);
let (intersection, union) = field_overlap(fields, &dataset.specs);
let threshold_met = if dataset.family == "docs" {
// A `docs` dataset's specs are the UNION of every document/config
- // the fold produced. `ensure_compatible` (run just above) already
- // requires every observed SCALAR field to be present in the frozen
- // specs — a field absent from the specs is tolerated ONLY when
- // `acc.n == 0` (pruned: e.g. a code file's object-valued `symbols`
- // sidecar, #580). So a document/config that reached here belongs to
- // the docs dataset regardless of how many other fields the union
- // carries; accept it. (The 0.7 Jaccard would wrongly reject a
- // document with fewer fields than the union, and `== fields.len()`
- // wrongly counts the pruned `symbols` key and aborted a new code
- // file — the #729 central case.) `let _` keeps the shared
- // `field_overlap` result in scope for the data-family branches.
+ // the fold produced. `ensure_compatible` (run just above) accepts a
+ // scalar field the frozen specs do not name yet — `widen_dataset`
+ // records it beside the specs, and the mapping update installs it —
+ // and still tolerates `acc.n == 0` (pruned: e.g. a
+ // code file's object-valued `symbols` sidecar, #580). So a
+ // document/config that reached here belongs to the docs dataset
+ // regardless of how many other fields the union carries; accept
+ // it. (The 0.7 Jaccard would wrongly reject a document with fewer
+ // fields than the union, and `== fields.len()` wrongly counts the
+ // pruned `symbols` key and aborted a new code file — the #729
+ // central case.) `let _` keeps the shared `field_overlap` result
+ // in scope for the data-family branches.
//
// #731 (3): this also silently admits a routed file with ZERO
// rendered fields (e.g. an unreadable/empty document) — harmless,
@@ -545,12 +687,16 @@ fn classify_new(
});
let Some((winner, winner_intersection, winner_union, winner_affinity)) = candidates.first()
else {
- anyhow::bail!(
+ let because = rejected
+ .as_deref()
+ .map(|detail| format!(" Last candidate refused: {detail}"))
+ .unwrap_or_default();
+ return Err(AssignFailure::File(format!(
"no frozen dataset accepts family {family}, group {:?}, and fields {:?}; \
- this requires unsupported dataset/schema evolution (use a new prefix)",
+ this requires unsupported dataset/schema evolution (use a new prefix).{because}",
group,
sorted_field_names(fields)
- );
+ )));
};
if let Some((runner_up, runner_up_intersection, runner_up_union, runner_up_affinity)) =
candidates.get(1)
@@ -559,21 +705,23 @@ fn classify_new(
// separate the top two. A pre-#1230 plan carries no scope (`None` for
// every dataset), so both affinities are `None`, they compare equal,
// and this fails closed exactly as before.
- anyhow::ensure!(
- ratio_cmp(
- *winner_intersection,
- *winner_union,
- *runner_up_intersection,
- *runner_up_union
- ) != Ordering::Equal
- || winner_affinity != runner_up_affinity,
- "new file matches frozen datasets {} and {} equally for family {family}, group {:?}, \
- and fields {:?}; refusing ambiguous assignment (use a new prefix)",
- winner.slug,
- runner_up.slug,
- group,
- sorted_field_names(fields)
- );
+ if ratio_cmp(
+ *winner_intersection,
+ *winner_union,
+ *runner_up_intersection,
+ *runner_up_union,
+ ) == Ordering::Equal
+ && winner_affinity == runner_up_affinity
+ {
+ return Err(AssignFailure::File(format!(
+ "new file matches frozen datasets {} and {} equally for family {family}, group {:?}, \
+ and fields {:?}; refusing ambiguous assignment (use a new prefix)",
+ winner.slug,
+ runner_up.slug,
+ group,
+ sorted_field_names(fields)
+ )));
+ }
}
Ok(winner.slug.clone())
}
@@ -605,7 +753,12 @@ fn field_overlap(fields: &HashMap<String, FieldAcc>, specs: &[FieldSpec]) -> (us
// Keep null-only and object-shaped fields in the schema fingerprint too.
// FieldAcc deliberately does not count those as scalar observations, but
// allowing an unknown one through would let the server create a dynamic
- // mapping outside the frozen plan.
+ // mapping outside the frozen plan. A scalar the specs do not name yet is
+ // also a mismatch here: `widen_dataset` adds it only after a file has
+ // already been assigned (a retained file, or a docs file the fold accepts).
+ // Dropping it from this ratio would make a dataset that lacks the field
+ // look identical to one that has it, and scope would then pick the worse
+ // one.
let observed: HashSet<&str> = fields.keys().map(String::as_str).collect();
let frozen: HashSet<&str> = specs.iter().map(|spec| spec.name.as_str()).collect();
(
@@ -618,48 +771,163 @@ fn ratio_cmp(left_n: usize, left_d: usize, right_n: usize, right_d: usize) -> Or
(left_n * right_d).cmp(&(right_n * left_d))
}
-fn ensure_compatible(fields: &HashMap<String, FieldAcc>, dataset: &PlanDataset) -> Result<()> {
- let specs: HashMap<&str, &FieldSpec> = dataset
+fn ensure_compatible(
+ fields: &HashMap<String, FieldAcc>,
+ dataset: &PlanDataset,
+ extra: &[FieldSpec],
+) -> Result<(), AssignFailure> {
+ let mut specs: HashMap<&str, &FieldSpec> = dataset
.specs
.iter()
.map(|spec| (spec.name.as_str(), spec))
.collect();
- for (name, acc) in fields {
- let Some(spec) = specs.get(name.as_str()) else {
+ for spec in extra {
+ specs.entry(spec.name.as_str()).or_insert(spec);
+ }
+ // Sorted so the reported field does not depend on hash-map order.
+ let mut names: Vec<&str> = fields.keys().map(String::as_str).collect();
+ names.sort_unstable();
+ for name in names {
+ let acc = &fields[name];
+ let Some(spec) = specs.get(name) else {
// A field observed with NO scalar values (`acc.n == 0`) — e.g. the
// code file's `symbols` sidecar (an array of objects, which
// `FieldAcc` deliberately does not scalar-type) — legitimately never
// received a frozen `FieldSpec` at genesis: it has nothing to type.
- // Its absence is therefore not drift; there is nothing to validate,
- // so mirror `compatible_with_spec`'s `acc.n == 0 => true` and skip
- // it. Without this, re-indexing ANY code file aborted the whole run
- // (#580), because `symbols` is observed on every generation but was
- // pruned from the frozen schema. A field that DOES carry scalar
- // values yet is absent is still a real incompatibility.
+ // Its absence is therefore not drift (#580).
//
- // #1237: `impl` and `end_line` on symbol records were added after
- // #579 froze the five symbol scalar fields; tolerate their absence
- // on pre-#1237 frozen `docs`/`code` datasets so incremental runs
- // over existing state dirs do not abort.
- let pre_1237_symbol_field = (name == "impl" || name == "end_line")
- && (dataset.family == "docs" || dataset.family == "code");
- anyhow::ensure!(
- acc.n == 0 || pre_1237_symbol_field,
- "field {name} is absent from frozen dataset {}",
+ // #1237: `impl` and `end_line` are installed on every docs/code
+ // mapping by `dataset_create_body`, outside the frozen specs.
+ // Re-inferring them here can elect `keyword` for a short `impl`
+ // and the server then refuses to change the existing `text` field.
+ //
+ // Any other absent scalar with a name the mapping can grow is
+ // added by `widen_dataset` (#2176). A name that cannot be added
+ // (empty, dotted, leading `_`) is a per-file refusal: letting it
+ // through would dynamic-map it.
+ if absent_field_is_tolerated(name, acc, dataset) {
+ continue;
+ }
+ if widenable_field_name(name) {
+ continue;
+ }
+ return Err(AssignFailure::File(format!(
+ "field {name} is absent from frozen dataset {} and cannot be added to its mapping",
dataset.slug
- );
- continue;
+ )));
};
- anyhow::ensure!(
- compatible_with_spec(acc, spec),
- "field {name} is incompatible with frozen {} mapping in dataset {}",
- spec.es_type,
- dataset.slug
- );
+ if !compatible_with_spec(acc, spec) {
+ return Err(AssignFailure::File(format!(
+ "field {name} is incompatible with frozen {} mapping in dataset {}",
+ spec.es_type, dataset.slug
+ )));
+ }
}
Ok(())
}
+/// `impl` / `end_line` and object-valued sidecars are already accounted for
+/// and must not be re-inferred onto the frozen specs.
+fn absent_field_is_tolerated(name: &str, acc: &FieldAcc, dataset: &PlanDataset) -> bool {
+ if acc.n == 0 {
+ return true;
+ }
+ (name == "impl" || name == "end_line") && (dataset.family == "docs" || dataset.family == "code")
+}
+
+/// A top-level field name `PUT /_mapping` can add without nesting or colliding
+/// with metadata. Dotted names become nested properties; a leading `_` is
+/// reserved. Anything else is refused per file instead of dynamic-mapped.
+fn widenable_field_name(name: &str) -> bool {
+ let mut chars = name.chars();
+ match chars.next() {
+ Some(first) if first.is_ascii_alphabetic() => {}
+ _ => return false,
+ }
+ chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
+}
+
+fn widen_dataset(
+ dataset: &PlanDataset,
+ added: &mut Vec<FieldSpec>,
+ fields: &HashMap<String, FieldAcc>,
+ records: u64,
+) -> Result<(), AssignFailure> {
+ let mut known: HashSet<&str> = dataset
+ .specs
+ .iter()
+ .map(|spec| spec.name.as_str())
+ .collect();
+ known.extend(added.iter().map(|spec| spec.name.as_str()));
+ let mut pending: HashMap<String, FieldAcc> = HashMap::new();
+ let mut names: Vec<&str> = fields.keys().map(String::as_str).collect();
+ names.sort_unstable();
+ for name in names {
+ let acc = &fields[name];
+ if known.contains(name) || absent_field_is_tolerated(name, acc, dataset) || acc.n == 0 {
+ continue;
+ }
+ if !widenable_field_name(name) {
+ return Err(AssignFailure::File(format!(
+ "field {name} is absent from frozen dataset {} and cannot be added to its mapping",
+ dataset.slug
+ )));
+ }
+ pending.insert(name.to_string(), acc.clone());
+ }
+ if pending.is_empty() {
+ return Ok(());
+ }
+ if dataset.specs.len() + added.len() + pending.len() > crate::infer::MAX_FIELDS_PER_DATASET {
+ let pending_len = pending.len();
+ return Err(AssignFailure::File(format!(
+ "dataset {} already maps {} fields; adding {pending_len} more would pass the {} field cap",
+ dataset.slug,
+ dataset.specs.len() + added.len(),
+ crate::infer::MAX_FIELDS_PER_DATASET
+ )));
+ }
+ // `no_semantic`: a new field must not elect `semantic_text`. That would
+ // change the dataset's embedding contract without updating `semantic_field`,
+ // and a fresh mapping install would then refuse or embed a field this
+ // generation does not declare as the body. Lexical text/keyword/long still
+ // search.
+ let mut inferred = crate::infer::infer_fields(&pending, records.max(1), true);
+ for spec in &mut inferred {
+ if spec.es_type == "semantic_text" {
+ spec.es_type = "text".into();
+ }
+ spec.notes
+ .push("added on incremental reconcile when the field first appeared".into());
+ }
+ added.extend(inferred);
+ Ok(())
+}
+
+fn rollback_widens(
+ fields: &mut BTreeMap<String, Vec<FieldSpec>>,
+ widened: &HashMap<String, usize>,
+) {
+ for (slug, len) in widened {
+ if let Some(specs) = fields.get_mut(slug) {
+ specs.truncate(*len);
+ }
+ }
+ fields.retain(|_, specs| !specs.is_empty());
+}
+
+fn file_schema_reason(detail: &str, retained: bool, rebuild_command: &str) -> String {
+ let fate = if retained {
+ "Documents already indexed for this file are dropped with this generation."
+ } else {
+ "Nothing was indexed for this file."
+ };
+ format!(
+ "{detail} This file was not indexed; every other file in this pass still is. {fate} \
+ Stop the watcher and rebuild the folder so the schema is inferred again: {rebuild_command}"
+ )
+}
+
fn compatible_with_spec(acc: &FieldAcc, spec: &FieldSpec) -> bool {
if acc.n == 0 {
return true;
@@ -697,6 +965,11 @@ mod tests {
use serde_json::json;
use std::path::PathBuf;
+ /// Printed verbatim by a per-file schema refusal. A reworded error that
+ /// drops this command is a failed recovery contract.
+ const REBUILD: &str =
+ "xerj autoindex /src --prefix p-reindex --state-dir /state-reindex --no-graph";
+
fn acc(values: &[serde_json::Value]) -> FieldAcc {
let mut acc = FieldAcc::default();
for value in values {
@@ -821,6 +1094,7 @@ mod tests {
&previous,
vec![scan(fields)],
50,
+ REBUILD,
)
.unwrap();
assert_eq!(
@@ -854,6 +1128,7 @@ mod tests {
&previous,
vec![scan(fields)],
50,
+ REBUILD,
)
.unwrap();
assert_eq!(plan.files["new"].assignments, vec![(None, "winner".into())]);
@@ -874,66 +1149,141 @@ mod tests {
("id".into(), acc(&[json!(7)])),
("body".into(), acc(&[json!("hello")])),
]);
- let error = reconcile_plan(
+ let plan = reconcile_plan(
&inventory("new.jsonl", "unix:6e", "new"),
&previous,
vec![scan(fields)],
50,
+ REBUILD,
)
- .unwrap_err();
- let message = format!("{error:#}");
+ .unwrap();
+ assert!(
+ plan.files.is_empty(),
+ "an equal tie must not assign a dataset"
+ );
+ assert_eq!(plan.junk_files.len(), 1);
+ let message = &plan.junk_files[0].reason;
assert!(
message.contains("refusing ambiguous assignment"),
"{message}"
);
assert!(message.contains("a-tie"), "{message}");
assert!(message.contains("z-tie"), "{message}");
+ assert!(message.contains(REBUILD), "{message}");
+ assert!(
+ message.contains("Nothing was indexed for this file."),
+ "{message}"
+ );
}
#[test]
- fn new_field_and_incompatible_type_fail_before_plan_changes() {
+ fn a_new_scalar_field_widens_and_an_unmappable_change_is_one_file() {
let previous = Plan {
datasets: vec![dataset("rows", vec![spec("id", "long")])],
..Plan::default()
};
+ // A file already assigned to this dataset grows a scalar the frozen
+ // specs do not name. The mapping is extended. It used to abort the plan.
+ let mut retained = previous.clone();
+ retained
+ .files
+ .insert("old".into(), assignment("rows.jsonl", "unix:6e", "rows"));
let unknown = HashMap::from([
("id".into(), acc(&[json!(7)])),
("surprise".into(), acc(&[json!("new mapping")])),
]);
- let error = reconcile_plan(
- &inventory("new.jsonl", "unix:6e", "new"),
- &previous,
+ let plan = reconcile_plan(
+ &inventory("rows.jsonl", "unix:6e", "new"),
+ &retained,
vec![scan(unknown)],
50,
+ REBUILD,
)
- .unwrap_err();
- assert!(format!("{error:#}").contains("unsupported dataset/schema evolution"));
+ .unwrap();
+ assert_eq!(plan.files["new"].assignments, vec![(None, "rows".into())]);
+ assert!(plan.junk_files.is_empty());
+ assert!(
+ plan.datasets[0].specs.iter().all(|spec| spec.name == "id"),
+ "frozen specs stay the generation identity"
+ );
+ assert!(
+ plan.incremental_fields["rows"]
+ .iter()
+ .any(|spec| spec.name == "surprise"),
+ "the new scalar is installed beside the frozen specs"
+ );
- // Object-shaped values are not counted as scalar FieldAcc evidence,
- // but their field name must still be rejected rather than dynamically
- // mapped by the server.
+ // The same extra field on a file with no committed assignment stays
+ // below the group-less overlap threshold, so it does not join — and
+ // it does not take the rest of the plan down with it.
+ let plan = reconcile_plan(
+ &inventory("new.jsonl", "unix:6f", "brand-new"),
+ &previous,
+ vec![scan(HashMap::from([
+ ("id".into(), acc(&[json!(7)])),
+ ("surprise".into(), acc(&[json!("new mapping")])),
+ ]))],
+ 50,
+ REBUILD,
+ )
+ .unwrap();
+ assert!(plan.files.is_empty());
+ assert!(
+ plan.junk_files[0]
+ .reason
+ .contains("unsupported dataset/schema evolution"),
+ "{}",
+ plan.junk_files[0].reason
+ );
+ assert!(plan.junk_files[0].reason.contains(REBUILD));
+ assert!(plan.datasets[0].specs.iter().all(|spec| spec.name == "id"));
+ assert!(plan.incremental_fields.is_empty());
+
+ // Object-shaped values are not scalar FieldAcc evidence. Their name
+ // stays in the overlap fingerprint, so a data file that introduces one
+ // still fails to join — and only that file is junked.
let object_field = HashMap::from([
("id".into(), acc(&[json!(7)])),
("object".into(), acc(&[json!({"nested": true})])),
]);
- let error = reconcile_plan(
+ let plan = reconcile_plan(
&inventory("new.jsonl", "unix:6e", "new"),
&previous,
vec![scan(object_field)],
50,
+ REBUILD,
)
- .unwrap_err();
- assert!(format!("{error:#}").contains("unsupported dataset/schema evolution"));
+ .unwrap();
+ assert!(plan.files.is_empty());
+ let message = &plan.junk_files[0].reason;
+ assert!(
+ message.contains("unsupported dataset/schema evolution"),
+ "{message}"
+ );
+ assert!(message.contains(REBUILD), "{message}");
+ assert!(
+ plan.datasets[0].specs.iter().all(|spec| spec.name == "id"),
+ "a refused file must not widen the frozen specs"
+ );
+ assert!(plan.incremental_fields.is_empty());
let wrong_type = HashMap::from([("id".into(), acc(&[json!("not-a-number")]))]);
- let error = reconcile_plan(
+ let plan = reconcile_plan(
&inventory("new.jsonl", "unix:6e", "new"),
&previous,
vec![scan(wrong_type)],
50,
+ REBUILD,
)
- .unwrap_err();
- assert!(format!("{error:#}").contains("unsupported dataset/schema evolution"));
+ .unwrap();
+ assert!(plan.files.is_empty());
+ let message = &plan.junk_files[0].reason;
+ assert!(message.contains("incompatible"), "{message}");
+ assert!(message.contains(REBUILD), "{message}");
+ assert!(
+ message.contains("Nothing was indexed for this file."),
+ "{message}"
+ );
}
/// A NEW document file (empty `key_fields`, no group — the shape a fresh
@@ -971,6 +1321,7 @@ mod tests {
&previous,
vec![file_scan],
50,
+ REBUILD,
)
.unwrap();
assert_eq!(
@@ -1031,7 +1382,7 @@ mod tests {
digests: vec!["digest-cfg".into()],
duplicates: vec![],
};
- let plan = reconcile_plan(&inv, &previous, vec![file_scan], 50).unwrap();
+ let plan = reconcile_plan(&inv, &previous, vec![file_scan], 50, REBUILD).unwrap();
assert_eq!(plan.files["cfg"].assignments, vec![(None, "docs".into())]);
assert!(
plan.files["cfg"].as_document,
@@ -1082,6 +1433,7 @@ mod tests {
&previous,
vec![file_scan],
50,
+ REBUILD,
)
.unwrap();
assert_eq!(plan.files["code"].assignments, vec![(None, "docs".into())]);
@@ -1091,6 +1443,93 @@ mod tests {
);
}
+ /// #2176: a code file indexed while it had no symbols freezes `docs` as
+ /// title/language/body. Adding a function emits scalar fields (`code`,
+ /// `name`, `defs`, …) the frozen specs do not name. The retained file must
+ /// stay on `docs` and those fields must be recorded beside the specs —
+ /// not written into them, which would move the generation identity.
+ /// `impl` and `end_line` stay out of that list too: the create body
+ /// already maps them, and re-inferring `impl` can elect `keyword`.
+ #[test]
+ fn retained_code_file_grows_a_function_and_docs_gains_code() {
+ let mut previous = Plan {
+ datasets: vec![PlanDataset {
+ family: "docs".into(),
+ ..dataset(
+ "docs",
+ vec![
+ spec("title", "text"),
+ spec("language", "keyword"),
+ spec("body", "text"),
+ ],
+ )
+ }],
+ ..Plan::default()
+ };
+ let mut prior = assignment("module.py", "unix:70", "docs");
+ prior.family = "code".into();
+ previous.files.insert("old-module".into(), prior);
+ let fields = HashMap::from([
+ ("title".to_string(), acc(&[json!("module.py")])),
+ ("language".to_string(), acc(&[json!("python")])),
+ (
+ "body".to_string(),
+ acc(&[json!("def cancel_request():\n return \"birchfunction\"")]),
+ ),
+ ("defs".to_string(), acc(&[json!("function cancel_request")])),
+ ("symbol_count".to_string(), acc(&[json!(1)])),
+ ("code".to_string(), acc(&[json!("def cancel_request():")])),
+ ("name".to_string(), acc(&[json!("cancel_request")])),
+ ("kind".to_string(), acc(&[json!("function")])),
+ ("line".to_string(), acc(&[json!(3)])),
+ (
+ "symbols".to_string(),
+ acc(&[json!({"name": "cancel_request"})]),
+ ),
+ ]);
+ let mut file_scan = scan(fields);
+ file_scan.sniffed.as_mut().unwrap().family = Family::Code;
+
+ let plan = reconcile_plan(
+ &inventory("module.py", "unix:70", "new-module"),
+ &previous,
+ vec![file_scan],
+ 50,
+ REBUILD,
+ )
+ .unwrap();
+ assert!(
+ plan.junk_files.is_empty(),
+ "adding a function must not junk the file: {:?}",
+ plan.junk_files
+ );
+ assert_eq!(
+ plan.files["new-module"].assignments,
+ vec![(None, "docs".into())]
+ );
+ let frozen: Vec<&str> = plan.datasets[0]
+ .specs
+ .iter()
+ .map(|spec| spec.name.as_str())
+ .collect();
+ assert_eq!(frozen, ["title", "language", "body"]);
+ let names: Vec<&str> = plan.incremental_fields["docs"]
+ .iter()
+ .map(|spec| spec.name.as_str())
+ .collect();
+ for field in ["code", "name", "kind", "line", "defs", "symbol_count"] {
+ assert!(names.contains(&field), "missing {field} in {names:?}");
+ }
+ assert!(!names.contains(&"impl"));
+ assert!(!names.contains(&"end_line"));
+ assert!(!names.contains(&"symbols"));
+ let reason = plan.files["new-module"].rel.as_str();
+ assert_eq!(reason, "module.py");
+ let message = file_schema_reason("field id is incompatible", true, REBUILD);
+ assert!(message.contains("Documents already indexed for this file are dropped"));
+ assert!(message.contains(REBUILD));
+ }
+
/// Two frozen scope-keyed `docs` datasets (one per nested repository) whose
/// genesis schemas are byte-identical. A code document is a group-less,
/// empty-key-field shape, so it routes to the `docs` family (see
@@ -1140,6 +1579,7 @@ mod tests {
&previous,
vec![fork_doc_scan()],
50,
+ REBUILD,
)
.expect("an adjacent file inside bitcoin-core must not abort the run");
assert_eq!(
@@ -1154,6 +1594,7 @@ mod tests {
&previous,
vec![fork_doc_scan()],
50,
+ REBUILD,
)
.expect("an adjacent file inside bitcoin-knots must not abort the run");
assert_eq!(
@@ -1196,6 +1637,7 @@ mod tests {
&previous,
vec![scan(fields)],
50,
+ REBUILD,
)
.unwrap();
assert_eq!(
@@ -1223,18 +1665,24 @@ mod tests {
],
..Plan::default()
};
- let error = reconcile_plan(
+ let plan = reconcile_plan(
&inventory("bitcoin-core/adjacent.ts", "unix:a4", "legacy-add"),
&previous,
vec![fork_doc_scan()],
50,
+ REBUILD,
)
- .unwrap_err();
- let message = format!("{error:#}");
+ .unwrap();
+ assert!(
+ plan.files.is_empty(),
+ "a legacy plan with no scope must not silently assign the tie"
+ );
+ let message = &plan.junk_files[0].reason;
assert!(
message.contains("refusing ambiguous assignment"),
"{message}"
);
+ assert!(message.contains(REBUILD), "{message}");
}
#[test]
@@ -1256,7 +1704,7 @@ mod tests {
bytes: 10,
});
let fields = HashMap::from([("id".into(), acc(&[json!(7)]))]);
- let plan = reconcile_plan(&current, &previous, vec![scan(fields)], 50).unwrap();
+ let plan = reconcile_plan(&current, &previous, vec![scan(fields)], 50, REBUILD).unwrap();
assert_eq!(plan.files["same"].rel, "renamed.jsonl");
assert_eq!(plan.datasets[0].file_count, 1);
assert_eq!(plan.duplicate_files.len(), 1);
@@ -1277,7 +1725,8 @@ mod tests {
let fields = || HashMap::from([("id".into(), acc(&[json!(7)]))]);
let conflict = inventory("a.jsonl", "unix:62", "content-a");
- let error = reconcile_plan(&conflict, &previous, vec![scan(fields())], 50).unwrap_err();
+ let error =
+ reconcile_plan(&conflict, &previous, vec![scan(fields())], 50, REBUILD).unwrap_err();
assert!(format!("{error:#}").contains("conflicting committed content and path owners"));
let duplicate = Inventory {
@@ -1291,6 +1740,7 @@ mod tests {
&previous,
vec![scan(fields()), scan(fields())],
50,
+ REBUILD,
)
.unwrap_err();
assert!(format!("{error:#}").contains("duplicate content identity"));
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index 0debae8c8..b9f308834 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -7,7 +7,7 @@ use anyhow::{Context, Result};
use fs2::FileExt;
use serde::{Deserialize, Serialize};
use serde_json::Value;
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::{BufRead, Write};
use std::path::{Path, PathBuf};
@@ -566,6 +566,18 @@ pub struct Plan {
/// a spurious "plan changed" generation.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub refused_datasets: Vec<RefusedDataset>,
+ /// Scalar fields installed on a frozen dataset after the first pass (#2176).
+ ///
+ /// Keyed by dataset slug. Empty — and omitted from the serialized plan —
+ /// until a watched file grows a field the frozen `specs` do not name.
+ /// Those specs stay frozen: `schema_identity`, `index_identity`, and the
+ /// generation-to-generation dataset equality check all read them, and
+ /// changing them aborts the next generation. The mapping update sends
+ /// this list on top of the frozen specs. The server accepts the additive
+ /// `PUT /_mapping`. A no-op re-run copies the list forward, so the plan
+ /// compares equal and commits nothing.
+ #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+ pub incremental_fields: BTreeMap<String, Vec<crate::infer::FieldSpec>>,
}
/// A dataset the plan inferred and the server refused to map (#929).