Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): name a changed option before sealing a --no-graph rerun (#2577) (#2683)

Jake Elbirt3be9949b371e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9b68a5439..39c45d495 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -153,6 +153,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`start_line` are unchanged. An unchanged file is not re-extracted, so re-index
with `xerj autoindex --fresh` to retitle documents already in an index.
+- A `--no-graph` re-index whose options differ from the committed generation
+ (for example `--sample` or `--follow-symlinks`) is refused before anything
+ is copied, and the error names the option. It used to copy and prepare the
+ whole folder, then report "could not derive generation N; the durable
+ generation state is not internally consistent", although the state was
+ intact and the original options still updated the corpus (#2577).
+
## [1.0.0-rc.100] - 2026-10-26
**rc.100: faster autoindex and search, shaped by your field reports.** This
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index a45845ab8..67df308fa 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -7392,3 +7392,68 @@ fn a_file_that_changes_size_during_snapshot_seal_is_skipped() {
"the rerun indexes the bytes the file settled on"
);
}
+
+/// #2577: a `--no-graph` run whose options differ from the committed
+/// generation is refused before anything is copied, and the refusal names the
+/// option. It used to seal the whole corpus and then report the durable state
+/// as "not internally consistent", which it was not: the original options
+/// still continue the corpus.
+#[test]
+fn changed_option_is_refused_before_sealing_and_named() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ let mut sample = config.clone();
+ sample.sample += 1;
+ let mut symlinks = config.clone();
+ symlinks.follow_symlinks = true;
+ let cases = [
+ (sample, "build fingerprints them differently (--sample"),
+ (symlinks, "--follow-symlinks was off and is now on"),
+ ];
+ for (round, (changed, named)) in cases.into_iter().enumerate() {
+ // Both reconcile arms: an unchanged folder and an edited one.
+ for edit in [false, true] {
+ if edit {
+ fs::write(
+ corpus.path().join("a.csv"),
+ format!("id,value\n1,alpha\n2,edit{round}\n"),
+ )
+ .unwrap();
+ }
+ let snapshots = snapshot_names(state_dir.path());
+ let bulks = endpoint.data_bulk_requests();
+ let begins = journal_events(state_dir.path(), "sync_begin");
+ let error = format!("{:#}", run_index(changed.clone()).unwrap_err());
+ let context = format!("edit={edit}: {error}");
+ assert!(
+ error.contains("execution configuration changed since the committed generation"),
+ "{context}"
+ );
+ assert!(error.contains(named), "{context}");
+ assert!(error.contains("Nothing was copied"), "{context}");
+ assert!(!error.contains("not internally consistent"), "{context}");
+ assert_eq!(snapshot_names(state_dir.path()), snapshots, "{context}");
+ assert_eq!(endpoint.data_bulk_requests(), bulks, "{context}");
+ assert_eq!(
+ journal_events(state_dir.path(), "sync_begin"),
+ begins,
+ "{context}"
+ );
+ }
+ // The committed state was never damaged: the original options
+ // continue it and publish the edit.
+ let commits = journal_events(state_dir.path(), "sync_commit");
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), commits + 1);
+ }
+ assert_eq!(endpoint.data_docs().len(), 2);
+}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 37a801e19..9d8ac5624 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -6261,6 +6261,111 @@ fn pending_for_phase_b(
.collect()
}
+/// #2577: what this `--no-graph` run's options change against the execution
+/// identity the committed generation pinned, one phrase per difference. Empty
+/// means the run may continue the committed corpus.
+///
+/// The record-preparation options are pinned only as a digest, so a change
+/// there names the options that feed it rather than the old values. The digest
+/// carries its contract version as a prefix, which tells a different XERJ
+/// build apart from a changed option.
+fn no_graph_option_changes(
+ expected: &sync::ExecutionIdentity,
+ cfg: &IndexCfg,
+ root_identity: &str,
+) -> Result<Vec<String>> {
+ let on_off = |on: bool| if on { "on" } else { "off" };
+ let mut changes = Vec::new();
+ if expected.root_identity != root_identity {
+ changes.push(format!(
+ "the folder was {} and is now {root_identity}",
+ expected.root_identity
+ ));
+ }
+ if expected.url != cfg.url {
+ changes.push(format!("--url was {} and is now {}", expected.url, cfg.url));
+ }
+ if expected.prefix != cfg.prefix {
+ changes.push(format!(
+ "--prefix was {} and is now {}",
+ expected.prefix, cfg.prefix
+ ));
+ }
+ if expected.follow_symlinks != cfg.follow_symlinks {
+ changes.push(format!(
+ "--follow-symlinks was {} and is now {}",
+ on_off(expected.follow_symlinks),
+ on_off(cfg.follow_symlinks)
+ ));
+ }
+ let chunker_identity = prepared_records_identity(cfg)?;
+ if expected.chunker_identity != chunker_identity {
+ let contract = |identity: &str| {
+ identity
+ .rsplit_once('-')
+ .map(|(contract, _)| contract.to_owned())
+ .unwrap_or_default()
+ };
+ let (was, now) = (
+ contract(&expected.chunker_identity),
+ contract(&chunker_identity),
+ );
+ changes.push(if was == now {
+ "one or more record-preparation options differ, or this XERJ build fingerprints \
+ them differently (--sample, --no-semantic, --pdf-timeout-secs, --label or the \
+ label file's contents, --max-file-gb, --max-file-mb, --max-file-bytes); only a \
+ fingerprint of them is saved, so the earlier values cannot be shown"
+ .to_owned()
+ } else {
+ format!("the record-preparation contract was {was} and this XERJ build uses {now}")
+ });
+ }
+ if expected.graph_enabled
+ || expected.brain != "disabled"
+ || expected.detector_identity != DETECTOR_DISABLED_IDENTITY
+ {
+ changes.push(format!(
+ "relationship detection was on (graph {}, --brain {}) and this run has it off",
+ on_off(expected.graph_enabled),
+ expected.brain
+ ));
+ }
+ Ok(changes)
+}
+
+/// #2577: the plan-derived half of the pinned execution identity. Dataset
+/// definitions are frozen across generations, so on the same XERJ build these
+/// match; a difference means the build derives different mappings for them.
+fn contract_identity_changes(
+ expected: &sync::ExecutionIdentity,
+ plan: &Plan,
+) -> Result<Vec<String>> {
+ let (schema_identity, index_identity) = generation_contract_identities(plan)?;
+ let mut changes = Vec::new();
+ if expected.schema_identity != schema_identity || expected.index_identity != index_identity {
+ changes.push(
+ "this XERJ build derives different schemas, index mappings or catalog mapping \
+ for the committed datasets than the build that committed them"
+ .to_owned(),
+ );
+ }
+ Ok(changes)
+}
+
+fn refuse_execution_changes(generation: u64, changes: Vec<String>) -> Result<()> {
+ if changes.is_empty() {
+ return Ok(());
+ }
+ anyhow::bail!(
+ "autoindex execution configuration changed since the committed generation \
+ {generation}: {}. Nothing was copied and no remote data was changed; generation \
+ {generation} and its state are intact. Re-run with the options (and the XERJ \
+ version) generation {generation} was built with to update it, or index the folder \
+ with a new --prefix and a new --state-dir to build it with the new options",
+ changes.join("; ")
+ )
+}
+
/// An unchanged re-index must not fail with a false "embedding execution
/// identity changed" message (#487). The identity changed only when a
/// comparable field differs from what the journal pinned. `resumable` is
@@ -9407,6 +9512,16 @@ fn run_index_report_inner(
.as_ref()
.context("generated journal lost its committed manifest")?
.clone();
+ // #2577: refuse a changed option before reconciling or sealing
+ // anything. Both arms below require these fields to match; the
+ // changed arm used to find out only after copying the corpus, as a
+ // false "state not internally consistent" error.
+ if let Some(expected) = &base.execution {
+ refuse_execution_changes(
+ base.generation,
+ no_graph_option_changes(expected, &cfg, &root_str)?,
+ )?;
+ }
let mut plan = project_reconcile_plan(
&inventory,
&base.plan,
@@ -9417,6 +9532,23 @@ fn run_index_report_inner(
tally,
)?;
record_unreadable_skips(&mut plan, &inventory.unreadable);
+ if let Some(expected) = &base.execution {
+ refuse_execution_changes(base.generation, contract_identity_changes(expected, &plan)?)?;
+ if plan
+ .datasets
+ .iter()
+ .any(|dataset| dataset.semantic_field.is_some())
+ {
+ let current = es.embedding_execution_identity()?;
+ ensure_embedding_execution_unchanged_and_resumable(
+ &current,
+ &expected.embedding_identity_sha256,
+ &expected.embedding_backend,
+ expected.embedding_dimension,
+ &expected.embedding_semantic_contract,
+ )?;
+ }
+ }
// Say out loud that the decision gate is not on this route. It is an
// incremental reconcile of an already-committed generation, so the work
// is the *changed* set and is published from a sealed snapshot rather
@@ -9434,37 +9566,6 @@ fn run_index_report_inner(
));
}
if serde_json::to_value(&plan)? == serde_json::to_value(&base.plan)? {
- if let Some(expected) = &base.execution {
- let (schema_identity, index_identity) = generation_contract_identities(&plan)?;
- anyhow::ensure!(
- expected.root_identity == root_str
- && expected.url == cfg.url
- && expected.prefix == cfg.prefix
- && expected.follow_symlinks == cfg.follow_symlinks
- && expected.chunker_identity == prepared_records_identity(&cfg)?
- && !expected.graph_enabled
- && expected.brain == "disabled"
- && expected.detector_identity == DETECTOR_DISABLED_IDENTITY
- && expected.schema_identity == schema_identity
- && expected.index_identity == index_identity,
- "autoindex execution configuration changed since the committed generation; \
- rebuild with a new --state-dir and a new --prefix"
- );
- if plan
- .datasets
- .iter()
- .any(|dataset| dataset.semantic_field.is_some())
- {
- let current = es.embedding_execution_identity()?;
- ensure_embedding_execution_unchanged_and_resumable(
- &current,
- &expected.embedding_identity_sha256,
- &expected.embedding_backend,
- expected.embedding_dimension,
- &expected.embedding_semantic_contract,
- )?;
- }
- }
let summary = finish_generated_run(&es, &mut journal, &cfg)?;
let code = generated_exit_code(&summary);
finish_generated_progress(&pr, code, &summary, dataless_skipped);