Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): shrinking file set on a junk-bearing generation no longer trips the manifest-digest invariant (#283)

xerj-orgfffc3ab444a5
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index ae70c788d..920ed55d2 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -1275,3 +1275,129 @@ fn a_generated_run_narrates_its_scan_and_closes_its_own_stream() {
String::from_utf8_lossy(&buffer.lock().unwrap())
);
}
+
+/// Two byte-identical junk files — two empty files are the everyday case —
+/// made the fresh plan carry a duplicate alias for content that has no
+/// `plan.files` entry, so the generation cutover failed its own
+/// alias-projection invariant and the whole run aborted with exit 1 (#283's
+/// reproducible sibling: junk plus duplicates diverging between the fresh plan
+/// and the incremental projection). The fresh plan now projects aliases the
+/// same way `reconcile_plan` does, so the folder indexes, a no-op re-run
+/// confirms instead of committing a spurious generation, and a shrinking file
+/// set on the junk-bearing generation reconciles.
+#[test]
+fn two_byte_identical_junk_files_commit_reconcile_and_survive_a_shrink() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n2,beta\n").unwrap();
+ fs::write(corpus.path().join("e1.dat"), "").unwrap();
+ fs::write(corpus.path().join("e2.dat"), "").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+
+ let (code, summary) = run_index_report(config.clone())
+ .expect("a folder with two byte-identical junk files must index, not abort");
+ let summary = summary.expect("the junk-bearing generation still commits");
+ assert_eq!(code, 3, "junk is recorded, never fatal (cli.rs EXIT CODES)");
+ assert_eq!(summary["generation"], 1);
+ assert_eq!(summary["records_total"], 2);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+
+ // The fresh plan and the incremental projection must agree byte-for-byte
+ // on how junk-content aliases project, or this no-op re-run would see a
+ // "changed" plan and commit generation 2 over nothing.
+ assert_eq!(run_index(config.clone()).unwrap(), 3);
+ assert_eq!(
+ journal_events(state_dir.path(), "sync_commit"),
+ 1,
+ "a no-op re-run over the junk-bearing generation confirms; it must not re-commit"
+ );
+
+ // The issue's headline shape: the file set shrinks on a generation that
+ // carries junk records.
+ fs::remove_file(corpus.path().join("a.csv")).unwrap();
+ assert_eq!(run_index(config.clone()).unwrap(), 3);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);
+ assert!(
+ paths(&endpoint.data_docs()).is_empty(),
+ "the deleted dataset's records must not survive the reconcile"
+ );
+
+ // And the junk files themselves disappearing reconciles back to clean.
+ fs::remove_file(corpus.path().join("e1.dat")).unwrap();
+ fs::remove_file(corpus.path().join("e2.dat")).unwrap();
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 3);
+}
+
+/// The #283 abort itself: a durable generation record that fails its own
+/// re-validation used to surface as a bare internal invariant ("desired
+/// manifest digest does not match sync_begin payload", exit 1) — text the user
+/// can do nothing with. However the journal got that way, the refusal must
+/// name the recovery route the way the exec-config guard already does, and
+/// keep the invariant attached as the cause.
+#[test]
+fn a_journal_that_fails_its_own_revalidation_names_the_rebuild_route() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let source = corpus.path().join("rows.csv");
+ fs::write(&source, "id,value\n1,first\n").unwrap();
+ fs::write(corpus.path().join("empty.csv"), "").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 3);
+
+ // Leave a durable pending generation behind: begin succeeds, the bulk is
+ // partially applied, the run fails before commit.
+ fs::write(&source, "id,value\n1,replaced\n2,second\n").unwrap();
+ endpoint
+ .state
+ .lock()
+ .unwrap()
+ .partially_apply_next_data_bulk = true;
+ assert!(run_index(config.clone()).is_err());
+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 2);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+
+ // Corrupt the pending sync_begin payload in a spot only the manifest
+ // digest covers: the junk record's byte count. This makes the next run
+ // hit exactly the #283 invariant.
+ let journal_path = state_dir.path().join("journal.ndjson");
+ let tampered: String = fs::read_to_string(&journal_path)
+ .unwrap()
+ .lines()
+ .map(|line| {
+ let mut record: Value = serde_json::from_str(line).unwrap();
+ if record.get("kind").and_then(Value::as_str) == Some("sync_begin")
+ && record
+ .pointer("/desired/generation")
+ .and_then(Value::as_u64)
+ == Some(2)
+ {
+ let bytes = record
+ .pointer_mut("/desired/plan/junk_files/0/bytes")
+ .expect("the junk-bearing desired manifest records its junk file");
+ *bytes = Value::from(bytes.as_u64().unwrap() + 1);
+ }
+ let mut line = serde_json::to_string(&record).unwrap();
+ line.push('\n');
+ line
+ })
+ .collect();
+ fs::write(&journal_path, tampered).unwrap();
+
+ let error = run_index(config).expect_err("a tampered generation journal must refuse to run");
+ let rendered = format!("{error:#}");
+ assert!(
+ rendered.contains("desired manifest digest does not match sync_begin payload"),
+ "the internal invariant must stay attached as the cause: {rendered}"
+ );
+ assert!(
+ rendered.contains("Rebuild with a new --state-dir and a new --prefix"),
+ "the refusal must name the recovery route, not just the invariant: {rendered}"
+ );
+}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 5fe5ce843..a0affb9b0 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -407,7 +407,20 @@ fn begin_non_graph_generation(
plan,
groups,
};
- let pending = sync::PendingSync::new(tx_id, &base, desired)?;
+ // A failure here is the generation machinery contradicting itself about
+ // durable state, and the invariant text alone gives the user nothing to
+ // act on (#283). Name the recovery route; the invariant stays attached as
+ // the cause so the report keeps its diagnostic value.
+ let pending = sync::PendingSync::new(tx_id, &base, desired).with_context(|| {
+ format!(
+ "autoindex could not derive generation {} from committed generation {}; the durable \
+ generation state in {} is not internally consistent, and re-running will not repair \
+ it. No remote data was changed. Rebuild with a new --state-dir and a new --prefix",
+ base.generation + 1,
+ base.generation,
+ state_dir.display()
+ )
+ })?;
journal.sync_begin(&pending)
}
@@ -1374,6 +1387,17 @@ fn build_phase_a(
file_count: c.members.len(),
});
}
+ // Aliases of junk/skipped content have no `plan.files` entry and therefore
+ // no manifest group to attach to, so keeping them made the generation
+ // cutover fail its own alias-projection invariant on any folder holding
+ // two byte-identical junk files — two empty files are enough (#283). The
+ // incremental projection already drops them (`reconcile_plan`'s
+ // `live_content` filter); the fresh plan must project the same way or a
+ // no-op re-run would see a changed plan and commit a spurious generation.
+ let duplicate_files = duplicate_files
+ .into_iter()
+ .filter(|alias| file_assignments.contains_key(&alias.file_key))
+ .collect();
let plan = Plan {
datasets,
files: file_assignments,
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index 5bd21e261..0ec49d909 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -565,7 +565,19 @@ fn read_plan_for_preflight(
}
}
Some(kind) if crate::sync::is_sync_record_kind(kind) => {
- crate::sync::replay_record(&value, &mut committed_manifest, &mut pending_sync)?;
+ // Same contract as the authoritative replay in
+ // `open_after_preflight` (#283): this preflight runs first, so
+ // an invariant that leaks here bare is the one the user sees.
+ crate::sync::replay_record(&value, &mut committed_manifest, &mut pending_sync)
+ .with_context(|| {
+ format!(
+ "the durable {kind} record at byte {record_start} in {} failed to \
+ replay; the generation journal is not internally consistent, \
+ and re-running will not repair it. No remote data was changed. \
+ Rebuild with a new --state-dir and a new --prefix",
+ path.display()
+ )
+ })?;
if matches!(kind, "sync_bootstrap" | "sync_commit") {
plan = committed_manifest
.as_ref()
@@ -838,11 +850,26 @@ impl Journal {
embedding_identity_resumable = resumable;
}
Some(kind) if crate::sync::is_sync_record_kind(kind) => {
+ // A durable record that no longer re-validates
+ // is unrepairable by re-running; surface the
+ // rebuild route instead of a bare internal
+ // invariant, keeping the invariant as the
+ // cause (#283).
crate::sync::replay_record(
&v,
&mut committed_manifest,
&mut pending_sync,
- )?;
+ )
+ .with_context(|| {
+ format!(
+ "the durable {kind} record at byte {record_start} in {} \
+ failed to replay; the generation journal is not \
+ internally consistent, and re-running will not repair \
+ it. No remote data was changed. Rebuild with a new \
+ --state-dir and a new --prefix",
+ jpath.display()
+ )
+ })?;
if matches!(kind, "sync_bootstrap" | "sync_commit") {
plan = committed_manifest
.as_ref()
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index 1a621a83c..3b0fc1926 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -677,7 +677,18 @@ pub fn replay_pending_operations(
.committed_manifest
.clone()
.context("pending corpus generation has no committed base")?;
- pending.validate_against(&base)?;
+ // Same contract as the journal-replay wrap in `state.rs` (#283): a pending
+ // generation that fails its own re-validation is unrepairable by
+ // re-running, and the internal invariant alone is not actionable.
+ pending.validate_against(&base).with_context(|| {
+ format!(
+ "the pending corpus generation {} no longer re-validates against committed \
+ generation {}; the generation journal is not internally consistent, and re-running \
+ will not repair it. No remote data was changed. Rebuild with a new --state-dir and \
+ a new --prefix",
+ pending.desired.generation, base.generation
+ )
+ })?;
let snapshot = open_snapshot(state_dir, &pending.tx_id)?;
verify_snapshot_binding(&pending, &snapshot)?;
backend.provision_generation(&pending.desired)?;