AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): shrinking file set on a junk-bearing generation no longer trips the manifest-digest invariant (#283)
xerj-orgfffc3ab444a5
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex ae70c788d..920ed55d2 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -1275,3 +1275,129 @@ fn a_generated_run_narrates_its_scan_and_closes_its_own_stream() {String::from_utf8_lossy(&buffer.lock().unwrap()));}++/// Two byte-identical junk files — two empty files are the everyday case —+/// made the fresh plan carry a duplicate alias for content that has no+/// `plan.files` entry, so the generation cutover failed its own+/// alias-projection invariant and the whole run aborted with exit 1 (#283's+/// reproducible sibling: junk plus duplicates diverging between the fresh plan+/// and the incremental projection). The fresh plan now projects aliases the+/// same way `reconcile_plan` does, so the folder indexes, a no-op re-run+/// confirms instead of committing a spurious generation, and a shrinking file+/// set on the junk-bearing generation reconciles.+#[test]+fn two_byte_identical_junk_files_commit_reconcile_and_survive_a_shrink() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n2,beta\n").unwrap();+ fs::write(corpus.path().join("e1.dat"), "").unwrap();+ fs::write(corpus.path().join("e2.dat"), "").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);++ let (code, summary) = run_index_report(config.clone())+ .expect("a folder with two byte-identical junk files must index, not abort");+ let summary = summary.expect("the junk-bearing generation still commits");+ assert_eq!(code, 3, "junk is recorded, never fatal (cli.rs EXIT CODES)");+ assert_eq!(summary["generation"], 1);+ assert_eq!(summary["records_total"], 2);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);++ // The fresh plan and the incremental projection must agree byte-for-byte+ // on how junk-content aliases project, or this no-op re-run would see a+ // "changed" plan and commit generation 2 over nothing.+ assert_eq!(run_index(config.clone()).unwrap(), 3);+ assert_eq!(+ journal_events(state_dir.path(), "sync_commit"),+ 1,+ "a no-op re-run over the junk-bearing generation confirms; it must not re-commit"+ );++ // The issue's headline shape: the file set shrinks on a generation that+ // carries junk records.+ fs::remove_file(corpus.path().join("a.csv")).unwrap();+ assert_eq!(run_index(config.clone()).unwrap(), 3);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);+ assert!(+ paths(&endpoint.data_docs()).is_empty(),+ "the deleted dataset's records must not survive the reconcile"+ );++ // And the junk files themselves disappearing reconciles back to clean.+ fs::remove_file(corpus.path().join("e1.dat")).unwrap();+ fs::remove_file(corpus.path().join("e2.dat")).unwrap();+ assert_eq!(run_index(config).unwrap(), 0);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 3);+}++/// The #283 abort itself: a durable generation record that fails its own+/// re-validation used to surface as a bare internal invariant ("desired+/// manifest digest does not match sync_begin payload", exit 1) — text the user+/// can do nothing with. However the journal got that way, the refusal must+/// name the recovery route the way the exec-config guard already does, and+/// keep the invariant attached as the cause.+#[test]+fn a_journal_that_fails_its_own_revalidation_names_the_rebuild_route() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap();+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ let source = corpus.path().join("rows.csv");+ fs::write(&source, "id,value\n1,first\n").unwrap();+ fs::write(corpus.path().join("empty.csv"), "").unwrap();+ let endpoint = HttpEndpoint::start();+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);+ assert_eq!(run_index(config.clone()).unwrap(), 3);++ // Leave a durable pending generation behind: begin succeeds, the bulk is+ // partially applied, the run fails before commit.+ fs::write(&source, "id,value\n1,replaced\n2,second\n").unwrap();+ endpoint+ .state+ .lock()+ .unwrap()+ .partially_apply_next_data_bulk = true;+ assert!(run_index(config.clone()).is_err());+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 2);+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);++ // Corrupt the pending sync_begin payload in a spot only the manifest+ // digest covers: the junk record's byte count. This makes the next run+ // hit exactly the #283 invariant.+ let journal_path = state_dir.path().join("journal.ndjson");+ let tampered: String = fs::read_to_string(&journal_path)+ .unwrap()+ .lines()+ .map(|line| {+ let mut record: Value = serde_json::from_str(line).unwrap();+ if record.get("kind").and_then(Value::as_str) == Some("sync_begin")+ && record+ .pointer("/desired/generation")+ .and_then(Value::as_u64)+ == Some(2)+ {+ let bytes = record+ .pointer_mut("/desired/plan/junk_files/0/bytes")+ .expect("the junk-bearing desired manifest records its junk file");+ *bytes = Value::from(bytes.as_u64().unwrap() + 1);+ }+ let mut line = serde_json::to_string(&record).unwrap();+ line.push('\n');+ line+ })+ .collect();+ fs::write(&journal_path, tampered).unwrap();++ let error = run_index(config).expect_err("a tampered generation journal must refuse to run");+ let rendered = format!("{error:#}");+ assert!(+ rendered.contains("desired manifest digest does not match sync_begin payload"),+ "the internal invariant must stay attached as the cause: {rendered}"+ );+ assert!(+ rendered.contains("Rebuild with a new --state-dir and a new --prefix"),+ "the refusal must name the recovery route, not just the invariant: {rendered}"+ );+}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 5fe5ce843..a0affb9b0 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -407,7 +407,20 @@ fn begin_non_graph_generation(plan,groups,};- let pending = sync::PendingSync::new(tx_id, &base, desired)?;+ // A failure here is the generation machinery contradicting itself about+ // durable state, and the invariant text alone gives the user nothing to+ // act on (#283). Name the recovery route; the invariant stays attached as+ // the cause so the report keeps its diagnostic value.+ let pending = sync::PendingSync::new(tx_id, &base, desired).with_context(|| {+ format!(+ "autoindex could not derive generation {} from committed generation {}; the durable \+ generation state in {} is not internally consistent, and re-running will not repair \+ it. No remote data was changed. Rebuild with a new --state-dir and a new --prefix",+ base.generation + 1,+ base.generation,+ state_dir.display()+ )+ })?;journal.sync_begin(&pending)}@@ -1374,6 +1387,17 @@ fn build_phase_a(file_count: c.members.len(),});}+ // Aliases of junk/skipped content have no `plan.files` entry and therefore+ // no manifest group to attach to, so keeping them made the generation+ // cutover fail its own alias-projection invariant on any folder holding+ // two byte-identical junk files — two empty files are enough (#283). The+ // incremental projection already drops them (`reconcile_plan`'s+ // `live_content` filter); the fresh plan must project the same way or a+ // no-op re-run would see a changed plan and commit a spurious generation.+ let duplicate_files = duplicate_files+ .into_iter()+ .filter(|alias| file_assignments.contains_key(&alias.file_key))+ .collect();let plan = Plan {datasets,files: file_assignments,diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex 5bd21e261..0ec49d909 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -565,7 +565,19 @@ fn read_plan_for_preflight(}}Some(kind) if crate::sync::is_sync_record_kind(kind) => {- crate::sync::replay_record(&value, &mut committed_manifest, &mut pending_sync)?;+ // Same contract as the authoritative replay in+ // `open_after_preflight` (#283): this preflight runs first, so+ // an invariant that leaks here bare is the one the user sees.+ crate::sync::replay_record(&value, &mut committed_manifest, &mut pending_sync)+ .with_context(|| {+ format!(+ "the durable {kind} record at byte {record_start} in {} failed to \+ replay; the generation journal is not internally consistent, \+ and re-running will not repair it. No remote data was changed. \+ Rebuild with a new --state-dir and a new --prefix",+ path.display()+ )+ })?;if matches!(kind, "sync_bootstrap" | "sync_commit") {plan = committed_manifest.as_ref()@@ -838,11 +850,26 @@ impl Journal {embedding_identity_resumable = resumable;}Some(kind) if crate::sync::is_sync_record_kind(kind) => {+ // A durable record that no longer re-validates+ // is unrepairable by re-running; surface the+ // rebuild route instead of a bare internal+ // invariant, keeping the invariant as the+ // cause (#283).crate::sync::replay_record(&v,&mut committed_manifest,&mut pending_sync,- )?;+ )+ .with_context(|| {+ format!(+ "the durable {kind} record at byte {record_start} in {} \+ failed to replay; the generation journal is not \+ internally consistent, and re-running will not repair \+ it. No remote data was changed. Rebuild with a new \+ --state-dir and a new --prefix",+ jpath.display()+ )+ })?;if matches!(kind, "sync_bootstrap" | "sync_commit") {plan = committed_manifest.as_ref()diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex 1a621a83c..3b0fc1926 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -677,7 +677,18 @@ pub fn replay_pending_operations(.committed_manifest.clone().context("pending corpus generation has no committed base")?;- pending.validate_against(&base)?;+ // Same contract as the journal-replay wrap in `state.rs` (#283): a pending+ // generation that fails its own re-validation is unrepairable by+ // re-running, and the internal invariant alone is not actionable.+ pending.validate_against(&base).with_context(|| {+ format!(+ "the pending corpus generation {} no longer re-validates against committed \+ generation {}; the generation journal is not internally consistent, and re-running \+ will not repair it. No remote data was changed. Rebuild with a new --state-dir and \+ a new --prefix",+ pending.desired.generation, base.generation+ )+ })?;let snapshot = open_snapshot(state_dir, &pending.tx_id)?;verify_snapshot_binding(&pending, &snapshot)?;backend.provision_generation(&pending.desired)?;