AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): define durable non-graph generations
bugerf77eaa324bc0
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex 4136e817a..e225b4732 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -72,8 +72,9 @@ pub enum Cmd {Help,}-const FRESH_HELP: &str = "ignore the existing journal and rebuild the plan in place\n\- (ids stay idempotent) — see RESUME POLICY";+const FRESH_HELP: &str =+ "start without resume state only when the selected state directory has no \+durable plan; an existing plan is refused and destination records are never reset";pub fn print_help() {println!(@@ -170,16 +171,14 @@ pub fn print_help() {\n\RESUME POLICY:\n\A durable plan supports no-op resume and same-path content replacement.\n\- Files added after the plan was frozen are reported as skipped and are not\n\- indexed; --fresh rebuilds the plan in place and picks them up.\n\- Removing an indexed file is refused before any remote mutation: its\n\- documents are already live and nothing here deletes them. Restore the file\n\- and rerun, or rebuild — in place by deleting the published indices and the\n\- state directory, or isolated under a new --state-dir, --prefix and --brain\n\- (or --no-graph), validated before you switch readers.\n\+ Added or removed content groups are refused before remote mutation.\n\+ An independent rebuild needs a new --state-dir, new --prefix, and, when\n\+ graph detection is enabled, new --brain (or --no-graph). Validate before\n\+ switching readers; the shared autoindex-catalog and old target require\n\+ explicit, validated cleanup.\n\\n\EXIT CODES: 0 complete; 3 completed-with-junk (junk recorded, never fatal);\n\- 2 usage; 1 endpoint/journal failure or a refused corpus removal\n",+ 2 usage; 1 endpoint/journal failure or unsupported corpus delta\n",fresh_help = FRESH_HELP);}@@ -477,10 +476,10 @@ mod tests {}#[test]- fn fresh_help_points_at_the_resume_policy_that_bounds_it() {- assert!(super::FRESH_HELP.contains("rebuild the plan in place"));- assert!(super::FRESH_HELP.contains("ids stay idempotent"));- assert!(super::FRESH_HELP.contains("RESUME POLICY"));+ fn fresh_help_warns_that_it_is_not_destination_reconciliation() {+ assert!(super::FRESH_HELP.contains("no durable plan"));+ assert!(super::FRESH_HELP.contains("existing plan is refused"));+ assert!(super::FRESH_HELP.contains("destination records are never reset"));}#[test]diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 6cc7a4f29..e3e750aa8 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -69,8 +69,6 @@ struct MockState {/// what `_count`/`_search` answer with, and the run verifies live data/// counts against the journal (#195). Catalog rows are not data rows.catalog_docs: HashMap<String, Value>,- /// Every (method, path) the run issued. The refusal tests assert on the- /// ABSENCE of remote mutations, which no document count can express.requests: Vec<(String, String)>,data_bulk_number: usize,fail_data_bulk: usize,@@ -202,6 +200,11 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {if response_delay_ms > 0 {thread::sleep(std::time::Duration::from_millis(response_delay_ms));}+ state+ .lock()+ .unwrap()+ .requests+ .push((method.to_owned(), path.to_owned()));let (status, response) = if method == "GET" && path == "/v1/embedding/identity" {let identity = state.lock().unwrap().embedding_identity_sha256.clone();@@ -532,15 +535,11 @@ fn assert_unsupported_delta_without_remote_mutation(let message = format!("{error:#}");assert!(message.contains("made no remote mutations"), "{message}");assert!(- message.contains("no longer exist in the folder"),- "{message}"- );- assert!(- message.contains("restore the removed file(s) and rerun"),+ message.contains("existing destination may already be partial or stale"),"{message}");assert!(- message.contains("rebuild in place by deleting the indices"),+ message.contains("`--fresh` is not recovery or destination reconciliation"),"{message}");assert!(@@ -562,58 +561,19 @@ fn assert_unsupported_delta_without_remote_mutation(}}-/// The documented headline workflow: point autoindex at a folder, add a file,-/// rerun. The rerun must not fail, must say plainly that the added file was-/// not indexed, and `--fresh` must then absorb it in place.#[test]-fn a_rerun_after_an_added_file_succeeds_and_fresh_absorbs_it() {+fn completed_plan_rejects_added_content_group_before_remote_mutation() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();let state_dir = tempfile::tempdir().unwrap();fs::write(corpus.path().join("first.csv"), "id,value\n1,first\n").unwrap();let endpoint = MockEndpoint::start(usize::MAX);- let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);assert_eq!(run_index(config.clone()).unwrap(), 0);fs::write(corpus.path().join("second.csv"), "id,value\n2,second\n").unwrap();- // 3 = completed-with-junk: the added file is reported as skipped, not- // indexed, because the frozen plan cannot absorb it.- let (code, report) = run_index_report(config.clone()).unwrap();- assert_eq!(code, 3);- let report = report.unwrap();- assert_eq!(report["files_junk"], 1, "{report}");- // `records_total` is the live server-side count (#195), so it still reports- // the record the first run published. What must be zero is the run-local- // counter: this rerun indexed nothing, because the frozen plan cannot- // absorb a file that appeared after it was written.- assert_eq!(report["records_submitted_this_run"], 0, "{report}");- assert_eq!(report["files_submitted_this_run"], 0, "{report}");- assert_eq!(report["records_total"], 1, "{report}");- {- let locked = endpoint.state.lock().unwrap();- let live: Vec<&str> = locked- .docs- .values()- .filter_map(|doc| doc["ax_path"].as_str())- .collect();- assert!(- live.iter().all(|path| *path == "first.csv"),- "the added file must not be published by a plan that predates it: {live:?}"- );- }-- // --fresh rebuilds the plan in place and picks the new file up.- config.fresh = true;- assert_eq!(run_index(config).unwrap(), 0);- let locked = endpoint.state.lock().unwrap();- let indexed: std::collections::HashSet<&str> = locked- .docs- .values()- .filter_map(|doc| doc["ax_path"].as_str())- .collect();- assert!(indexed.contains("first.csv"), "{indexed:?}");- assert!(indexed.contains("second.csv"), "{indexed:?}");+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &["second.csv"], &[]);}#[test]@@ -678,127 +638,8 @@ fn completed_plan_rejects_mixed_membership_delta_in_stable_order() {assert_eq!(vanished, ["a-old.csv", "b-old.csv"]);}-/// An in-place EDIT is a replacement, not a removal. `--fresh` is the route-/// the refusal itself recommends for picking up added and changed files, so-/// the gate must not classify the edited file's superseded content key as a-/// vanished group — the path is still there, and the pipeline republishes it.-#[test]-fn fresh_absorbs_an_edited_file_instead_of_calling_it_a_removal() {- let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();- let corpus = tempfile::tempdir().unwrap();- let state_dir = tempfile::tempdir().unwrap();- fs::write(corpus.path().join("notes.csv"), "id,value\n1,before\n").unwrap();- let endpoint = MockEndpoint::start(usize::MAX);- let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);- assert_eq!(run_index(config.clone()).unwrap(), 0);-- fs::write(corpus.path().join("notes.csv"), "id,value\n1,after\n").unwrap();- config.fresh = true;- assert_eq!(- run_index(config).unwrap(),- 0,- "an edited file is a same-path replacement, not a vanished content group"- );- let locked = endpoint.state.lock().unwrap();- assert!(- locked- .docs- .values()- .any(|doc| doc["value"].as_str() == Some("after")),- "the new content is live: {:?}",- locked.docs- );- // Documented, and NOT what this gate is for: `--fresh` rebuilds the plan,- // it does not reconcile the destination. Document ids derive from the- // content key (`ids::doc_id`), so the pre-edit record stays live beside- // the new one — the same outcome `--fresh` has always had. An ordinary- // rerun is the clean route for an edit: it keeps the planned key and runs- // a delete-before-replace transaction on it. What must never happen is- // this run being REFUSED by a message claiming a file that is sitting in- // the folder has vanished.- assert!(- locked- .docs- .values()- .any(|doc| doc["value"].as_str() == Some("before")),- "known --fresh gap: superseded records are not deleted: {:?}",- locked.docs- );-}--/// The clean route for the same edit: an ordinary rerun keeps the planned key-/// and replaces its records, so nothing is stranded. This is the contrast that-/// makes the `--fresh` gap above a documented trade-off rather than a silent-/// one.-#[test]-fn an_ordinary_rerun_replaces_an_edited_file_without_stranding_records() {- let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();- let corpus = tempfile::tempdir().unwrap();- let state_dir = tempfile::tempdir().unwrap();- fs::write(corpus.path().join("notes.csv"), "id,value\n1,before\n").unwrap();- let endpoint = MockEndpoint::start(usize::MAX);- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);- assert_eq!(run_index(config.clone()).unwrap(), 0);-- fs::write(corpus.path().join("notes.csv"), "id,value\n1,after\n").unwrap();- assert_eq!(run_index(config).unwrap(), 0);- let locked = endpoint.state.lock().unwrap();- let values: Vec<&str> = locked- .docs- .values()- .filter_map(|doc| doc["value"].as_str())- .collect();- assert_eq!(values, ["after"], "the superseded record is replaced");-}--/// The composite workflow the run's own stderr recommends: index, then add one-/// file and edit another, rerun (added file reported and skipped by the frozen-/// plan), then `--fresh` to absorb both.#[test]-fn fresh_absorbs_an_addition_and_an_edit_after_a_reported_rerun() {- let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();- let corpus = tempfile::tempdir().unwrap();- let state_dir = tempfile::tempdir().unwrap();- fs::write(corpus.path().join("first.csv"), "id,value\n1,before\n").unwrap();- let endpoint = MockEndpoint::start(usize::MAX);- let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);- assert_eq!(run_index(config.clone()).unwrap(), 0);-- fs::write(corpus.path().join("first.csv"), "id,value\n1,after\n").unwrap();- fs::write(corpus.path().join("second.csv"), "id,value\n2,second\n").unwrap();- // 3 = completed-with-junk: the frozen plan cannot absorb the added file,- // so the rerun reports it as skipped rather than refusing.- assert_eq!(run_index(config.clone()).unwrap(), 3);-- config.fresh = true;- assert_eq!(- run_index(config).unwrap(),- 0,- "--fresh is the documented way out of exactly this state"- );- let locked = endpoint.state.lock().unwrap();- let live: std::collections::HashSet<&str> = locked- .docs- .values()- .filter_map(|doc| doc["ax_path"].as_str())- .collect();- assert!(live.contains("first.csv"), "{live:?}");- assert!(live.contains("second.csv"), "{live:?}");- assert!(- locked- .docs- .values()- .any(|doc| doc["value"].as_str() == Some("after")),- "the edited file's new content is live: {:?}",- locked.docs- );-}--#[test]-fn fresh_cannot_erase_the_plan_and_bypass_the_removal_gate() {+fn nonempty_fresh_cannot_erase_plan_and_bypass_membership_gate() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();@@ -815,7 +656,7 @@ fn fresh_cannot_erase_the_plan_and_bypass_the_removal_gate() {}#[test]-fn deleting_one_path_of_a_duplicate_pair_is_not_a_removed_content_group() {+fn fresh_rejects_same_content_canonical_promotion_even_when_group_key_matches() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();@@ -824,24 +665,14 @@ fn deleting_one_path_of_a_duplicate_pair_is_not_a_removed_content_group() {fs::write(corpus.path().join("a.csv"), bytes).unwrap();fs::write(corpus.path().join("b.csv"), bytes).unwrap();let endpoint = MockEndpoint::start(usize::MAX);- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);assert_eq!(run_index(config.clone()).unwrap(), 0);- // b.csv becomes canonical under the same content key: the group survives- // the deletion, so no document is stranded and the rerun is allowed.+ // b.csv becomes canonical with the same content key. A key-only delta is+ // empty, but fresh would discard the alias/path cleanup knowledge.fs::remove_file(corpus.path().join("a.csv")).unwrap();- let result = run_index(config);- assert!(- result.is_ok(),- "a surviving content group must not trip the removal gate: {result:?}"- );- let locked = endpoint.state.lock().unwrap();- let live: Vec<&str> = locked- .docs- .values()- .filter_map(|doc| doc["ax_path"].as_str())- .collect();- assert_eq!(live, ["b.csv"], "canonical path follows the surviving file");+ config.fresh = true;+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &[], &[]);}#[test]@@ -867,7 +698,7 @@ fn unchanged_planned_junk_is_not_reported_as_added_content() {}#[test]-fn deleted_planned_junk_is_swept_rather_than_refused() {+fn deleted_planned_junk_fails_closed_before_catalog_can_stay_stale() {let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();let corpus = tempfile::tempdir().unwrap();@@ -882,33 +713,8 @@ fn deleted_planned_junk_is_swept_rather_than_refused() {let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);assert_eq!(run_index(config.clone()).unwrap(), 3);- // A junk file publishes exactly one catalog row and nothing else, and the- // #238 sweep deletes that row. Nothing is stranded, so the removal gate- // must not fire — refusing here would block a case the pipeline handles- // completely. Only a file that published DOCUMENTS refuses a rerun.fs::remove_file(corpus.path().join("opaque.bin")).unwrap();- assert_eq!(run_index(config.clone()).unwrap(), 0);- {- let locked = endpoint.state.lock().unwrap();- assert!(- locked- .catalog_docs- .values()- .all(|doc| doc["path"].as_str() != Some("opaque.bin")),- "the deleted junk file's catalog row must be swept, not left immortal"- );- assert!(- locked- .catalog_docs- .values()- .any(|doc| doc["path"].as_str() == Some("rows.csv")),- "the surviving indexed file keeps its entry"- );- }-- // Deleting the INDEXED file is still refused: its documents are live.- fs::remove_file(corpus.path().join("rows.csv")).unwrap();- assert_unsupported_delta_without_remote_mutation(&endpoint, config, &[], &["rows.csv"]);+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &[], &["opaque.bin"]);}#[test]@@ -923,7 +729,6 @@ fn completed_plan_rejects_empty_current_folder_before_remote_mutation() {assert_eq!(run_index(config.clone()).unwrap(), 0);fs::remove_file(corpus.path().join("only.csv")).unwrap();- assert_unsupported_delta_without_remote_mutation(&endpoint, config.clone(), &[], &["only.csv"]);// Even `--fresh` must not erase the only durable inventory evidence and// then report success while the destination still contains the document.config.fresh = true;@@ -1725,12 +1530,12 @@ fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {).unwrap();fs::write(corpus.path().join("b.csv"), original).unwrap();- assert_eq!(run_index(config.clone()).unwrap(), 3);+ assert_unsupported_delta_without_remote_mutation(&endpoint, config.clone(), &["b.csv"], &[]);{let locked = endpoint.state.lock().unwrap();assert_eq!(locked.docs.len(), 2);assert!(locked.docs.values().all(|doc| {- doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("rewritten")+ doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("original")}));}let replay = state::Journal::open(@@ -1745,24 +1550,20 @@ fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {assert!(replay.pending_replacements.is_empty());assert_eq!(replay.done.len(), 1);let plan = replay.plan.clone().unwrap();- assert_eq!(plan.junk_files.len(), 1);- assert_eq!(plan.junk_files[0].rel, "b.csv");- assert!(plan.junk_files[0]- .reason- .contains("key ownership exclusive"));+ assert!(plan.junk_files.is_empty());drop(replay);- // The divergence is durable and deterministic: an identical rerun keeps- // the same owner, appends no new plan, and changes no documents.+ // The refusal is deterministic: an identical rerun keeps the old owner,+ // appends no new plan, and changes no documents.let plans_before = event_count(state_dir.path(), "plan");- assert_eq!(run_index(config).unwrap(), 3);+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &["b.csv"], &[]);assert_eq!(event_count(state_dir.path(), "plan"), plans_before);let locked = endpoint.state.lock().unwrap();assert_eq!(locked.docs.len(), 2);assert!(locked.docs.values()- .all(|doc| doc["value"].as_str() == Some("rewritten")));+ .all(|doc| doc["value"].as_str() == Some("original")));}/// #238: a file added after the plan was frozen is skipped and reported in thediff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 4563c6ada..bfdf50142 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -19,6 +19,7 @@ pub mod progress;pub mod resources;pub mod sniff;pub mod state;+mod sync;pub mod walk;use anyhow::{Context, Result};@@ -1040,8 +1041,10 @@ fn select_resume_plan_keys(for (key, assignment) in &plan.files {if let Some(previous) = planned_by_rel.insert(&assignment.rel, key) {anyhow::bail!(- "resume plan assigns path {} to both {} and {}; use --fresh after verifying the \- existing index",+ "resume plan assigns path {} to both {} and {}; refusing to discard history under \+ the same destination. For an isolated rebuild use a new --state-dir, new \+ --prefix, and new --brain when graph detection is enabled (or --no-graph); \+ explicitly validate and clean the shared catalog and old target",assignment.rel,previous,key@@ -1050,8 +1053,11 @@ fn select_resume_plan_keys(if !assignment.path_id.is_empty() {if let Some(previous) = planned_by_path_id.insert(&assignment.path_id, key) {anyhow::bail!(- "resume plan assigns one native path identity to both {} and {}; use --fresh \- after verifying the existing index",+ "resume plan assigns one native path identity to both {} and {}; refusing to \+ discard history under the same destination. For an isolated rebuild use a \+ new --state-dir, new --prefix, and new --brain when graph detection is \+ enabled (or --no-graph); explicitly validate and clean the shared catalog \+ and old target",previous,key);@@ -1089,9 +1095,11 @@ fn select_resume_plan_keys(anyhow::bail!("{} collides with legacy resume key {} already owned by {}. No documents \were changed; remove or move one of these two files out of the corpus \- and rerun — every other file keeps its resume state. Deleting the \- journal at {} (or rerunning with --fresh) also clears the collision, \- but re-extracts and re-embeds the entire corpus",+ and rerun — every other file keeps its resume state. Refusing to discard \+ history under the same destination. For an isolated rebuild use a new \+ --state-dir, new --prefix, and new --brain when graph detection is \+ enabled (or --no-graph); explicitly validate and clean the shared \+ catalog and old target. Journal: {}",file.rel,legacy_key,assignment.rel,@@ -1135,95 +1143,10 @@ struct UnsupportedInventoryDelta {vanished_content_groups: Vec<InventoryDeltaEntry>,}-/// Everything the refusal needs to name the destination it is protecting.-/// Collected at the gate so the message can list the exact indices and state-/// directory an operator has to act on, instead of describing them abstractly.-#[derive(Debug, Clone, PartialEq, Eq)]-struct RefusalTargets {- state_dir: String,- data_indices: Vec<String>,- edges_index: Option<String>,-}--impl RefusalTargets {- fn describe(cfg: &IndexCfg, state_dir: &Path, plan: &Plan) -> Self {- let mut data_indices: Vec<String> = plan.datasets.iter().map(|d| d.index.clone()).collect();- data_indices.sort();- data_indices.dedup();- let edges_index = (!cfg.no_graph).then(|| {- let brain = cfg- .brain- .clone()- .unwrap_or_else(|| derive_brain_name(&cfg.root));- detect::edges_index_name(&brain)- });- Self {- state_dir: state_dir.display().to_string(),- data_indices,- edges_index,- }- }-- fn indices_phrase(&self) -> String {- if self.data_indices.is_empty() {- "the indices this plan publishes".to_string()- } else {- self.data_indices.join(", ")- }- }-- fn edges_note(&self) -> String {- match &self.edges_index {- Some(edges) => format!(- " Graph edges taught by the removed file(s) stay live in {edges} until that \- index is deleted too."- ),- None => String::new(),- }- }-}--/// The #195 zero-live verification failure, as a type rather than a string.-///-/// The journal claims records were made visible; the destination answers with-/// none. That is always a failure, but not always the *same* failure: for a-/// composing caller like `xerj brain` a resume journal that outlived a wiped-/// data directory has a specific, executable recovery (`--fresh` in place),-/// while a write-blocked or unreachable server does not. Only a typed error-/// lets the caller tell them apart — `anyhow::bail!` forces it to either-/// string-match the prose or reprint it verbatim.-#[derive(Debug, Clone, PartialEq, Eq)]-pub struct ZeroLiveVerificationError {- /// Records the resume journal says were published.- pub journal_records: u64,- /// Completed files those records came from.- pub files_done_journaled: usize,- /// Dataset indices that answered with zero live documents.- pub dataset_indices: usize,-}--impl std::fmt::Display for ZeroLiveVerificationError {- fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(- formatter,- "autoindex verification failed: the resume journal records {} \- record(s) from {} completed file(s), but 0 documents are \- live across the {} dataset index(es). The indices exist and look healthy but \- hold nothing — a server-side write rejection (e.g. a disk flood-stage or \- index write block), deleted indices, or an unreadable server can all cause \- this. Fix the server-side condition, then rerun with --fresh to re-index \- from scratch",- self.journal_records, self.files_done_journaled, self.dataset_indices- )- }-}--impl std::error::Error for ZeroLiveVerificationError {}-#[derive(Debug)]struct UnsupportedInventoryDeltaError {delta: UnsupportedInventoryDelta,- targets: RefusalTargets,+ fresh_existing_plan: bool,}impl UnsupportedInventoryDeltaError {@@ -1231,22 +1154,21 @@ impl UnsupportedInventoryDeltaError {json!({"schema": "xerj.autoindex.unsupported_sync_delta.v1","status": "error",- "error": "unsupported_content_group_removal",- "message": "this attempt made no remote mutations. Files that were indexed under this resume plan no longer exist in the folder, and their documents are still live in the destination; removing files from an indexed folder is not reconciled yet",- "vanished_content_groups": self.delta.vanished_content_groups,- // Context, not the reason for the refusal: a rerun over a frozen- // plan does not index files added after the plan was frozen.+ "error": if self.fresh_existing_plan {+ "unsafe_fresh_existing_plan"+ } else {+ "unsupported_inventory_delta"+ },+ "message": if self.fresh_existing_plan {+ "this attempt made no remote mutations; --fresh cannot discard an existing plan under the same destination because alias, path, graph, and stale-record cleanup knowledge would be lost; the existing destination may already be partial or stale"+ } else {+ "this attempt made no remote mutations because this autoindex plan cannot safely reconcile corpus membership changes; the existing destination may already be partial or stale"+ },"added_content_groups": self.delta.added_content_groups,+ "vanished_content_groups": self.delta.vanished_content_groups,"recovery": {- "restore_removed_files": "put the listed file(s) back and rerun; every other file keeps its resume state",- "rebuild_in_place": format!(- "delete the indices this plan publishes ({}) and the state directory {}, then rerun. This re-extracts and re-embeds the whole corpus.{}",- self.targets.indices_phrase(),- self.targets.state_dir,- self.targets.edges_note().trim_start_matches(' ')- ),- "rebuild_isolated": "index with a new --state-dir, new --prefix, and (when graph detection is enabled) new --brain; alternatively add --no-graph. Validate the isolated target before switching readers, then clean the old one",- "fresh_warning": "--fresh re-extracts the current folder in place and does pick up added and changed files, but it never deletes documents already published for removed files, so it is refused here"+ "exact_rebuild": "index with a new --state-dir, new --prefix, and (when graph detection is enabled) new --brain; alternatively add --no-graph. Validate the isolated target before switching readers",+ "warning": "--fresh is not recovery or destination reconciliation. The global autoindex-catalog and old target are not cleaned automatically; validate and clean them explicitly"}})}@@ -1254,52 +1176,30 @@ impl UnsupportedInventoryDeltaError {impl std::fmt::Display for UnsupportedInventoryDeltaError {fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- // Capped like the duplicate and unplanned-file listings above; see- // REFUSAL_LIST_CAP. The machine-readable `--json` rendering keeps the- // full lists, so nothing is lost — only the prose is bounded.let render = |entries: &[InventoryDeltaEntry]| {- let mut rendered = entries+ entries.iter()- .take(REFUSAL_LIST_CAP).map(|entry| format!("{} ({})", entry.path, entry.file_key)).collect::<Vec<_>>()- .join(", ");- let remaining = entries.len().saturating_sub(REFUSAL_LIST_CAP);- if remaining > 0 {- rendered.push_str(&format!(", … and {remaining} more"));- }- rendered+ .join(", ")+ };+ let reason = if self.fresh_existing_plan {+ "`--fresh` cannot discard an existing plan under the same destination because alias, \+ path, graph, and stale-record cleanup knowledge would be lost"+ } else {+ "corpus membership synchronization is not yet supported"};write!(formatter,- "{} file(s) indexed under this resume plan no longer exist in the folder, and their \- documents are still live in the destination. Removing files from an indexed folder \- is not reconciled yet, so this attempt made no remote mutations — no documents, \- aliases, graph edges or catalog entries were written. Removed content groups [{}].",- self.delta.vanished_content_groups.len(),+ "this attempt made no remote mutations; the existing destination may already be \+ partial or stale. {reason}. Added \+ content groups [{}]; vanished content groups [{}]. For an exact rebuild, index the \+ current folder with a new --state-dir, a new --prefix, and, when graph detection is \+ enabled, a new --brain (or use --no-graph). Validate the isolated target before \+ switching readers. `--fresh` is not recovery or destination reconciliation; the \+ global autoindex-catalog and old target require explicit validated cleanup",+ render(&self.delta.added_content_groups),render(&self.delta.vanished_content_groups)- )?;- if !self.delta.added_content_groups.is_empty() {- write!(- formatter,- " Also present but not in the frozen resume plan, so not indexed by this run \- either [{}].",- render(&self.delta.added_content_groups)- )?;- }- write!(- formatter,- " Recovery, cheapest first: (1) restore the removed file(s) and rerun — every other \- file keeps its resume state; (2) rebuild in place by deleting the indices this plan \- publishes ({}) and the state directory {}, then rerunning — this re-extracts and \- re-embeds the whole corpus.{} (3) rebuild isolated with a new --state-dir, a new \- --prefix and, when graph detection is enabled, a new --brain (or --no-graph), \- validate it, switch readers, then clean the old target. `--fresh` picks up added \- and changed files in place but never deletes documents for removed files, so it is \- refused here too",- self.targets.indices_phrase(),- self.targets.state_dir,- self.targets.edges_note())}}@@ -1319,27 +1219,6 @@ impl UnsupportedInventoryDelta {.map(String::as_str).chain(plan.junk_files.iter().map(|junk| junk.file_key.as_str())).collect();- // Path identity, not just content identity. An in-place EDIT gives the- // same file a new content key, so a key-only comparison reads the- // superseded key as vanished and the new one as added — the SAME path- // listed as both removed and added. That is a replacement, and the file- // is still there to be republished, so it must never be refused.- // Ordinary resume already reaches this conclusion by mapping each file- // onto its planned key (`select_resume_plan_keys`); doing it here makes- // `--fresh`, which has no plan to map through, agree — without making- // the gate more fatal than the open it precedes.- let current_rels: std::collections::HashSet<&str> =- files.iter().map(|file| file.rel.as_str()).collect();- let current_path_ids: std::collections::HashSet<&str> = files- .iter()- .map(|file| file.rel_id.as_str())- .filter(|id| !id.is_empty())- .collect();- let path_survives = |assignment: &FileAssignment| {- current_rels.contains(assignment.rel.as_str())- || (!assignment.path_id.is_empty()- && current_path_ids.contains(assignment.path_id.as_str()))- };let mut added_content_groups: Vec<InventoryDeltaEntry> = files.iter()@@ -1353,22 +1232,21 @@ impl UnsupportedInventoryDelta {let mut vanished_content_groups: Vec<InventoryDeltaEntry> = plan.files.iter()- .filter(|(key, assignment)| {- !current_keys.contains(key.as_str()) && !path_survives(assignment)- })+ .filter(|(key, _)| !current_keys.contains(key.as_str())).map(|(key, assignment)| InventoryDeltaEntry {file_key: key.clone(),path: assignment.rel.clone(),}).collect();- // Deliberately NOT extended with `plan.junk_files`. A junk/skipped file- // published no documents, no aliases and no graph edges — its entire- // live footprint is one `file:{key}` catalog row, and the stale- // junk-catalog sweep below (#238) deletes that row before dropping the- // plan entry. Removing it therefore strands nothing, so refusing the- // rerun would block a case the pipeline now handles completely. Junk- // keys stay in `durable_keys` above: an unchanged skipped file is still- // not an addition.+ vanished_content_groups.extend(+ plan.junk_files+ .iter()+ .filter(|junk| !current_keys.contains(junk.file_key.as_str()))+ .map(|junk| InventoryDeltaEntry {+ file_key: junk.file_key.clone(),+ path: junk.rel.clone(),+ }),+ );let stable_order = |left: &InventoryDeltaEntry, right: &InventoryDeltaEntry| {left.path@@ -1383,19 +1261,22 @@ impl UnsupportedInventoryDelta {}}- /// A rerun is refused only when a content group the plan published has- /// vanished from the folder: those documents stay live and searchable with- /// no source file behind them, and nothing in this pipeline removes them.- /// Additions are not refused — they are skipped by the frozen plan exactly- /// as before and `--fresh` rebuilds the plan in place to include them.- fn refuses(&self) -> bool {- !self.vanished_content_groups.is_empty()+ fn is_empty(&self) -> bool {+ self.added_content_groups.is_empty() && self.vanished_content_groups.is_empty()+ }++ fn into_error(self) -> anyhow::Error {+ UnsupportedInventoryDeltaError {+ delta: self,+ fresh_existing_plan: false,+ }+ .into()}- fn into_error(self, targets: RefusalTargets) -> anyhow::Error {+ fn into_fresh_error(self) -> anyhow::Error {UnsupportedInventoryDeltaError {delta: self,- targets,+ fresh_existing_plan: true,}.into()}@@ -1589,20 +1470,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {// Acquire state authority before discovery as well as hashing. A waiter// must never classify a path snapshot taken while another owner was// publishing or replacing the durable plan.- let preflight =- state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix, cfg.fresh)?;- if let Some(reason) = preflight.unreadable_plan.as_deref() {- // Only reachable under --fresh; without it the preflight would have- // returned this as an error. It goes through the progress surface, not- // a raw eprintln!, so `--progress none` (which `--quiet` selects) stays- // silent and `--progress json` stays one parseable stream (#241).- pr.note(&format!(- "autoindex: --fresh: the durable resume plan in {} could not be read ({reason}); \- rebuilding it from the current folder. Documents published for files that are no \- longer present cannot be identified from an unreadable plan and are not deleted.",- state_dir.join("journal.ndjson").display()- ));- }+ let preflight = state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix)?;// Totals are unknown until the walk returns, so this phase honestly// reports `pct=unknown` and proves liveness with the clock alone.pr.phase("walk", 0, 0);@@ -1614,6 +1482,9 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {discovered_bytes / (1 << 20),root_str));+ // An empty folder is only "nothing to do" when there is also no durable+ // state: with a journal present, zero files is a deletion of the whole+ // corpus and has to be reconciled, not shrugged off.if discovered_files.is_empty() && !preflight.journal_exists {println!("no files found under {}", cfg.root.display());pr.finish(true, 0, "no-files", &[]);@@ -1644,13 +1515,16 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {.map(|(planned, current)| planned.unwrap_or_else(|| current.clone())).collect()};- // `--fresh` is checked here too: discarding the plan does not delete- // the documents already published for a file that is now gone, so a- // removal is unsafe in place whether or not the plan is kept.let delta =UnsupportedInventoryDelta::between(&inventory.files, &comparison_keys, prior_plan);- if delta.refuses() {- return Err(delta.into_error(RefusalTargets::describe(&cfg, &state_dir, prior_plan)));+ if cfg.fresh {+ // Even an apparently unchanged content-key set can have alias,+ // path, graph, catalog, or partial-publication history that only+ // the old plan can reconcile. Route 1 cannot safely discard it.+ return Err(delta.into_fresh_error());+ }+ if !delta.is_empty() {+ return Err(delta.into_error());}}let mut journal = state::Journal::open_after_preflight(@@ -1878,19 +1752,17 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {plan};- // Second gate, after key selection has settled: fail before index- // creation/mapping, replacement intent, graph invalidation,- // delete-by-query, bulk publication, refresh, or catalog writes when a- // canonical content group the plan published is gone from the folder.- // Continuing would leave those documents searchable with no source file.- // Additions, same-path replacement and crash repair remain supported.- //- // It runs before the #238 junk sweep below on purpose: a refused rerun- // must compute nothing and mutate nothing.+ // A durable plan is a crash-resume boundary, not a folder-sync+ // generation. Fail before index creation/mapping, replacement intent,+ // graph invalidation, delete-by-query, bulk publication, refresh, or+ // catalog writes when the current inventory adds or removes an entire+ // canonical content group. Continuing would either silently skip new+ // data or leave vanished source records searchable. Existing planned-key+ // content replacement and crash repair remain supported.if resumed_with_plan {let delta = UnsupportedInventoryDelta::between(&files, &keys, &plan);- if delta.refuses() {- return Err(delta.into_error(RefusalTargets::describe(&cfg, &state_dir, &plan)));+ if !delta.is_empty() {+ return Err(delta.into_error());}}@@ -2066,9 +1938,9 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {rel: files[i].rel.clone(),format: "unknown".into(),status: "skipped".into(),- reason: "not in the frozen resume plan, so nothing was published for it. Re-run \- with --fresh to rebuild the plan in place and include it (ids stay \- idempotent), or index it under a new --state-dir and --prefix"+ reason: "not in the frozen resume plan; no destination change was made. For an \+ exact rebuild use a new --state-dir, a new --prefix, and, when graph \+ detection is enabled, a new --brain (or use --no-graph)".into(),bytes: files[i].size,});@@ -3794,43 +3666,17 @@ mod inventory_delta_tests {gzip: false,content_digest: Some(format!("digest:{path}")),assignments: vec![(None, "rows".into())],- as_document: false,- }- }-- fn targets() -> RefusalTargets {- RefusalTargets {- state_dir: "/state".into(),- data_indices: vec!["ax-rows".into()],- edges_index: Some(".xerj-memory-corpus-edges".into()),}}#[test]- fn only_a_removed_content_group_refuses_a_rerun() {+ fn classifier_is_empty_for_noop_and_sorts_added_and_vanished_groups() {let mut plan = Plan::default();plan.files.insert("keep".into(), assignment("keep.csv"));assert!(- !UnsupportedInventoryDelta::between(&[file("keep.csv")], &["keep".into()], &plan)- .refuses()- );- // An added file is skipped by the frozen plan, not a refusal: the- // documented rerun-then---fresh workflow has to keep working.- let added = UnsupportedInventoryDelta::between(- &[file("keep.csv"), file("new.csv")],- &["keep".into(), "new".into()],- &plan,+ UnsupportedInventoryDelta::between(&[file("keep.csv")], &["keep".into()], &plan)+ .is_empty());- assert_eq!(added.added_content_groups.len(), 1);- assert!(!added.refuses(), "an addition alone must not fail the run");- // A removal leaves live documents with no source file behind them.- assert!(UnsupportedInventoryDelta::between(&[], &[], &plan).refuses());- }-- #[test]- fn classifier_sorts_added_and_vanished_groups() {- let mut plan = Plan::default();- plan.files.insert("keep".into(), assignment("keep.csv"));plan.junk_files.push(JunkFile {file_key: "junk".into(),rel: "broken.pdf".into(),@@ -3840,13 +3686,13 @@ mod inventory_delta_tests {bytes: 1,});assert!(- !UnsupportedInventoryDelta::between(+ UnsupportedInventoryDelta::between(&[file("keep.csv"), file("broken.pdf")],&["keep".into(), "junk".into()],&plan,)- .refuses(),- "unchanged durable junk is neither added nor vanished"+ .is_empty(),+ "unchanged durable junk is not newly added content");plan.files.insert("old-z".into(), assignment("z-old.csv"));@@ -3879,97 +3725,23 @@ mod inventory_delta_tests {);}- #[test]- fn refusal_names_the_removed_files_and_every_recovery_route() {- let mut plan = Plan::default();- plan.files.insert("gone".into(), assignment("gone.csv"));- let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());- let message = format!("{error:#}");- assert!(message.contains("gone.csv"), "{message}");- assert!(- message.contains("no longer exist in the folder"),- "{message}"- );- assert!(message.contains("made no remote mutations"), "{message}");- assert!(message.contains("restore the removed file(s)"), "{message}");- assert!(message.contains("ax-rows"), "{message}");- assert!(message.contains("/state"), "{message}");- assert!(message.contains(".xerj-memory-corpus-edges"), "{message}");- assert!(message.contains("new --state-dir"), "{message}");- assert!(message.contains("`--fresh`"), "{message}");- }-- /// The prose refusal is bounded by REFUSAL_LIST_CAP, not by the corpus: an- /// unmounted bind mount under an indexed root vanishes every group at once,- /// and rendering 82k entries into one String is megabytes of stderr in the- /// one path whose job is to be read. `--json` still carries all of them.- #[test]- fn refusal_prose_caps_its_listings_while_json_keeps_every_entry() {- let mut plan = Plan::default();- let total = REFUSAL_LIST_CAP * 3;- for i in 0..total {- plan.files.insert(- format!("gone{i:03}"),- assignment(&format!("gone{i:03}.csv")),- );- }- let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());- let message = format!("{error:#}");- assert!(message.contains("gone000.csv"), "{message}");- assert!(- message.contains(&format!(", … and {} more", total - REFUSAL_LIST_CAP)),- "{message}"- );- // The tail past the cap must not be rendered at all, not merely elided.- let last = format!("gone{:03}.csv", total - 1);- assert!(!message.contains(&last), "{message}");- assert_eq!(- message.matches(".csv (").count(),- REFUSAL_LIST_CAP,- "{message}"- );-- let stdout = route_cli_error(&error, true)- .stdout- .expect("the typed refusal renders as JSON under --json");- let value: Value = serde_json::from_str(&stdout).unwrap();- assert_eq!(- value["vanished_content_groups"].as_array().unwrap().len(),- total- );- }-- #[test]- fn refusal_prose_below_the_cap_has_no_more_tail() {- let mut plan = Plan::default();- plan.files.insert("gone".into(), assignment("gone.csv"));- let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());- let message = format!("{error:#}");- assert!(!message.contains("… and "), "{message}");- }-#[test]fn cli_error_routing_separates_typed_json_from_unrelated_human_errors() {let typed = UnsupportedInventoryDelta {- added_content_groups: Vec::new(),- vanished_content_groups: vec![InventoryDeltaEntry {+ added_content_groups: vec![InventoryDeltaEntry {file_key: "key".into(),- path: "gone.csv".into(),+ path: "new.csv".into(),}],+ vanished_content_groups: Vec::new(),}- .into_error(targets());+ .into_error();let route = route_cli_error(&typed, true);assert_eq!(route.exit_code, 1);assert!(route.stderr.is_none());let stdout = route.stdout.unwrap();let value: Value = serde_json::from_str(&stdout).unwrap();assert_eq!(value["schema"], "xerj.autoindex.unsupported_sync_delta.v1");- assert_eq!(value["error"], "unsupported_content_group_removal");- assert_eq!(value["vanished_content_groups"][0]["path"], "gone.csv");- assert!(value["recovery"]["rebuild_in_place"]- .as_str()- .unwrap()- .contains("ax-rows"));+ assert_eq!(value["error"], "unsupported_inventory_delta");let unrelated = anyhow::anyhow!("endpoint unavailable");let route = route_cli_error(&unrelated, true);@@ -4029,11 +3801,11 @@ mod duplicate_integration_tests {.unwrap_err();let message = format!("{error:#}");assert!(message.contains("collides with legacy resume key"));- // Recovery advice must stay scoped to the two colliding files and be- // honest that discarding the journal re-embeds the whole corpus.+ // Recovery advice must stay scoped to the two colliding files and+ // keep an exact rebuild isolated from the old destination.assert!(message.contains("remove or move one of these two files"));assert!(message.contains("/state/journal.ndjson"));- assert!(message.contains("re-extracts and re-embeds the entire corpus"));+ assert!(message.contains("new --state-dir, new --prefix, and new --brain"));}#[test]diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rsindex 992d79530..4abde0d7c 100644--- a/engine/crates/xerj-autoindex/src/state.rs+++ b/engine/crates/xerj-autoindex/src/state.rs@@ -14,14 +14,23 @@ use std::path::{Path, PathBuf};#[cfg(test)]static FILE_DONE_IO_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);#[cfg(test)]+static SYNC_IO_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);+#[cfg(test)]pub(crate) static FILE_DONE_IO_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> =std::sync::Mutex::new(());+#[cfg(test)]+static SYNC_IO_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());#[cfg(test)]pub(crate) fn fail_next_file_done_io(boundary: u8) {FILE_DONE_IO_FAILPOINT.store(boundary, std::sync::atomic::Ordering::SeqCst);}+#[cfg(test)]+fn fail_next_sync_io(boundary: u8) {+ SYNC_IO_FAILPOINT.store(boundary, std::sync::atomic::Ordering::SeqCst);+}+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct PlanDataset {pub slug: String,@@ -177,6 +186,19 @@ mod tests {"{tail}");}+ #[test]+ fn preflight_holds_the_same_exclusive_lock_through_authoritative_open() {+ let dir = tempfile::tempdir().unwrap();+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix").unwrap();+ let error = Journal::open(dir.path(), "root", "url", "prefix", 300, false)+ .err()+ .expect("a second opener must not pass the held preflight lock");+ assert!(format!("{error:#}").contains("already in use"));++ let journal =+ Journal::open_after_preflight(preflight, "root", "url", "prefix", 300, false).unwrap();+ drop(journal);+ assert!(Journal::open(dir.path(), "root", "url", "prefix", 300, false).is_ok());}/// The preflight must never be more fatal than the open it precedes.@@ -351,6 +373,11 @@ pub struct Journal {pub plan: Option<Plan>,pub embedding_identity_sha256: Option<String>,pub embedding_identity_resumable: Option<bool>,+ /// Newest corpus reconciliation transaction without a durable+ /// `sync_commit`. Its desired plan is deliberately not authoritative yet.+ pub committed_manifest: Option<crate::sync::CommittedManifest>,+ pub pending_sync: Option<crate::sync::PendingSync>,+ pub legacy_migration_reasons: Vec<String>,}pub fn default_state_dir(root: &str, url: &str, prefix: &str) -> PathBuf {@@ -361,6 +388,56 @@ pub fn default_state_dir(root: &str, url: &str, prefix: &str) -> PathBuf {.join(crate::ids::state_key(root, url, prefix))}+fn apply_legacy_plan(+ plan: Plan,+ committed: &mut Option<crate::sync::CommittedManifest>,+ migration_reasons: &mut Vec<String>,+) -> Result<()> {+ anyhow::ensure!(+ committed+ .as_ref()+ .is_none_or(|manifest| manifest.generation == 0),+ "legacy plan record cannot follow a committed generated manifest"+ );+ match crate::sync::CommittedManifest::bootstrap_legacy(plan)? {+ crate::sync::LegacyBootstrap::Ready(manifest) => {+ *committed = Some(*manifest);+ migration_reasons.clear();+ }+ crate::sync::LegacyBootstrap::MigrationRequired { reasons } => {+ *committed = None;+ *migration_reasons = reasons;+ }+ }+ Ok(())+}++#[cfg(test)]+fn consume_io_failpoint(what: &str, boundary: u8) -> bool {+ let failpoint = if what == "file_done" {+ &FILE_DONE_IO_FAILPOINT+ } else if what.starts_with("sync_") {+ &SYNC_IO_FAILPOINT+ } else {+ return false;+ };+ failpoint+ .compare_exchange(+ boundary,+ 0,+ std::sync::atomic::Ordering::SeqCst,+ std::sync::atomic::Ordering::SeqCst,+ )+ .is_ok()+}++type PreflightReplay = (+ Option<Plan>,+ Option<crate::sync::CommittedManifest>,+ Option<crate::sync::PendingSync>,+ Vec<String>,+);+/// Read the last durable plan without locking, repairing, truncating or/// appending to the journal. This is a preflight hint only: `Journal::open`/// remains the authority after the caller passes its fail-closed inventory@@ -370,14 +447,19 @@ fn read_plan_for_preflight(root: &str,url: &str,prefix: &str,-) -> Result<Option<Plan>> {+) -> Result<PreflightReplay> {let file = match std::fs::File::open(path) {Ok(file) => file,- Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {+ return Ok((None, None, None, Vec::new()));+ }Err(error) => return Err(error.into()),};let mut reader = std::io::BufReader::new(file);let mut plan = None;+ let mut committed_manifest: Option<crate::sync::CommittedManifest> = None;+ let mut pending_sync: Option<crate::sync::PendingSync> = None;+ let mut legacy_migration_reasons = Vec::new();let mut offset = 0u64;loop {let record_start = offset;@@ -399,9 +481,10 @@ fn read_plan_for_preflight("journal corruption at byte {record_start} in {}: malformed \newline-terminated record. Refusing to discard later records. Restore \the journal from a backup, or truncate it to exactly {record_start} \- bytes to keep every completion recorded before the corruption. Deleting \- the whole journal (or rerunning with --fresh) also recovers, but \- re-extracts and re-embeds the entire corpus",+ bytes to keep every completion recorded before the corruption. For an \+ isolated rebuild use a new --state-dir, new --prefix, and new --brain \+ when graph detection is enabled (or --no-graph); explicitly validate \+ and clean the shared catalog and old target",path.display())})?;@@ -424,43 +507,59 @@ fn read_plan_for_preflight();}Some("plan") => {+ anyhow::ensure!(+ pending_sync.is_none(),+ "legacy plan record cannot appear while a sync is pending"+ );if let Some(encoded) = value.get("plan") {- plan =- Some(serde_json::from_value(encoded.clone()).with_context(|| {- format!("decode durable plan in {}", path.display())- })?);+ let decoded: Plan = serde_json::from_value(encoded.clone())+ .with_context(|| format!("decode durable plan in {}", path.display()))?;+ plan = Some(decoded.clone());+ apply_legacy_plan(+ decoded,+ &mut committed_manifest,+ &mut legacy_migration_reasons,+ )?;}}+ Some(kind) if crate::sync::is_sync_record_kind(kind) => {+ crate::sync::replay_record(&value, &mut committed_manifest, &mut pending_sync)?;+ if kind == "sync_commit" {+ plan = committed_manifest+ .as_ref()+ .map(|manifest| manifest.plan.clone());+ }+ }+ Some(kind) if kind.starts_with("sync_") => {+ anyhow::bail!("unknown durable sync record kind {kind}");+ }_ => {}}}- Ok(plan)+ Ok((+ plan,+ committed_manifest,+ pending_sync,+ legacy_migration_reasons,+ ))}pub struct JournalPreflight {state_dir: PathBuf,state_lock: std::fs::File,pub plan: Option<Plan>,+ pub committed_manifest: Option<crate::sync::CommittedManifest>,+ pub pending_sync: Option<crate::sync::PendingSync>,+ pub legacy_migration_reasons: Vec<String>,pub journal_exists: bool,- /// Set only under `--fresh`, when the durable plan could not be read and- /// the run is about to discard the journal anyway. The caller owns the- /// decision to print it (autoindex honours `--quiet`).- pub unreadable_plan: Option<String>,}impl Journal {- /// `fresh` is not a formality here. `open_after_preflight` removes the- /// journal before replaying it when `--fresh` is set, so nothing inside a- /// journal can be fatal to that path — and a preflight that hard-errors on- /// a corrupt record or a root/url/prefix mismatch would make `--fresh`- /// unreachable in exactly the cases whose error text recommends it. The- /// preflight must therefore never be more fatal than the open it precedes.pub fn preflight(state_dir: &Path,root: &str,url: &str,prefix: &str,- fresh: bool,) -> Result<JournalPreflight> {std::fs::create_dir_all(state_dir).with_context(|| format!("create state dir {}", state_dir.display()))?;@@ -480,22 +579,16 @@ impl Journal {})?;let journal_path = state_dir.join("journal.ndjson");let journal_exists = journal_path.exists();- let (plan, unreadable_plan) =- match read_plan_for_preflight(&journal_path, root, url, prefix) {- Ok(plan) => (plan, None),- // The journal is about to be deleted unread. Losing the plan here- // costs the removed-file gate its comparison basis, which is- // exactly what a full in-place rebuild accepts; the alternative is- // a state directory that no supported flag can recover.- Err(error) if fresh => (None, Some(format!("{error:#}"))),- Err(error) => return Err(error),- };+ let (plan, committed_manifest, pending_sync, legacy_migration_reasons) =+ read_plan_for_preflight(&journal_path, root, url, prefix)?;Ok(JournalPreflight {state_dir: state_dir.to_owned(),state_lock,plan,+ committed_manifest,+ pending_sync,+ legacy_migration_reasons,journal_exists,- unreadable_plan,})}@@ -507,7 +600,7 @@ impl Journal {bulk_timeout_secs: u64,fresh: bool,) -> Result<Journal> {- let preflight = Self::preflight(state_dir, root, url, prefix, fresh)?;+ let preflight = Self::preflight(state_dir, root, url, prefix)?;Self::open_after_preflight(preflight, root, url, prefix, bulk_timeout_secs, fresh)}@@ -549,6 +642,9 @@ impl Journal {let mut plan = None;let mut embedding_identity_sha256 = None;let mut embedding_identity_resumable = None;+ let mut committed_manifest: Option<crate::sync::CommittedManifest> = None;+ let mut pending_sync: Option<crate::sync::PendingSync> = None;+ let mut legacy_migration_reasons = Vec::new();let mut run_id = None;let mut resumed = false;if jpath.exists() {@@ -580,9 +676,10 @@ impl Journal {anyhow::bail!("journal at {} was created for root={jr} url={ju} prefix={jp}; \current run has root={root} url={url} prefix={prefix}. \- Use --state-dir for separate state, or --fresh to rebuild the \- plan in place — note that --fresh never deletes documents \- already published under the other root, url or prefix.",+ Refusing to discard history under the same destination. For an \+ isolated rebuild use a new --state-dir, new --prefix, and new \+ --brain when graph detection is enabled (or --no-graph); \+ explicitly validate and clean the shared catalog and old target.",jpath.display());}@@ -595,10 +692,32 @@ impl Journal {resumed = true;}Some("plan") => {+ anyhow::ensure!(+ pending_sync.is_none(),+ "legacy plan record at byte {record_start} cannot appear while \+ a sync is pending"+ );if let Some(p) = v.get("plan") {- if let Ok(p) = serde_json::from_value::<Plan>(p.clone()) {- plan = Some(p);- }+ let decoded: Plan = serde_json::from_value(p.clone())+ .with_context(|| {+ format!(+ "decode durable plan record at byte {record_start} \+ in {}",+ jpath.display()+ )+ })?;+ plan = Some(decoded.clone());+ apply_legacy_plan(+ decoded,+ &mut committed_manifest,+ &mut legacy_migration_reasons,+ )?;+ } else {+ anyhow::bail!(+ "durable plan record at byte {record_start} in {} has no \+ plan payload",+ jpath.display()+ );}}Some("embedding_identity") => {@@ -644,6 +763,25 @@ impl Journal {embedding_identity_sha256 = digest.map(str::to_owned);embedding_identity_resumable = resumable;}+ Some(kind) if crate::sync::is_sync_record_kind(kind) => {+ crate::sync::replay_record(+ &v,+ &mut committed_manifest,+ &mut pending_sync,+ )?;+ if kind == "sync_commit" {+ plan = committed_manifest+ .as_ref()+ .map(|manifest| manifest.plan.clone());+ }+ }+ Some(kind) if kind.starts_with("sync_") => {+ anyhow::bail!(+ "unknown durable sync record kind {kind} at byte \+ {record_start} in {}",+ jpath.display()+ );+ }Some("file_done") => {if let Ok(fd) = serde_json::from_value::<FileDone>(v.clone()) {let commits_pending = pending_replacements@@ -691,9 +829,10 @@ impl Journal {after corruption. Restore the journal from a backup, or truncate it \to exactly {record_start} bytes to keep every completion recorded \before the corruption (files journaled after that point are \- re-verified, re-indexed and re-embedded on the next run). Deleting \- the whole journal (or rerunning with --fresh) also recovers, but \- re-extracts and re-embeds the entire corpus",+ re-verified, re-indexed and re-embedded on the next run). For an \+ isolated rebuild use a new --state-dir, new --prefix, and new --brain \+ when graph detection is enabled (or --no-graph); explicitly validate \+ and clean the shared catalog and old target",jpath.display());}@@ -723,6 +862,9 @@ impl Journal {plan,embedding_identity_sha256,embedding_identity_resumable,+ committed_manifest,+ pending_sync,+ legacy_migration_reasons,};if is_new {j.append_transaction(@@ -805,28 +947,11 @@ impl Journal {line.push('\n');let start = self.file.metadata()?.len();#[cfg(test)]- if what == "file_done"- && FILE_DONE_IO_FAILPOINT- .compare_exchange(- 1,- 0,- std::sync::atomic::Ordering::SeqCst,- std::sync::atomic::Ordering::SeqCst,- )- .is_ok()- {- anyhow::bail!("injected file_done append failure before any bytes");+ if consume_io_failpoint(what, 1) {+ anyhow::bail!("injected {what} append failure before any bytes");}#[cfg(test)]- let partial_write = what == "file_done"- && FILE_DONE_IO_FAILPOINT- .compare_exchange(- 3,- 0,- std::sync::atomic::Ordering::SeqCst,- std::sync::atomic::Ordering::SeqCst,- )- .is_ok();+ let partial_write = consume_io_failpoint(what, 3);#[cfg(not(test))]let partial_write = false;let write_result = if partial_write {@@ -835,7 +960,7 @@ impl Journal {.write_all(&line.as_bytes()[..written]).and_then(|_| {Err(std::io::Error::other(format!(- "injected partial file_done write after {written} bytes"+ "injected partial {what} write after {written} bytes")))})} else {@@ -845,19 +970,13 @@ impl Journal {return self.rollback_transaction(start, what, "append", error);}#[cfg(test)]- let injected_sync_failure = what == "file_done"- && FILE_DONE_IO_FAILPOINT- .compare_exchange(- 2,- 0,- std::sync::atomic::Ordering::SeqCst,- std::sync::atomic::Ordering::SeqCst,- )- .is_ok();+ let injected_sync_failure = consume_io_failpoint(what, 2);#[cfg(not(test))]let injected_sync_failure = false;let sync_result = if injected_sync_failure {- Err(std::io::Error::other("injected file_done fsync failure"))+ Err(std::io::Error::other(format!(+ "injected {what} fsync failure"+ )))} else {self.file.sync_data()};@@ -888,8 +1007,134 @@ impl Journal {}pub fn write_plan(&mut self, plan: &Plan) -> Result<()> {+ anyhow::ensure!(+ self.pending_sync.is_none(),+ "legacy plan write cannot occur while a sync is pending"+ );+ anyhow::ensure!(+ self.committed_manifest+ .as_ref()+ .is_none_or(|manifest| manifest.generation == 0),+ "legacy plan write cannot follow a committed generated manifest"+ );+ let mut committed = self.committed_manifest.clone();+ let mut migration_reasons = self.legacy_migration_reasons.clone();+ apply_legacy_plan(plan.clone(), &mut committed, &mut migration_reasons)?;self.append_transaction(&serde_json::json!({"kind": "plan", "plan": plan}), "plan")?;self.plan = Some(plan.clone());+ self.committed_manifest = committed;+ self.legacy_migration_reasons = migration_reasons;+ Ok(())+ }++ pub fn sync_begin(&mut self, sync: &crate::sync::PendingSync) -> Result<()> {+ anyhow::ensure!(+ self.pending_sync.is_none(),+ "cannot begin sync {} while {} is still pending",+ sync.tx_id,+ self.pending_sync+ .as_ref()+ .map(|pending| pending.tx_id.as_str())+ .unwrap_or("another sync")+ );+ let committed = self+ .committed_manifest+ .as_ref()+ .context("cannot begin sync before a committed plan exists")?;+ let mut pending = None;+ crate::sync::apply_begin(committed, &mut pending, sync.clone())?;+ let mut value = serde_json::to_value(sync)?;+ value["kind"] = Value::String("sync_begin".into());+ self.append_transaction(&value, "sync_begin")?;+ self.pending_sync = pending;+ Ok(())+ }++ pub fn sync_operation_state(+ &mut self,+ operation_id: &str,+ state: crate::sync::SyncOperationState,+ ) -> Result<()> {+ let pending = self+ .pending_sync+ .as_ref()+ .context("cannot record operation state without sync_begin")?;+ let tx_id = pending.tx_id.clone();+ let mut candidate = pending.clone();+ candidate.apply_operation_state(&tx_id, operation_id, state.clone())?;+ self.append_transaction(+ &serde_json::json!({+ "kind": "sync_operation_state",+ "tx_id": tx_id,+ "operation_id": operation_id,+ "state": state,+ }),+ "sync_operation_state",+ )?;+ self.pending_sync = Some(candidate);+ Ok(())+ }++ pub fn sync_validated(&mut self) -> Result<()> {+ let pending = self+ .pending_sync+ .as_ref()+ .context("cannot validate sync without sync_begin")?;+ let tx_id = pending.tx_id.clone();+ let digest = pending.desired_manifest_digest.clone();+ let mut candidate = pending.clone();+ candidate.apply_validated(&tx_id, &digest)?;+ self.append_transaction(+ &serde_json::json!({+ "kind": "sync_validated",+ "tx_id": tx_id,+ "desired_manifest_digest": digest,+ }),+ "sync_validated",+ )?;+ self.pending_sync = Some(candidate);+ Ok(())+ }++ pub fn sync_commit(&mut self) -> Result<()> {+ let pending = self+ .pending_sync+ .as_ref()+ .context("cannot commit sync without sync_begin")?;+ let tx_id = pending.tx_id.clone();+ let digest = pending.desired_manifest_digest.clone();+ let committed = pending.clone().apply_commit(&tx_id, &digest)?;+ self.append_transaction(+ &serde_json::json!({+ "kind": "sync_commit",+ "tx_id": tx_id,+ "desired_manifest_digest": digest,+ }),+ "sync_commit",+ )?;+ self.pending_sync = None;+ self.plan = Some(committed.plan.clone());+ self.committed_manifest = Some(committed);+ Ok(())+ }++ pub fn sync_abort(&mut self, reason: &str) -> Result<()> {+ let pending = self+ .pending_sync+ .as_ref()+ .context("cannot abort without pending sync")?;+ let tx_id = pending.tx_id.clone();+ let digest = pending.desired_manifest_digest.clone();+ self.append_transaction(+ &serde_json::json!({+ "kind": "sync_abort",+ "tx_id": tx_id,+ "desired_manifest_digest": digest,+ "reason": reason,+ }),+ "sync_abort",+ )?;+ self.pending_sync = None;Ok(())}@@ -947,6 +1192,401 @@ impl Journal {}}+#[cfg(test)]+mod sync_journal_tests {+ use super::*;+ use crate::sync::{+ ExecutionIdentity, GenerationManifest, ManifestGroup, ManifestPath, PendingSync,+ SourceExecutionPolicy, SyncOperationState, EXECUTION_IDENTITY_VERSION,+ };++ fn journal_with_plan(dir: &Path) -> Journal {+ let mut journal = Journal::open(dir, "root", "url", "prefix", 300, false).unwrap();+ let mut plan = Plan::default();+ plan.datasets.push(PlanDataset {+ slug: "reports".into(),+ index: "prefix-reports".into(),+ family: "pdf".into(),+ group: None,+ specs: Vec::new(),+ time_field: None,+ semantic_field: None,+ sampled_records: 1,+ file_count: 0,+ });+ journal.write_plan(&plan).unwrap();+ journal+ }++ fn pending(journal: &Journal) -> PendingSync {+ let base = journal.committed_manifest.as_ref().unwrap();+ let mut plan = base.plan.clone();+ plan.datasets[0].file_count = 1;+ plan.files.insert(+ "content-a".into(),+ FileAssignment {+ rel: "a.pdf".into(),+ path_id: "unix:61".into(),+ family: "pdf".into(),+ gzip: false,+ content_digest: Some("digest-a".into()),+ assignments: vec![(None, "reports".into())],+ },+ );+ PendingSync::new(+ "sync-1".into(),+ base,+ GenerationManifest {+ generation: base.generation + 1,+ execution: Some(ExecutionIdentity {+ version: EXECUTION_IDENTITY_VERSION,+ root_identity: "unix:root".into(),+ url: "url".into(),+ prefix: "prefix".into(),+ follow_symlinks: false,+ chunker_identity: "chunker-v1".into(),+ embedding_backend: "lexical".into(),+ embedding_model: "feature-hash".into(),+ embedding_tokenizer: "builtin".into(),+ embedding_dimension: 384,+ graph_enabled: false,+ brain: "none".into(),+ detector_identity: "detectors-v1".into(),+ schema_identity: "schema-v1".into(),+ index_identity: "index-v1".into(),+ source_policy: SourceExecutionPolicy::AbortOnSourceChange {+ inventory_digest: "inventory-v1".into(),+ },+ }),+ plan,+ groups: vec![ManifestGroup {+ group_id: "group-a".into(),+ content_id: "content-a".into(),+ content_digest: "digest-a".into(),+ content_size: 10,+ canonical: ManifestPath {+ path_id: "unix:61".into(),+ rel: "a.pdf".into(),+ is_symlink: false,+ },+ aliases: Vec::new(),+ dataset_slugs: vec!["reports".into()],+ expected_records: 1,+ expected_passages: 1,+ expected_vectors: 1,+ }],+ },+ )+ .unwrap()+ }++ #[test]+ fn sync_begin_is_pending_and_never_promotes_the_committed_manifest() {+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let dir = tempfile::tempdir().unwrap();+ let mut journal = journal_with_plan(dir.path());+ let desired = pending(&journal);+ journal.sync_begin(&desired).unwrap();+ let before = std::fs::read(dir.path().join("journal.ndjson")).unwrap();+ assert!(journal.write_plan(&Plan::default()).is_err());+ assert_eq!(+ std::fs::read(dir.path().join("journal.ndjson")).unwrap(),+ before+ );+ assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 0);+ assert!(!journal.plan.as_ref().unwrap().alias_paths_indexed);+ drop(journal);++ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);+ assert!(!reopened.plan.as_ref().unwrap().alias_paths_indexed);+ assert_eq!(reopened.pending_sync.as_ref().unwrap().tx_id, "sync-1");+ }++ #[test]+ fn replay_rejects_sync_begin_followed_by_legacy_plan_without_changing_authority() {+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let dir = tempfile::tempdir().unwrap();+ let mut journal = journal_with_plan(dir.path());+ journal.sync_begin(&pending(&journal)).unwrap();+ let committed_digest = journal+ .committed_manifest+ .as_ref()+ .unwrap()+ .manifest_digest+ .clone();+ drop(journal);+ let path = dir.path().join("journal.ndjson");+ let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();+ file.write_all(+ format!(+ "{}\n",+ serde_json::json!({"kind": "plan", "plan": Plan::default()})+ )+ .as_bytes(),+ )+ .unwrap();+ file.sync_data().unwrap();+ drop(file);+ let error = Journal::open(dir.path(), "root", "url", "prefix", 300, false)+ .err()+ .unwrap();+ assert!(format!("{error:#}").contains("while a sync is pending"));+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix")+ .err()+ .unwrap();+ assert!(format!("{preflight:#}").contains("while a sync is pending"));+ // The rejected record cannot manufacture a sync_commit; the durable+ // committed digest remains present only in the earlier sync_begin base.+ let text = std::fs::read_to_string(dir.path().join("journal.ndjson")).unwrap();+ assert!(text.contains(&committed_digest));+ assert!(!text.contains("\"kind\":\"sync_commit\""));+ }++ #[test]+ fn only_validated_all_committed_sync_advances_authority_after_restart() {+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let dir = tempfile::tempdir().unwrap();+ let mut journal = journal_with_plan(dir.path());+ journal.sync_begin(&pending(&journal)).unwrap();+ assert!(journal.sync_validated().is_err());+ assert!(journal+ .sync_operation_state("upsert:group-a", SyncOperationState::Committed)+ .is_err());+ journal+ .sync_operation_state("upsert:group-a", SyncOperationState::Started)+ .unwrap();+ journal+ .sync_operation_state("upsert:group-a", SyncOperationState::Committed)+ .unwrap();+ journal.sync_validated().unwrap();+ journal.sync_commit().unwrap();+ assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 1);+ assert!(journal+ .plan+ .as_ref()+ .unwrap()+ .files+ .contains_key("content-a"));+ assert!(journal.pending_sync.is_none());+ let before = std::fs::read(dir.path().join("journal.ndjson")).unwrap();+ assert!(journal.write_plan(&Plan::default()).is_err());+ assert_eq!(+ std::fs::read(dir.path().join("journal.ndjson")).unwrap(),+ before+ );+ drop(journal);++ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 1);+ assert!(reopened+ .plan+ .as_ref()+ .unwrap()+ .files+ .contains_key("content-a"));+ assert!(reopened.pending_sync.is_none());+ }++ #[test]+ fn digest_bound_abort_discards_pending_without_advancing_authority() {+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let dir = tempfile::tempdir().unwrap();+ let mut journal = journal_with_plan(dir.path());+ journal.sync_begin(&pending(&journal)).unwrap();+ journal.sync_abort("source changed").unwrap();+ assert!(journal.pending_sync.is_none());+ assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 0);+ drop(journal);+ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ assert!(reopened.pending_sync.is_none());+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);+ }++ #[test]+ fn unknown_sync_namespace_record_fails_closed_but_preflight_exposes_pending() {+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ let dir = tempfile::tempdir().unwrap();+ let mut journal = journal_with_plan(dir.path());+ journal.sync_begin(&pending(&journal)).unwrap();+ drop(journal);+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix").unwrap();+ assert_eq!(preflight.committed_manifest.as_ref().unwrap().generation, 0);+ assert_eq!(preflight.pending_sync.as_ref().unwrap().tx_id, "sync-1");+ drop(preflight);++ let path = dir.path().join("journal.ndjson");+ let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();+ file.write_all(b"{\"kind\":\"sync_future_v99\"}\n").unwrap();+ file.sync_data().unwrap();+ drop(file);+ assert!(Journal::open(dir.path(), "root", "url", "prefix", 300, false).is_err());+ }++ #[test]+ fn out_of_order_or_malformed_known_sync_records_fail_closed() {+ for record in [+ "{\"kind\":\"sync_commit\",\"tx_id\":\"none\",\"desired_manifest_digest\":\"none\"}\n",+ "{\"kind\":\"sync_begin\"}\n",+ ] {+ let dir = tempfile::tempdir().unwrap();+ drop(journal_with_plan(dir.path()));+ let path = dir.path().join("journal.ndjson");+ let mut file = std::fs::OpenOptions::new()+ .append(true)+ .open(&path)+ .unwrap();+ file.write_all(record.as_bytes()).unwrap();+ file.sync_data().unwrap();+ let error = Journal::open(dir.path(), "root", "url", "prefix", 300, false)+ .err()+ .unwrap();+ let message = format!("{error:#}");+ assert!(+ message.contains("pending sync") || message.contains("sync_begin"),+ "{message}"+ );+ }+ }++ #[test]+ fn torn_sync_tail_is_removed_without_creating_a_pending_generation() {+ let dir = tempfile::tempdir().unwrap();+ drop(journal_with_plan(dir.path()));+ let path = dir.path().join("journal.ndjson");+ let durable_len = std::fs::metadata(&path).unwrap().len();+ let mut file = std::fs::OpenOptions::new()+ .append(true)+ .open(&path)+ .unwrap();+ file.write_all(br#"{"kind":"sync_begin","tx_id":"torn""#)+ .unwrap();+ file.sync_data().unwrap();+ drop(file);++ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ assert!(reopened.pending_sync.is_none());+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);+ // Reopen appends one resume record, but the torn bytes are absent.+ let contents = std::fs::read_to_string(path).unwrap();+ assert!(!contents.contains("\"tx_id\":\"torn\""));+ assert!(contents.len() as u64 > durable_len);+ }++ #[test]+ fn complete_middle_corruption_is_not_truncated_or_skipped() {+ let dir = tempfile::tempdir().unwrap();+ drop(journal_with_plan(dir.path()));+ let path = dir.path().join("journal.ndjson");+ let before = std::fs::metadata(&path).unwrap().len();+ let mut file = std::fs::OpenOptions::new()+ .append(true)+ .open(&path)+ .unwrap();+ file.write_all(b"{malformed sync record}\n").unwrap();+ file.write_all(b"{\"kind\":\"unknown-after-corruption\"}\n")+ .unwrap();+ file.sync_data().unwrap();+ drop(file);+ assert!(Journal::open(dir.path(), "root", "url", "prefix", 300, false).is_err());+ assert!(std::fs::metadata(path).unwrap().len() > before);+ }++ #[test]+ fn unknown_future_records_do_not_change_legacy_completion_replay() {+ let dir = tempfile::tempdir().unwrap();+ let mut journal = journal_with_plan(dir.path());+ journal+ .file_done(&FileDone {+ file_key: "legacy".into(),+ path: "legacy.pdf".into(),+ records: 1,+ junk: 0,+ bytes: 10,+ generation: None,+ })+ .unwrap();+ drop(journal);+ let path = dir.path().join("journal.ndjson");+ let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();+ file.write_all(b"{\"kind\":\"extension_future_v99\",\"payload\":{\"x\":1}}\n")+ .unwrap();+ file.sync_data().unwrap();+ drop(file);++ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ assert!(reopened.done.contains_key("legacy"));+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);+ }++ #[test]+ fn sync_append_and_fsync_failures_roll_back_without_phantom_pending_state() {+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ for boundary in [1, 2, 3] {+ let dir = tempfile::tempdir().unwrap();+ let mut journal = journal_with_plan(dir.path());+ let desired = pending(&journal);+ let before = std::fs::read(dir.path().join("journal.ndjson")).unwrap();+ fail_next_sync_io(boundary);+ assert!(journal.sync_begin(&desired).is_err());+ assert!(journal.pending_sync.is_none());+ drop(journal);+ assert_eq!(+ std::fs::read(dir.path().join("journal.ndjson")).unwrap(),+ before+ );+ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ assert!(reopened.pending_sync.is_none());+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);+ }+ }++ #[test]+ fn every_later_sync_record_rolls_back_append_partial_and_fsync_failures() {+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();+ for stage in ["operation", "validated", "commit", "abort"] {+ for boundary in [1, 2, 3] {+ let dir = tempfile::tempdir().unwrap();+ let mut journal = journal_with_plan(dir.path());+ journal.sync_begin(&pending(&journal)).unwrap();+ if stage != "operation" {+ journal+ .sync_operation_state("upsert:group-a", SyncOperationState::Started)+ .unwrap();+ journal+ .sync_operation_state("upsert:group-a", SyncOperationState::Committed)+ .unwrap();+ }+ if matches!(stage, "commit") {+ journal.sync_validated().unwrap();+ }+ let before = std::fs::read(dir.path().join("journal.ndjson")).unwrap();+ fail_next_sync_io(boundary);+ let result = match stage {+ "operation" => {+ journal.sync_operation_state("upsert:group-a", SyncOperationState::Started)+ }+ "validated" => journal.sync_validated(),+ "commit" => journal.sync_commit(),+ "abort" => journal.sync_abort("test abort"),+ _ => unreachable!(),+ };+ assert!(result.is_err(), "{stage} boundary {boundary}");+ assert_eq!(+ std::fs::read(dir.path().join("journal.ndjson")).unwrap(),+ before,+ "{stage} boundary {boundary}"+ );+ drop(journal);+ let reopened =+ Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);+ assert!(reopened.pending_sync.is_some());+ }+ }+ }+}+#[cfg(test)]mod compatibility_tests {use super::*;@@ -1123,9 +1763,10 @@ mod compatibility_tests {let message = format!("{error:#}");assert!(message.contains("journal corruption"));// Recovery guidance must stay scoped and honest: byte-exact truncation- // keeps prior completions; discarding the journal re-embeds everything.+ // keeps prior completions; an exact rebuild needs isolated state and+ // destination identities.assert!(message.contains("truncate it to exactly"));- assert!(message.contains("re-extracts and re-embeds the entire corpus"));+ assert!(message.contains("new --state-dir, new --prefix, and new --brain"));}#[test]diff --git a/engine/crates/xerj-autoindex/src/sync.rs b/engine/crates/xerj-autoindex/src/sync.rsnew file mode 100644index 000000000..bebce6e84--- /dev/null+++ b/engine/crates/xerj-autoindex/src/sync.rs@@ -0,0 +1,1197 @@+//! Pure, inert corpus-generation planning and durable replay.+//!+//! Nothing in this module performs HTTP I/O or changes autoindex behaviour.+//! It defines the complete state an eventual synchronization executor must+//! durably bind before publication.++use crate::state::{DuplicateFile, Plan};+use anyhow::{Context, Result};+use serde::{Deserialize, Serialize};+use serde_json::Value;+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};++pub const EXECUTION_IDENTITY_VERSION: u32 = 1;++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]+pub struct ManifestPath {+ /// Reversible platform-native path identity (`unix:`/`windows:` encoding).+ pub path_id: String,+ pub rel: String,+ /// Canonical election ranks real paths before followed symlinks.+ pub is_symlink: bool,+}++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct ManifestGroup {+ /// Stable logical identity across rename/replacement.+ pub group_id: String,+ /// Collision-safe document identity assigned only after byte verification.+ pub content_id: String,+ /// Raw full-content digest. Never treated as globally unique by itself.+ pub content_digest: String,+ pub content_size: u64,+ pub canonical: ManifestPath,+ #[serde(default)]+ pub aliases: Vec<ManifestPath>,+ #[serde(default)]+ pub dataset_slugs: Vec<String>,+ pub expected_records: u64,+ pub expected_passages: u64,+ pub expected_vectors: u64,+}++impl ManifestGroup {+ fn all_paths(&self) -> impl Iterator<Item = &ManifestPath> {+ std::iter::once(&self.canonical).chain(self.aliases.iter())+ }+}++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+#[serde(rename_all = "snake_case", tag = "policy")]+pub enum SourceExecutionPolicy {+ DurableSnapshot {+ reference: String,+ snapshot_digest: String,+ },+ AbortOnSourceChange {+ inventory_digest: String,+ },+}++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct ExecutionIdentity {+ pub version: u32,+ pub root_identity: String,+ pub url: String,+ pub prefix: String,+ pub follow_symlinks: bool,+ pub chunker_identity: String,+ pub embedding_backend: String,+ pub embedding_model: String,+ pub embedding_tokenizer: String,+ pub embedding_dimension: usize,+ pub graph_enabled: bool,+ pub brain: String,+ pub detector_identity: String,+ pub schema_identity: String,+ pub index_identity: String,+ pub source_policy: SourceExecutionPolicy,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct GenerationManifest {+ pub generation: u64,+ /// Missing only for a legacy generation-0 plan.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub execution: Option<ExecutionIdentity>,+ pub plan: Plan,+ pub groups: Vec<ManifestGroup>,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct CommittedManifest {+ pub generation: u64,+ pub manifest_digest: String,+ pub plan: Plan,+ pub groups: Vec<ManifestGroup>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub execution: Option<ExecutionIdentity>,+}++#[derive(Debug, Clone)]+pub enum LegacyBootstrap {+ Ready(Box<CommittedManifest>),+ MigrationRequired { reasons: Vec<String> },+}++impl CommittedManifest {+ pub fn bootstrap_legacy(plan: Plan) -> Result<LegacyBootstrap> {+ let mut reasons = Vec::new();+ for (file_key, file) in &plan.files {+ if file.content_digest.is_none() {+ reasons.push(format!("{file_key}: missing full content digest"));+ }+ if !native_path_id_supported(&file.path_id) {+ reasons.push(format!(+ "{file_key}: missing or lossy native canonical path identity"+ ));+ }+ // Legacy plans did not persist whether the canonical path was a+ // followed symlink, so canonical rank cannot be reconstructed.+ reasons.push(format!(+ "{file_key}: legacy plan did not persist canonical symlink rank"+ ));+ }+ for alias in &plan.duplicate_files {+ if !native_path_id_supported(&alias.path_id) {+ reasons.push(format!(+ "{}: missing or lossy native alias path identity",+ alias.rel+ ));+ }+ }+ reasons.sort();+ reasons.dedup();+ if !reasons.is_empty() {+ return Ok(LegacyBootstrap::MigrationRequired { reasons });+ }+ let manifest = GenerationManifest {+ generation: 0,+ execution: None,+ plan: plan.clone(),+ groups: Vec::new(),+ };+ Ok(LegacyBootstrap::Ready(Box::new(Self {+ generation: 0,+ manifest_digest: manifest_digest(&manifest)?,+ plan,+ groups: Vec::new(),+ execution: None,+ })))+ }++ fn from_generation(manifest: GenerationManifest, digest: String) -> Self {+ Self {+ generation: manifest.generation,+ manifest_digest: digest,+ plan: manifest.plan,+ groups: manifest.groups,+ execution: manifest.execution,+ }+ }+}++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]+#[serde(rename_all = "snake_case")]+pub enum SyncOperationKind {+ Upsert,+ Delete,+ Metadata,+}++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]+pub struct SyncOperation {+ pub operation_id: String,+ pub kind: SyncOperationKind,+ pub group_id: String,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub desired_content_id: Option<String>,+}++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+#[serde(rename_all = "snake_case")]+pub enum SyncOperationState {+ Started,+ Committed,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct PendingSync {+ pub tx_id: String,+ pub base_generation: u64,+ pub base_manifest_digest: String,+ pub desired_manifest_digest: String,+ pub operation_hash: String,+ pub desired: GenerationManifest,+ pub operations: Vec<SyncOperation>,+ #[serde(default)]+ pub operation_states: BTreeMap<String, SyncOperationState>,+ #[serde(default)]+ pub validated: bool,+}++impl PendingSync {+ pub fn new(+ tx_id: String,+ base: &CommittedManifest,+ mut desired: GenerationManifest,+ ) -> Result<Self> {+ canonicalize_manifest_order(&mut desired);+ validate_manifest(&desired, false)?;+ validate_supported_manifest_delta(base, &desired)?;+ let operations = plan_operations(&base.groups, &desired.groups);+ let pending = Self {+ tx_id,+ base_generation: base.generation,+ base_manifest_digest: base.manifest_digest.clone(),+ desired_manifest_digest: manifest_digest(&desired)?,+ operation_hash: operation_hash(&operations)?,+ desired,+ operations,+ operation_states: BTreeMap::new(),+ validated: false,+ };+ pending.validate_against(base)?;+ Ok(pending)+ }++ pub fn validate_against(&self, base: &CommittedManifest) -> Result<()> {+ anyhow::ensure!(+ self.base_generation == base.generation+ && self.base_manifest_digest == base.manifest_digest,+ "sync_begin base does not match committed authority"+ );+ anyhow::ensure!(+ self.desired.generation == self.base_generation + 1,+ "desired generation must immediately follow its base"+ );+ validate_manifest(&self.desired, false)?;+ validate_supported_manifest_delta(base, &self.desired)?;+ anyhow::ensure!(+ self.desired_manifest_digest == manifest_digest(&self.desired)?,+ "desired manifest digest does not match sync_begin payload"+ );+ let expected = plan_operations(&base.groups, &self.desired.groups);+ anyhow::ensure!(+ self.operations == expected,+ "sync operation list was not derived exactly from base and desired manifests"+ );+ anyhow::ensure!(+ self.operation_hash == operation_hash(&expected)?,+ "sync operation hash does not match derived operation list"+ );+ let operation_ids: HashSet<&str> = expected+ .iter()+ .map(|operation| operation.operation_id.as_str())+ .collect();+ anyhow::ensure!(+ self.operation_states+ .keys()+ .all(|operation_id| operation_ids.contains(operation_id.as_str())),+ "sync state references an operation absent from sync_begin"+ );+ Ok(())+ }++ pub fn apply_operation_state(+ &mut self,+ tx_id: &str,+ operation_id: &str,+ state: SyncOperationState,+ ) -> Result<()> {+ anyhow::ensure!(+ self.tx_id == tx_id,+ "operation state belongs to another sync"+ );+ anyhow::ensure!(+ self.operations+ .iter()+ .any(|operation| operation.operation_id == operation_id),+ "operation state references unknown operation {operation_id}"+ );+ let legal = match self.operation_states.get(operation_id) {+ None => state == SyncOperationState::Started,+ Some(previous) => {+ previous == &state+ || previous == &SyncOperationState::Started+ && state == SyncOperationState::Committed+ }+ };+ anyhow::ensure!(+ legal,+ "illegal operation transition {:?} -> {state:?}",+ self.operation_states.get(operation_id)+ );+ self.operation_states+ .insert(operation_id.to_string(), state);+ Ok(())+ }++ pub fn all_operations_committed(&self) -> bool {+ self.operations.iter().all(|operation| {+ self.operation_states.get(&operation.operation_id)+ == Some(&SyncOperationState::Committed)+ })+ }++ pub fn apply_validated(&mut self, tx_id: &str, digest: &str) -> Result<()> {+ anyhow::ensure!(self.tx_id == tx_id, "validation belongs to another sync");+ anyhow::ensure!(+ self.desired_manifest_digest == digest,+ "validation manifest digest does not match sync_begin"+ );+ anyhow::ensure!(+ self.all_operations_committed(),+ "cannot validate sync with unfinished operations"+ );+ self.validated = true;+ Ok(())+ }++ pub fn apply_commit(self, tx_id: &str, digest: &str) -> Result<CommittedManifest> {+ anyhow::ensure!(self.tx_id == tx_id, "commit belongs to another sync");+ anyhow::ensure!(+ self.desired_manifest_digest == digest,+ "commit manifest digest does not match sync_begin"+ );+ anyhow::ensure!(self.validated, "cannot commit an unvalidated sync");+ anyhow::ensure!(+ self.all_operations_committed(),+ "cannot commit sync with unfinished operations"+ );+ Ok(CommittedManifest::from_generation(+ self.desired,+ self.desired_manifest_digest,+ ))+ }+}++fn validate_supported_manifest_delta(+ base: &CommittedManifest,+ desired: &GenerationManifest,+) -> Result<()> {+ // Binding complete identity while migrating the legacy generation zero is+ // the only identity transition this vocabulary can represent. Once bound,+ // backend/model/source-policy changes require a future explicit operation.+ if base.generation > 0 || base.execution.is_some() {+ anyhow::ensure!(+ base.execution == desired.execution,+ "execution identity or source policy changed without a supported operation"+ );+ }+ anyhow::ensure!(+ base.plan.alias_paths_indexed == desired.plan.alias_paths_indexed,+ "plan flags changed without a supported operation"+ );+ anyhow::ensure!(+ canonical_json_bytes(&base.plan.junk_files)?+ == canonical_json_bytes(&desired.plan.junk_files)?,+ "junk plan changed without a supported operation"+ );+ let normalize_datasets = |datasets: &[crate::state::PlanDataset]| {+ let mut datasets = datasets.to_vec();+ for dataset in &mut datasets {+ // Membership operations legitimately change this projection.+ dataset.file_count = 0;+ }+ datasets.sort_by(|left, right| {+ left.slug+ .cmp(&right.slug)+ .then_with(|| left.index.cmp(&right.index))+ .then_with(|| left.group.cmp(&right.group))+ });+ for dataset in &mut datasets {+ dataset+ .specs+ .sort_by(|left, right| left.name.cmp(&right.name));+ }+ datasets+ };+ anyhow::ensure!(+ canonical_json_bytes(&normalize_datasets(&base.plan.datasets))?+ == canonical_json_bytes(&normalize_datasets(&desired.plan.datasets))?,+ "dataset schema/index plan changed without a schema-generation operation"+ );+ Ok(())+}++pub fn plan_operations(base: &[ManifestGroup], desired: &[ManifestGroup]) -> Vec<SyncOperation> {+ let base: BTreeMap<&str, &ManifestGroup> = base+ .iter()+ .map(|group| (group.group_id.as_str(), group))+ .collect();+ let desired: BTreeMap<&str, &ManifestGroup> = desired+ .iter()+ .map(|group| (group.group_id.as_str(), group))+ .collect();+ let mut operations = Vec::new();+ for (group_id, old) in &base {+ match desired.get(group_id) {+ None => operations.push(operation(SyncOperationKind::Delete, group_id, None)),+ Some(new)+ if old.content_id != new.content_id+ || old.content_digest != new.content_digest+ || old.content_size != new.content_size =>+ {+ operations.push(operation(+ SyncOperationKind::Upsert,+ group_id,+ Some(new.content_id.clone()),+ ));+ }+ Some(new) if *old != *new => operations.push(operation(+ SyncOperationKind::Metadata,+ group_id,+ Some(new.content_id.clone()),+ )),+ Some(_) => {}+ }+ }+ for (group_id, new) in desired {+ if !base.contains_key(group_id) {+ operations.push(operation(+ SyncOperationKind::Upsert,+ group_id,+ Some(new.content_id.clone()),+ ));+ }+ }+ operations.sort();+ operations+}++fn operation(+ kind: SyncOperationKind,+ group_id: &str,+ desired_content_id: Option<String>,+) -> SyncOperation {+ SyncOperation {+ operation_id: format!("{kind:?}:{group_id}").to_ascii_lowercase(),+ kind,+ group_id: group_id.to_string(),+ desired_content_id,+ }+}++#[derive(Debug, Clone)]+#[cfg_attr(not(test), allow(dead_code))]+pub struct DesiredContentGroup {+ pub content_id: String,+ pub content_digest: String,+ pub content_size: u64,+ pub paths: Vec<ManifestPath>,+ pub dataset_slugs: Vec<String>,+ pub expected_records: u64,+ pub expected_passages: u64,+ pub expected_vectors: u64,+}++/// Content identity is collision-safe and byte-verified by the inventory+/// layer. Raw digest equality alone never merges groups.+#[cfg_attr(not(test), allow(dead_code))]+pub fn reconcile_groups(+ previous: &[ManifestGroup],+ mut desired: Vec<DesiredContentGroup>,+) -> Result<Vec<ManifestGroup>> {+ validate_groups(previous)?;+ desired.sort_by(|left, right| {+ left.content_id+ .cmp(&right.content_id)+ .then_with(|| canonical_path(&left.paths).cmp(&canonical_path(&right.paths)))+ });+ let old_by_content: HashMap<&str, &ManifestGroup> = previous+ .iter()+ .map(|group| (group.content_id.as_str(), group))+ .collect();+ anyhow::ensure!(+ old_by_content.len() == previous.len(),+ "collision-safe content IDs must be unique"+ );+ let surviving_content: HashSet<String> = desired+ .iter()+ .map(|group| group.content_id.clone())+ .collect();+ let mut old_by_path = HashMap::new();+ for group in previous {+ for path in group.all_paths() {+ anyhow::ensure!(+ old_by_path.insert(path.path_id.as_str(), group).is_none(),+ "native path identity belongs to multiple committed groups"+ );+ }+ }+ let mut claimed = HashSet::new();+ let mut seen_paths = HashSet::new();+ let mut result = Vec::new();+ for mut candidate in desired {+ normalize_paths(&mut candidate.paths, &mut seen_paths)?;+ let canonical = candidate.paths.remove(0);+ let aliases = candidate.paths;+ let content_owner = old_by_content.get(candidate.content_id.as_str()).copied();+ let mut path_owners: BTreeSet<&str> = std::iter::once(&canonical)+ .chain(aliases.iter())+ .filter_map(|path| old_by_path.get(path.path_id.as_str()).copied())+ .filter(|owner| !surviving_content.contains(owner.content_id.as_str()))+ .map(|owner| owner.group_id.as_str())+ .collect();+ let group_id = if let Some(owner) = content_owner {+ anyhow::ensure!(+ path_owners.is_empty() || path_owners.contains(owner.group_id.as_str()),+ "desired group has conflicting content and path identities"+ );+ owner.group_id.clone()+ } else if path_owners.len() == 1 {+ path_owners.pop_first().unwrap().to_string()+ } else if path_owners.is_empty() {+ format!(+ "axg1-{:032x}",+ xxhash_rust::xxh3::xxh3_128(+ format!("{}\0{}", candidate.content_id, canonical.path_id).as_bytes()+ )+ )+ } else {+ anyhow::bail!("desired content merges multiple historical groups");+ };+ anyhow::ensure!(claimed.insert(group_id.clone()), "group claimed twice");+ candidate.dataset_slugs.sort();+ candidate.dataset_slugs.dedup();+ result.push(ManifestGroup {+ group_id,+ content_id: candidate.content_id,+ content_digest: candidate.content_digest,+ content_size: candidate.content_size,+ canonical,+ aliases,+ dataset_slugs: candidate.dataset_slugs,+ expected_records: candidate.expected_records,+ expected_passages: candidate.expected_passages,+ expected_vectors: candidate.expected_vectors,+ });+ }+ result.sort_by(|left, right| left.group_id.cmp(&right.group_id));+ Ok(result)+}++#[cfg_attr(not(test), allow(dead_code))]+fn canonical_path(paths: &[ManifestPath]) -> (bool, &str) {+ paths+ .iter()+ .map(|path| (path.is_symlink, path.path_id.as_str()))+ .min()+ .unwrap_or((true, ""))+}++#[cfg_attr(not(test), allow(dead_code))]+fn normalize_paths(paths: &mut [ManifestPath], seen: &mut HashSet<String>) -> Result<()> {+ anyhow::ensure!(!paths.is_empty(), "desired content group has no paths");+ for path in paths.iter() {+ anyhow::ensure!(+ native_path_id_supported(&path.path_id),+ "path {} has no supported reversible native identity",+ path.rel+ );+ anyhow::ensure!(+ seen.insert(path.path_id.clone()),+ "native path identity occurs more than once"+ );+ }+ paths.sort_by(|left, right| {+ (left.is_symlink, left.path_id.as_str()).cmp(&(right.is_symlink, right.path_id.as_str()))+ });+ Ok(())+}++fn native_path_id_supported(path_id: &str) -> bool {+ let valid_hex = |hex: &str, width: usize| {+ !hex.is_empty()+ && hex.len().is_multiple_of(width)+ && hex.bytes().all(|byte| byte.is_ascii_hexdigit())+ };+ path_id+ .strip_prefix("unix:")+ .is_some_and(|hex| valid_hex(hex, 2))+ || path_id+ .strip_prefix("windows:")+ .is_some_and(|hex| valid_hex(hex, 4))+}++fn validate_manifest(manifest: &GenerationManifest, legacy: bool) -> Result<()> {+ if manifest.generation == 0 && legacy {+ anyhow::ensure!(+ manifest.execution.is_none(),+ "legacy generation zero cannot invent execution identity"+ );+ } else {+ let execution = manifest+ .execution+ .as_ref()+ .context("new generation requires complete execution identity")?;+ anyhow::ensure!(+ execution.version == EXECUTION_IDENTITY_VERSION,+ "unsupported execution identity version"+ );+ for (name, value) in [+ ("root_identity", execution.root_identity.as_str()),+ ("url", execution.url.as_str()),+ ("prefix", execution.prefix.as_str()),+ ("chunker_identity", execution.chunker_identity.as_str()),+ ("embedding_backend", execution.embedding_backend.as_str()),+ ("embedding_model", execution.embedding_model.as_str()),+ (+ "embedding_tokenizer",+ execution.embedding_tokenizer.as_str(),+ ),+ ("detector_identity", execution.detector_identity.as_str()),+ ("schema_identity", execution.schema_identity.as_str()),+ ("index_identity", execution.index_identity.as_str()),+ ] {+ anyhow::ensure!(!value.is_empty(), "execution identity {name} is empty");+ }+ match &execution.source_policy {+ SourceExecutionPolicy::DurableSnapshot {+ reference,+ snapshot_digest,+ } => anyhow::ensure!(+ !reference.is_empty() && !snapshot_digest.is_empty(),+ "durable source snapshot identity is incomplete"+ ),+ SourceExecutionPolicy::AbortOnSourceChange { inventory_digest } => {+ anyhow::ensure!(+ !inventory_digest.is_empty(),+ "abort-on-change policy needs inventory digest"+ )+ }+ }+ }+ validate_groups(&manifest.groups)?;+ validate_plan_projection(&manifest.plan, &manifest.groups)+}++fn validate_groups(groups: &[ManifestGroup]) -> Result<()> {+ let mut group_ids = HashSet::new();+ let mut content_ids = HashSet::new();+ let mut path_ids = HashSet::new();+ for group in groups {+ anyhow::ensure!(group_ids.insert(&group.group_id), "duplicate group ID");+ anyhow::ensure!(+ content_ids.insert(&group.content_id),+ "duplicate collision-safe content ID"+ );+ anyhow::ensure!(+ !group.content_digest.is_empty(),+ "group has no raw content digest"+ );+ anyhow::ensure!(+ native_path_id_supported(&group.canonical.path_id),+ "canonical path identity is unsupported"+ );+ anyhow::ensure!(+ !group.canonical.is_symlink || group.aliases.iter().all(|p| p.is_symlink),+ "a symlink cannot be canonical while a real path survives"+ );+ anyhow::ensure!(+ group.aliases.iter().all(|alias| {+ (group.canonical.is_symlink, group.canonical.path_id.as_str())+ < (alias.is_symlink, alias.path_id.as_str())+ }),+ "canonical path does not have the lowest canonical rank"+ );+ for path in group.all_paths() {+ anyhow::ensure!(+ path_ids.insert(&path.path_id),+ "native path identity occurs in multiple groups"+ );+ }+ anyhow::ensure!(+ group.aliases.windows(2).all(|pair| {+ (pair[0].is_symlink, pair[0].path_id.as_str())+ < (pair[1].is_symlink, pair[1].path_id.as_str())+ }),+ "aliases are not in canonical rank order"+ );+ anyhow::ensure!(+ group.dataset_slugs.windows(2).all(|pair| pair[0] < pair[1]),+ "dataset assignments are not canonical"+ );+ }+ anyhow::ensure!(+ groups+ .windows(2)+ .all(|pair| pair[0].group_id < pair[1].group_id),+ "manifest groups are not canonical"+ );+ Ok(())+}++fn validate_plan_projection(plan: &Plan, groups: &[ManifestGroup]) -> Result<()> {+ let dataset_slugs: HashSet<&str> = plan+ .datasets+ .iter()+ .map(|dataset| dataset.slug.as_str())+ .collect();+ anyhow::ensure!(+ dataset_slugs.len() == plan.datasets.len(),+ "plan has duplicate dataset slugs"+ );+ let groups_by_content: HashMap<&str, &ManifestGroup> = groups+ .iter()+ .map(|group| (group.content_id.as_str(), group))+ .collect();+ anyhow::ensure!(+ plan.files.len() == groups.len(),+ "plan files and manifest groups disagree"+ );+ for (content_id, assignment) in &plan.files {+ let group = groups_by_content+ .get(content_id.as_str())+ .context("plan file has no manifest group")?;+ anyhow::ensure!(+ assignment.rel == group.canonical.rel+ && assignment.path_id == group.canonical.path_id+ && assignment.content_digest.as_deref() == Some(group.content_digest.as_str()),+ "plan canonical projection disagrees with manifest group"+ );+ let mut assigned: Vec<String> = assignment+ .assignments+ .iter()+ .map(|(_, slug)| slug.clone())+ .collect();+ assigned.sort();+ assigned.dedup();+ anyhow::ensure!(+ assigned == group.dataset_slugs,+ "plan dataset projection disagrees with manifest group"+ );+ anyhow::ensure!(+ assigned+ .iter()+ .all(|slug| dataset_slugs.contains(slug.as_str())),+ "group references absent dataset schema"+ );+ }+ let expected_aliases: BTreeSet<(String, String, String, String)> = groups+ .iter()+ .flat_map(|group| {+ group.aliases.iter().map(|alias| {+ (+ group.content_id.clone(),+ alias.rel.clone(),+ alias.path_id.clone(),+ group.canonical.rel.clone(),+ )+ })+ })+ .collect();+ let actual_aliases: BTreeSet<(String, String, String, String)> = plan+ .duplicate_files+ .iter()+ .map(|alias: &DuplicateFile| {+ (+ alias.file_key.clone(),+ alias.rel.clone(),+ alias.path_id.clone(),+ alias.duplicate_of.clone(),+ )+ })+ .collect();+ anyhow::ensure!(+ actual_aliases == expected_aliases,+ "plan alias projection disagrees with manifest groups"+ );+ Ok(())+}++fn canonicalize_manifest_order(manifest: &mut GenerationManifest) {+ manifest.plan.datasets.sort_by(|left, right| {+ left.slug+ .cmp(&right.slug)+ .then_with(|| left.index.cmp(&right.index))+ .then_with(|| left.group.cmp(&right.group))+ });+ for dataset in &mut manifest.plan.datasets {+ dataset+ .specs+ .sort_by(|left, right| left.name.cmp(&right.name));+ }+ for assignment in manifest.plan.files.values_mut() {+ assignment.assignments.sort();+ assignment.assignments.dedup();+ }+ manifest.plan.junk_files.sort_by(|left, right| {+ left.file_key+ .cmp(&right.file_key)+ .then_with(|| left.rel.cmp(&right.rel))+ });+ manifest.plan.duplicate_files.sort_by(|left, right| {+ left.file_key+ .cmp(&right.file_key)+ .then_with(|| left.path_id.cmp(&right.path_id))+ .then_with(|| left.rel.cmp(&right.rel))+ });+ for group in &mut manifest.groups {+ group.aliases.sort_by(|left, right| {+ (left.is_symlink, left.path_id.as_str())+ .cmp(&(right.is_symlink, right.path_id.as_str()))+ });+ group.dataset_slugs.sort();+ group.dataset_slugs.dedup();+ }+ manifest+ .groups+ .sort_by(|left, right| left.group_id.cmp(&right.group_id));+}++pub fn manifest_digest(manifest: &GenerationManifest) -> Result<String> {+ let mut normalized = manifest.clone();+ canonicalize_manifest_order(&mut normalized);+ let encoded = canonical_json_bytes(&normalized)?;+ Ok(format!(+ "axm1-{:032x}",+ xxhash_rust::xxh3::xxh3_128(&encoded)+ ))+}++pub fn operation_hash(operations: &[SyncOperation]) -> Result<String> {+ let mut normalized = operations.to_vec();+ normalized.sort();+ let encoded = canonical_json_bytes(&normalized)?;+ Ok(format!(+ "axo1-{:032x}",+ xxhash_rust::xxh3::xxh3_128(&encoded)+ ))+}++fn canonical_json_bytes<T: Serialize + ?Sized>(value: &T) -> Result<Vec<u8>> {+ fn canonicalize(value: Value) -> Value {+ match value {+ Value::Object(object) => {+ let sorted: BTreeMap<String, Value> = object+ .into_iter()+ .map(|(key, value)| (key, canonicalize(value)))+ .collect();+ Value::Object(sorted.into_iter().collect())+ }+ Value::Array(values) => Value::Array(values.into_iter().map(canonicalize).collect()),+ scalar => scalar,+ }+ }+ Ok(serde_json::to_vec(&canonicalize(serde_json::to_value(+ value,+ )?))?)+}++pub fn apply_begin(+ committed: &CommittedManifest,+ pending: &mut Option<PendingSync>,+ begin: PendingSync,+) -> Result<()> {+ anyhow::ensure!(pending.is_none(), "another sync is already pending");+ begin.validate_against(committed)?;+ *pending = Some(begin);+ Ok(())+}++pub fn is_sync_record_kind(kind: &str) -> bool {+ matches!(+ kind,+ "sync_begin" | "sync_operation_state" | "sync_validated" | "sync_commit" | "sync_abort"+ )+}++pub fn replay_record(+ record: &Value,+ committed: &mut Option<CommittedManifest>,+ pending: &mut Option<PendingSync>,+) -> Result<()> {+ let kind = record_str(record, "kind")?;+ match kind {+ "sync_begin" => {+ let begin: PendingSync =+ serde_json::from_value(record.clone()).context("decode durable sync_begin")?;+ apply_begin(+ committed+ .as_ref()+ .context("sync_begin has no committed base manifest")?,+ pending,+ begin,+ )+ }+ "sync_operation_state" => {+ let tx = record_str(record, "tx_id")?;+ let operation = record_str(record, "operation_id")?;+ let state = serde_json::from_value(+ record+ .get("state")+ .cloned()+ .context("sync operation state is missing")?,+ )?;+ pending+ .as_mut()+ .context("operation state has no pending sync")?+ .apply_operation_state(tx, operation, state)+ }+ "sync_validated" => pending+ .as_mut()+ .context("validation has no pending sync")?+ .apply_validated(+ record_str(record, "tx_id")?,+ record_str(record, "desired_manifest_digest")?,+ ),+ "sync_commit" => {+ let active = pending.take().context("commit has no pending sync")?;+ *committed = Some(active.apply_commit(+ record_str(record, "tx_id")?,+ record_str(record, "desired_manifest_digest")?,+ )?);+ Ok(())+ }+ "sync_abort" => {+ let active = pending.as_ref().context("abort has no pending sync")?;+ anyhow::ensure!(+ active.tx_id == record_str(record, "tx_id")?+ && active.desired_manifest_digest+ == record_str(record, "desired_manifest_digest")?,+ "abort does not bind the pending transaction and manifest"+ );+ *pending = None;+ Ok(())+ }+ _ => anyhow::bail!("unknown sync record kind {kind}"),+ }+}++fn record_str<'a>(record: &'a Value, field: &str) -> Result<&'a str> {+ record+ .get(field)+ .and_then(Value::as_str)+ .ok_or_else(|| anyhow::anyhow!("sync record has no {field}"))+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::state::{FileAssignment, PlanDataset};++ fn path(id_hex: &str, symlink: bool) -> ManifestPath {+ ManifestPath {+ path_id: format!("unix:{id_hex}"),+ rel: format!("{id_hex}.pdf"),+ is_symlink: symlink,+ }+ }++ fn group(id: &str, content: &str, path: ManifestPath) -> ManifestGroup {+ ManifestGroup {+ group_id: id.into(),+ content_id: content.into(),+ content_digest: format!("digest-{content}"),+ content_size: 10,+ canonical: path,+ aliases: Vec::new(),+ dataset_slugs: vec!["reports".into()],+ expected_records: 1,+ expected_passages: 1,+ expected_vectors: 1,+ }+ }++ fn desired(content: &str, paths: Vec<ManifestPath>) -> DesiredContentGroup {+ DesiredContentGroup {+ content_id: content.into(),+ content_digest: format!("digest-{content}"),+ content_size: 10,+ paths,+ dataset_slugs: vec!["reports".into()],+ expected_records: 1,+ expected_passages: 1,+ expected_vectors: 1,+ }+ }++ #[test]+ fn canonical_prefers_real_then_native_bytes_including_non_utf8_identity() {+ let groups = reconcile_groups(+ &[],+ vec![desired(+ "content",+ vec![path("ff", false), path("01", true), path("80", false)],+ )],+ )+ .unwrap();+ assert_eq!(groups[0].canonical.path_id, "unix:80");+ assert!(!groups[0].canonical.is_symlink);+ }++ #[test]+ fn equal_raw_digest_does_not_merge_distinct_collision_safe_content_ids() {+ let mut a = desired("verified-a", vec![path("61", false)]);+ let mut b = desired("verified-b", vec![path("62", false)]);+ b.content_digest = a.content_digest.clone();+ a.content_size = 9;+ b.content_size = 9;+ let groups = reconcile_groups(&[], vec![a, b]).unwrap();+ assert_eq!(groups.len(), 2);+ assert_ne!(groups[0].content_id, groups[1].content_id);+ }++ #[test]+ fn changed_former_canonical_cannot_steal_surviving_content_group() {+ let mut old = group("logical", "old-content", path("61", false));+ old.aliases.push(path("62", false));+ let result = reconcile_groups(+ &[old],+ vec![+ desired("new-content", vec![path("61", false)]),+ desired("old-content", vec![path("62", false)]),+ ],+ )+ .unwrap();+ assert_eq!(+ result+ .iter()+ .find(|group| group.content_id == "old-content")+ .unwrap()+ .group_id,+ "logical"+ );+ }++ #[test]+ fn operation_planner_covers_add_delete_metadata_and_mixed() {+ let unchanged = group("same", "same", path("01", false));+ let deleted = group("gone", "gone", path("02", false));+ let metadata_old = group("meta", "meta", path("03", false));+ let mut metadata_new = metadata_old.clone();+ metadata_new.expected_records = 2;+ let added = group("new", "new", path("04", false));+ let operations = plan_operations(+ &[deleted, metadata_old, unchanged.clone()],+ &[added, metadata_new, unchanged],+ );+ assert_eq!(+ operations+ .iter()+ .map(|operation| operation.kind.clone())+ .collect::<BTreeSet<_>>(),+ BTreeSet::from([+ SyncOperationKind::Delete,+ SyncOperationKind::Metadata,+ SyncOperationKind::Upsert+ ])+ );+ }++ #[test]+ fn legacy_nonempty_plan_requires_explicit_migration() {+ let mut plan = Plan::default();+ plan.files.insert(+ "key".into(),+ FileAssignment {+ rel: "a.pdf".into(),+ path_id: "unix:61".into(),+ family: "pdf".into(),+ gzip: false,+ content_digest: Some("digest".into()),+ assignments: Vec::new(),+ },+ );+ assert!(matches!(+ CommittedManifest::bootstrap_legacy(plan).unwrap(),+ LegacyBootstrap::MigrationRequired { .. }+ ));+ }++ #[test]+ fn canonical_digest_ignores_hashmap_and_semantic_group_order() {+ let mut first = Plan::default();+ let mut second = Plan::default();+ for (plan, order) in [(&mut first, ["a", "b"]), (&mut second, ["b", "a"])] {+ for key in order {+ plan.files.insert(+ key.into(),+ FileAssignment {+ rel: format!("{key}.pdf"),+ path_id: format!("unix:{:02x}", key.as_bytes()[0]),+ family: "pdf".into(),+ gzip: false,+ content_digest: Some(format!("digest-{key}")),+ assignments: Vec::new(),+ },+ );+ }+ }+ let group_a = group("a", "a", path("61", false));+ let group_b = group("b", "b", path("62", false));+ let a = GenerationManifest {+ generation: 0,+ execution: None,+ plan: first,+ groups: vec![group_a.clone(), group_b.clone()],+ };+ let b = GenerationManifest {+ generation: 0,+ execution: None,+ plan: second,+ groups: vec![group_b, group_a],+ };+ assert_eq!(manifest_digest(&a).unwrap(), manifest_digest(&b).unwrap());+ }++ fn identity(inventory: &str) -> ExecutionIdentity {+ ExecutionIdentity {+ version: EXECUTION_IDENTITY_VERSION,+ root_identity: "unix:root".into(),+ url: "url".into(),+ prefix: "prefix".into(),+ follow_symlinks: false,+ chunker_identity: "chunker-v1".into(),+ embedding_backend: "lexical".into(),+ embedding_model: "feature-hash".into(),+ embedding_tokenizer: "builtin".into(),+ embedding_dimension: 384,+ graph_enabled: false,+ brain: "none".into(),+ detector_identity: "detectors-v1".into(),+ schema_identity: "schema-v1".into(),+ index_identity: "index-v1".into(),+ source_policy: SourceExecutionPolicy::AbortOnSourceChange {+ inventory_digest: inventory.into(),+ },+ }+ }++ #[test]+ fn no_op_plan_or_bound_execution_change_is_refused() {+ let mut legacy_plan = Plan::default();+ legacy_plan.datasets.push(PlanDataset {+ slug: "reports".into(),+ index: "prefix-reports".into(),+ family: "pdf".into(),+ group: None,+ specs: Vec::new(),+ time_field: None,+ semantic_field: None,+ sampled_records: 0,+ file_count: 0,+ });+ let LegacyBootstrap::Ready(legacy) =+ CommittedManifest::bootstrap_legacy(legacy_plan.clone()).unwrap()+ else {+ panic!("empty legacy membership is migratable");+ };+ let first = GenerationManifest {+ generation: 1,+ execution: Some(identity("inventory-a")),+ plan: legacy_plan.clone(),+ groups: Vec::new(),+ };+ let pending = PendingSync::new("bind".into(), &legacy, first.clone()).unwrap();++ let committed =+ CommittedManifest::from_generation(first.clone(), pending.desired_manifest_digest);+ let mut changed_execution = first.clone();+ changed_execution.generation = 2;+ changed_execution.execution = Some(identity("inventory-b"));+ assert!(+ PendingSync::new("changed-execution".into(), &committed, changed_execution).is_err()+ );++ let mut changed_plan = first;+ changed_plan.plan.alias_paths_indexed = true;+ assert!(PendingSync::new("changed-plan".into(), &legacy, changed_plan).is_err());+ }++ #[test]+ fn serialized_malicious_operation_list_is_rejected_against_manifests() {+ let LegacyBootstrap::Ready(base) =+ CommittedManifest::bootstrap_legacy(Plan::default()).unwrap()+ else {+ panic!("empty legacy plan is ready");+ };+ let desired = GenerationManifest {+ generation: 1,+ execution: Some(identity("inventory-a")),+ plan: Plan::default(),+ groups: Vec::new(),+ };+ let valid = PendingSync::new("tx".into(), &base, desired).unwrap();+ let mut encoded = serde_json::to_value(valid).unwrap();+ encoded["operations"] = serde_json::json!([{+ "operation_id": "delete:invented",+ "kind": "delete",+ "group_id": "invented"+ }]);+ let malicious: PendingSync = serde_json::from_value(encoded).unwrap();+ assert!(malicious.validate_against(&base).is_err());+ }+}