Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): define durable non-graph generations

bugerf77eaa324bc0
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index 4136e817a..e225b4732 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -72,8 +72,9 @@ pub enum Cmd {
Help,
}
-const FRESH_HELP: &str = "ignore the existing journal and rebuild the plan in place\n\
- (ids stay idempotent) — see RESUME POLICY";
+const FRESH_HELP: &str =
+ "start without resume state only when the selected state directory has no \
+durable plan; an existing plan is refused and destination records are never reset";
pub fn print_help() {
println!(
@@ -170,16 +171,14 @@ pub fn print_help() {
\n\
RESUME POLICY:\n\
A durable plan supports no-op resume and same-path content replacement.\n\
- Files added after the plan was frozen are reported as skipped and are not\n\
- indexed; --fresh rebuilds the plan in place and picks them up.\n\
- Removing an indexed file is refused before any remote mutation: its\n\
- documents are already live and nothing here deletes them. Restore the file\n\
- and rerun, or rebuild — in place by deleting the published indices and the\n\
- state directory, or isolated under a new --state-dir, --prefix and --brain\n\
- (or --no-graph), validated before you switch readers.\n\
+ Added or removed content groups are refused before remote mutation.\n\
+ An independent rebuild needs a new --state-dir, new --prefix, and, when\n\
+ graph detection is enabled, new --brain (or --no-graph). Validate before\n\
+ switching readers; the shared autoindex-catalog and old target require\n\
+ explicit, validated cleanup.\n\
\n\
EXIT CODES: 0 complete; 3 completed-with-junk (junk recorded, never fatal);\n\
- 2 usage; 1 endpoint/journal failure or a refused corpus removal\n",
+ 2 usage; 1 endpoint/journal failure or unsupported corpus delta\n",
fresh_help = FRESH_HELP
);
}
@@ -477,10 +476,10 @@ mod tests {
}
#[test]
- fn fresh_help_points_at_the_resume_policy_that_bounds_it() {
- assert!(super::FRESH_HELP.contains("rebuild the plan in place"));
- assert!(super::FRESH_HELP.contains("ids stay idempotent"));
- assert!(super::FRESH_HELP.contains("RESUME POLICY"));
+ fn fresh_help_warns_that_it_is_not_destination_reconciliation() {
+ assert!(super::FRESH_HELP.contains("no durable plan"));
+ assert!(super::FRESH_HELP.contains("existing plan is refused"));
+ assert!(super::FRESH_HELP.contains("destination records are never reset"));
}
#[test]
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 6cc7a4f29..e3e750aa8 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -69,8 +69,6 @@ struct MockState {
/// what `_count`/`_search` answer with, and the run verifies live data
/// counts against the journal (#195). Catalog rows are not data rows.
catalog_docs: HashMap<String, Value>,
- /// Every (method, path) the run issued. The refusal tests assert on the
- /// ABSENCE of remote mutations, which no document count can express.
requests: Vec<(String, String)>,
data_bulk_number: usize,
fail_data_bulk: usize,
@@ -202,6 +200,11 @@ fn handle(mut stream: TcpStream, state: &Arc<Mutex<MockState>>) {
if response_delay_ms > 0 {
thread::sleep(std::time::Duration::from_millis(response_delay_ms));
}
+ state
+ .lock()
+ .unwrap()
+ .requests
+ .push((method.to_owned(), path.to_owned()));
let (status, response) = if method == "GET" && path == "/v1/embedding/identity" {
let identity = state.lock().unwrap().embedding_identity_sha256.clone();
@@ -532,15 +535,11 @@ fn assert_unsupported_delta_without_remote_mutation(
let message = format!("{error:#}");
assert!(message.contains("made no remote mutations"), "{message}");
assert!(
- message.contains("no longer exist in the folder"),
- "{message}"
- );
- assert!(
- message.contains("restore the removed file(s) and rerun"),
+ message.contains("existing destination may already be partial or stale"),
"{message}"
);
assert!(
- message.contains("rebuild in place by deleting the indices"),
+ message.contains("`--fresh` is not recovery or destination reconciliation"),
"{message}"
);
assert!(
@@ -562,58 +561,19 @@ fn assert_unsupported_delta_without_remote_mutation(
}
}
-/// The documented headline workflow: point autoindex at a folder, add a file,
-/// rerun. The rerun must not fail, must say plainly that the added file was
-/// not indexed, and `--fresh` must then absorb it in place.
#[test]
-fn a_rerun_after_an_added_file_succeeds_and_fresh_absorbs_it() {
+fn completed_plan_rejects_added_content_group_before_remote_mutation() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
let state_dir = tempfile::tempdir().unwrap();
fs::write(corpus.path().join("first.csv"), "id,value\n1,first\n").unwrap();
let endpoint = MockEndpoint::start(usize::MAX);
- let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
assert_eq!(run_index(config.clone()).unwrap(), 0);
fs::write(corpus.path().join("second.csv"), "id,value\n2,second\n").unwrap();
- // 3 = completed-with-junk: the added file is reported as skipped, not
- // indexed, because the frozen plan cannot absorb it.
- let (code, report) = run_index_report(config.clone()).unwrap();
- assert_eq!(code, 3);
- let report = report.unwrap();
- assert_eq!(report["files_junk"], 1, "{report}");
- // `records_total` is the live server-side count (#195), so it still reports
- // the record the first run published. What must be zero is the run-local
- // counter: this rerun indexed nothing, because the frozen plan cannot
- // absorb a file that appeared after it was written.
- assert_eq!(report["records_submitted_this_run"], 0, "{report}");
- assert_eq!(report["files_submitted_this_run"], 0, "{report}");
- assert_eq!(report["records_total"], 1, "{report}");
- {
- let locked = endpoint.state.lock().unwrap();
- let live: Vec<&str> = locked
- .docs
- .values()
- .filter_map(|doc| doc["ax_path"].as_str())
- .collect();
- assert!(
- live.iter().all(|path| *path == "first.csv"),
- "the added file must not be published by a plan that predates it: {live:?}"
- );
- }
-
- // --fresh rebuilds the plan in place and picks the new file up.
- config.fresh = true;
- assert_eq!(run_index(config).unwrap(), 0);
- let locked = endpoint.state.lock().unwrap();
- let indexed: std::collections::HashSet<&str> = locked
- .docs
- .values()
- .filter_map(|doc| doc["ax_path"].as_str())
- .collect();
- assert!(indexed.contains("first.csv"), "{indexed:?}");
- assert!(indexed.contains("second.csv"), "{indexed:?}");
+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &["second.csv"], &[]);
}
#[test]
@@ -678,127 +638,8 @@ fn completed_plan_rejects_mixed_membership_delta_in_stable_order() {
assert_eq!(vanished, ["a-old.csv", "b-old.csv"]);
}
-/// An in-place EDIT is a replacement, not a removal. `--fresh` is the route
-/// the refusal itself recommends for picking up added and changed files, so
-/// the gate must not classify the edited file's superseded content key as a
-/// vanished group — the path is still there, and the pipeline republishes it.
-#[test]
-fn fresh_absorbs_an_edited_file_instead_of_calling_it_a_removal() {
- let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
- let corpus = tempfile::tempdir().unwrap();
- let state_dir = tempfile::tempdir().unwrap();
- fs::write(corpus.path().join("notes.csv"), "id,value\n1,before\n").unwrap();
- let endpoint = MockEndpoint::start(usize::MAX);
- let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
- assert_eq!(run_index(config.clone()).unwrap(), 0);
-
- fs::write(corpus.path().join("notes.csv"), "id,value\n1,after\n").unwrap();
- config.fresh = true;
- assert_eq!(
- run_index(config).unwrap(),
- 0,
- "an edited file is a same-path replacement, not a vanished content group"
- );
- let locked = endpoint.state.lock().unwrap();
- assert!(
- locked
- .docs
- .values()
- .any(|doc| doc["value"].as_str() == Some("after")),
- "the new content is live: {:?}",
- locked.docs
- );
- // Documented, and NOT what this gate is for: `--fresh` rebuilds the plan,
- // it does not reconcile the destination. Document ids derive from the
- // content key (`ids::doc_id`), so the pre-edit record stays live beside
- // the new one — the same outcome `--fresh` has always had. An ordinary
- // rerun is the clean route for an edit: it keeps the planned key and runs
- // a delete-before-replace transaction on it. What must never happen is
- // this run being REFUSED by a message claiming a file that is sitting in
- // the folder has vanished.
- assert!(
- locked
- .docs
- .values()
- .any(|doc| doc["value"].as_str() == Some("before")),
- "known --fresh gap: superseded records are not deleted: {:?}",
- locked.docs
- );
-}
-
-/// The clean route for the same edit: an ordinary rerun keeps the planned key
-/// and replaces its records, so nothing is stranded. This is the contrast that
-/// makes the `--fresh` gap above a documented trade-off rather than a silent
-/// one.
-#[test]
-fn an_ordinary_rerun_replaces_an_edited_file_without_stranding_records() {
- let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
- let corpus = tempfile::tempdir().unwrap();
- let state_dir = tempfile::tempdir().unwrap();
- fs::write(corpus.path().join("notes.csv"), "id,value\n1,before\n").unwrap();
- let endpoint = MockEndpoint::start(usize::MAX);
- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
- assert_eq!(run_index(config.clone()).unwrap(), 0);
-
- fs::write(corpus.path().join("notes.csv"), "id,value\n1,after\n").unwrap();
- assert_eq!(run_index(config).unwrap(), 0);
- let locked = endpoint.state.lock().unwrap();
- let values: Vec<&str> = locked
- .docs
- .values()
- .filter_map(|doc| doc["value"].as_str())
- .collect();
- assert_eq!(values, ["after"], "the superseded record is replaced");
-}
-
-/// The composite workflow the run's own stderr recommends: index, then add one
-/// file and edit another, rerun (added file reported and skipped by the frozen
-/// plan), then `--fresh` to absorb both.
#[test]
-fn fresh_absorbs_an_addition_and_an_edit_after_a_reported_rerun() {
- let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
- let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
- let corpus = tempfile::tempdir().unwrap();
- let state_dir = tempfile::tempdir().unwrap();
- fs::write(corpus.path().join("first.csv"), "id,value\n1,before\n").unwrap();
- let endpoint = MockEndpoint::start(usize::MAX);
- let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
- assert_eq!(run_index(config.clone()).unwrap(), 0);
-
- fs::write(corpus.path().join("first.csv"), "id,value\n1,after\n").unwrap();
- fs::write(corpus.path().join("second.csv"), "id,value\n2,second\n").unwrap();
- // 3 = completed-with-junk: the frozen plan cannot absorb the added file,
- // so the rerun reports it as skipped rather than refusing.
- assert_eq!(run_index(config.clone()).unwrap(), 3);
-
- config.fresh = true;
- assert_eq!(
- run_index(config).unwrap(),
- 0,
- "--fresh is the documented way out of exactly this state"
- );
- let locked = endpoint.state.lock().unwrap();
- let live: std::collections::HashSet<&str> = locked
- .docs
- .values()
- .filter_map(|doc| doc["ax_path"].as_str())
- .collect();
- assert!(live.contains("first.csv"), "{live:?}");
- assert!(live.contains("second.csv"), "{live:?}");
- assert!(
- locked
- .docs
- .values()
- .any(|doc| doc["value"].as_str() == Some("after")),
- "the edited file's new content is live: {:?}",
- locked.docs
- );
-}
-
-#[test]
-fn fresh_cannot_erase_the_plan_and_bypass_the_removal_gate() {
+fn nonempty_fresh_cannot_erase_plan_and_bypass_membership_gate() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
@@ -815,7 +656,7 @@ fn fresh_cannot_erase_the_plan_and_bypass_the_removal_gate() {
}
#[test]
-fn deleting_one_path_of_a_duplicate_pair_is_not_a_removed_content_group() {
+fn fresh_rejects_same_content_canonical_promotion_even_when_group_key_matches() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
@@ -824,24 +665,14 @@ fn deleting_one_path_of_a_duplicate_pair_is_not_a_removed_content_group() {
fs::write(corpus.path().join("a.csv"), bytes).unwrap();
fs::write(corpus.path().join("b.csv"), bytes).unwrap();
let endpoint = MockEndpoint::start(usize::MAX);
- let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
assert_eq!(run_index(config.clone()).unwrap(), 0);
- // b.csv becomes canonical under the same content key: the group survives
- // the deletion, so no document is stranded and the rerun is allowed.
+ // b.csv becomes canonical with the same content key. A key-only delta is
+ // empty, but fresh would discard the alias/path cleanup knowledge.
fs::remove_file(corpus.path().join("a.csv")).unwrap();
- let result = run_index(config);
- assert!(
- result.is_ok(),
- "a surviving content group must not trip the removal gate: {result:?}"
- );
- let locked = endpoint.state.lock().unwrap();
- let live: Vec<&str> = locked
- .docs
- .values()
- .filter_map(|doc| doc["ax_path"].as_str())
- .collect();
- assert_eq!(live, ["b.csv"], "canonical path follows the surviving file");
+ config.fresh = true;
+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &[], &[]);
}
#[test]
@@ -867,7 +698,7 @@ fn unchanged_planned_junk_is_not_reported_as_added_content() {
}
#[test]
-fn deleted_planned_junk_is_swept_rather_than_refused() {
+fn deleted_planned_junk_fails_closed_before_catalog_can_stay_stale() {
let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();
let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
let corpus = tempfile::tempdir().unwrap();
@@ -882,33 +713,8 @@ fn deleted_planned_junk_is_swept_rather_than_refused() {
let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
assert_eq!(run_index(config.clone()).unwrap(), 3);
- // A junk file publishes exactly one catalog row and nothing else, and the
- // #238 sweep deletes that row. Nothing is stranded, so the removal gate
- // must not fire — refusing here would block a case the pipeline handles
- // completely. Only a file that published DOCUMENTS refuses a rerun.
fs::remove_file(corpus.path().join("opaque.bin")).unwrap();
- assert_eq!(run_index(config.clone()).unwrap(), 0);
- {
- let locked = endpoint.state.lock().unwrap();
- assert!(
- locked
- .catalog_docs
- .values()
- .all(|doc| doc["path"].as_str() != Some("opaque.bin")),
- "the deleted junk file's catalog row must be swept, not left immortal"
- );
- assert!(
- locked
- .catalog_docs
- .values()
- .any(|doc| doc["path"].as_str() == Some("rows.csv")),
- "the surviving indexed file keeps its entry"
- );
- }
-
- // Deleting the INDEXED file is still refused: its documents are live.
- fs::remove_file(corpus.path().join("rows.csv")).unwrap();
- assert_unsupported_delta_without_remote_mutation(&endpoint, config, &[], &["rows.csv"]);
+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &[], &["opaque.bin"]);
}
#[test]
@@ -923,7 +729,6 @@ fn completed_plan_rejects_empty_current_folder_before_remote_mutation() {
assert_eq!(run_index(config.clone()).unwrap(), 0);
fs::remove_file(corpus.path().join("only.csv")).unwrap();
- assert_unsupported_delta_without_remote_mutation(&endpoint, config.clone(), &[], &["only.csv"]);
// Even `--fresh` must not erase the only durable inventory evidence and
// then report success while the destination still contains the document.
config.fresh = true;
@@ -1725,12 +1530,12 @@ fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {
)
.unwrap();
fs::write(corpus.path().join("b.csv"), original).unwrap();
- assert_eq!(run_index(config.clone()).unwrap(), 3);
+ assert_unsupported_delta_without_remote_mutation(&endpoint, config.clone(), &["b.csv"], &[]);
{
let locked = endpoint.state.lock().unwrap();
assert_eq!(locked.docs.len(), 2);
assert!(locked.docs.values().all(|doc| {
- doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("rewritten")
+ doc["ax_path"].as_str() == Some("a.csv") && doc["value"].as_str() == Some("original")
}));
}
let replay = state::Journal::open(
@@ -1745,24 +1550,20 @@ fn a_planned_key_never_gains_two_owners_when_old_content_moves_paths() {
assert!(replay.pending_replacements.is_empty());
assert_eq!(replay.done.len(), 1);
let plan = replay.plan.clone().unwrap();
- assert_eq!(plan.junk_files.len(), 1);
- assert_eq!(plan.junk_files[0].rel, "b.csv");
- assert!(plan.junk_files[0]
- .reason
- .contains("key ownership exclusive"));
+ assert!(plan.junk_files.is_empty());
drop(replay);
- // The divergence is durable and deterministic: an identical rerun keeps
- // the same owner, appends no new plan, and changes no documents.
+ // The refusal is deterministic: an identical rerun keeps the old owner,
+ // appends no new plan, and changes no documents.
let plans_before = event_count(state_dir.path(), "plan");
- assert_eq!(run_index(config).unwrap(), 3);
+ assert_unsupported_delta_without_remote_mutation(&endpoint, config, &["b.csv"], &[]);
assert_eq!(event_count(state_dir.path(), "plan"), plans_before);
let locked = endpoint.state.lock().unwrap();
assert_eq!(locked.docs.len(), 2);
assert!(locked
.docs
.values()
- .all(|doc| doc["value"].as_str() == Some("rewritten")));
+ .all(|doc| doc["value"].as_str() == Some("original")));
}
/// #238: a file added after the plan was frozen is skipped and reported in the
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 4563c6ada..bfdf50142 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -19,6 +19,7 @@ pub mod progress;
pub mod resources;
pub mod sniff;
pub mod state;
+mod sync;
pub mod walk;
use anyhow::{Context, Result};
@@ -1040,8 +1041,10 @@ fn select_resume_plan_keys(
for (key, assignment) in &plan.files {
if let Some(previous) = planned_by_rel.insert(&assignment.rel, key) {
anyhow::bail!(
- "resume plan assigns path {} to both {} and {}; use --fresh after verifying the \
- existing index",
+ "resume plan assigns path {} to both {} and {}; refusing to discard history under \
+ the same destination. For an isolated rebuild use a new --state-dir, new \
+ --prefix, and new --brain when graph detection is enabled (or --no-graph); \
+ explicitly validate and clean the shared catalog and old target",
assignment.rel,
previous,
key
@@ -1050,8 +1053,11 @@ fn select_resume_plan_keys(
if !assignment.path_id.is_empty() {
if let Some(previous) = planned_by_path_id.insert(&assignment.path_id, key) {
anyhow::bail!(
- "resume plan assigns one native path identity to both {} and {}; use --fresh \
- after verifying the existing index",
+ "resume plan assigns one native path identity to both {} and {}; refusing to \
+ discard history under the same destination. For an isolated rebuild use a \
+ new --state-dir, new --prefix, and new --brain when graph detection is \
+ enabled (or --no-graph); explicitly validate and clean the shared catalog \
+ and old target",
previous,
key
);
@@ -1089,9 +1095,11 @@ fn select_resume_plan_keys(
anyhow::bail!(
"{} collides with legacy resume key {} already owned by {}. No documents \
were changed; remove or move one of these two files out of the corpus \
- and rerun — every other file keeps its resume state. Deleting the \
- journal at {} (or rerunning with --fresh) also clears the collision, \
- but re-extracts and re-embeds the entire corpus",
+ and rerun — every other file keeps its resume state. Refusing to discard \
+ history under the same destination. For an isolated rebuild use a new \
+ --state-dir, new --prefix, and new --brain when graph detection is \
+ enabled (or --no-graph); explicitly validate and clean the shared \
+ catalog and old target. Journal: {}",
file.rel,
legacy_key,
assignment.rel,
@@ -1135,95 +1143,10 @@ struct UnsupportedInventoryDelta {
vanished_content_groups: Vec<InventoryDeltaEntry>,
}
-/// Everything the refusal needs to name the destination it is protecting.
-/// Collected at the gate so the message can list the exact indices and state
-/// directory an operator has to act on, instead of describing them abstractly.
-#[derive(Debug, Clone, PartialEq, Eq)]
-struct RefusalTargets {
- state_dir: String,
- data_indices: Vec<String>,
- edges_index: Option<String>,
-}
-
-impl RefusalTargets {
- fn describe(cfg: &IndexCfg, state_dir: &Path, plan: &Plan) -> Self {
- let mut data_indices: Vec<String> = plan.datasets.iter().map(|d| d.index.clone()).collect();
- data_indices.sort();
- data_indices.dedup();
- let edges_index = (!cfg.no_graph).then(|| {
- let brain = cfg
- .brain
- .clone()
- .unwrap_or_else(|| derive_brain_name(&cfg.root));
- detect::edges_index_name(&brain)
- });
- Self {
- state_dir: state_dir.display().to_string(),
- data_indices,
- edges_index,
- }
- }
-
- fn indices_phrase(&self) -> String {
- if self.data_indices.is_empty() {
- "the indices this plan publishes".to_string()
- } else {
- self.data_indices.join(", ")
- }
- }
-
- fn edges_note(&self) -> String {
- match &self.edges_index {
- Some(edges) => format!(
- " Graph edges taught by the removed file(s) stay live in {edges} until that \
- index is deleted too."
- ),
- None => String::new(),
- }
- }
-}
-
-/// The #195 zero-live verification failure, as a type rather than a string.
-///
-/// The journal claims records were made visible; the destination answers with
-/// none. That is always a failure, but not always the *same* failure: for a
-/// composing caller like `xerj brain` a resume journal that outlived a wiped
-/// data directory has a specific, executable recovery (`--fresh` in place),
-/// while a write-blocked or unreachable server does not. Only a typed error
-/// lets the caller tell them apart — `anyhow::bail!` forces it to either
-/// string-match the prose or reprint it verbatim.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ZeroLiveVerificationError {
- /// Records the resume journal says were published.
- pub journal_records: u64,
- /// Completed files those records came from.
- pub files_done_journaled: usize,
- /// Dataset indices that answered with zero live documents.
- pub dataset_indices: usize,
-}
-
-impl std::fmt::Display for ZeroLiveVerificationError {
- fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(
- formatter,
- "autoindex verification failed: the resume journal records {} \
- record(s) from {} completed file(s), but 0 documents are \
- live across the {} dataset index(es). The indices exist and look healthy but \
- hold nothing — a server-side write rejection (e.g. a disk flood-stage or \
- index write block), deleted indices, or an unreadable server can all cause \
- this. Fix the server-side condition, then rerun with --fresh to re-index \
- from scratch",
- self.journal_records, self.files_done_journaled, self.dataset_indices
- )
- }
-}
-
-impl std::error::Error for ZeroLiveVerificationError {}
-
#[derive(Debug)]
struct UnsupportedInventoryDeltaError {
delta: UnsupportedInventoryDelta,
- targets: RefusalTargets,
+ fresh_existing_plan: bool,
}
impl UnsupportedInventoryDeltaError {
@@ -1231,22 +1154,21 @@ impl UnsupportedInventoryDeltaError {
json!({
"schema": "xerj.autoindex.unsupported_sync_delta.v1",
"status": "error",
- "error": "unsupported_content_group_removal",
- "message": "this attempt made no remote mutations. Files that were indexed under this resume plan no longer exist in the folder, and their documents are still live in the destination; removing files from an indexed folder is not reconciled yet",
- "vanished_content_groups": self.delta.vanished_content_groups,
- // Context, not the reason for the refusal: a rerun over a frozen
- // plan does not index files added after the plan was frozen.
+ "error": if self.fresh_existing_plan {
+ "unsafe_fresh_existing_plan"
+ } else {
+ "unsupported_inventory_delta"
+ },
+ "message": if self.fresh_existing_plan {
+ "this attempt made no remote mutations; --fresh cannot discard an existing plan under the same destination because alias, path, graph, and stale-record cleanup knowledge would be lost; the existing destination may already be partial or stale"
+ } else {
+ "this attempt made no remote mutations because this autoindex plan cannot safely reconcile corpus membership changes; the existing destination may already be partial or stale"
+ },
"added_content_groups": self.delta.added_content_groups,
+ "vanished_content_groups": self.delta.vanished_content_groups,
"recovery": {
- "restore_removed_files": "put the listed file(s) back and rerun; every other file keeps its resume state",
- "rebuild_in_place": format!(
- "delete the indices this plan publishes ({}) and the state directory {}, then rerun. This re-extracts and re-embeds the whole corpus.{}",
- self.targets.indices_phrase(),
- self.targets.state_dir,
- self.targets.edges_note().trim_start_matches(' ')
- ),
- "rebuild_isolated": "index with a new --state-dir, new --prefix, and (when graph detection is enabled) new --brain; alternatively add --no-graph. Validate the isolated target before switching readers, then clean the old one",
- "fresh_warning": "--fresh re-extracts the current folder in place and does pick up added and changed files, but it never deletes documents already published for removed files, so it is refused here"
+ "exact_rebuild": "index with a new --state-dir, new --prefix, and (when graph detection is enabled) new --brain; alternatively add --no-graph. Validate the isolated target before switching readers",
+ "warning": "--fresh is not recovery or destination reconciliation. The global autoindex-catalog and old target are not cleaned automatically; validate and clean them explicitly"
}
})
}
@@ -1254,52 +1176,30 @@ impl UnsupportedInventoryDeltaError {
impl std::fmt::Display for UnsupportedInventoryDeltaError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- // Capped like the duplicate and unplanned-file listings above; see
- // REFUSAL_LIST_CAP. The machine-readable `--json` rendering keeps the
- // full lists, so nothing is lost — only the prose is bounded.
let render = |entries: &[InventoryDeltaEntry]| {
- let mut rendered = entries
+ entries
.iter()
- .take(REFUSAL_LIST_CAP)
.map(|entry| format!("{} ({})", entry.path, entry.file_key))
.collect::<Vec<_>>()
- .join(", ");
- let remaining = entries.len().saturating_sub(REFUSAL_LIST_CAP);
- if remaining > 0 {
- rendered.push_str(&format!(", … and {remaining} more"));
- }
- rendered
+ .join(", ")
+ };
+ let reason = if self.fresh_existing_plan {
+ "`--fresh` cannot discard an existing plan under the same destination because alias, \
+ path, graph, and stale-record cleanup knowledge would be lost"
+ } else {
+ "corpus membership synchronization is not yet supported"
};
write!(
formatter,
- "{} file(s) indexed under this resume plan no longer exist in the folder, and their \
- documents are still live in the destination. Removing files from an indexed folder \
- is not reconciled yet, so this attempt made no remote mutations — no documents, \
- aliases, graph edges or catalog entries were written. Removed content groups [{}].",
- self.delta.vanished_content_groups.len(),
+ "this attempt made no remote mutations; the existing destination may already be \
+ partial or stale. {reason}. Added \
+ content groups [{}]; vanished content groups [{}]. For an exact rebuild, index the \
+ current folder with a new --state-dir, a new --prefix, and, when graph detection is \
+ enabled, a new --brain (or use --no-graph). Validate the isolated target before \
+ switching readers. `--fresh` is not recovery or destination reconciliation; the \
+ global autoindex-catalog and old target require explicit validated cleanup",
+ render(&self.delta.added_content_groups),
render(&self.delta.vanished_content_groups)
- )?;
- if !self.delta.added_content_groups.is_empty() {
- write!(
- formatter,
- " Also present but not in the frozen resume plan, so not indexed by this run \
- either [{}].",
- render(&self.delta.added_content_groups)
- )?;
- }
- write!(
- formatter,
- " Recovery, cheapest first: (1) restore the removed file(s) and rerun — every other \
- file keeps its resume state; (2) rebuild in place by deleting the indices this plan \
- publishes ({}) and the state directory {}, then rerunning — this re-extracts and \
- re-embeds the whole corpus.{} (3) rebuild isolated with a new --state-dir, a new \
- --prefix and, when graph detection is enabled, a new --brain (or --no-graph), \
- validate it, switch readers, then clean the old target. `--fresh` picks up added \
- and changed files in place but never deletes documents for removed files, so it is \
- refused here too",
- self.targets.indices_phrase(),
- self.targets.state_dir,
- self.targets.edges_note()
)
}
}
@@ -1319,27 +1219,6 @@ impl UnsupportedInventoryDelta {
.map(String::as_str)
.chain(plan.junk_files.iter().map(|junk| junk.file_key.as_str()))
.collect();
- // Path identity, not just content identity. An in-place EDIT gives the
- // same file a new content key, so a key-only comparison reads the
- // superseded key as vanished and the new one as added — the SAME path
- // listed as both removed and added. That is a replacement, and the file
- // is still there to be republished, so it must never be refused.
- // Ordinary resume already reaches this conclusion by mapping each file
- // onto its planned key (`select_resume_plan_keys`); doing it here makes
- // `--fresh`, which has no plan to map through, agree — without making
- // the gate more fatal than the open it precedes.
- let current_rels: std::collections::HashSet<&str> =
- files.iter().map(|file| file.rel.as_str()).collect();
- let current_path_ids: std::collections::HashSet<&str> = files
- .iter()
- .map(|file| file.rel_id.as_str())
- .filter(|id| !id.is_empty())
- .collect();
- let path_survives = |assignment: &FileAssignment| {
- current_rels.contains(assignment.rel.as_str())
- || (!assignment.path_id.is_empty()
- && current_path_ids.contains(assignment.path_id.as_str()))
- };
let mut added_content_groups: Vec<InventoryDeltaEntry> = files
.iter()
@@ -1353,22 +1232,21 @@ impl UnsupportedInventoryDelta {
let mut vanished_content_groups: Vec<InventoryDeltaEntry> = plan
.files
.iter()
- .filter(|(key, assignment)| {
- !current_keys.contains(key.as_str()) && !path_survives(assignment)
- })
+ .filter(|(key, _)| !current_keys.contains(key.as_str()))
.map(|(key, assignment)| InventoryDeltaEntry {
file_key: key.clone(),
path: assignment.rel.clone(),
})
.collect();
- // Deliberately NOT extended with `plan.junk_files`. A junk/skipped file
- // published no documents, no aliases and no graph edges — its entire
- // live footprint is one `file:{key}` catalog row, and the stale
- // junk-catalog sweep below (#238) deletes that row before dropping the
- // plan entry. Removing it therefore strands nothing, so refusing the
- // rerun would block a case the pipeline now handles completely. Junk
- // keys stay in `durable_keys` above: an unchanged skipped file is still
- // not an addition.
+ vanished_content_groups.extend(
+ plan.junk_files
+ .iter()
+ .filter(|junk| !current_keys.contains(junk.file_key.as_str()))
+ .map(|junk| InventoryDeltaEntry {
+ file_key: junk.file_key.clone(),
+ path: junk.rel.clone(),
+ }),
+ );
let stable_order = |left: &InventoryDeltaEntry, right: &InventoryDeltaEntry| {
left.path
@@ -1383,19 +1261,22 @@ impl UnsupportedInventoryDelta {
}
}
- /// A rerun is refused only when a content group the plan published has
- /// vanished from the folder: those documents stay live and searchable with
- /// no source file behind them, and nothing in this pipeline removes them.
- /// Additions are not refused — they are skipped by the frozen plan exactly
- /// as before and `--fresh` rebuilds the plan in place to include them.
- fn refuses(&self) -> bool {
- !self.vanished_content_groups.is_empty()
+ fn is_empty(&self) -> bool {
+ self.added_content_groups.is_empty() && self.vanished_content_groups.is_empty()
+ }
+
+ fn into_error(self) -> anyhow::Error {
+ UnsupportedInventoryDeltaError {
+ delta: self,
+ fresh_existing_plan: false,
+ }
+ .into()
}
- fn into_error(self, targets: RefusalTargets) -> anyhow::Error {
+ fn into_fresh_error(self) -> anyhow::Error {
UnsupportedInventoryDeltaError {
delta: self,
- targets,
+ fresh_existing_plan: true,
}
.into()
}
@@ -1589,20 +1470,7 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
// Acquire state authority before discovery as well as hashing. A waiter
// must never classify a path snapshot taken while another owner was
// publishing or replacing the durable plan.
- let preflight =
- state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix, cfg.fresh)?;
- if let Some(reason) = preflight.unreadable_plan.as_deref() {
- // Only reachable under --fresh; without it the preflight would have
- // returned this as an error. It goes through the progress surface, not
- // a raw eprintln!, so `--progress none` (which `--quiet` selects) stays
- // silent and `--progress json` stays one parseable stream (#241).
- pr.note(&format!(
- "autoindex: --fresh: the durable resume plan in {} could not be read ({reason}); \
- rebuilding it from the current folder. Documents published for files that are no \
- longer present cannot be identified from an unreadable plan and are not deleted.",
- state_dir.join("journal.ndjson").display()
- ));
- }
+ let preflight = state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix)?;
// Totals are unknown until the walk returns, so this phase honestly
// reports `pct=unknown` and proves liveness with the clock alone.
pr.phase("walk", 0, 0);
@@ -1614,6 +1482,9 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
discovered_bytes / (1 << 20),
root_str
));
+ // An empty folder is only "nothing to do" when there is also no durable
+ // state: with a journal present, zero files is a deletion of the whole
+ // corpus and has to be reconciled, not shrugged off.
if discovered_files.is_empty() && !preflight.journal_exists {
println!("no files found under {}", cfg.root.display());
pr.finish(true, 0, "no-files", &[]);
@@ -1644,13 +1515,16 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
.map(|(planned, current)| planned.unwrap_or_else(|| current.clone()))
.collect()
};
- // `--fresh` is checked here too: discarding the plan does not delete
- // the documents already published for a file that is now gone, so a
- // removal is unsafe in place whether or not the plan is kept.
let delta =
UnsupportedInventoryDelta::between(&inventory.files, &comparison_keys, prior_plan);
- if delta.refuses() {
- return Err(delta.into_error(RefusalTargets::describe(&cfg, &state_dir, prior_plan)));
+ if cfg.fresh {
+ // Even an apparently unchanged content-key set can have alias,
+ // path, graph, catalog, or partial-publication history that only
+ // the old plan can reconcile. Route 1 cannot safely discard it.
+ return Err(delta.into_fresh_error());
+ }
+ if !delta.is_empty() {
+ return Err(delta.into_error());
}
}
let mut journal = state::Journal::open_after_preflight(
@@ -1878,19 +1752,17 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
plan
};
- // Second gate, after key selection has settled: fail before index
- // creation/mapping, replacement intent, graph invalidation,
- // delete-by-query, bulk publication, refresh, or catalog writes when a
- // canonical content group the plan published is gone from the folder.
- // Continuing would leave those documents searchable with no source file.
- // Additions, same-path replacement and crash repair remain supported.
- //
- // It runs before the #238 junk sweep below on purpose: a refused rerun
- // must compute nothing and mutate nothing.
+ // A durable plan is a crash-resume boundary, not a folder-sync
+ // generation. Fail before index creation/mapping, replacement intent,
+ // graph invalidation, delete-by-query, bulk publication, refresh, or
+ // catalog writes when the current inventory adds or removes an entire
+ // canonical content group. Continuing would either silently skip new
+ // data or leave vanished source records searchable. Existing planned-key
+ // content replacement and crash repair remain supported.
if resumed_with_plan {
let delta = UnsupportedInventoryDelta::between(&files, &keys, &plan);
- if delta.refuses() {
- return Err(delta.into_error(RefusalTargets::describe(&cfg, &state_dir, &plan)));
+ if !delta.is_empty() {
+ return Err(delta.into_error());
}
}
@@ -2066,9 +1938,9 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
rel: files[i].rel.clone(),
format: "unknown".into(),
status: "skipped".into(),
- reason: "not in the frozen resume plan, so nothing was published for it. Re-run \
- with --fresh to rebuild the plan in place and include it (ids stay \
- idempotent), or index it under a new --state-dir and --prefix"
+ reason: "not in the frozen resume plan; no destination change was made. For an \
+ exact rebuild use a new --state-dir, a new --prefix, and, when graph \
+ detection is enabled, a new --brain (or use --no-graph)"
.into(),
bytes: files[i].size,
});
@@ -3794,43 +3666,17 @@ mod inventory_delta_tests {
gzip: false,
content_digest: Some(format!("digest:{path}")),
assignments: vec![(None, "rows".into())],
- as_document: false,
- }
- }
-
- fn targets() -> RefusalTargets {
- RefusalTargets {
- state_dir: "/state".into(),
- data_indices: vec!["ax-rows".into()],
- edges_index: Some(".xerj-memory-corpus-edges".into()),
}
}
#[test]
- fn only_a_removed_content_group_refuses_a_rerun() {
+ fn classifier_is_empty_for_noop_and_sorts_added_and_vanished_groups() {
let mut plan = Plan::default();
plan.files.insert("keep".into(), assignment("keep.csv"));
assert!(
- !UnsupportedInventoryDelta::between(&[file("keep.csv")], &["keep".into()], &plan)
- .refuses()
- );
- // An added file is skipped by the frozen plan, not a refusal: the
- // documented rerun-then---fresh workflow has to keep working.
- let added = UnsupportedInventoryDelta::between(
- &[file("keep.csv"), file("new.csv")],
- &["keep".into(), "new".into()],
- &plan,
+ UnsupportedInventoryDelta::between(&[file("keep.csv")], &["keep".into()], &plan)
+ .is_empty()
);
- assert_eq!(added.added_content_groups.len(), 1);
- assert!(!added.refuses(), "an addition alone must not fail the run");
- // A removal leaves live documents with no source file behind them.
- assert!(UnsupportedInventoryDelta::between(&[], &[], &plan).refuses());
- }
-
- #[test]
- fn classifier_sorts_added_and_vanished_groups() {
- let mut plan = Plan::default();
- plan.files.insert("keep".into(), assignment("keep.csv"));
plan.junk_files.push(JunkFile {
file_key: "junk".into(),
rel: "broken.pdf".into(),
@@ -3840,13 +3686,13 @@ mod inventory_delta_tests {
bytes: 1,
});
assert!(
- !UnsupportedInventoryDelta::between(
+ UnsupportedInventoryDelta::between(
&[file("keep.csv"), file("broken.pdf")],
&["keep".into(), "junk".into()],
&plan,
)
- .refuses(),
- "unchanged durable junk is neither added nor vanished"
+ .is_empty(),
+ "unchanged durable junk is not newly added content"
);
plan.files.insert("old-z".into(), assignment("z-old.csv"));
@@ -3879,97 +3725,23 @@ mod inventory_delta_tests {
);
}
- #[test]
- fn refusal_names_the_removed_files_and_every_recovery_route() {
- let mut plan = Plan::default();
- plan.files.insert("gone".into(), assignment("gone.csv"));
- let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());
- let message = format!("{error:#}");
- assert!(message.contains("gone.csv"), "{message}");
- assert!(
- message.contains("no longer exist in the folder"),
- "{message}"
- );
- assert!(message.contains("made no remote mutations"), "{message}");
- assert!(message.contains("restore the removed file(s)"), "{message}");
- assert!(message.contains("ax-rows"), "{message}");
- assert!(message.contains("/state"), "{message}");
- assert!(message.contains(".xerj-memory-corpus-edges"), "{message}");
- assert!(message.contains("new --state-dir"), "{message}");
- assert!(message.contains("`--fresh`"), "{message}");
- }
-
- /// The prose refusal is bounded by REFUSAL_LIST_CAP, not by the corpus: an
- /// unmounted bind mount under an indexed root vanishes every group at once,
- /// and rendering 82k entries into one String is megabytes of stderr in the
- /// one path whose job is to be read. `--json` still carries all of them.
- #[test]
- fn refusal_prose_caps_its_listings_while_json_keeps_every_entry() {
- let mut plan = Plan::default();
- let total = REFUSAL_LIST_CAP * 3;
- for i in 0..total {
- plan.files.insert(
- format!("gone{i:03}"),
- assignment(&format!("gone{i:03}.csv")),
- );
- }
- let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());
- let message = format!("{error:#}");
- assert!(message.contains("gone000.csv"), "{message}");
- assert!(
- message.contains(&format!(", … and {} more", total - REFUSAL_LIST_CAP)),
- "{message}"
- );
- // The tail past the cap must not be rendered at all, not merely elided.
- let last = format!("gone{:03}.csv", total - 1);
- assert!(!message.contains(&last), "{message}");
- assert_eq!(
- message.matches(".csv (").count(),
- REFUSAL_LIST_CAP,
- "{message}"
- );
-
- let stdout = route_cli_error(&error, true)
- .stdout
- .expect("the typed refusal renders as JSON under --json");
- let value: Value = serde_json::from_str(&stdout).unwrap();
- assert_eq!(
- value["vanished_content_groups"].as_array().unwrap().len(),
- total
- );
- }
-
- #[test]
- fn refusal_prose_below_the_cap_has_no_more_tail() {
- let mut plan = Plan::default();
- plan.files.insert("gone".into(), assignment("gone.csv"));
- let error = UnsupportedInventoryDelta::between(&[], &[], &plan).into_error(targets());
- let message = format!("{error:#}");
- assert!(!message.contains("… and "), "{message}");
- }
-
#[test]
fn cli_error_routing_separates_typed_json_from_unrelated_human_errors() {
let typed = UnsupportedInventoryDelta {
- added_content_groups: Vec::new(),
- vanished_content_groups: vec![InventoryDeltaEntry {
+ added_content_groups: vec![InventoryDeltaEntry {
file_key: "key".into(),
- path: "gone.csv".into(),
+ path: "new.csv".into(),
}],
+ vanished_content_groups: Vec::new(),
}
- .into_error(targets());
+ .into_error();
let route = route_cli_error(&typed, true);
assert_eq!(route.exit_code, 1);
assert!(route.stderr.is_none());
let stdout = route.stdout.unwrap();
let value: Value = serde_json::from_str(&stdout).unwrap();
assert_eq!(value["schema"], "xerj.autoindex.unsupported_sync_delta.v1");
- assert_eq!(value["error"], "unsupported_content_group_removal");
- assert_eq!(value["vanished_content_groups"][0]["path"], "gone.csv");
- assert!(value["recovery"]["rebuild_in_place"]
- .as_str()
- .unwrap()
- .contains("ax-rows"));
+ assert_eq!(value["error"], "unsupported_inventory_delta");
let unrelated = anyhow::anyhow!("endpoint unavailable");
let route = route_cli_error(&unrelated, true);
@@ -4029,11 +3801,11 @@ mod duplicate_integration_tests {
.unwrap_err();
let message = format!("{error:#}");
assert!(message.contains("collides with legacy resume key"));
- // Recovery advice must stay scoped to the two colliding files and be
- // honest that discarding the journal re-embeds the whole corpus.
+ // Recovery advice must stay scoped to the two colliding files and
+ // keep an exact rebuild isolated from the old destination.
assert!(message.contains("remove or move one of these two files"));
assert!(message.contains("/state/journal.ndjson"));
- assert!(message.contains("re-extracts and re-embeds the entire corpus"));
+ assert!(message.contains("new --state-dir, new --prefix, and new --brain"));
}
#[test]
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index 992d79530..4abde0d7c 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -14,14 +14,23 @@ use std::path::{Path, PathBuf};
#[cfg(test)]
static FILE_DONE_IO_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
#[cfg(test)]
+static SYNC_IO_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
+#[cfg(test)]
pub(crate) static FILE_DONE_IO_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> =
std::sync::Mutex::new(());
+#[cfg(test)]
+static SYNC_IO_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
#[cfg(test)]
pub(crate) fn fail_next_file_done_io(boundary: u8) {
FILE_DONE_IO_FAILPOINT.store(boundary, std::sync::atomic::Ordering::SeqCst);
}
+#[cfg(test)]
+fn fail_next_sync_io(boundary: u8) {
+ SYNC_IO_FAILPOINT.store(boundary, std::sync::atomic::Ordering::SeqCst);
+}
+
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PlanDataset {
pub slug: String,
@@ -177,6 +186,19 @@ mod tests {
"{tail}"
);
}
+ #[test]
+ fn preflight_holds_the_same_exclusive_lock_through_authoritative_open() {
+ let dir = tempfile::tempdir().unwrap();
+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix").unwrap();
+ let error = Journal::open(dir.path(), "root", "url", "prefix", 300, false)
+ .err()
+ .expect("a second opener must not pass the held preflight lock");
+ assert!(format!("{error:#}").contains("already in use"));
+
+ let journal =
+ Journal::open_after_preflight(preflight, "root", "url", "prefix", 300, false).unwrap();
+ drop(journal);
+ assert!(Journal::open(dir.path(), "root", "url", "prefix", 300, false).is_ok());
}
/// The preflight must never be more fatal than the open it precedes.
@@ -351,6 +373,11 @@ pub struct Journal {
pub plan: Option<Plan>,
pub embedding_identity_sha256: Option<String>,
pub embedding_identity_resumable: Option<bool>,
+ /// Newest corpus reconciliation transaction without a durable
+ /// `sync_commit`. Its desired plan is deliberately not authoritative yet.
+ pub committed_manifest: Option<crate::sync::CommittedManifest>,
+ pub pending_sync: Option<crate::sync::PendingSync>,
+ pub legacy_migration_reasons: Vec<String>,
}
pub fn default_state_dir(root: &str, url: &str, prefix: &str) -> PathBuf {
@@ -361,6 +388,56 @@ pub fn default_state_dir(root: &str, url: &str, prefix: &str) -> PathBuf {
.join(crate::ids::state_key(root, url, prefix))
}
+fn apply_legacy_plan(
+ plan: Plan,
+ committed: &mut Option<crate::sync::CommittedManifest>,
+ migration_reasons: &mut Vec<String>,
+) -> Result<()> {
+ anyhow::ensure!(
+ committed
+ .as_ref()
+ .is_none_or(|manifest| manifest.generation == 0),
+ "legacy plan record cannot follow a committed generated manifest"
+ );
+ match crate::sync::CommittedManifest::bootstrap_legacy(plan)? {
+ crate::sync::LegacyBootstrap::Ready(manifest) => {
+ *committed = Some(*manifest);
+ migration_reasons.clear();
+ }
+ crate::sync::LegacyBootstrap::MigrationRequired { reasons } => {
+ *committed = None;
+ *migration_reasons = reasons;
+ }
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+fn consume_io_failpoint(what: &str, boundary: u8) -> bool {
+ let failpoint = if what == "file_done" {
+ &FILE_DONE_IO_FAILPOINT
+ } else if what.starts_with("sync_") {
+ &SYNC_IO_FAILPOINT
+ } else {
+ return false;
+ };
+ failpoint
+ .compare_exchange(
+ boundary,
+ 0,
+ std::sync::atomic::Ordering::SeqCst,
+ std::sync::atomic::Ordering::SeqCst,
+ )
+ .is_ok()
+}
+
+type PreflightReplay = (
+ Option<Plan>,
+ Option<crate::sync::CommittedManifest>,
+ Option<crate::sync::PendingSync>,
+ Vec<String>,
+);
+
/// Read the last durable plan without locking, repairing, truncating or
/// appending to the journal. This is a preflight hint only: `Journal::open`
/// remains the authority after the caller passes its fail-closed inventory
@@ -370,14 +447,19 @@ fn read_plan_for_preflight(
root: &str,
url: &str,
prefix: &str,
-) -> Result<Option<Plan>> {
+) -> Result<PreflightReplay> {
let file = match std::fs::File::open(path) {
Ok(file) => file,
- Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ return Ok((None, None, None, Vec::new()));
+ }
Err(error) => return Err(error.into()),
};
let mut reader = std::io::BufReader::new(file);
let mut plan = None;
+ let mut committed_manifest: Option<crate::sync::CommittedManifest> = None;
+ let mut pending_sync: Option<crate::sync::PendingSync> = None;
+ let mut legacy_migration_reasons = Vec::new();
let mut offset = 0u64;
loop {
let record_start = offset;
@@ -399,9 +481,10 @@ fn read_plan_for_preflight(
"journal corruption at byte {record_start} in {}: malformed \
newline-terminated record. Refusing to discard later records. Restore \
the journal from a backup, or truncate it to exactly {record_start} \
- bytes to keep every completion recorded before the corruption. Deleting \
- the whole journal (or rerunning with --fresh) also recovers, but \
- re-extracts and re-embeds the entire corpus",
+ bytes to keep every completion recorded before the corruption. For an \
+ isolated rebuild use a new --state-dir, new --prefix, and new --brain \
+ when graph detection is enabled (or --no-graph); explicitly validate \
+ and clean the shared catalog and old target",
path.display()
)
})?;
@@ -424,43 +507,59 @@ fn read_plan_for_preflight(
);
}
Some("plan") => {
+ anyhow::ensure!(
+ pending_sync.is_none(),
+ "legacy plan record cannot appear while a sync is pending"
+ );
if let Some(encoded) = value.get("plan") {
- plan =
- Some(serde_json::from_value(encoded.clone()).with_context(|| {
- format!("decode durable plan in {}", path.display())
- })?);
+ let decoded: Plan = serde_json::from_value(encoded.clone())
+ .with_context(|| format!("decode durable plan in {}", path.display()))?;
+ plan = Some(decoded.clone());
+ apply_legacy_plan(
+ decoded,
+ &mut committed_manifest,
+ &mut legacy_migration_reasons,
+ )?;
}
}
+ Some(kind) if crate::sync::is_sync_record_kind(kind) => {
+ crate::sync::replay_record(&value, &mut committed_manifest, &mut pending_sync)?;
+ if kind == "sync_commit" {
+ plan = committed_manifest
+ .as_ref()
+ .map(|manifest| manifest.plan.clone());
+ }
+ }
+ Some(kind) if kind.starts_with("sync_") => {
+ anyhow::bail!("unknown durable sync record kind {kind}");
+ }
_ => {}
}
}
- Ok(plan)
+ Ok((
+ plan,
+ committed_manifest,
+ pending_sync,
+ legacy_migration_reasons,
+ ))
}
pub struct JournalPreflight {
state_dir: PathBuf,
state_lock: std::fs::File,
pub plan: Option<Plan>,
+ pub committed_manifest: Option<crate::sync::CommittedManifest>,
+ pub pending_sync: Option<crate::sync::PendingSync>,
+ pub legacy_migration_reasons: Vec<String>,
pub journal_exists: bool,
- /// Set only under `--fresh`, when the durable plan could not be read and
- /// the run is about to discard the journal anyway. The caller owns the
- /// decision to print it (autoindex honours `--quiet`).
- pub unreadable_plan: Option<String>,
}
impl Journal {
- /// `fresh` is not a formality here. `open_after_preflight` removes the
- /// journal before replaying it when `--fresh` is set, so nothing inside a
- /// journal can be fatal to that path — and a preflight that hard-errors on
- /// a corrupt record or a root/url/prefix mismatch would make `--fresh`
- /// unreachable in exactly the cases whose error text recommends it. The
- /// preflight must therefore never be more fatal than the open it precedes.
pub fn preflight(
state_dir: &Path,
root: &str,
url: &str,
prefix: &str,
- fresh: bool,
) -> Result<JournalPreflight> {
std::fs::create_dir_all(state_dir)
.with_context(|| format!("create state dir {}", state_dir.display()))?;
@@ -480,22 +579,16 @@ impl Journal {
})?;
let journal_path = state_dir.join("journal.ndjson");
let journal_exists = journal_path.exists();
- let (plan, unreadable_plan) =
- match read_plan_for_preflight(&journal_path, root, url, prefix) {
- Ok(plan) => (plan, None),
- // The journal is about to be deleted unread. Losing the plan here
- // costs the removed-file gate its comparison basis, which is
- // exactly what a full in-place rebuild accepts; the alternative is
- // a state directory that no supported flag can recover.
- Err(error) if fresh => (None, Some(format!("{error:#}"))),
- Err(error) => return Err(error),
- };
+ let (plan, committed_manifest, pending_sync, legacy_migration_reasons) =
+ read_plan_for_preflight(&journal_path, root, url, prefix)?;
Ok(JournalPreflight {
state_dir: state_dir.to_owned(),
state_lock,
plan,
+ committed_manifest,
+ pending_sync,
+ legacy_migration_reasons,
journal_exists,
- unreadable_plan,
})
}
@@ -507,7 +600,7 @@ impl Journal {
bulk_timeout_secs: u64,
fresh: bool,
) -> Result<Journal> {
- let preflight = Self::preflight(state_dir, root, url, prefix, fresh)?;
+ let preflight = Self::preflight(state_dir, root, url, prefix)?;
Self::open_after_preflight(preflight, root, url, prefix, bulk_timeout_secs, fresh)
}
@@ -549,6 +642,9 @@ impl Journal {
let mut plan = None;
let mut embedding_identity_sha256 = None;
let mut embedding_identity_resumable = None;
+ let mut committed_manifest: Option<crate::sync::CommittedManifest> = None;
+ let mut pending_sync: Option<crate::sync::PendingSync> = None;
+ let mut legacy_migration_reasons = Vec::new();
let mut run_id = None;
let mut resumed = false;
if jpath.exists() {
@@ -580,9 +676,10 @@ impl Journal {
anyhow::bail!(
"journal at {} was created for root={jr} url={ju} prefix={jp}; \
current run has root={root} url={url} prefix={prefix}. \
- Use --state-dir for separate state, or --fresh to rebuild the \
- plan in place — note that --fresh never deletes documents \
- already published under the other root, url or prefix.",
+ Refusing to discard history under the same destination. For an \
+ isolated rebuild use a new --state-dir, new --prefix, and new \
+ --brain when graph detection is enabled (or --no-graph); \
+ explicitly validate and clean the shared catalog and old target.",
jpath.display()
);
}
@@ -595,10 +692,32 @@ impl Journal {
resumed = true;
}
Some("plan") => {
+ anyhow::ensure!(
+ pending_sync.is_none(),
+ "legacy plan record at byte {record_start} cannot appear while \
+ a sync is pending"
+ );
if let Some(p) = v.get("plan") {
- if let Ok(p) = serde_json::from_value::<Plan>(p.clone()) {
- plan = Some(p);
- }
+ let decoded: Plan = serde_json::from_value(p.clone())
+ .with_context(|| {
+ format!(
+ "decode durable plan record at byte {record_start} \
+ in {}",
+ jpath.display()
+ )
+ })?;
+ plan = Some(decoded.clone());
+ apply_legacy_plan(
+ decoded,
+ &mut committed_manifest,
+ &mut legacy_migration_reasons,
+ )?;
+ } else {
+ anyhow::bail!(
+ "durable plan record at byte {record_start} in {} has no \
+ plan payload",
+ jpath.display()
+ );
}
}
Some("embedding_identity") => {
@@ -644,6 +763,25 @@ impl Journal {
embedding_identity_sha256 = digest.map(str::to_owned);
embedding_identity_resumable = resumable;
}
+ Some(kind) if crate::sync::is_sync_record_kind(kind) => {
+ crate::sync::replay_record(
+ &v,
+ &mut committed_manifest,
+ &mut pending_sync,
+ )?;
+ if kind == "sync_commit" {
+ plan = committed_manifest
+ .as_ref()
+ .map(|manifest| manifest.plan.clone());
+ }
+ }
+ Some(kind) if kind.starts_with("sync_") => {
+ anyhow::bail!(
+ "unknown durable sync record kind {kind} at byte \
+ {record_start} in {}",
+ jpath.display()
+ );
+ }
Some("file_done") => {
if let Ok(fd) = serde_json::from_value::<FileDone>(v.clone()) {
let commits_pending = pending_replacements
@@ -691,9 +829,10 @@ impl Journal {
after corruption. Restore the journal from a backup, or truncate it \
to exactly {record_start} bytes to keep every completion recorded \
before the corruption (files journaled after that point are \
- re-verified, re-indexed and re-embedded on the next run). Deleting \
- the whole journal (or rerunning with --fresh) also recovers, but \
- re-extracts and re-embeds the entire corpus",
+ re-verified, re-indexed and re-embedded on the next run). For an \
+ isolated rebuild use a new --state-dir, new --prefix, and new --brain \
+ when graph detection is enabled (or --no-graph); explicitly validate \
+ and clean the shared catalog and old target",
jpath.display()
);
}
@@ -723,6 +862,9 @@ impl Journal {
plan,
embedding_identity_sha256,
embedding_identity_resumable,
+ committed_manifest,
+ pending_sync,
+ legacy_migration_reasons,
};
if is_new {
j.append_transaction(
@@ -805,28 +947,11 @@ impl Journal {
line.push('\n');
let start = self.file.metadata()?.len();
#[cfg(test)]
- if what == "file_done"
- && FILE_DONE_IO_FAILPOINT
- .compare_exchange(
- 1,
- 0,
- std::sync::atomic::Ordering::SeqCst,
- std::sync::atomic::Ordering::SeqCst,
- )
- .is_ok()
- {
- anyhow::bail!("injected file_done append failure before any bytes");
+ if consume_io_failpoint(what, 1) {
+ anyhow::bail!("injected {what} append failure before any bytes");
}
#[cfg(test)]
- let partial_write = what == "file_done"
- && FILE_DONE_IO_FAILPOINT
- .compare_exchange(
- 3,
- 0,
- std::sync::atomic::Ordering::SeqCst,
- std::sync::atomic::Ordering::SeqCst,
- )
- .is_ok();
+ let partial_write = consume_io_failpoint(what, 3);
#[cfg(not(test))]
let partial_write = false;
let write_result = if partial_write {
@@ -835,7 +960,7 @@ impl Journal {
.write_all(&line.as_bytes()[..written])
.and_then(|_| {
Err(std::io::Error::other(format!(
- "injected partial file_done write after {written} bytes"
+ "injected partial {what} write after {written} bytes"
)))
})
} else {
@@ -845,19 +970,13 @@ impl Journal {
return self.rollback_transaction(start, what, "append", error);
}
#[cfg(test)]
- let injected_sync_failure = what == "file_done"
- && FILE_DONE_IO_FAILPOINT
- .compare_exchange(
- 2,
- 0,
- std::sync::atomic::Ordering::SeqCst,
- std::sync::atomic::Ordering::SeqCst,
- )
- .is_ok();
+ let injected_sync_failure = consume_io_failpoint(what, 2);
#[cfg(not(test))]
let injected_sync_failure = false;
let sync_result = if injected_sync_failure {
- Err(std::io::Error::other("injected file_done fsync failure"))
+ Err(std::io::Error::other(format!(
+ "injected {what} fsync failure"
+ )))
} else {
self.file.sync_data()
};
@@ -888,8 +1007,134 @@ impl Journal {
}
pub fn write_plan(&mut self, plan: &Plan) -> Result<()> {
+ anyhow::ensure!(
+ self.pending_sync.is_none(),
+ "legacy plan write cannot occur while a sync is pending"
+ );
+ anyhow::ensure!(
+ self.committed_manifest
+ .as_ref()
+ .is_none_or(|manifest| manifest.generation == 0),
+ "legacy plan write cannot follow a committed generated manifest"
+ );
+ let mut committed = self.committed_manifest.clone();
+ let mut migration_reasons = self.legacy_migration_reasons.clone();
+ apply_legacy_plan(plan.clone(), &mut committed, &mut migration_reasons)?;
self.append_transaction(&serde_json::json!({"kind": "plan", "plan": plan}), "plan")?;
self.plan = Some(plan.clone());
+ self.committed_manifest = committed;
+ self.legacy_migration_reasons = migration_reasons;
+ Ok(())
+ }
+
+ pub fn sync_begin(&mut self, sync: &crate::sync::PendingSync) -> Result<()> {
+ anyhow::ensure!(
+ self.pending_sync.is_none(),
+ "cannot begin sync {} while {} is still pending",
+ sync.tx_id,
+ self.pending_sync
+ .as_ref()
+ .map(|pending| pending.tx_id.as_str())
+ .unwrap_or("another sync")
+ );
+ let committed = self
+ .committed_manifest
+ .as_ref()
+ .context("cannot begin sync before a committed plan exists")?;
+ let mut pending = None;
+ crate::sync::apply_begin(committed, &mut pending, sync.clone())?;
+ let mut value = serde_json::to_value(sync)?;
+ value["kind"] = Value::String("sync_begin".into());
+ self.append_transaction(&value, "sync_begin")?;
+ self.pending_sync = pending;
+ Ok(())
+ }
+
+ pub fn sync_operation_state(
+ &mut self,
+ operation_id: &str,
+ state: crate::sync::SyncOperationState,
+ ) -> Result<()> {
+ let pending = self
+ .pending_sync
+ .as_ref()
+ .context("cannot record operation state without sync_begin")?;
+ let tx_id = pending.tx_id.clone();
+ let mut candidate = pending.clone();
+ candidate.apply_operation_state(&tx_id, operation_id, state.clone())?;
+ self.append_transaction(
+ &serde_json::json!({
+ "kind": "sync_operation_state",
+ "tx_id": tx_id,
+ "operation_id": operation_id,
+ "state": state,
+ }),
+ "sync_operation_state",
+ )?;
+ self.pending_sync = Some(candidate);
+ Ok(())
+ }
+
+ pub fn sync_validated(&mut self) -> Result<()> {
+ let pending = self
+ .pending_sync
+ .as_ref()
+ .context("cannot validate sync without sync_begin")?;
+ let tx_id = pending.tx_id.clone();
+ let digest = pending.desired_manifest_digest.clone();
+ let mut candidate = pending.clone();
+ candidate.apply_validated(&tx_id, &digest)?;
+ self.append_transaction(
+ &serde_json::json!({
+ "kind": "sync_validated",
+ "tx_id": tx_id,
+ "desired_manifest_digest": digest,
+ }),
+ "sync_validated",
+ )?;
+ self.pending_sync = Some(candidate);
+ Ok(())
+ }
+
+ pub fn sync_commit(&mut self) -> Result<()> {
+ let pending = self
+ .pending_sync
+ .as_ref()
+ .context("cannot commit sync without sync_begin")?;
+ let tx_id = pending.tx_id.clone();
+ let digest = pending.desired_manifest_digest.clone();
+ let committed = pending.clone().apply_commit(&tx_id, &digest)?;
+ self.append_transaction(
+ &serde_json::json!({
+ "kind": "sync_commit",
+ "tx_id": tx_id,
+ "desired_manifest_digest": digest,
+ }),
+ "sync_commit",
+ )?;
+ self.pending_sync = None;
+ self.plan = Some(committed.plan.clone());
+ self.committed_manifest = Some(committed);
+ Ok(())
+ }
+
+ pub fn sync_abort(&mut self, reason: &str) -> Result<()> {
+ let pending = self
+ .pending_sync
+ .as_ref()
+ .context("cannot abort without pending sync")?;
+ let tx_id = pending.tx_id.clone();
+ let digest = pending.desired_manifest_digest.clone();
+ self.append_transaction(
+ &serde_json::json!({
+ "kind": "sync_abort",
+ "tx_id": tx_id,
+ "desired_manifest_digest": digest,
+ "reason": reason,
+ }),
+ "sync_abort",
+ )?;
+ self.pending_sync = None;
Ok(())
}
@@ -947,6 +1192,401 @@ impl Journal {
}
}
+#[cfg(test)]
+mod sync_journal_tests {
+ use super::*;
+ use crate::sync::{
+ ExecutionIdentity, GenerationManifest, ManifestGroup, ManifestPath, PendingSync,
+ SourceExecutionPolicy, SyncOperationState, EXECUTION_IDENTITY_VERSION,
+ };
+
+ fn journal_with_plan(dir: &Path) -> Journal {
+ let mut journal = Journal::open(dir, "root", "url", "prefix", 300, false).unwrap();
+ let mut plan = Plan::default();
+ plan.datasets.push(PlanDataset {
+ slug: "reports".into(),
+ index: "prefix-reports".into(),
+ family: "pdf".into(),
+ group: None,
+ specs: Vec::new(),
+ time_field: None,
+ semantic_field: None,
+ sampled_records: 1,
+ file_count: 0,
+ });
+ journal.write_plan(&plan).unwrap();
+ journal
+ }
+
+ fn pending(journal: &Journal) -> PendingSync {
+ let base = journal.committed_manifest.as_ref().unwrap();
+ let mut plan = base.plan.clone();
+ plan.datasets[0].file_count = 1;
+ plan.files.insert(
+ "content-a".into(),
+ FileAssignment {
+ rel: "a.pdf".into(),
+ path_id: "unix:61".into(),
+ family: "pdf".into(),
+ gzip: false,
+ content_digest: Some("digest-a".into()),
+ assignments: vec![(None, "reports".into())],
+ },
+ );
+ PendingSync::new(
+ "sync-1".into(),
+ base,
+ GenerationManifest {
+ generation: base.generation + 1,
+ execution: Some(ExecutionIdentity {
+ version: EXECUTION_IDENTITY_VERSION,
+ root_identity: "unix:root".into(),
+ url: "url".into(),
+ prefix: "prefix".into(),
+ follow_symlinks: false,
+ chunker_identity: "chunker-v1".into(),
+ embedding_backend: "lexical".into(),
+ embedding_model: "feature-hash".into(),
+ embedding_tokenizer: "builtin".into(),
+ embedding_dimension: 384,
+ graph_enabled: false,
+ brain: "none".into(),
+ detector_identity: "detectors-v1".into(),
+ schema_identity: "schema-v1".into(),
+ index_identity: "index-v1".into(),
+ source_policy: SourceExecutionPolicy::AbortOnSourceChange {
+ inventory_digest: "inventory-v1".into(),
+ },
+ }),
+ plan,
+ groups: vec![ManifestGroup {
+ group_id: "group-a".into(),
+ content_id: "content-a".into(),
+ content_digest: "digest-a".into(),
+ content_size: 10,
+ canonical: ManifestPath {
+ path_id: "unix:61".into(),
+ rel: "a.pdf".into(),
+ is_symlink: false,
+ },
+ aliases: Vec::new(),
+ dataset_slugs: vec!["reports".into()],
+ expected_records: 1,
+ expected_passages: 1,
+ expected_vectors: 1,
+ }],
+ },
+ )
+ .unwrap()
+ }
+
+ #[test]
+ fn sync_begin_is_pending_and_never_promotes_the_committed_manifest() {
+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = journal_with_plan(dir.path());
+ let desired = pending(&journal);
+ journal.sync_begin(&desired).unwrap();
+ let before = std::fs::read(dir.path().join("journal.ndjson")).unwrap();
+ assert!(journal.write_plan(&Plan::default()).is_err());
+ assert_eq!(
+ std::fs::read(dir.path().join("journal.ndjson")).unwrap(),
+ before
+ );
+ assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 0);
+ assert!(!journal.plan.as_ref().unwrap().alias_paths_indexed);
+ drop(journal);
+
+ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);
+ assert!(!reopened.plan.as_ref().unwrap().alias_paths_indexed);
+ assert_eq!(reopened.pending_sync.as_ref().unwrap().tx_id, "sync-1");
+ }
+
+ #[test]
+ fn replay_rejects_sync_begin_followed_by_legacy_plan_without_changing_authority() {
+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = journal_with_plan(dir.path());
+ journal.sync_begin(&pending(&journal)).unwrap();
+ let committed_digest = journal
+ .committed_manifest
+ .as_ref()
+ .unwrap()
+ .manifest_digest
+ .clone();
+ drop(journal);
+ let path = dir.path().join("journal.ndjson");
+ let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();
+ file.write_all(
+ format!(
+ "{}\n",
+ serde_json::json!({"kind": "plan", "plan": Plan::default()})
+ )
+ .as_bytes(),
+ )
+ .unwrap();
+ file.sync_data().unwrap();
+ drop(file);
+ let error = Journal::open(dir.path(), "root", "url", "prefix", 300, false)
+ .err()
+ .unwrap();
+ assert!(format!("{error:#}").contains("while a sync is pending"));
+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix")
+ .err()
+ .unwrap();
+ assert!(format!("{preflight:#}").contains("while a sync is pending"));
+ // The rejected record cannot manufacture a sync_commit; the durable
+ // committed digest remains present only in the earlier sync_begin base.
+ let text = std::fs::read_to_string(dir.path().join("journal.ndjson")).unwrap();
+ assert!(text.contains(&committed_digest));
+ assert!(!text.contains("\"kind\":\"sync_commit\""));
+ }
+
+ #[test]
+ fn only_validated_all_committed_sync_advances_authority_after_restart() {
+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = journal_with_plan(dir.path());
+ journal.sync_begin(&pending(&journal)).unwrap();
+ assert!(journal.sync_validated().is_err());
+ assert!(journal
+ .sync_operation_state("upsert:group-a", SyncOperationState::Committed)
+ .is_err());
+ journal
+ .sync_operation_state("upsert:group-a", SyncOperationState::Started)
+ .unwrap();
+ journal
+ .sync_operation_state("upsert:group-a", SyncOperationState::Committed)
+ .unwrap();
+ journal.sync_validated().unwrap();
+ journal.sync_commit().unwrap();
+ assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 1);
+ assert!(journal
+ .plan
+ .as_ref()
+ .unwrap()
+ .files
+ .contains_key("content-a"));
+ assert!(journal.pending_sync.is_none());
+ let before = std::fs::read(dir.path().join("journal.ndjson")).unwrap();
+ assert!(journal.write_plan(&Plan::default()).is_err());
+ assert_eq!(
+ std::fs::read(dir.path().join("journal.ndjson")).unwrap(),
+ before
+ );
+ drop(journal);
+
+ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 1);
+ assert!(reopened
+ .plan
+ .as_ref()
+ .unwrap()
+ .files
+ .contains_key("content-a"));
+ assert!(reopened.pending_sync.is_none());
+ }
+
+ #[test]
+ fn digest_bound_abort_discards_pending_without_advancing_authority() {
+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = journal_with_plan(dir.path());
+ journal.sync_begin(&pending(&journal)).unwrap();
+ journal.sync_abort("source changed").unwrap();
+ assert!(journal.pending_sync.is_none());
+ assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 0);
+ drop(journal);
+ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ assert!(reopened.pending_sync.is_none());
+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);
+ }
+
+ #[test]
+ fn unknown_sync_namespace_record_fails_closed_but_preflight_exposes_pending() {
+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = journal_with_plan(dir.path());
+ journal.sync_begin(&pending(&journal)).unwrap();
+ drop(journal);
+ let preflight = Journal::preflight(dir.path(), "root", "url", "prefix").unwrap();
+ assert_eq!(preflight.committed_manifest.as_ref().unwrap().generation, 0);
+ assert_eq!(preflight.pending_sync.as_ref().unwrap().tx_id, "sync-1");
+ drop(preflight);
+
+ let path = dir.path().join("journal.ndjson");
+ let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();
+ file.write_all(b"{\"kind\":\"sync_future_v99\"}\n").unwrap();
+ file.sync_data().unwrap();
+ drop(file);
+ assert!(Journal::open(dir.path(), "root", "url", "prefix", 300, false).is_err());
+ }
+
+ #[test]
+ fn out_of_order_or_malformed_known_sync_records_fail_closed() {
+ for record in [
+ "{\"kind\":\"sync_commit\",\"tx_id\":\"none\",\"desired_manifest_digest\":\"none\"}\n",
+ "{\"kind\":\"sync_begin\"}\n",
+ ] {
+ let dir = tempfile::tempdir().unwrap();
+ drop(journal_with_plan(dir.path()));
+ let path = dir.path().join("journal.ndjson");
+ let mut file = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&path)
+ .unwrap();
+ file.write_all(record.as_bytes()).unwrap();
+ file.sync_data().unwrap();
+ let error = Journal::open(dir.path(), "root", "url", "prefix", 300, false)
+ .err()
+ .unwrap();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("pending sync") || message.contains("sync_begin"),
+ "{message}"
+ );
+ }
+ }
+
+ #[test]
+ fn torn_sync_tail_is_removed_without_creating_a_pending_generation() {
+ let dir = tempfile::tempdir().unwrap();
+ drop(journal_with_plan(dir.path()));
+ let path = dir.path().join("journal.ndjson");
+ let durable_len = std::fs::metadata(&path).unwrap().len();
+ let mut file = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&path)
+ .unwrap();
+ file.write_all(br#"{"kind":"sync_begin","tx_id":"torn""#)
+ .unwrap();
+ file.sync_data().unwrap();
+ drop(file);
+
+ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ assert!(reopened.pending_sync.is_none());
+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);
+ // Reopen appends one resume record, but the torn bytes are absent.
+ let contents = std::fs::read_to_string(path).unwrap();
+ assert!(!contents.contains("\"tx_id\":\"torn\""));
+ assert!(contents.len() as u64 > durable_len);
+ }
+
+ #[test]
+ fn complete_middle_corruption_is_not_truncated_or_skipped() {
+ let dir = tempfile::tempdir().unwrap();
+ drop(journal_with_plan(dir.path()));
+ let path = dir.path().join("journal.ndjson");
+ let before = std::fs::metadata(&path).unwrap().len();
+ let mut file = std::fs::OpenOptions::new()
+ .append(true)
+ .open(&path)
+ .unwrap();
+ file.write_all(b"{malformed sync record}\n").unwrap();
+ file.write_all(b"{\"kind\":\"unknown-after-corruption\"}\n")
+ .unwrap();
+ file.sync_data().unwrap();
+ drop(file);
+ assert!(Journal::open(dir.path(), "root", "url", "prefix", 300, false).is_err());
+ assert!(std::fs::metadata(path).unwrap().len() > before);
+ }
+
+ #[test]
+ fn unknown_future_records_do_not_change_legacy_completion_replay() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = journal_with_plan(dir.path());
+ journal
+ .file_done(&FileDone {
+ file_key: "legacy".into(),
+ path: "legacy.pdf".into(),
+ records: 1,
+ junk: 0,
+ bytes: 10,
+ generation: None,
+ })
+ .unwrap();
+ drop(journal);
+ let path = dir.path().join("journal.ndjson");
+ let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();
+ file.write_all(b"{\"kind\":\"extension_future_v99\",\"payload\":{\"x\":1}}\n")
+ .unwrap();
+ file.sync_data().unwrap();
+ drop(file);
+
+ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ assert!(reopened.done.contains_key("legacy"));
+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);
+ }
+
+ #[test]
+ fn sync_append_and_fsync_failures_roll_back_without_phantom_pending_state() {
+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ for boundary in [1, 2, 3] {
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = journal_with_plan(dir.path());
+ let desired = pending(&journal);
+ let before = std::fs::read(dir.path().join("journal.ndjson")).unwrap();
+ fail_next_sync_io(boundary);
+ assert!(journal.sync_begin(&desired).is_err());
+ assert!(journal.pending_sync.is_none());
+ drop(journal);
+ assert_eq!(
+ std::fs::read(dir.path().join("journal.ndjson")).unwrap(),
+ before
+ );
+ let reopened = Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ assert!(reopened.pending_sync.is_none());
+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);
+ }
+ }
+
+ #[test]
+ fn every_later_sync_record_rolls_back_append_partial_and_fsync_failures() {
+ let _guard = SYNC_IO_FAILPOINT_TEST_LOCK.lock().unwrap();
+ for stage in ["operation", "validated", "commit", "abort"] {
+ for boundary in [1, 2, 3] {
+ let dir = tempfile::tempdir().unwrap();
+ let mut journal = journal_with_plan(dir.path());
+ journal.sync_begin(&pending(&journal)).unwrap();
+ if stage != "operation" {
+ journal
+ .sync_operation_state("upsert:group-a", SyncOperationState::Started)
+ .unwrap();
+ journal
+ .sync_operation_state("upsert:group-a", SyncOperationState::Committed)
+ .unwrap();
+ }
+ if matches!(stage, "commit") {
+ journal.sync_validated().unwrap();
+ }
+ let before = std::fs::read(dir.path().join("journal.ndjson")).unwrap();
+ fail_next_sync_io(boundary);
+ let result = match stage {
+ "operation" => {
+ journal.sync_operation_state("upsert:group-a", SyncOperationState::Started)
+ }
+ "validated" => journal.sync_validated(),
+ "commit" => journal.sync_commit(),
+ "abort" => journal.sync_abort("test abort"),
+ _ => unreachable!(),
+ };
+ assert!(result.is_err(), "{stage} boundary {boundary}");
+ assert_eq!(
+ std::fs::read(dir.path().join("journal.ndjson")).unwrap(),
+ before,
+ "{stage} boundary {boundary}"
+ );
+ drop(journal);
+ let reopened =
+ Journal::open(dir.path(), "root", "url", "prefix", 300, false).unwrap();
+ assert_eq!(reopened.committed_manifest.as_ref().unwrap().generation, 0);
+ assert!(reopened.pending_sync.is_some());
+ }
+ }
+ }
+}
+
#[cfg(test)]
mod compatibility_tests {
use super::*;
@@ -1123,9 +1763,10 @@ mod compatibility_tests {
let message = format!("{error:#}");
assert!(message.contains("journal corruption"));
// Recovery guidance must stay scoped and honest: byte-exact truncation
- // keeps prior completions; discarding the journal re-embeds everything.
+ // keeps prior completions; an exact rebuild needs isolated state and
+ // destination identities.
assert!(message.contains("truncate it to exactly"));
- assert!(message.contains("re-extracts and re-embeds the entire corpus"));
+ assert!(message.contains("new --state-dir, new --prefix, and new --brain"));
}
#[test]
diff --git a/engine/crates/xerj-autoindex/src/sync.rs b/engine/crates/xerj-autoindex/src/sync.rs
new file mode 100644
index 000000000..bebce6e84
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/sync.rs
@@ -0,0 +1,1197 @@
+//! Pure, inert corpus-generation planning and durable replay.
+//!
+//! Nothing in this module performs HTTP I/O or changes autoindex behaviour.
+//! It defines the complete state an eventual synchronization executor must
+//! durably bind before publication.
+
+use crate::state::{DuplicateFile, Plan};
+use anyhow::{Context, Result};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+
+pub const EXECUTION_IDENTITY_VERSION: u32 = 1;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ManifestPath {
+ /// Reversible platform-native path identity (`unix:`/`windows:` encoding).
+ pub path_id: String,
+ pub rel: String,
+ /// Canonical election ranks real paths before followed symlinks.
+ pub is_symlink: bool,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ManifestGroup {
+ /// Stable logical identity across rename/replacement.
+ pub group_id: String,
+ /// Collision-safe document identity assigned only after byte verification.
+ pub content_id: String,
+ /// Raw full-content digest. Never treated as globally unique by itself.
+ pub content_digest: String,
+ pub content_size: u64,
+ pub canonical: ManifestPath,
+ #[serde(default)]
+ pub aliases: Vec<ManifestPath>,
+ #[serde(default)]
+ pub dataset_slugs: Vec<String>,
+ pub expected_records: u64,
+ pub expected_passages: u64,
+ pub expected_vectors: u64,
+}
+
+impl ManifestGroup {
+ fn all_paths(&self) -> impl Iterator<Item = &ManifestPath> {
+ std::iter::once(&self.canonical).chain(self.aliases.iter())
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case", tag = "policy")]
+pub enum SourceExecutionPolicy {
+ DurableSnapshot {
+ reference: String,
+ snapshot_digest: String,
+ },
+ AbortOnSourceChange {
+ inventory_digest: String,
+ },
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ExecutionIdentity {
+ pub version: u32,
+ pub root_identity: String,
+ pub url: String,
+ pub prefix: String,
+ pub follow_symlinks: bool,
+ pub chunker_identity: String,
+ pub embedding_backend: String,
+ pub embedding_model: String,
+ pub embedding_tokenizer: String,
+ pub embedding_dimension: usize,
+ pub graph_enabled: bool,
+ pub brain: String,
+ pub detector_identity: String,
+ pub schema_identity: String,
+ pub index_identity: String,
+ pub source_policy: SourceExecutionPolicy,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GenerationManifest {
+ pub generation: u64,
+ /// Missing only for a legacy generation-0 plan.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub execution: Option<ExecutionIdentity>,
+ pub plan: Plan,
+ pub groups: Vec<ManifestGroup>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CommittedManifest {
+ pub generation: u64,
+ pub manifest_digest: String,
+ pub plan: Plan,
+ pub groups: Vec<ManifestGroup>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub execution: Option<ExecutionIdentity>,
+}
+
+#[derive(Debug, Clone)]
+pub enum LegacyBootstrap {
+ Ready(Box<CommittedManifest>),
+ MigrationRequired { reasons: Vec<String> },
+}
+
+impl CommittedManifest {
+ pub fn bootstrap_legacy(plan: Plan) -> Result<LegacyBootstrap> {
+ let mut reasons = Vec::new();
+ for (file_key, file) in &plan.files {
+ if file.content_digest.is_none() {
+ reasons.push(format!("{file_key}: missing full content digest"));
+ }
+ if !native_path_id_supported(&file.path_id) {
+ reasons.push(format!(
+ "{file_key}: missing or lossy native canonical path identity"
+ ));
+ }
+ // Legacy plans did not persist whether the canonical path was a
+ // followed symlink, so canonical rank cannot be reconstructed.
+ reasons.push(format!(
+ "{file_key}: legacy plan did not persist canonical symlink rank"
+ ));
+ }
+ for alias in &plan.duplicate_files {
+ if !native_path_id_supported(&alias.path_id) {
+ reasons.push(format!(
+ "{}: missing or lossy native alias path identity",
+ alias.rel
+ ));
+ }
+ }
+ reasons.sort();
+ reasons.dedup();
+ if !reasons.is_empty() {
+ return Ok(LegacyBootstrap::MigrationRequired { reasons });
+ }
+ let manifest = GenerationManifest {
+ generation: 0,
+ execution: None,
+ plan: plan.clone(),
+ groups: Vec::new(),
+ };
+ Ok(LegacyBootstrap::Ready(Box::new(Self {
+ generation: 0,
+ manifest_digest: manifest_digest(&manifest)?,
+ plan,
+ groups: Vec::new(),
+ execution: None,
+ })))
+ }
+
+ fn from_generation(manifest: GenerationManifest, digest: String) -> Self {
+ Self {
+ generation: manifest.generation,
+ manifest_digest: digest,
+ plan: manifest.plan,
+ groups: manifest.groups,
+ execution: manifest.execution,
+ }
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(rename_all = "snake_case")]
+pub enum SyncOperationKind {
+ Upsert,
+ Delete,
+ Metadata,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct SyncOperation {
+ pub operation_id: String,
+ pub kind: SyncOperationKind,
+ pub group_id: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub desired_content_id: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum SyncOperationState {
+ Started,
+ Committed,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PendingSync {
+ pub tx_id: String,
+ pub base_generation: u64,
+ pub base_manifest_digest: String,
+ pub desired_manifest_digest: String,
+ pub operation_hash: String,
+ pub desired: GenerationManifest,
+ pub operations: Vec<SyncOperation>,
+ #[serde(default)]
+ pub operation_states: BTreeMap<String, SyncOperationState>,
+ #[serde(default)]
+ pub validated: bool,
+}
+
+impl PendingSync {
+ pub fn new(
+ tx_id: String,
+ base: &CommittedManifest,
+ mut desired: GenerationManifest,
+ ) -> Result<Self> {
+ canonicalize_manifest_order(&mut desired);
+ validate_manifest(&desired, false)?;
+ validate_supported_manifest_delta(base, &desired)?;
+ let operations = plan_operations(&base.groups, &desired.groups);
+ let pending = Self {
+ tx_id,
+ base_generation: base.generation,
+ base_manifest_digest: base.manifest_digest.clone(),
+ desired_manifest_digest: manifest_digest(&desired)?,
+ operation_hash: operation_hash(&operations)?,
+ desired,
+ operations,
+ operation_states: BTreeMap::new(),
+ validated: false,
+ };
+ pending.validate_against(base)?;
+ Ok(pending)
+ }
+
+ pub fn validate_against(&self, base: &CommittedManifest) -> Result<()> {
+ anyhow::ensure!(
+ self.base_generation == base.generation
+ && self.base_manifest_digest == base.manifest_digest,
+ "sync_begin base does not match committed authority"
+ );
+ anyhow::ensure!(
+ self.desired.generation == self.base_generation + 1,
+ "desired generation must immediately follow its base"
+ );
+ validate_manifest(&self.desired, false)?;
+ validate_supported_manifest_delta(base, &self.desired)?;
+ anyhow::ensure!(
+ self.desired_manifest_digest == manifest_digest(&self.desired)?,
+ "desired manifest digest does not match sync_begin payload"
+ );
+ let expected = plan_operations(&base.groups, &self.desired.groups);
+ anyhow::ensure!(
+ self.operations == expected,
+ "sync operation list was not derived exactly from base and desired manifests"
+ );
+ anyhow::ensure!(
+ self.operation_hash == operation_hash(&expected)?,
+ "sync operation hash does not match derived operation list"
+ );
+ let operation_ids: HashSet<&str> = expected
+ .iter()
+ .map(|operation| operation.operation_id.as_str())
+ .collect();
+ anyhow::ensure!(
+ self.operation_states
+ .keys()
+ .all(|operation_id| operation_ids.contains(operation_id.as_str())),
+ "sync state references an operation absent from sync_begin"
+ );
+ Ok(())
+ }
+
+ pub fn apply_operation_state(
+ &mut self,
+ tx_id: &str,
+ operation_id: &str,
+ state: SyncOperationState,
+ ) -> Result<()> {
+ anyhow::ensure!(
+ self.tx_id == tx_id,
+ "operation state belongs to another sync"
+ );
+ anyhow::ensure!(
+ self.operations
+ .iter()
+ .any(|operation| operation.operation_id == operation_id),
+ "operation state references unknown operation {operation_id}"
+ );
+ let legal = match self.operation_states.get(operation_id) {
+ None => state == SyncOperationState::Started,
+ Some(previous) => {
+ previous == &state
+ || previous == &SyncOperationState::Started
+ && state == SyncOperationState::Committed
+ }
+ };
+ anyhow::ensure!(
+ legal,
+ "illegal operation transition {:?} -> {state:?}",
+ self.operation_states.get(operation_id)
+ );
+ self.operation_states
+ .insert(operation_id.to_string(), state);
+ Ok(())
+ }
+
+ pub fn all_operations_committed(&self) -> bool {
+ self.operations.iter().all(|operation| {
+ self.operation_states.get(&operation.operation_id)
+ == Some(&SyncOperationState::Committed)
+ })
+ }
+
+ pub fn apply_validated(&mut self, tx_id: &str, digest: &str) -> Result<()> {
+ anyhow::ensure!(self.tx_id == tx_id, "validation belongs to another sync");
+ anyhow::ensure!(
+ self.desired_manifest_digest == digest,
+ "validation manifest digest does not match sync_begin"
+ );
+ anyhow::ensure!(
+ self.all_operations_committed(),
+ "cannot validate sync with unfinished operations"
+ );
+ self.validated = true;
+ Ok(())
+ }
+
+ pub fn apply_commit(self, tx_id: &str, digest: &str) -> Result<CommittedManifest> {
+ anyhow::ensure!(self.tx_id == tx_id, "commit belongs to another sync");
+ anyhow::ensure!(
+ self.desired_manifest_digest == digest,
+ "commit manifest digest does not match sync_begin"
+ );
+ anyhow::ensure!(self.validated, "cannot commit an unvalidated sync");
+ anyhow::ensure!(
+ self.all_operations_committed(),
+ "cannot commit sync with unfinished operations"
+ );
+ Ok(CommittedManifest::from_generation(
+ self.desired,
+ self.desired_manifest_digest,
+ ))
+ }
+}
+
+fn validate_supported_manifest_delta(
+ base: &CommittedManifest,
+ desired: &GenerationManifest,
+) -> Result<()> {
+ // Binding complete identity while migrating the legacy generation zero is
+ // the only identity transition this vocabulary can represent. Once bound,
+ // backend/model/source-policy changes require a future explicit operation.
+ if base.generation > 0 || base.execution.is_some() {
+ anyhow::ensure!(
+ base.execution == desired.execution,
+ "execution identity or source policy changed without a supported operation"
+ );
+ }
+ anyhow::ensure!(
+ base.plan.alias_paths_indexed == desired.plan.alias_paths_indexed,
+ "plan flags changed without a supported operation"
+ );
+ anyhow::ensure!(
+ canonical_json_bytes(&base.plan.junk_files)?
+ == canonical_json_bytes(&desired.plan.junk_files)?,
+ "junk plan changed without a supported operation"
+ );
+ let normalize_datasets = |datasets: &[crate::state::PlanDataset]| {
+ let mut datasets = datasets.to_vec();
+ for dataset in &mut datasets {
+ // Membership operations legitimately change this projection.
+ dataset.file_count = 0;
+ }
+ datasets.sort_by(|left, right| {
+ left.slug
+ .cmp(&right.slug)
+ .then_with(|| left.index.cmp(&right.index))
+ .then_with(|| left.group.cmp(&right.group))
+ });
+ for dataset in &mut datasets {
+ dataset
+ .specs
+ .sort_by(|left, right| left.name.cmp(&right.name));
+ }
+ datasets
+ };
+ anyhow::ensure!(
+ canonical_json_bytes(&normalize_datasets(&base.plan.datasets))?
+ == canonical_json_bytes(&normalize_datasets(&desired.plan.datasets))?,
+ "dataset schema/index plan changed without a schema-generation operation"
+ );
+ Ok(())
+}
+
+pub fn plan_operations(base: &[ManifestGroup], desired: &[ManifestGroup]) -> Vec<SyncOperation> {
+ let base: BTreeMap<&str, &ManifestGroup> = base
+ .iter()
+ .map(|group| (group.group_id.as_str(), group))
+ .collect();
+ let desired: BTreeMap<&str, &ManifestGroup> = desired
+ .iter()
+ .map(|group| (group.group_id.as_str(), group))
+ .collect();
+ let mut operations = Vec::new();
+ for (group_id, old) in &base {
+ match desired.get(group_id) {
+ None => operations.push(operation(SyncOperationKind::Delete, group_id, None)),
+ Some(new)
+ if old.content_id != new.content_id
+ || old.content_digest != new.content_digest
+ || old.content_size != new.content_size =>
+ {
+ operations.push(operation(
+ SyncOperationKind::Upsert,
+ group_id,
+ Some(new.content_id.clone()),
+ ));
+ }
+ Some(new) if *old != *new => operations.push(operation(
+ SyncOperationKind::Metadata,
+ group_id,
+ Some(new.content_id.clone()),
+ )),
+ Some(_) => {}
+ }
+ }
+ for (group_id, new) in desired {
+ if !base.contains_key(group_id) {
+ operations.push(operation(
+ SyncOperationKind::Upsert,
+ group_id,
+ Some(new.content_id.clone()),
+ ));
+ }
+ }
+ operations.sort();
+ operations
+}
+
+fn operation(
+ kind: SyncOperationKind,
+ group_id: &str,
+ desired_content_id: Option<String>,
+) -> SyncOperation {
+ SyncOperation {
+ operation_id: format!("{kind:?}:{group_id}").to_ascii_lowercase(),
+ kind,
+ group_id: group_id.to_string(),
+ desired_content_id,
+ }
+}
+
+#[derive(Debug, Clone)]
+#[cfg_attr(not(test), allow(dead_code))]
+pub struct DesiredContentGroup {
+ pub content_id: String,
+ pub content_digest: String,
+ pub content_size: u64,
+ pub paths: Vec<ManifestPath>,
+ pub dataset_slugs: Vec<String>,
+ pub expected_records: u64,
+ pub expected_passages: u64,
+ pub expected_vectors: u64,
+}
+
+/// Content identity is collision-safe and byte-verified by the inventory
+/// layer. Raw digest equality alone never merges groups.
+#[cfg_attr(not(test), allow(dead_code))]
+pub fn reconcile_groups(
+ previous: &[ManifestGroup],
+ mut desired: Vec<DesiredContentGroup>,
+) -> Result<Vec<ManifestGroup>> {
+ validate_groups(previous)?;
+ desired.sort_by(|left, right| {
+ left.content_id
+ .cmp(&right.content_id)
+ .then_with(|| canonical_path(&left.paths).cmp(&canonical_path(&right.paths)))
+ });
+ let old_by_content: HashMap<&str, &ManifestGroup> = previous
+ .iter()
+ .map(|group| (group.content_id.as_str(), group))
+ .collect();
+ anyhow::ensure!(
+ old_by_content.len() == previous.len(),
+ "collision-safe content IDs must be unique"
+ );
+ let surviving_content: HashSet<String> = desired
+ .iter()
+ .map(|group| group.content_id.clone())
+ .collect();
+ let mut old_by_path = HashMap::new();
+ for group in previous {
+ for path in group.all_paths() {
+ anyhow::ensure!(
+ old_by_path.insert(path.path_id.as_str(), group).is_none(),
+ "native path identity belongs to multiple committed groups"
+ );
+ }
+ }
+ let mut claimed = HashSet::new();
+ let mut seen_paths = HashSet::new();
+ let mut result = Vec::new();
+ for mut candidate in desired {
+ normalize_paths(&mut candidate.paths, &mut seen_paths)?;
+ let canonical = candidate.paths.remove(0);
+ let aliases = candidate.paths;
+ let content_owner = old_by_content.get(candidate.content_id.as_str()).copied();
+ let mut path_owners: BTreeSet<&str> = std::iter::once(&canonical)
+ .chain(aliases.iter())
+ .filter_map(|path| old_by_path.get(path.path_id.as_str()).copied())
+ .filter(|owner| !surviving_content.contains(owner.content_id.as_str()))
+ .map(|owner| owner.group_id.as_str())
+ .collect();
+ let group_id = if let Some(owner) = content_owner {
+ anyhow::ensure!(
+ path_owners.is_empty() || path_owners.contains(owner.group_id.as_str()),
+ "desired group has conflicting content and path identities"
+ );
+ owner.group_id.clone()
+ } else if path_owners.len() == 1 {
+ path_owners.pop_first().unwrap().to_string()
+ } else if path_owners.is_empty() {
+ format!(
+ "axg1-{:032x}",
+ xxhash_rust::xxh3::xxh3_128(
+ format!("{}\0{}", candidate.content_id, canonical.path_id).as_bytes()
+ )
+ )
+ } else {
+ anyhow::bail!("desired content merges multiple historical groups");
+ };
+ anyhow::ensure!(claimed.insert(group_id.clone()), "group claimed twice");
+ candidate.dataset_slugs.sort();
+ candidate.dataset_slugs.dedup();
+ result.push(ManifestGroup {
+ group_id,
+ content_id: candidate.content_id,
+ content_digest: candidate.content_digest,
+ content_size: candidate.content_size,
+ canonical,
+ aliases,
+ dataset_slugs: candidate.dataset_slugs,
+ expected_records: candidate.expected_records,
+ expected_passages: candidate.expected_passages,
+ expected_vectors: candidate.expected_vectors,
+ });
+ }
+ result.sort_by(|left, right| left.group_id.cmp(&right.group_id));
+ Ok(result)
+}
+
+#[cfg_attr(not(test), allow(dead_code))]
+fn canonical_path(paths: &[ManifestPath]) -> (bool, &str) {
+ paths
+ .iter()
+ .map(|path| (path.is_symlink, path.path_id.as_str()))
+ .min()
+ .unwrap_or((true, ""))
+}
+
+#[cfg_attr(not(test), allow(dead_code))]
+fn normalize_paths(paths: &mut [ManifestPath], seen: &mut HashSet<String>) -> Result<()> {
+ anyhow::ensure!(!paths.is_empty(), "desired content group has no paths");
+ for path in paths.iter() {
+ anyhow::ensure!(
+ native_path_id_supported(&path.path_id),
+ "path {} has no supported reversible native identity",
+ path.rel
+ );
+ anyhow::ensure!(
+ seen.insert(path.path_id.clone()),
+ "native path identity occurs more than once"
+ );
+ }
+ paths.sort_by(|left, right| {
+ (left.is_symlink, left.path_id.as_str()).cmp(&(right.is_symlink, right.path_id.as_str()))
+ });
+ Ok(())
+}
+
+fn native_path_id_supported(path_id: &str) -> bool {
+ let valid_hex = |hex: &str, width: usize| {
+ !hex.is_empty()
+ && hex.len().is_multiple_of(width)
+ && hex.bytes().all(|byte| byte.is_ascii_hexdigit())
+ };
+ path_id
+ .strip_prefix("unix:")
+ .is_some_and(|hex| valid_hex(hex, 2))
+ || path_id
+ .strip_prefix("windows:")
+ .is_some_and(|hex| valid_hex(hex, 4))
+}
+
+fn validate_manifest(manifest: &GenerationManifest, legacy: bool) -> Result<()> {
+ if manifest.generation == 0 && legacy {
+ anyhow::ensure!(
+ manifest.execution.is_none(),
+ "legacy generation zero cannot invent execution identity"
+ );
+ } else {
+ let execution = manifest
+ .execution
+ .as_ref()
+ .context("new generation requires complete execution identity")?;
+ anyhow::ensure!(
+ execution.version == EXECUTION_IDENTITY_VERSION,
+ "unsupported execution identity version"
+ );
+ for (name, value) in [
+ ("root_identity", execution.root_identity.as_str()),
+ ("url", execution.url.as_str()),
+ ("prefix", execution.prefix.as_str()),
+ ("chunker_identity", execution.chunker_identity.as_str()),
+ ("embedding_backend", execution.embedding_backend.as_str()),
+ ("embedding_model", execution.embedding_model.as_str()),
+ (
+ "embedding_tokenizer",
+ execution.embedding_tokenizer.as_str(),
+ ),
+ ("detector_identity", execution.detector_identity.as_str()),
+ ("schema_identity", execution.schema_identity.as_str()),
+ ("index_identity", execution.index_identity.as_str()),
+ ] {
+ anyhow::ensure!(!value.is_empty(), "execution identity {name} is empty");
+ }
+ match &execution.source_policy {
+ SourceExecutionPolicy::DurableSnapshot {
+ reference,
+ snapshot_digest,
+ } => anyhow::ensure!(
+ !reference.is_empty() && !snapshot_digest.is_empty(),
+ "durable source snapshot identity is incomplete"
+ ),
+ SourceExecutionPolicy::AbortOnSourceChange { inventory_digest } => {
+ anyhow::ensure!(
+ !inventory_digest.is_empty(),
+ "abort-on-change policy needs inventory digest"
+ )
+ }
+ }
+ }
+ validate_groups(&manifest.groups)?;
+ validate_plan_projection(&manifest.plan, &manifest.groups)
+}
+
+fn validate_groups(groups: &[ManifestGroup]) -> Result<()> {
+ let mut group_ids = HashSet::new();
+ let mut content_ids = HashSet::new();
+ let mut path_ids = HashSet::new();
+ for group in groups {
+ anyhow::ensure!(group_ids.insert(&group.group_id), "duplicate group ID");
+ anyhow::ensure!(
+ content_ids.insert(&group.content_id),
+ "duplicate collision-safe content ID"
+ );
+ anyhow::ensure!(
+ !group.content_digest.is_empty(),
+ "group has no raw content digest"
+ );
+ anyhow::ensure!(
+ native_path_id_supported(&group.canonical.path_id),
+ "canonical path identity is unsupported"
+ );
+ anyhow::ensure!(
+ !group.canonical.is_symlink || group.aliases.iter().all(|p| p.is_symlink),
+ "a symlink cannot be canonical while a real path survives"
+ );
+ anyhow::ensure!(
+ group.aliases.iter().all(|alias| {
+ (group.canonical.is_symlink, group.canonical.path_id.as_str())
+ < (alias.is_symlink, alias.path_id.as_str())
+ }),
+ "canonical path does not have the lowest canonical rank"
+ );
+ for path in group.all_paths() {
+ anyhow::ensure!(
+ path_ids.insert(&path.path_id),
+ "native path identity occurs in multiple groups"
+ );
+ }
+ anyhow::ensure!(
+ group.aliases.windows(2).all(|pair| {
+ (pair[0].is_symlink, pair[0].path_id.as_str())
+ < (pair[1].is_symlink, pair[1].path_id.as_str())
+ }),
+ "aliases are not in canonical rank order"
+ );
+ anyhow::ensure!(
+ group.dataset_slugs.windows(2).all(|pair| pair[0] < pair[1]),
+ "dataset assignments are not canonical"
+ );
+ }
+ anyhow::ensure!(
+ groups
+ .windows(2)
+ .all(|pair| pair[0].group_id < pair[1].group_id),
+ "manifest groups are not canonical"
+ );
+ Ok(())
+}
+
+fn validate_plan_projection(plan: &Plan, groups: &[ManifestGroup]) -> Result<()> {
+ let dataset_slugs: HashSet<&str> = plan
+ .datasets
+ .iter()
+ .map(|dataset| dataset.slug.as_str())
+ .collect();
+ anyhow::ensure!(
+ dataset_slugs.len() == plan.datasets.len(),
+ "plan has duplicate dataset slugs"
+ );
+ let groups_by_content: HashMap<&str, &ManifestGroup> = groups
+ .iter()
+ .map(|group| (group.content_id.as_str(), group))
+ .collect();
+ anyhow::ensure!(
+ plan.files.len() == groups.len(),
+ "plan files and manifest groups disagree"
+ );
+ for (content_id, assignment) in &plan.files {
+ let group = groups_by_content
+ .get(content_id.as_str())
+ .context("plan file has no manifest group")?;
+ anyhow::ensure!(
+ assignment.rel == group.canonical.rel
+ && assignment.path_id == group.canonical.path_id
+ && assignment.content_digest.as_deref() == Some(group.content_digest.as_str()),
+ "plan canonical projection disagrees with manifest group"
+ );
+ let mut assigned: Vec<String> = assignment
+ .assignments
+ .iter()
+ .map(|(_, slug)| slug.clone())
+ .collect();
+ assigned.sort();
+ assigned.dedup();
+ anyhow::ensure!(
+ assigned == group.dataset_slugs,
+ "plan dataset projection disagrees with manifest group"
+ );
+ anyhow::ensure!(
+ assigned
+ .iter()
+ .all(|slug| dataset_slugs.contains(slug.as_str())),
+ "group references absent dataset schema"
+ );
+ }
+ let expected_aliases: BTreeSet<(String, String, String, String)> = groups
+ .iter()
+ .flat_map(|group| {
+ group.aliases.iter().map(|alias| {
+ (
+ group.content_id.clone(),
+ alias.rel.clone(),
+ alias.path_id.clone(),
+ group.canonical.rel.clone(),
+ )
+ })
+ })
+ .collect();
+ let actual_aliases: BTreeSet<(String, String, String, String)> = plan
+ .duplicate_files
+ .iter()
+ .map(|alias: &DuplicateFile| {
+ (
+ alias.file_key.clone(),
+ alias.rel.clone(),
+ alias.path_id.clone(),
+ alias.duplicate_of.clone(),
+ )
+ })
+ .collect();
+ anyhow::ensure!(
+ actual_aliases == expected_aliases,
+ "plan alias projection disagrees with manifest groups"
+ );
+ Ok(())
+}
+
+fn canonicalize_manifest_order(manifest: &mut GenerationManifest) {
+ manifest.plan.datasets.sort_by(|left, right| {
+ left.slug
+ .cmp(&right.slug)
+ .then_with(|| left.index.cmp(&right.index))
+ .then_with(|| left.group.cmp(&right.group))
+ });
+ for dataset in &mut manifest.plan.datasets {
+ dataset
+ .specs
+ .sort_by(|left, right| left.name.cmp(&right.name));
+ }
+ for assignment in manifest.plan.files.values_mut() {
+ assignment.assignments.sort();
+ assignment.assignments.dedup();
+ }
+ manifest.plan.junk_files.sort_by(|left, right| {
+ left.file_key
+ .cmp(&right.file_key)
+ .then_with(|| left.rel.cmp(&right.rel))
+ });
+ manifest.plan.duplicate_files.sort_by(|left, right| {
+ left.file_key
+ .cmp(&right.file_key)
+ .then_with(|| left.path_id.cmp(&right.path_id))
+ .then_with(|| left.rel.cmp(&right.rel))
+ });
+ for group in &mut manifest.groups {
+ group.aliases.sort_by(|left, right| {
+ (left.is_symlink, left.path_id.as_str())
+ .cmp(&(right.is_symlink, right.path_id.as_str()))
+ });
+ group.dataset_slugs.sort();
+ group.dataset_slugs.dedup();
+ }
+ manifest
+ .groups
+ .sort_by(|left, right| left.group_id.cmp(&right.group_id));
+}
+
+pub fn manifest_digest(manifest: &GenerationManifest) -> Result<String> {
+ let mut normalized = manifest.clone();
+ canonicalize_manifest_order(&mut normalized);
+ let encoded = canonical_json_bytes(&normalized)?;
+ Ok(format!(
+ "axm1-{:032x}",
+ xxhash_rust::xxh3::xxh3_128(&encoded)
+ ))
+}
+
+pub fn operation_hash(operations: &[SyncOperation]) -> Result<String> {
+ let mut normalized = operations.to_vec();
+ normalized.sort();
+ let encoded = canonical_json_bytes(&normalized)?;
+ Ok(format!(
+ "axo1-{:032x}",
+ xxhash_rust::xxh3::xxh3_128(&encoded)
+ ))
+}
+
+fn canonical_json_bytes<T: Serialize + ?Sized>(value: &T) -> Result<Vec<u8>> {
+ fn canonicalize(value: Value) -> Value {
+ match value {
+ Value::Object(object) => {
+ let sorted: BTreeMap<String, Value> = object
+ .into_iter()
+ .map(|(key, value)| (key, canonicalize(value)))
+ .collect();
+ Value::Object(sorted.into_iter().collect())
+ }
+ Value::Array(values) => Value::Array(values.into_iter().map(canonicalize).collect()),
+ scalar => scalar,
+ }
+ }
+ Ok(serde_json::to_vec(&canonicalize(serde_json::to_value(
+ value,
+ )?))?)
+}
+
+pub fn apply_begin(
+ committed: &CommittedManifest,
+ pending: &mut Option<PendingSync>,
+ begin: PendingSync,
+) -> Result<()> {
+ anyhow::ensure!(pending.is_none(), "another sync is already pending");
+ begin.validate_against(committed)?;
+ *pending = Some(begin);
+ Ok(())
+}
+
+pub fn is_sync_record_kind(kind: &str) -> bool {
+ matches!(
+ kind,
+ "sync_begin" | "sync_operation_state" | "sync_validated" | "sync_commit" | "sync_abort"
+ )
+}
+
+pub fn replay_record(
+ record: &Value,
+ committed: &mut Option<CommittedManifest>,
+ pending: &mut Option<PendingSync>,
+) -> Result<()> {
+ let kind = record_str(record, "kind")?;
+ match kind {
+ "sync_begin" => {
+ let begin: PendingSync =
+ serde_json::from_value(record.clone()).context("decode durable sync_begin")?;
+ apply_begin(
+ committed
+ .as_ref()
+ .context("sync_begin has no committed base manifest")?,
+ pending,
+ begin,
+ )
+ }
+ "sync_operation_state" => {
+ let tx = record_str(record, "tx_id")?;
+ let operation = record_str(record, "operation_id")?;
+ let state = serde_json::from_value(
+ record
+ .get("state")
+ .cloned()
+ .context("sync operation state is missing")?,
+ )?;
+ pending
+ .as_mut()
+ .context("operation state has no pending sync")?
+ .apply_operation_state(tx, operation, state)
+ }
+ "sync_validated" => pending
+ .as_mut()
+ .context("validation has no pending sync")?
+ .apply_validated(
+ record_str(record, "tx_id")?,
+ record_str(record, "desired_manifest_digest")?,
+ ),
+ "sync_commit" => {
+ let active = pending.take().context("commit has no pending sync")?;
+ *committed = Some(active.apply_commit(
+ record_str(record, "tx_id")?,
+ record_str(record, "desired_manifest_digest")?,
+ )?);
+ Ok(())
+ }
+ "sync_abort" => {
+ let active = pending.as_ref().context("abort has no pending sync")?;
+ anyhow::ensure!(
+ active.tx_id == record_str(record, "tx_id")?
+ && active.desired_manifest_digest
+ == record_str(record, "desired_manifest_digest")?,
+ "abort does not bind the pending transaction and manifest"
+ );
+ *pending = None;
+ Ok(())
+ }
+ _ => anyhow::bail!("unknown sync record kind {kind}"),
+ }
+}
+
+fn record_str<'a>(record: &'a Value, field: &str) -> Result<&'a str> {
+ record
+ .get(field)
+ .and_then(Value::as_str)
+ .ok_or_else(|| anyhow::anyhow!("sync record has no {field}"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::state::{FileAssignment, PlanDataset};
+
+ fn path(id_hex: &str, symlink: bool) -> ManifestPath {
+ ManifestPath {
+ path_id: format!("unix:{id_hex}"),
+ rel: format!("{id_hex}.pdf"),
+ is_symlink: symlink,
+ }
+ }
+
+ fn group(id: &str, content: &str, path: ManifestPath) -> ManifestGroup {
+ ManifestGroup {
+ group_id: id.into(),
+ content_id: content.into(),
+ content_digest: format!("digest-{content}"),
+ content_size: 10,
+ canonical: path,
+ aliases: Vec::new(),
+ dataset_slugs: vec!["reports".into()],
+ expected_records: 1,
+ expected_passages: 1,
+ expected_vectors: 1,
+ }
+ }
+
+ fn desired(content: &str, paths: Vec<ManifestPath>) -> DesiredContentGroup {
+ DesiredContentGroup {
+ content_id: content.into(),
+ content_digest: format!("digest-{content}"),
+ content_size: 10,
+ paths,
+ dataset_slugs: vec!["reports".into()],
+ expected_records: 1,
+ expected_passages: 1,
+ expected_vectors: 1,
+ }
+ }
+
+ #[test]
+ fn canonical_prefers_real_then_native_bytes_including_non_utf8_identity() {
+ let groups = reconcile_groups(
+ &[],
+ vec![desired(
+ "content",
+ vec![path("ff", false), path("01", true), path("80", false)],
+ )],
+ )
+ .unwrap();
+ assert_eq!(groups[0].canonical.path_id, "unix:80");
+ assert!(!groups[0].canonical.is_symlink);
+ }
+
+ #[test]
+ fn equal_raw_digest_does_not_merge_distinct_collision_safe_content_ids() {
+ let mut a = desired("verified-a", vec![path("61", false)]);
+ let mut b = desired("verified-b", vec![path("62", false)]);
+ b.content_digest = a.content_digest.clone();
+ a.content_size = 9;
+ b.content_size = 9;
+ let groups = reconcile_groups(&[], vec![a, b]).unwrap();
+ assert_eq!(groups.len(), 2);
+ assert_ne!(groups[0].content_id, groups[1].content_id);
+ }
+
+ #[test]
+ fn changed_former_canonical_cannot_steal_surviving_content_group() {
+ let mut old = group("logical", "old-content", path("61", false));
+ old.aliases.push(path("62", false));
+ let result = reconcile_groups(
+ &[old],
+ vec![
+ desired("new-content", vec![path("61", false)]),
+ desired("old-content", vec![path("62", false)]),
+ ],
+ )
+ .unwrap();
+ assert_eq!(
+ result
+ .iter()
+ .find(|group| group.content_id == "old-content")
+ .unwrap()
+ .group_id,
+ "logical"
+ );
+ }
+
+ #[test]
+ fn operation_planner_covers_add_delete_metadata_and_mixed() {
+ let unchanged = group("same", "same", path("01", false));
+ let deleted = group("gone", "gone", path("02", false));
+ let metadata_old = group("meta", "meta", path("03", false));
+ let mut metadata_new = metadata_old.clone();
+ metadata_new.expected_records = 2;
+ let added = group("new", "new", path("04", false));
+ let operations = plan_operations(
+ &[deleted, metadata_old, unchanged.clone()],
+ &[added, metadata_new, unchanged],
+ );
+ assert_eq!(
+ operations
+ .iter()
+ .map(|operation| operation.kind.clone())
+ .collect::<BTreeSet<_>>(),
+ BTreeSet::from([
+ SyncOperationKind::Delete,
+ SyncOperationKind::Metadata,
+ SyncOperationKind::Upsert
+ ])
+ );
+ }
+
+ #[test]
+ fn legacy_nonempty_plan_requires_explicit_migration() {
+ let mut plan = Plan::default();
+ plan.files.insert(
+ "key".into(),
+ FileAssignment {
+ rel: "a.pdf".into(),
+ path_id: "unix:61".into(),
+ family: "pdf".into(),
+ gzip: false,
+ content_digest: Some("digest".into()),
+ assignments: Vec::new(),
+ },
+ );
+ assert!(matches!(
+ CommittedManifest::bootstrap_legacy(plan).unwrap(),
+ LegacyBootstrap::MigrationRequired { .. }
+ ));
+ }
+
+ #[test]
+ fn canonical_digest_ignores_hashmap_and_semantic_group_order() {
+ let mut first = Plan::default();
+ let mut second = Plan::default();
+ for (plan, order) in [(&mut first, ["a", "b"]), (&mut second, ["b", "a"])] {
+ for key in order {
+ plan.files.insert(
+ key.into(),
+ FileAssignment {
+ rel: format!("{key}.pdf"),
+ path_id: format!("unix:{:02x}", key.as_bytes()[0]),
+ family: "pdf".into(),
+ gzip: false,
+ content_digest: Some(format!("digest-{key}")),
+ assignments: Vec::new(),
+ },
+ );
+ }
+ }
+ let group_a = group("a", "a", path("61", false));
+ let group_b = group("b", "b", path("62", false));
+ let a = GenerationManifest {
+ generation: 0,
+ execution: None,
+ plan: first,
+ groups: vec![group_a.clone(), group_b.clone()],
+ };
+ let b = GenerationManifest {
+ generation: 0,
+ execution: None,
+ plan: second,
+ groups: vec![group_b, group_a],
+ };
+ assert_eq!(manifest_digest(&a).unwrap(), manifest_digest(&b).unwrap());
+ }
+
+ fn identity(inventory: &str) -> ExecutionIdentity {
+ ExecutionIdentity {
+ version: EXECUTION_IDENTITY_VERSION,
+ root_identity: "unix:root".into(),
+ url: "url".into(),
+ prefix: "prefix".into(),
+ follow_symlinks: false,
+ chunker_identity: "chunker-v1".into(),
+ embedding_backend: "lexical".into(),
+ embedding_model: "feature-hash".into(),
+ embedding_tokenizer: "builtin".into(),
+ embedding_dimension: 384,
+ graph_enabled: false,
+ brain: "none".into(),
+ detector_identity: "detectors-v1".into(),
+ schema_identity: "schema-v1".into(),
+ index_identity: "index-v1".into(),
+ source_policy: SourceExecutionPolicy::AbortOnSourceChange {
+ inventory_digest: inventory.into(),
+ },
+ }
+ }
+
+ #[test]
+ fn no_op_plan_or_bound_execution_change_is_refused() {
+ let mut legacy_plan = Plan::default();
+ legacy_plan.datasets.push(PlanDataset {
+ slug: "reports".into(),
+ index: "prefix-reports".into(),
+ family: "pdf".into(),
+ group: None,
+ specs: Vec::new(),
+ time_field: None,
+ semantic_field: None,
+ sampled_records: 0,
+ file_count: 0,
+ });
+ let LegacyBootstrap::Ready(legacy) =
+ CommittedManifest::bootstrap_legacy(legacy_plan.clone()).unwrap()
+ else {
+ panic!("empty legacy membership is migratable");
+ };
+ let first = GenerationManifest {
+ generation: 1,
+ execution: Some(identity("inventory-a")),
+ plan: legacy_plan.clone(),
+ groups: Vec::new(),
+ };
+ let pending = PendingSync::new("bind".into(), &legacy, first.clone()).unwrap();
+
+ let committed =
+ CommittedManifest::from_generation(first.clone(), pending.desired_manifest_digest);
+ let mut changed_execution = first.clone();
+ changed_execution.generation = 2;
+ changed_execution.execution = Some(identity("inventory-b"));
+ assert!(
+ PendingSync::new("changed-execution".into(), &committed, changed_execution).is_err()
+ );
+
+ let mut changed_plan = first;
+ changed_plan.plan.alias_paths_indexed = true;
+ assert!(PendingSync::new("changed-plan".into(), &legacy, changed_plan).is_err());
+ }
+
+ #[test]
+ fn serialized_malicious_operation_list_is_rejected_against_manifests() {
+ let LegacyBootstrap::Ready(base) =
+ CommittedManifest::bootstrap_legacy(Plan::default()).unwrap()
+ else {
+ panic!("empty legacy plan is ready");
+ };
+ let desired = GenerationManifest {
+ generation: 1,
+ execution: Some(identity("inventory-a")),
+ plan: Plan::default(),
+ groups: Vec::new(),
+ };
+ let valid = PendingSync::new("tx".into(), &base, desired).unwrap();
+ let mut encoded = serde_json::to_value(valid).unwrap();
+ encoded["operations"] = serde_json::json!([{
+ "operation_id": "delete:invented",
+ "kind": "delete",
+ "group_id": "invented"
+ }]);
+ let malicious: PendingSync = serde_json::from_value(encoded).unwrap();
+ assert!(malicious.validate_against(&base).is_err());
+ }
+}