Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): seal bounded generation replay

buger14567df51cd7
diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rs
index 581e423b7..924ad2570 100644
--- a/engine/crates/xerj-autoindex/src/content.rs
+++ b/engine/crates/xerj-autoindex/src/content.rs
@@ -106,6 +106,7 @@ fn resolve_with_digests(files: Vec<FileEntry>, digests: Vec<String>) -> Result<I
file_key: resolved_keys[canonical].clone(),
rel: files[index].rel.clone(),
path_id: files[index].rel_id.clone(),
+ is_symlink: Some(files[index].is_symlink),
duplicate_of: files[canonical].rel.clone(),
bytes: files[index].size,
});
@@ -343,6 +344,7 @@ mod tests {
let inv = resolve(crate::walk::walk(dir.path(), false).unwrap()).unwrap();
assert_eq!(inv.files.len(), 1);
assert_eq!(inv.duplicates.len(), 1);
+ assert_eq!(inv.duplicates[0].is_symlink, Some(false));
}
#[cfg(unix)]
@@ -379,5 +381,6 @@ mod tests {
let inv = resolve(crate::walk::walk(dir.path(), true).unwrap()).unwrap();
assert_eq!(inv.files[0].rel, "z-real");
assert_eq!(inv.duplicates[0].rel, "a-link");
+ assert_eq!(inv.duplicates[0].is_symlink, Some(true));
}
}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index bfdf50142..ee89b8cf7 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -20,6 +20,7 @@ pub mod resources;
pub mod sniff;
pub mod state;
mod sync;
+mod sync_executor;
pub mod walk;
use anyhow::{Context, Result};
@@ -950,6 +951,7 @@ fn build_phase_a(
.or_insert_with(|| FileAssignment {
rel: files[m].rel.clone(),
path_id: files[m].rel_id.clone(),
+ is_symlink: Some(files[m].is_symlink),
family: family.as_str().to_string(),
gzip,
content_digest: Some(digests[m].clone()),
@@ -1471,6 +1473,30 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
// must never classify a path snapshot taken while another owner was
// publishing or replacing the durable plan.
let preflight = state::Journal::preflight(&state_dir, &root_str, &cfg.url, &cfg.prefix)?;
+ // A durable sync_begin owns the desired generation. Never rediscover and
+ // replan from a mutable source tree while that transaction is pending.
+ // Operation handlers are deliberately not enabled by this foundation
+ // slice; fail with the exact durable transaction rather than accidentally
+ // executing a different folder snapshot.
+ if preflight.pending_sync.is_some() {
+ let mut journal = state::Journal::open_after_preflight(
+ preflight,
+ &root_str,
+ &cfg.url,
+ &cfg.prefix,
+ cfg.bulk_timeout_secs,
+ cfg.fresh,
+ )?;
+ pr.phase("replay", 0, 0);
+ let mut backend = sync_executor::EsSyncBackend::new(&es, &state_dir, cfg.bulk_mb << 20);
+ sync_executor::replay_pending_operations(&state_dir, &mut journal, &mut backend)?;
+ // Through the progress surface, never a bare `eprintln!`: stderr
+ // belongs to that surface, so `--progress none` stays silent and
+ // `--progress json` stays one parseable stream (#241).
+ pr.note("autoindex: resumed and committed pending corpus generation from durable source");
+ pr.finish(true, 0, "completed", &[]);
+ return Ok((0, None));
+ }
// Totals are unknown until the walk returns, so this phase honestly
// reports `pct=unknown` and proves liveness with the clock alone.
pr.phase("walk", 0, 0);
@@ -3662,6 +3688,7 @@ mod inventory_delta_tests {
FileAssignment {
rel: path.to_owned(),
path_id: format!("id:{path}"),
+ is_symlink: Some(false),
family: "csv".into(),
gzip: false,
content_digest: Some(format!("digest:{path}")),
@@ -3762,6 +3789,7 @@ mod duplicate_integration_tests {
FileAssignment {
rel: rel.to_string(),
path_id: String::new(),
+ is_symlink: None,
family: "txt".to_string(),
gzip: false,
content_digest: None,
@@ -3852,6 +3880,7 @@ mod duplicate_integration_tests {
file_key: file_key.to_string(),
rel: rel.to_string(),
path_id: format!("id:{rel}"),
+ is_symlink: Some(false),
duplicate_of: format!("{file_key}.txt"),
bytes: 10,
};
diff --git a/engine/crates/xerj-autoindex/src/state.rs b/engine/crates/xerj-autoindex/src/state.rs
index 4abde0d7c..ddb559fa2 100644
--- a/engine/crates/xerj-autoindex/src/state.rs
+++ b/engine/crates/xerj-autoindex/src/state.rs
@@ -19,7 +19,7 @@ static SYNC_IO_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::Atomi
pub(crate) static FILE_DONE_IO_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> =
std::sync::Mutex::new(());
#[cfg(test)]
-static SYNC_IO_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+pub(crate) static SYNC_IO_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
#[cfg(test)]
pub(crate) fn fail_next_file_done_io(boundary: u8) {
@@ -289,6 +289,9 @@ pub struct FileAssignment {
/// Reversible path identity used for deterministic resume matching.
#[serde(default, skip_serializing_if = "String::is_empty")]
pub path_id: String,
+ /// Canonical path rank input. `None` identifies a legacy plan.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub is_symlink: Option<bool>,
pub family: String,
pub gzip: bool,
/// Full-content digest used to validate durable identity across resumes.
@@ -321,6 +324,10 @@ pub struct DuplicateFile {
pub rel: String,
#[serde(default)]
pub path_id: String,
+ /// Whether this alias was reached through a followed symlink. `None`
+ /// identifies legacy plans which cannot prove canonical rank.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub is_symlink: Option<bool>,
/// Root-relative canonical path whose records represent this content.
pub duplicate_of: String,
pub bytes: u64,
@@ -1227,6 +1234,7 @@ mod sync_journal_tests {
FileAssignment {
rel: "a.pdf".into(),
path_id: "unix:61".into(),
+ is_symlink: Some(false),
family: "pdf".into(),
gzip: false,
content_digest: Some("digest-a".into()),
diff --git a/engine/crates/xerj-autoindex/src/sync.rs b/engine/crates/xerj-autoindex/src/sync.rs
index bebce6e84..c60f4c151 100644
--- a/engine/crates/xerj-autoindex/src/sync.rs
+++ b/engine/crates/xerj-autoindex/src/sync.rs
@@ -116,10 +116,15 @@ impl CommittedManifest {
"{file_key}: missing or lossy native canonical path identity"
));
}
- // Legacy plans did not persist whether the canonical path was a
- // followed symlink, so canonical rank cannot be reconstructed.
+ if file.is_symlink.is_none() {
+ reasons.push(format!(
+ "{file_key}: legacy plan did not persist canonical symlink rank"
+ ));
+ }
+ // A Plan is not itself a generation manifest: it has no stable
+ // group ID, content byte length, or validated output counts.
reasons.push(format!(
- "{file_key}: legacy plan did not persist canonical symlink rank"
+ "{file_key}: legacy plan has no complete manifest group"
));
}
for alias in &plan.duplicate_files {
@@ -129,6 +134,12 @@ impl CommittedManifest {
alias.rel
));
}
+ if alias.is_symlink.is_none() {
+ reasons.push(format!(
+ "{}: legacy plan did not persist alias symlink rank",
+ alias.rel
+ ));
+ }
}
reasons.sort();
reasons.dedup();
@@ -341,11 +352,32 @@ fn validate_supported_manifest_delta(
desired: &GenerationManifest,
) -> Result<()> {
// Binding complete identity while migrating the legacy generation zero is
- // the only identity transition this vocabulary can represent. Once bound,
- // backend/model/source-policy changes require a future explicit operation.
+ // the only configuration transition this vocabulary can represent. A
+ // durable snapshot reference and digest are generation inputs, not engine
+ // configuration, so they must advance with each desired generation.
if base.generation > 0 || base.execution.is_some() {
+ let same_execution_configuration = |left: &ExecutionIdentity, right: &ExecutionIdentity| {
+ let mut left = left.clone();
+ let mut right = right.clone();
+ if let (
+ SourceExecutionPolicy::DurableSnapshot { .. },
+ SourceExecutionPolicy::DurableSnapshot { .. },
+ ) = (&left.source_policy, &right.source_policy)
+ {
+ left.source_policy = SourceExecutionPolicy::DurableSnapshot {
+ reference: String::new(),
+ snapshot_digest: String::new(),
+ };
+ right.source_policy = left.source_policy.clone();
+ }
+ left == right
+ };
anyhow::ensure!(
- base.execution == desired.execution,
+ match (&base.execution, &desired.execution) {
+ (Some(left), Some(right)) => same_execution_configuration(left, right),
+ (None, None) => true,
+ _ => false,
+ },
"execution identity or source policy changed without a supported operation"
);
}
@@ -718,6 +750,7 @@ fn validate_plan_projection(plan: &Plan, groups: &[ManifestGroup]) -> Result<()>
anyhow::ensure!(
assignment.rel == group.canonical.rel
&& assignment.path_id == group.canonical.path_id
+ && assignment.is_symlink == Some(group.canonical.is_symlink)
&& assignment.content_digest.as_deref() == Some(group.content_digest.as_str()),
"plan canonical projection disagrees with manifest group"
);
@@ -739,7 +772,7 @@ fn validate_plan_projection(plan: &Plan, groups: &[ManifestGroup]) -> Result<()>
"group references absent dataset schema"
);
}
- let expected_aliases: BTreeSet<(String, String, String, String)> = groups
+ let expected_aliases: BTreeSet<(String, String, String, bool, String)> = groups
.iter()
.flat_map(|group| {
group.aliases.iter().map(|alias| {
@@ -747,23 +780,27 @@ fn validate_plan_projection(plan: &Plan, groups: &[ManifestGroup]) -> Result<()>
group.content_id.clone(),
alias.rel.clone(),
alias.path_id.clone(),
+ alias.is_symlink,
group.canonical.rel.clone(),
)
})
})
.collect();
- let actual_aliases: BTreeSet<(String, String, String, String)> = plan
+ let actual_aliases: BTreeSet<(String, String, String, bool, String)> = plan
.duplicate_files
.iter()
- .map(|alias: &DuplicateFile| {
- (
+ .map(|alias: &DuplicateFile| -> Result<_> {
+ Ok((
alias.file_key.clone(),
alias.rel.clone(),
alias.path_id.clone(),
+ alias
+ .is_symlink
+ .context("plan alias has no persisted symlink rank")?,
alias.duplicate_of.clone(),
- )
+ ))
})
- .collect();
+ .collect::<Result<_>>()?;
anyhow::ensure!(
actual_aliases == expected_aliases,
"plan alias projection disagrees with manifest groups"
@@ -1059,6 +1096,7 @@ mod tests {
FileAssignment {
rel: "a.pdf".into(),
path_id: "unix:61".into(),
+ is_symlink: None,
family: "pdf".into(),
gzip: false,
content_digest: Some("digest".into()),
@@ -1082,6 +1120,7 @@ mod tests {
FileAssignment {
rel: format!("{key}.pdf"),
path_id: format!("unix:{:02x}", key.as_bytes()[0]),
+ is_symlink: Some(false),
family: "pdf".into(),
gzip: false,
content_digest: Some(format!("digest-{key}")),
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
new file mode 100644
index 000000000..1c1ab4541
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -0,0 +1,1545 @@
+//! Durable source preparation for incremental corpus reconciliation.
+//!
+//! This slice deliberately performs no remote mutation. It projects a
+//! byte-verified inventory into generation groups and makes desired bytes
+//! immutable before a later executor writes `sync_begin`.
+
+use crate::content::Inventory;
+use crate::state::{Journal, Plan};
+use crate::sync::{
+ self, CommittedManifest, DesiredContentGroup, GenerationManifest, ManifestGroup, ManifestPath,
+ PendingSync, SourceExecutionPolicy, SyncOperation, SyncOperationState,
+};
+use anyhow::{Context, Result};
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+use std::collections::HashMap;
+use std::fs::{File, OpenOptions};
+use std::io::{BufReader, BufWriter, Read, Write};
+use std::path::Path;
+
+const SNAPSHOT_VERSION: u32 = 1;
+
+#[cfg(test)]
+static SNAPSHOT_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
+#[cfg(test)]
+static SNAPSHOT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+#[cfg(test)]
+static REPLAY_FAIL_AFTER_APPLY: std::sync::atomic::AtomicBool =
+ std::sync::atomic::AtomicBool::new(false);
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SnapshotFile {
+ pub content_id: String,
+ pub content_digest: String,
+ pub content_size: u64,
+ pub relative_blob: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub prepared: Option<PreparedArtifact>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct PreparedArtifact {
+ pub relative_ndjson: String,
+ pub records: u64,
+ pub passages: u64,
+ pub vectors: u64,
+ pub bytes: u64,
+ pub digest: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SourceSnapshot {
+ pub version: u32,
+ pub tx_id: String,
+ pub files: Vec<SnapshotFile>,
+ pub snapshot_digest: String,
+}
+
+/// Remote mutations must be convergent: calling `apply` twice for one
+/// operation after an accepted-but-unrecorded response must produce exactly
+/// the same live state. `validate` is the final generation-wide barrier.
+#[cfg_attr(not(test), allow(dead_code))]
+pub trait SyncOperationBackend {
+ fn apply(
+ &mut self,
+ operation: &SyncOperation,
+ base: &CommittedManifest,
+ desired: &GenerationManifest,
+ snapshot: &SourceSnapshot,
+ ) -> Result<()>;
+
+ fn validate(
+ &mut self,
+ base: &CommittedManifest,
+ desired: &GenerationManifest,
+ snapshot: &SourceSnapshot,
+ ) -> Result<()>;
+}
+
+/// Production ES-compatible operation backend for graph-disabled generations.
+///
+/// Upserts stream sealed index actions after removing both replaced and
+/// retry-partial content. Metadata operations read deterministic IDs from the
+/// committed snapshot and issue partial updates containing provenance only,
+/// so unchanged semantic fields are never re-embedded.
+#[cfg_attr(not(test), allow(dead_code))]
+pub struct EsSyncBackend<'a> {
+ es: &'a crate::esclient::Es,
+ state_dir: &'a Path,
+ bulk_bytes: usize,
+}
+
+#[cfg_attr(not(test), allow(dead_code))]
+impl<'a> EsSyncBackend<'a> {
+ pub fn new(es: &'a crate::esclient::Es, state_dir: &'a Path, bulk_bytes: usize) -> Self {
+ Self {
+ es,
+ state_dir,
+ bulk_bytes: bulk_bytes.max(64 * 1024),
+ }
+ }
+
+ fn delete_group(&self, group: &ManifestGroup, plan: &Plan) -> Result<()> {
+ for index in group_indices(group, plan)? {
+ self.es.delete_by_query(
+ &index,
+ &serde_json::json!({"term": {
+ "ax_file": &group.content_id
+ }}),
+ )?;
+ }
+ Ok(())
+ }
+
+ fn replay_prepared(&self, snapshot: &SourceSnapshot, content_id: &str) -> Result<()> {
+ let prepared = prepared_for(snapshot, content_id)?;
+ let snapshot_dir = self.state_dir.join("sync-snapshots").join(&snapshot.tx_id);
+ let file = File::open(snapshot_dir.join(&prepared.relative_ndjson))?;
+ stream_ndjson_pairs(BufReader::new(file), self.bulk_bytes, |body| {
+ checked_bulk(self.es, body)
+ })
+ }
+
+ fn replay_metadata(
+ &self,
+ base: &CommittedManifest,
+ desired_group: &ManifestGroup,
+ ) -> Result<()> {
+ let base_snapshot = open_committed_snapshot(self.state_dir, base)?;
+ let old_group = base
+ .groups
+ .iter()
+ .find(|group| group.group_id == desired_group.group_id)
+ .context("metadata operation has no committed group")?;
+ let prepared = prepared_for(&base_snapshot, &old_group.content_id)?;
+ let snapshot_dir = self
+ .state_dir
+ .join("sync-snapshots")
+ .join(&base_snapshot.tx_id);
+ let reader = BufReader::new(File::open(snapshot_dir.join(&prepared.relative_ndjson))?);
+ let paths: Vec<Value> = std::iter::once(&desired_group.canonical)
+ .chain(desired_group.aliases.iter())
+ .map(|path| Value::String(path.rel.clone()))
+ .collect();
+ stream_metadata_updates(
+ reader,
+ self.bulk_bytes,
+ &desired_group.canonical.rel,
+ &paths,
+ |body| checked_bulk(self.es, body),
+ )
+ }
+
+ fn exact_group_count(&self, group: &ManifestGroup, plan: &Plan) -> Result<u64> {
+ let mut total = 0u64;
+ for index in group_indices(group, plan)? {
+ let response = self.es.search(
+ &index,
+ &serde_json::json!({
+ "size": 0,
+ "track_total_hits": true,
+ "query": {"term": {"ax_file": &group.content_id}}
+ }),
+ )?;
+ total = total
+ .checked_add(
+ response
+ .pointer("/hits/total/value")
+ .and_then(Value::as_u64)
+ .context("exact validation response has no total hit count")?,
+ )
+ .context("exact validation count overflow")?;
+ }
+ Ok(total)
+ }
+}
+
+impl SyncOperationBackend for EsSyncBackend<'_> {
+ fn apply(
+ &mut self,
+ operation: &SyncOperation,
+ base: &CommittedManifest,
+ desired: &GenerationManifest,
+ snapshot: &SourceSnapshot,
+ ) -> Result<()> {
+ anyhow::ensure!(
+ desired
+ .execution
+ .as_ref()
+ .is_some_and(|execution| !execution.graph_enabled),
+ "production incremental graph reconciliation is not enabled yet"
+ );
+ let old = base
+ .groups
+ .iter()
+ .find(|group| group.group_id == operation.group_id);
+ let new = desired
+ .groups
+ .iter()
+ .find(|group| group.group_id == operation.group_id);
+ match operation.kind {
+ crate::sync::SyncOperationKind::Delete => self.delete_group(
+ old.context("delete operation has no committed group")?,
+ &base.plan,
+ ),
+ crate::sync::SyncOperationKind::Upsert => {
+ if let Some(old) = old {
+ self.delete_group(old, &base.plan)?;
+ }
+ let new = new.context("upsert operation has no desired group")?;
+ // Remove a partial prior retry of the desired identity too.
+ self.delete_group(new, &desired.plan)?;
+ self.replay_prepared(snapshot, &new.content_id)
+ }
+ crate::sync::SyncOperationKind::Metadata => self.replay_metadata(
+ base,
+ new.context("metadata operation has no desired group")?,
+ ),
+ }
+ }
+
+ fn validate(
+ &mut self,
+ base: &CommittedManifest,
+ desired: &GenerationManifest,
+ _snapshot: &SourceSnapshot,
+ ) -> Result<()> {
+ for dataset in &desired.plan.datasets {
+ self.es.refresh(&dataset.index)?;
+ }
+ for group in &desired.groups {
+ anyhow::ensure!(
+ self.exact_group_count(group, &desired.plan)? == group.expected_records,
+ "live record count disagrees with desired group {}",
+ group.group_id
+ );
+ }
+ let desired_ids: std::collections::HashSet<&str> = desired
+ .groups
+ .iter()
+ .map(|group| group.content_id.as_str())
+ .collect();
+ for old in &base.groups {
+ if !desired_ids.contains(old.content_id.as_str()) {
+ anyhow::ensure!(
+ self.exact_group_count(old, &base.plan)? == 0,
+ "replaced or deleted content {} remains live",
+ old.content_id
+ );
+ }
+ }
+ Ok(())
+ }
+}
+
+/// Resume a journaled generation strictly from its verified snapshot.
+///
+/// The durable order is Started -> convergent remote apply -> Committed.
+/// Therefore a crash after an accepted response repeats the same operation;
+/// a crash after Committed skips it. Only an exact backend validation allows
+/// `sync_validated` and the minimal authority switch in `sync_commit`.
+#[cfg_attr(not(test), allow(dead_code))]
+pub fn replay_pending_operations(
+ state_dir: &Path,
+ journal: &mut Journal,
+ backend: &mut impl SyncOperationBackend,
+) -> Result<()> {
+ let pending = journal
+ .pending_sync
+ .clone()
+ .context("cannot replay without a pending corpus generation")?;
+ let base = journal
+ .committed_manifest
+ .clone()
+ .context("pending corpus generation has no committed base")?;
+ pending.validate_against(&base)?;
+ let snapshot = open_snapshot(state_dir, &pending.tx_id)?;
+ verify_snapshot_binding(&pending, &snapshot)?;
+
+ for operation in &pending.operations {
+ let state = journal
+ .pending_sync
+ .as_ref()
+ .and_then(|sync| sync.operation_states.get(&operation.operation_id))
+ .cloned();
+ if state == Some(SyncOperationState::Committed) {
+ continue;
+ }
+ if state.is_none() {
+ journal.sync_operation_state(&operation.operation_id, SyncOperationState::Started)?;
+ }
+ backend.apply(operation, &base, &pending.desired, &snapshot)?;
+ replay_fail_after_apply()?;
+ journal.sync_operation_state(&operation.operation_id, SyncOperationState::Committed)?;
+ }
+ backend.validate(&base, &pending.desired, &snapshot)?;
+ journal.sync_validated()?;
+ journal.sync_commit()
+}
+
+/// Test helper proving a pending generation binds before mutable discovery.
+/// Production calls `replay_pending_operations` at this boundary.
+#[cfg(test)]
+pub fn require_resumable_pending_source(
+ state_dir: &Path,
+ pending: Option<&PendingSync>,
+) -> Result<()> {
+ let Some(pending) = pending else {
+ return Ok(());
+ };
+ let snapshot = open_snapshot(state_dir, &pending.tx_id).with_context(|| {
+ format!(
+ "pending corpus generation {} must resume before source discovery",
+ pending.tx_id
+ )
+ })?;
+ verify_snapshot_binding(pending, &snapshot)?;
+ anyhow::bail!(
+ "corpus generation {} is pending with verified durable source snapshot {} ({} files); \
+ incremental operation replay is not enabled by this executor slice, so no source \
+ discovery or remote mutation was attempted",
+ pending.tx_id,
+ snapshot.snapshot_digest,
+ snapshot.files.len()
+ )
+}
+
+fn verify_snapshot_binding(pending: &PendingSync, snapshot: &SourceSnapshot) -> Result<()> {
+ let execution = pending
+ .desired
+ .execution
+ .as_ref()
+ .context("pending generation has no execution identity")?;
+ let SourceExecutionPolicy::DurableSnapshot {
+ reference,
+ snapshot_digest,
+ } = &execution.source_policy
+ else {
+ anyhow::bail!("pending generation is not bound to a durable source snapshot");
+ };
+ anyhow::ensure!(
+ reference == &format!("sync-snapshots/{}", pending.tx_id)
+ && snapshot_digest == &snapshot.snapshot_digest,
+ "pending generation source snapshot binding does not match verified snapshot"
+ );
+ Ok(())
+}
+
+fn open_committed_snapshot(
+ state_dir: &Path,
+ committed: &CommittedManifest,
+) -> Result<SourceSnapshot> {
+ let execution = committed
+ .execution
+ .as_ref()
+ .context("committed generation has no execution identity")?;
+ let SourceExecutionPolicy::DurableSnapshot {
+ reference,
+ snapshot_digest,
+ } = &execution.source_policy
+ else {
+ anyhow::bail!("committed generation is not bound to a durable snapshot");
+ };
+ let tx_id = reference
+ .strip_prefix("sync-snapshots/")
+ .context("committed snapshot reference is not state-relative")?;
+ let snapshot = open_snapshot(state_dir, tx_id)?;
+ anyhow::ensure!(
+ &snapshot.snapshot_digest == snapshot_digest,
+ "committed snapshot digest mismatch"
+ );
+ Ok(snapshot)
+}
+
+fn prepared_for<'a>(
+ snapshot: &'a SourceSnapshot,
+ content_id: &str,
+) -> Result<&'a PreparedArtifact> {
+ snapshot
+ .files
+ .iter()
+ .find(|file| file.content_id == content_id)
+ .and_then(|file| file.prepared.as_ref())
+ .with_context(|| format!("content {content_id} has no sealed prepared artifact"))
+}
+
+fn group_indices(group: &ManifestGroup, plan: &Plan) -> Result<Vec<String>> {
+ let by_slug: HashMap<&str, &str> = plan
+ .datasets
+ .iter()
+ .map(|dataset| (dataset.slug.as_str(), dataset.index.as_str()))
+ .collect();
+ let mut indices = group
+ .dataset_slugs
+ .iter()
+ .map(|slug| {
+ by_slug
+ .get(slug.as_str())
+ .map(|index| (*index).to_string())
+ .with_context(|| format!("group references absent dataset {slug}"))
+ })
+ .collect::<Result<Vec<_>>>()?;
+ indices.sort();
+ indices.dedup();
+ Ok(indices)
+}
+
+fn checked_bulk(es: &crate::esclient::Es, body: Vec<u8>) -> Result<()> {
+ if body.is_empty() {
+ return Ok(());
+ }
+ let outcome = es.bulk(body)?;
+ anyhow::ensure!(
+ outcome.item_errors == 0,
+ "prepared bulk contained {} rejected items: {}",
+ outcome.item_errors,
+ outcome
+ .first_error
+ .unwrap_or_else(|| "unknown error".into())
+ );
+ Ok(())
+}
+
+fn stream_ndjson_pairs(
+ mut reader: impl std::io::BufRead,
+ bulk_bytes: usize,
+ mut send: impl FnMut(Vec<u8>) -> Result<()>,
+) -> Result<()> {
+ let mut body = Vec::with_capacity(bulk_bytes);
+ loop {
+ let mut action = Vec::new();
+ if reader.read_until(b'\n', &mut action)? == 0 {
+ break;
+ }
+ let mut document = Vec::new();
+ anyhow::ensure!(
+ reader.read_until(b'\n', &mut document)? > 0,
+ "prepared NDJSON ended without a document"
+ );
+ if !body.is_empty() && body.len() + action.len() + document.len() > bulk_bytes {
+ send(std::mem::take(&mut body))?;
+ body.reserve(bulk_bytes);
+ }
+ body.extend_from_slice(&action);
+ body.extend_from_slice(&document);
+ }
+ send(body)
+}
+
+fn stream_metadata_updates(
+ mut reader: impl std::io::BufRead,
+ bulk_bytes: usize,
+ canonical: &str,
+ paths: &[Value],
+ mut send: impl FnMut(Vec<u8>) -> Result<()>,
+) -> Result<()> {
+ let mut body = Vec::with_capacity(bulk_bytes);
+ loop {
+ let mut action_line = String::new();
+ if reader.read_line(&mut action_line)? == 0 {
+ break;
+ }
+ let mut ignored_document = String::new();
+ anyhow::ensure!(
+ reader.read_line(&mut ignored_document)? > 0,
+ "prepared NDJSON ended without a document"
+ );
+ let action: Value = serde_json::from_str(&action_line)?;
+ let index = action
+ .pointer("/index/_index")
+ .and_then(Value::as_str)
+ .context("prepared action has no index")?;
+ let id = action
+ .pointer("/index/_id")
+ .and_then(Value::as_str)
+ .context("prepared action has no ID")?;
+ let update =
+ serde_json::to_vec(&serde_json::json!({"update": {"_index": index, "_id": id}}))?;
+ let patch = serde_json::to_vec(&serde_json::json!({"doc": {
+ "ax_path": canonical,
+ "ax_paths": paths
+ }}))?;
+ let required = update.len() + patch.len() + 2;
+ if !body.is_empty() && body.len() + required > bulk_bytes {
+ send(std::mem::take(&mut body))?;
+ body.reserve(bulk_bytes);
+ }
+ body.extend_from_slice(&update);
+ body.push(b'\n');
+ body.extend_from_slice(&patch);
+ body.push(b'\n');
+ }
+ send(body)
+}
+
+/// Build desired groups without guessing assignments. Output counts survive
+/// only when the committed content identity is byte-identical.
+#[cfg_attr(not(test), allow(dead_code))]
+pub fn groups_from_inventory(
+ inventory: &Inventory,
+ plan: &Plan,
+ previous: &[ManifestGroup],
+) -> Result<Vec<ManifestGroup>> {
+ ensure_inventory_lengths(inventory)?;
+ let previous_by_content: HashMap<&str, &ManifestGroup> = previous
+ .iter()
+ .map(|group| (group.content_id.as_str(), group))
+ .collect();
+ let mut aliases_by_key: HashMap<&str, Vec<ManifestPath>> = HashMap::new();
+ for alias in &inventory.duplicates {
+ aliases_by_key
+ .entry(alias.file_key.as_str())
+ .or_default()
+ .push(ManifestPath {
+ path_id: alias.path_id.clone(),
+ rel: alias.rel.clone(),
+ is_symlink: alias.is_symlink.with_context(|| {
+ format!("alias {} has no persisted symlink rank", alias.rel)
+ })?,
+ });
+ }
+ let desired = inventory
+ .files
+ .iter()
+ .zip(&inventory.keys)
+ .zip(&inventory.digests)
+ .map(|((file, content_id), content_digest)| {
+ let assignment = plan.files.get(content_id).with_context(|| {
+ format!("typed plan has no assignment for content {content_id}")
+ })?;
+ anyhow::ensure!(
+ assignment.rel == file.rel
+ && assignment.path_id == file.rel_id
+ && assignment.content_digest.as_deref() == Some(content_digest.as_str()),
+ "typed plan canonical projection disagrees with inventory for {}",
+ file.rel
+ );
+ let mut dataset_slugs: Vec<String> = assignment
+ .assignments
+ .iter()
+ .map(|(_, slug)| slug.clone())
+ .collect();
+ dataset_slugs.sort();
+ dataset_slugs.dedup();
+ let prior = previous_by_content.get(content_id.as_str()).copied();
+ Ok(DesiredContentGroup {
+ content_id: content_id.clone(),
+ content_digest: content_digest.clone(),
+ content_size: file.size,
+ paths: std::iter::once(ManifestPath {
+ path_id: file.rel_id.clone(),
+ rel: file.rel.clone(),
+ is_symlink: file.is_symlink,
+ })
+ .chain(
+ aliases_by_key
+ .get(content_id.as_str())
+ .into_iter()
+ .flatten()
+ .cloned(),
+ )
+ .collect(),
+ dataset_slugs,
+ expected_records: prior.map_or(0, |group| group.expected_records),
+ expected_passages: prior.map_or(0, |group| group.expected_passages),
+ expected_vectors: prior.map_or(0, |group| group.expected_vectors),
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ sync::reconcile_groups(previous, desired)
+}
+
+/// Bind exact prepared output cardinalities into the desired manifest before
+/// `sync_begin`. Content without a prepared artifact is rejected; unchanged
+/// groups should retain their already committed counts instead.
+#[cfg_attr(not(test), allow(dead_code))]
+pub fn bind_prepared_counts(
+ groups: &mut [ManifestGroup],
+ snapshot: &SourceSnapshot,
+ prepared_content: &[String],
+) -> Result<()> {
+ let prepared_content: std::collections::HashSet<&str> =
+ prepared_content.iter().map(String::as_str).collect();
+ let by_content: HashMap<&str, &PreparedArtifact> = snapshot
+ .files
+ .iter()
+ .filter_map(|file| {
+ file.prepared
+ .as_ref()
+ .map(|prepared| (file.content_id.as_str(), prepared))
+ })
+ .collect();
+ for group in groups {
+ if !prepared_content.contains(group.content_id.as_str()) {
+ continue;
+ }
+ let prepared = by_content
+ .get(group.content_id.as_str())
+ .context("desired content has no prepared artifact")?;
+ group.expected_records = prepared.records;
+ group.expected_passages = prepared.passages;
+ group.expected_vectors = prepared.vectors;
+ }
+ Ok(())
+}
+
+/// Fsync blobs and manifest in a private directory, then atomically publish
+/// the transaction snapshot. Retries reuse a verified final snapshot or
+/// replace only that transaction's incomplete staging directory.
+#[cfg_attr(not(test), allow(dead_code))]
+pub fn create_snapshot(
+ state_dir: &Path,
+ tx_id: &str,
+ inventory: &Inventory,
+) -> Result<SourceSnapshot> {
+ create_snapshot_inner(state_dir, tx_id, inventory, None)
+}
+
+/// Seal deterministic bulk actions together with source bytes. Extraction and
+/// coercion happen before `sync_begin`; retries stream this artifact and never
+/// re-extract or re-embed unchanged content.
+#[cfg_attr(not(test), allow(dead_code))]
+pub fn create_prepared_snapshot(
+ state_dir: &Path,
+ tx_id: &str,
+ inventory: &Inventory,
+ plan: &Plan,
+) -> Result<SourceSnapshot> {
+ create_snapshot_inner(state_dir, tx_id, inventory, Some(plan))
+}
+
+fn create_snapshot_inner(
+ state_dir: &Path,
+ tx_id: &str,
+ inventory: &Inventory,
+ plan: Option<&Plan>,
+) -> Result<SourceSnapshot> {
+ validate_tx_id(tx_id)?;
+ ensure_inventory_lengths(inventory)?;
+ let root = state_dir.join("sync-snapshots");
+ std::fs::create_dir_all(&root)?;
+ let final_dir = root.join(tx_id);
+ if final_dir.exists() {
+ return open_snapshot(state_dir, tx_id);
+ }
+ let staging = root.join(format!(".{tx_id}.partial"));
+ if staging.exists() {
+ std::fs::remove_dir_all(&staging)?;
+ }
+ std::fs::create_dir(&staging)?;
+ let blobs = staging.join("blobs");
+ std::fs::create_dir(&blobs)?;
+ let prepared_dir = staging.join("prepared");
+ if plan.is_some() {
+ std::fs::create_dir(&prepared_dir)?;
+ }
+ let mut files = Vec::with_capacity(inventory.files.len());
+ for (ordinal, ((source, content_id), content_digest)) in inventory
+ .files
+ .iter()
+ .zip(&inventory.keys)
+ .zip(&inventory.digests)
+ .enumerate()
+ {
+ crate::content::verify(&source.path, source.size, content_digest)?;
+ let relative_blob = format!("blobs/{ordinal:08}");
+ let destination = staging.join(&relative_blob);
+ copy_synced(&source.path, &destination)?;
+ crate::content::verify(&destination, source.size, content_digest)?;
+ let prepared = plan
+ .map(|plan| prepare_artifact(&staging, ordinal, source, content_id, &destination, plan))
+ .transpose()?;
+ files.push(SnapshotFile {
+ content_id: content_id.clone(),
+ content_digest: content_digest.clone(),
+ content_size: source.size,
+ relative_blob,
+ prepared,
+ });
+ }
+ snapshot_failpoint(1)?;
+ let snapshot = SourceSnapshot {
+ version: SNAPSHOT_VERSION,
+ tx_id: tx_id.to_string(),
+ snapshot_digest: snapshot_digest(tx_id, &files)?,
+ files,
+ };
+ write_synced_json(&staging.join("manifest.json"), &snapshot)?;
+ sync_dir(&blobs)?;
+ if plan.is_some() {
+ sync_dir(&prepared_dir)?;
+ }
+ sync_dir(&staging)?;
+ snapshot_failpoint(2)?;
+ std::fs::rename(&staging, &final_dir)?;
+ sync_dir(&root)?;
+ snapshot_failpoint(3)?;
+ open_snapshot(state_dir, tx_id)
+}
+
+pub fn open_snapshot(state_dir: &Path, tx_id: &str) -> Result<SourceSnapshot> {
+ validate_tx_id(tx_id)?;
+ let dir = state_dir.join("sync-snapshots").join(tx_id);
+ let snapshot: SourceSnapshot =
+ serde_json::from_slice(&std::fs::read(dir.join("manifest.json"))?)?;
+ anyhow::ensure!(
+ snapshot.version == SNAPSHOT_VERSION && snapshot.tx_id == tx_id,
+ "source snapshot identity mismatch"
+ );
+ anyhow::ensure!(
+ snapshot.snapshot_digest == snapshot_digest(tx_id, &snapshot.files)?,
+ "source snapshot manifest digest mismatch"
+ );
+ for file in &snapshot.files {
+ anyhow::ensure!(
+ file.relative_blob.starts_with("blobs/")
+ && !file.relative_blob.contains("..")
+ && !Path::new(&file.relative_blob).is_absolute(),
+ "invalid source snapshot blob path"
+ );
+ crate::content::verify(
+ &dir.join(&file.relative_blob),
+ file.content_size,
+ &file.content_digest,
+ )?;
+ if let Some(prepared) = &file.prepared {
+ validate_relative_path(&prepared.relative_ndjson, "prepared/")?;
+ let path = dir.join(&prepared.relative_ndjson);
+ let metadata = std::fs::metadata(&path)?;
+ anyhow::ensure!(
+ metadata.len() == prepared.bytes
+ && stream_digest(&path, "axp1")? == prepared.digest,
+ "prepared artifact digest or size mismatch"
+ );
+ }
+ }
+ Ok(snapshot)
+}
+
+fn prepare_artifact(
+ staging: &Path,
+ ordinal: usize,
+ source: &crate::walk::FileEntry,
+ content_id: &str,
+ snapshot_blob: &Path,
+ plan: &Plan,
+) -> Result<PreparedArtifact> {
+ let assignment = plan
+ .files
+ .get(content_id)
+ .with_context(|| format!("plan has no assignment for {}", source.rel))?;
+ let datasets: HashMap<&str, &crate::state::PlanDataset> = plan
+ .datasets
+ .iter()
+ .map(|dataset| (dataset.slug.as_str(), dataset))
+ .collect();
+ let coercions: HashMap<&str, HashMap<String, crate::coerce::Coerce>> = plan
+ .datasets
+ .iter()
+ .map(|dataset| {
+ (
+ dataset.slug.as_str(),
+ crate::coerce::plan_from_specs(&dataset.specs),
+ )
+ })
+ .collect();
+ let mut paths = vec![assignment.rel.clone()];
+ paths.extend(
+ plan.duplicate_files
+ .iter()
+ .filter(|alias| alias.file_key == content_id)
+ .map(|alias| alias.rel.clone()),
+ );
+ paths.sort();
+ paths.dedup();
+ let assignments: HashMap<Option<String>, &str> = assignment
+ .assignments
+ .iter()
+ .map(|(group, slug)| (group.clone(), slug.as_str()))
+ .collect();
+ let sniffed = crate::sniff::sniff(&source.path)
+ .with_context(|| format!("sniff {} for durable preparation", source.rel))?;
+ let relative_ndjson = format!("prepared/{ordinal:08}.ndjson");
+ let path = staging.join(&relative_ndjson);
+ let file = OpenOptions::new()
+ .create_new(true)
+ .write(true)
+ .open(&path)?;
+ let mut writer = BufWriter::new(file);
+ let mut records = 0u64;
+ let mut passages = 0u64;
+ let mut vectors = 0u64;
+ let mut sink_error: Option<anyhow::Error> = None;
+ let mut sink = |record: crate::extract::RawRecord| -> bool {
+ let Some(slug) = assignments
+ .get(&record.group)
+ .or_else(|| assignments.get(&None))
+ .copied()
+ else {
+ sink_error = Some(anyhow::anyhow!(
+ "record {} has no stable dataset assignment",
+ record.locator
+ ));
+ return false;
+ };
+ let Some(dataset) = datasets.get(slug).copied() else {
+ sink_error = Some(anyhow::anyhow!(
+ "dataset {slug} is absent from prepared plan"
+ ));
+ return false;
+ };
+ let mut fields: Map<String, Value> = record.fields;
+ let Some(coercion) = coercions.get(slug) else {
+ sink_error = Some(anyhow::anyhow!("dataset {slug} has no coercion plan"));
+ return false;
+ };
+ crate::coerce::coerce_record(&mut fields, coercion);
+ fields.insert("ax_path".into(), Value::String(assignment.rel.clone()));
+ fields.insert(
+ "ax_paths".into(),
+ Value::Array(paths.iter().cloned().map(Value::String).collect()),
+ );
+ fields.insert("ax_file".into(), Value::String(content_id.to_string()));
+ fields.insert("ax_locator".into(), Value::String(record.locator.clone()));
+ fields.insert("ax_dataset".into(), Value::String(slug.to_string()));
+ fields.insert("ax_run".into(), Value::String("generation-pending".into()));
+ fields.insert(
+ "ax_format".into(),
+ Value::String(source.path.extension().map_or_else(
+ || "unknown".into(),
+ |extension| extension.to_string_lossy().to_ascii_lowercase(),
+ )),
+ );
+ let id = crate::ids::doc_id(slug, content_id, &record.locator);
+ let action = serde_json::json!({"index": {"_index": dataset.index, "_id": id}});
+ if writeln!(writer, "{action}")
+ .and_then(|_| writeln!(writer, "{}", Value::Object(fields.clone())))
+ .is_err()
+ {
+ sink_error = Some(anyhow::anyhow!("write prepared NDJSON"));
+ return false;
+ }
+ records += 1;
+ if dataset
+ .semantic_field
+ .as_ref()
+ .is_some_and(|field| fields.get(field).is_some())
+ {
+ passages += 1;
+ vectors += 1;
+ }
+ true
+ };
+ let stats = crate::extract::extract(snapshot_blob, &sniffed, None, &mut sink)
+ .with_context(|| format!("extract {} into durable preparation", source.rel))?;
+ anyhow::ensure!(
+ stats.junk == 0,
+ "durable preparation of {} produced {} junk records",
+ source.rel,
+ stats.junk
+ );
+ if let Some(error) = sink_error {
+ return Err(error);
+ }
+ writer.flush()?;
+ writer.get_ref().sync_all()?;
+ drop(writer);
+ let bytes = std::fs::metadata(&path)?.len();
+ Ok(PreparedArtifact {
+ relative_ndjson,
+ records,
+ passages,
+ vectors,
+ bytes,
+ digest: stream_digest(&path, "axp1")?,
+ })
+}
+
+fn validate_relative_path(path: &str, prefix: &str) -> Result<()> {
+ anyhow::ensure!(
+ path.starts_with(prefix) && !path.contains("..") && !Path::new(path).is_absolute(),
+ "invalid snapshot artifact path"
+ );
+ Ok(())
+}
+
+fn stream_digest(path: &Path, prefix: &str) -> Result<String> {
+ let mut reader = BufReader::new(File::open(path)?);
+ let mut hash = xxhash_rust::xxh3::Xxh3::new();
+ let mut buffer = [0u8; 64 * 1024];
+ loop {
+ let read = reader.read(&mut buffer)?;
+ if read == 0 {
+ break;
+ }
+ hash.update(&buffer[..read]);
+ }
+ Ok(format!("{prefix}-{:016x}", hash.digest()))
+}
+
+fn ensure_inventory_lengths(inventory: &Inventory) -> Result<()> {
+ anyhow::ensure!(
+ inventory.files.len() == inventory.keys.len()
+ && inventory.keys.len() == inventory.digests.len(),
+ "inventory vectors have inconsistent lengths"
+ );
+ Ok(())
+}
+
+fn snapshot_digest(tx_id: &str, files: &[SnapshotFile]) -> Result<String> {
+ let mut files = files.to_vec();
+ files.sort_by(|left, right| {
+ left.content_id
+ .cmp(&right.content_id)
+ .then_with(|| left.relative_blob.cmp(&right.relative_blob))
+ });
+ let encoded = serde_json::to_vec(&(SNAPSHOT_VERSION, tx_id, files))?;
+ Ok(format!(
+ "axs1-{:032x}",
+ xxhash_rust::xxh3::xxh3_128(&encoded)
+ ))
+}
+
+fn validate_tx_id(tx_id: &str) -> Result<()> {
+ anyhow::ensure!(
+ !tx_id.is_empty()
+ && tx_id.len() <= 128
+ && tx_id
+ .bytes()
+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')),
+ "invalid source snapshot transaction ID"
+ );
+ Ok(())
+}
+
+fn copy_synced(source: &Path, destination: &Path) -> Result<()> {
+ let mut input = File::open(source)?;
+ let mut output = OpenOptions::new()
+ .create_new(true)
+ .write(true)
+ .open(destination)?;
+ std::io::copy(&mut input, &mut output)?;
+ output.sync_all()?;
+ Ok(())
+}
+
+fn write_synced_json(path: &Path, value: &impl Serialize) -> Result<()> {
+ let mut file = OpenOptions::new().create_new(true).write(true).open(path)?;
+ serde_json::to_writer(&mut file, value)?;
+ file.write_all(b"\n")?;
+ file.sync_all()?;
+ Ok(())
+}
+
+fn sync_dir(path: &Path) -> Result<()> {
+ File::open(path)?.sync_all()?;
+ Ok(())
+}
+
+#[cfg(test)]
+fn fail_next_snapshot(boundary: u8) {
+ SNAPSHOT_FAILPOINT.store(boundary, std::sync::atomic::Ordering::SeqCst);
+}
+
+#[cfg(test)]
+fn snapshot_failpoint(boundary: u8) -> Result<()> {
+ if SNAPSHOT_FAILPOINT.compare_exchange(
+ boundary,
+ 0,
+ std::sync::atomic::Ordering::SeqCst,
+ std::sync::atomic::Ordering::SeqCst,
+ ) == Ok(boundary)
+ {
+ anyhow::bail!("injected snapshot failure at boundary {boundary}");
+ }
+ Ok(())
+}
+
+#[cfg(not(test))]
+fn snapshot_failpoint(_boundary: u8) -> Result<()> {
+ Ok(())
+}
+
+#[cfg(test)]
+fn fail_replay_after_next_apply() {
+ REPLAY_FAIL_AFTER_APPLY.store(true, std::sync::atomic::Ordering::SeqCst);
+}
+
+#[cfg(test)]
+fn replay_fail_after_apply() -> Result<()> {
+ if REPLAY_FAIL_AFTER_APPLY.swap(false, std::sync::atomic::Ordering::SeqCst) {
+ anyhow::bail!("injected crash after accepted operation");
+ }
+ Ok(())
+}
+
+#[cfg(not(test))]
+fn replay_fail_after_apply() -> Result<()> {
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::state::{FileAssignment, PlanDataset};
+ use crate::sync::{
+ ExecutionIdentity, GenerationManifest, SourceExecutionPolicy, SyncOperationKind,
+ EXECUTION_IDENTITY_VERSION,
+ };
+ use crate::walk;
+ use std::collections::BTreeMap;
+
+ fn inventory(root: &Path) -> Inventory {
+ crate::content::resolve(walk::walk(root, false).unwrap()).unwrap()
+ }
+
+ fn plan_for(inventory: &Inventory) -> Plan {
+ let key = inventory.keys[0].clone();
+ let file = &inventory.files[0];
+ Plan {
+ datasets: vec![PlanDataset {
+ slug: "docs".into(),
+ index: "ax-docs".into(),
+ family: "text".into(),
+ group: None,
+ specs: vec![],
+ time_field: None,
+ semantic_field: None,
+ sampled_records: 1,
+ file_count: 1,
+ }],
+ files: HashMap::from([(
+ key,
+ FileAssignment {
+ rel: file.rel.clone(),
+ path_id: file.rel_id.clone(),
+ is_symlink: Some(file.is_symlink),
+ family: "text".into(),
+ gzip: false,
+ content_digest: Some(inventory.digests[0].clone()),
+ assignments: vec![(None, "docs".into())],
+ },
+ )]),
+ alias_paths_indexed: false,
+ ..Plan::default()
+ }
+ }
+
+ fn execution(tx_id: &str, digest: &str) -> ExecutionIdentity {
+ ExecutionIdentity {
+ version: EXECUTION_IDENTITY_VERSION,
+ root_identity: "root".into(),
+ url: "http://engine".into(),
+ prefix: "ax".into(),
+ follow_symlinks: false,
+ chunker_identity: "chunker-v1".into(),
+ embedding_backend: "lexical".into(),
+ embedding_model: "feature-hash".into(),
+ embedding_tokenizer: "none".into(),
+ embedding_dimension: 384,
+ graph_enabled: false,
+ brain: "brain".into(),
+ detector_identity: "none".into(),
+ schema_identity: "schema-v1".into(),
+ index_identity: "index-v1".into(),
+ source_policy: SourceExecutionPolicy::DurableSnapshot {
+ reference: format!("sync-snapshots/{tx_id}"),
+ snapshot_digest: digest.into(),
+ },
+ }
+ }
+
+ fn desired(
+ generation: u64,
+ tx_id: &str,
+ snapshot: &SourceSnapshot,
+ plan: Plan,
+ groups: Vec<ManifestGroup>,
+ ) -> GenerationManifest {
+ GenerationManifest {
+ generation,
+ execution: Some(execution(tx_id, &snapshot.snapshot_digest)),
+ plan,
+ groups,
+ }
+ }
+
+ #[derive(Debug, Clone, PartialEq, Eq)]
+ struct LiveGroup {
+ content_id: String,
+ canonical: String,
+ aliases: Vec<String>,
+ }
+
+ #[derive(Default)]
+ struct FakeBackend {
+ live: BTreeMap<String, LiveGroup>,
+ applications: Vec<String>,
+ }
+
+ impl SyncOperationBackend for FakeBackend {
+ fn apply(
+ &mut self,
+ operation: &SyncOperation,
+ _base: &CommittedManifest,
+ desired: &GenerationManifest,
+ _snapshot: &SourceSnapshot,
+ ) -> Result<()> {
+ self.applications.push(operation.operation_id.clone());
+ match operation.kind {
+ SyncOperationKind::Delete => {
+ self.live.remove(&operation.group_id);
+ }
+ SyncOperationKind::Upsert | SyncOperationKind::Metadata => {
+ let group = desired
+ .groups
+ .iter()
+ .find(|group| group.group_id == operation.group_id)
+ .context("desired operation group is absent")?;
+ self.live.insert(
+ group.group_id.clone(),
+ LiveGroup {
+ content_id: group.content_id.clone(),
+ canonical: group.canonical.path_id.clone(),
+ aliases: group
+ .aliases
+ .iter()
+ .map(|alias| alias.path_id.clone())
+ .collect(),
+ },
+ );
+ }
+ }
+ Ok(())
+ }
+
+ fn validate(
+ &mut self,
+ _base: &CommittedManifest,
+ desired: &GenerationManifest,
+ _snapshot: &SourceSnapshot,
+ ) -> Result<()> {
+ let expected: BTreeMap<String, LiveGroup> = desired
+ .groups
+ .iter()
+ .map(|group| {
+ (
+ group.group_id.clone(),
+ LiveGroup {
+ content_id: group.content_id.clone(),
+ canonical: group.canonical.path_id.clone(),
+ aliases: group
+ .aliases
+ .iter()
+ .map(|alias| alias.path_id.clone())
+ .collect(),
+ },
+ )
+ })
+ .collect();
+ anyhow::ensure!(self.live == expected, "fake live generation mismatch");
+ Ok(())
+ }
+ }
+
+ fn begin(
+ journal: &mut Journal,
+ tx_id: &str,
+ snapshot: &SourceSnapshot,
+ plan: Plan,
+ groups: Vec<ManifestGroup>,
+ ) {
+ let base = journal.committed_manifest.as_ref().unwrap();
+ let pending = PendingSync::new(
+ tx_id.into(),
+ base,
+ desired(base.generation + 1, tx_id, snapshot, plan, groups),
+ )
+ .unwrap();
+ journal.sync_begin(&pending).unwrap();
+ }
+
+ fn empty_base_for(plan: &Plan) -> Plan {
+ let mut base = plan.clone();
+ base.files.clear();
+ base.duplicate_files.clear();
+ for dataset in &mut base.datasets {
+ dataset.file_count = 0;
+ }
+ base
+ }
+
+ #[test]
+ fn inventory_bridge_retains_counts_only_for_identical_content() {
+ let corpus = tempfile::tempdir().unwrap();
+ std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
+ let inventory = inventory(corpus.path());
+ let plan = plan_for(&inventory);
+ let first = groups_from_inventory(&inventory, &plan, &[]).unwrap();
+ assert_eq!(first[0].expected_records, 0);
+ let mut committed = first;
+ committed[0].expected_records = 7;
+ committed[0].expected_passages = 3;
+ committed[0].expected_vectors = 3;
+ let resumed = groups_from_inventory(&inventory, &plan, &committed).unwrap();
+ assert_eq!(
+ (
+ resumed[0].expected_records,
+ resumed[0].expected_passages,
+ resumed[0].expected_vectors
+ ),
+ (7, 3, 3)
+ );
+ }
+
+ #[test]
+ fn snapshot_is_immutable_and_detects_blob_corruption() {
+ let _guard = SNAPSHOT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
+ let inventory = inventory(corpus.path());
+ let snapshot = create_snapshot(state.path(), "tx-1", &inventory).unwrap();
+ std::fs::write(corpus.path().join("a.txt"), "changed source").unwrap();
+ assert_eq!(open_snapshot(state.path(), "tx-1").unwrap(), snapshot);
+ let blob = state
+ .path()
+ .join("sync-snapshots/tx-1")
+ .join(&snapshot.files[0].relative_blob);
+ std::fs::write(blob, "corrupt").unwrap();
+ assert!(open_snapshot(state.path(), "tx-1").is_err());
+ }
+
+ #[test]
+ fn prepared_snapshot_streams_exact_actions_counts_and_digest() {
+ let _guard = SNAPSHOT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ std::fs::write(
+ corpus.path().join("a.jsonl"),
+ "{\"message\":\"alpha\"}\n{\"message\":\"beta\"}\n",
+ )
+ .unwrap();
+ let inventory = inventory(corpus.path());
+ let plan = plan_for(&inventory);
+ let snapshot =
+ create_prepared_snapshot(state.path(), "tx-prepared", &inventory, &plan).unwrap();
+ let prepared = snapshot.files[0].prepared.as_ref().unwrap();
+ assert_eq!(prepared.records, 2);
+ assert!(prepared.bytes > 0);
+ let ndjson = std::fs::read_to_string(
+ state
+ .path()
+ .join("sync-snapshots/tx-prepared")
+ .join(&prepared.relative_ndjson),
+ )
+ .unwrap();
+ assert_eq!(ndjson.lines().count(), 4);
+ assert!(ndjson.contains("\"ax_file\""));
+ assert!(ndjson.contains("\"_index\":\"ax-docs\""));
+
+ let mut groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
+ bind_prepared_counts(&mut groups, &snapshot, &inventory.keys).unwrap();
+ assert_eq!(groups[0].expected_records, 2);
+ let artifact_path = state
+ .path()
+ .join("sync-snapshots/tx-prepared")
+ .join(&prepared.relative_ndjson);
+ std::fs::write(artifact_path, "corrupt").unwrap();
+ assert!(open_snapshot(state.path(), "tx-prepared").is_err());
+ }
+
+ #[test]
+ fn prepared_and_metadata_streams_preserve_pairs_under_tiny_budgets() {
+ let input = concat!(
+ "{\"index\":{\"_index\":\"docs\",\"_id\":\"a\"}}\n",
+ "{\"body\":\"alpha\"}\n",
+ "{\"index\":{\"_index\":\"docs\",\"_id\":\"b\"}}\n",
+ "{\"body\":\"beta\"}\n"
+ );
+ let mut index_chunks = Vec::new();
+ stream_ndjson_pairs(input.as_bytes(), 1, |body| {
+ index_chunks.push(String::from_utf8(body).unwrap());
+ Ok(())
+ })
+ .unwrap();
+ assert_eq!(index_chunks.len(), 2);
+ assert!(index_chunks.iter().all(|chunk| chunk.lines().count() == 2));
+
+ let mut update_chunks = Vec::new();
+ stream_metadata_updates(
+ input.as_bytes(),
+ 1,
+ "renamed.txt",
+ &[Value::String("renamed.txt".into())],
+ |body| {
+ update_chunks.push(String::from_utf8(body).unwrap());
+ Ok(())
+ },
+ )
+ .unwrap();
+ assert_eq!(update_chunks.len(), 2);
+ assert!(update_chunks.iter().all(|chunk| {
+ chunk.lines().count() == 2
+ && chunk.contains("\"update\"")
+ && chunk.contains("\"ax_path\":\"renamed.txt\"")
+ && !chunk.contains("\"body\"")
+ }));
+
+ let es = crate::esclient::Es::new("http://127.0.0.1:1", None).unwrap();
+ let state = tempfile::tempdir().unwrap();
+ let _backend = EsSyncBackend::new(&es, state.path(), 1);
+ }
+
+ #[test]
+ fn every_snapshot_crash_boundary_restarts_deterministically() {
+ let _guard = SNAPSHOT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ for boundary in 1..=3 {
+ let corpus = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
+ let inventory = inventory(corpus.path());
+ fail_next_snapshot(boundary);
+ assert!(create_snapshot(state.path(), "tx-restart", &inventory).is_err());
+ let recovered = create_snapshot(state.path(), "tx-restart", &inventory).unwrap();
+ assert_eq!(
+ open_snapshot(state.path(), "tx-restart").unwrap(),
+ recovered
+ );
+ }
+ }
+
+ #[test]
+ fn pending_generation_is_bound_before_mutable_source_replanning() {
+ let _guard = SNAPSHOT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let state = tempfile::tempdir().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
+ let inventory = inventory(corpus.path());
+ let snapshot = create_snapshot(state.path(), "tx-pending", &inventory).unwrap();
+ let pending: PendingSync = serde_json::from_value(serde_json::json!({
+ "tx_id": "tx-pending",
+ "base_generation": 0,
+ "base_manifest_digest": "base",
+ "desired_manifest_digest": "desired",
+ "operation_hash": "operations",
+ "desired": {
+ "generation": 1,
+ "execution": {
+ "version": 1,
+ "root_identity": "root",
+ "url": "http://engine",
+ "prefix": "ax",
+ "follow_symlinks": false,
+ "chunker_identity": "chunker",
+ "embedding_backend": "lexical",
+ "embedding_model": "feature-hash",
+ "embedding_tokenizer": "none",
+ "embedding_dimension": 384,
+ "graph_enabled": false,
+ "brain": "brain",
+ "detector_identity": "none",
+ "schema_identity": "schema",
+ "index_identity": "index",
+ "source_policy": {
+ "policy": "durable_snapshot",
+ "reference": "sync-snapshots/tx-pending",
+ "snapshot_digest": snapshot.snapshot_digest
+ }
+ },
+ "plan": {
+ "datasets": [],
+ "files": {},
+ "junk_files": [],
+ "duplicate_files": [],
+ "alias_paths_indexed": false
+ },
+ "groups": []
+ },
+ "operations": []
+ }))
+ .unwrap();
+ let error = require_resumable_pending_source(state.path(), Some(&pending))
+ .unwrap_err()
+ .to_string();
+ std::fs::write(corpus.path().join("a.txt"), "mutated after sync_begin").unwrap();
+ let repeated = require_resumable_pending_source(state.path(), Some(&pending))
+ .unwrap_err()
+ .to_string();
+ assert_eq!(error, repeated);
+ assert!(error.contains("verified durable source snapshot"));
+ }
+
+ #[test]
+ fn accepted_upsert_replays_without_duplicate_live_state() {
+ let _journal_guard = crate::state::SYNC_IO_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let _guard = SNAPSHOT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
+ let inventory = inventory(corpus.path());
+ let plan = plan_for(&inventory);
+ let groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
+ let snapshot = create_snapshot(state.path(), "tx-add", &inventory).unwrap();
+ let mut journal =
+ Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
+ journal.write_plan(&empty_base_for(&plan)).unwrap();
+ begin(&mut journal, "tx-add", &snapshot, plan, groups);
+ let mut backend = FakeBackend::default();
+
+ fail_replay_after_next_apply();
+ assert!(replay_pending_operations(state.path(), &mut journal, &mut backend).is_err());
+ assert_eq!(backend.live.len(), 1);
+ drop(journal);
+
+ let mut journal =
+ Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
+ replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
+ assert_eq!(backend.live.len(), 1);
+ assert_eq!(backend.applications.len(), 2);
+ assert!(journal.pending_sync.is_none());
+ assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 1);
+ }
+
+ #[test]
+ fn replay_converges_change_metadata_and_delete_without_resurrection() {
+ let _journal_guard = crate::state::SYNC_IO_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let _guard = SNAPSHOT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ let state = tempfile::tempdir().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
+ let first_inventory = inventory(corpus.path());
+ let first_plan = plan_for(&first_inventory);
+ let first_groups = groups_from_inventory(&first_inventory, &first_plan, &[]).unwrap();
+ let first_snapshot = create_snapshot(state.path(), "tx-first", &first_inventory).unwrap();
+ let mut journal =
+ Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
+ journal.write_plan(&empty_base_for(&first_plan)).unwrap();
+ begin(
+ &mut journal,
+ "tx-first",
+ &first_snapshot,
+ first_plan,
+ first_groups,
+ );
+ let mut backend = FakeBackend::default();
+ replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
+ let stable_group_id = journal.committed_manifest.as_ref().unwrap().groups[0]
+ .group_id
+ .clone();
+
+ std::fs::write(corpus.path().join("a.txt"), "beta").unwrap();
+ let changed_inventory = inventory(corpus.path());
+ let changed_plan = plan_for(&changed_inventory);
+ let changed_groups = groups_from_inventory(
+ &changed_inventory,
+ &changed_plan,
+ &journal.committed_manifest.as_ref().unwrap().groups,
+ )
+ .unwrap();
+ assert_eq!(changed_groups[0].group_id, stable_group_id);
+ let changed_snapshot =
+ create_snapshot(state.path(), "tx-change", &changed_inventory).unwrap();
+ begin(
+ &mut journal,
+ "tx-change",
+ &changed_snapshot,
+ changed_plan,
+ changed_groups,
+ );
+ fail_replay_after_next_apply();
+ assert!(replay_pending_operations(state.path(), &mut journal, &mut backend).is_err());
+ replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
+ let changed_content = backend.live[&stable_group_id].content_id.clone();
+
+ let mut renamed_plan = journal.committed_manifest.as_ref().unwrap().plan.clone();
+ let content_id = renamed_plan.files.keys().next().unwrap().clone();
+ let assignment = renamed_plan.files.get_mut(&content_id).unwrap();
+ assignment.rel = "renamed.txt".into();
+ assignment.path_id = "unix:72656e616d65642e747874".into();
+ let mut renamed_groups = journal.committed_manifest.as_ref().unwrap().groups.clone();
+ renamed_groups[0].canonical.rel = "renamed.txt".into();
+ renamed_groups[0].canonical.path_id = "unix:72656e616d65642e747874".into();
+ let renamed_snapshot =
+ create_snapshot(state.path(), "tx-metadata", &changed_inventory).unwrap();
+ begin(
+ &mut journal,
+ "tx-metadata",
+ &renamed_snapshot,
+ renamed_plan,
+ renamed_groups,
+ );
+ replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
+ assert_eq!(backend.live[&stable_group_id].content_id, changed_content);
+ assert_eq!(
+ backend.live[&stable_group_id].canonical,
+ "unix:72656e616d65642e747874"
+ );
+
+ let empty = Inventory {
+ files: vec![],
+ keys: vec![],
+ digests: vec![],
+ duplicates: vec![],
+ };
+ let empty_snapshot = create_snapshot(state.path(), "tx-delete", &empty).unwrap();
+ let mut empty_plan = journal.committed_manifest.as_ref().unwrap().plan.clone();
+ empty_plan.files.clear();
+ empty_plan.datasets[0].file_count = 0;
+ begin(
+ &mut journal,
+ "tx-delete",
+ &empty_snapshot,
+ empty_plan,
+ vec![],
+ );
+ fail_replay_after_next_apply();
+ assert!(replay_pending_operations(state.path(), &mut journal, &mut backend).is_err());
+ assert!(backend.live.is_empty());
+ replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
+ assert!(backend.live.is_empty());
+ assert!(journal
+ .committed_manifest
+ .as_ref()
+ .unwrap()
+ .groups
+ .is_empty());
+ }
+}