Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

311 KB
Raw Blame
//! Durable source preparation for incremental corpus reconciliation.
//!
//! This slice deliberately performs no remote mutation. It projects a
//! byte-verified inventory into generation groups and makes desired bytes
//! immutable before a later executor writes `sync_begin`.

use crate::content::Inventory;
use crate::state::{Journal, Plan};
use crate::sync::{
    self, CommittedManifest, DesiredContentGroup, GenerationManifest, ManifestGroup, ManifestPath,
    PendingSync, SourceExecutionPolicy, SyncOperation, SyncOperationState,
};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::fs::{File, OpenOptions};
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::Path;

const SNAPSHOT_VERSION: u32 = 2;

#[cfg(test)]
thread_local! {
    // Test failpoints belong to the thread that arms them: an unrelated test
    // running in parallel must not consume another test's one-shot failure.
    static SNAPSHOT_FAILPOINT: std::cell::Cell<u8> = const { std::cell::Cell::new(0) };
    static REPLAY_FAIL_AFTER_APPLY: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}
#[cfg(test)]
static SNAPSHOT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
#[cfg(test)]
static POST_SEAL_SOURCE_REPLACEMENT: std::sync::Mutex<Option<(std::path::PathBuf, Vec<u8>)>> =
    std::sync::Mutex::new(None);
#[cfg(test)]
pub(crate) static REPLAY_FAILPOINT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
#[cfg(test)]
static GC_FAIL_AFTER_RENAME: std::sync::atomic::AtomicBool =
    std::sync::atomic::AtomicBool::new(false);

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct SnapshotFile {
    pub content_id: String,
    pub content_digest: String,
    pub content_size: u64,
    pub relative_blob: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub prepared: Option<PreparedArtifact>,
    /// #971: identity of every input `prepare_artifact` consumes for THIS file
    /// (chunker identity, document-id identity, the file's assignment, its
    /// alias paths, the datasets it routes to). A later generation whose file
    /// carries the same content digest AND the same identity reuses the sealed
    /// blob and prepared artifact via hardlink instead of re-copying,
    /// re-verifying and re-extracting, which is what makes a one-file change
    /// cost O(changed) rather than O(corpus). Skip-serialized so a manifest
    /// sealed before this field existed keeps hashing to the digest the journal
    /// recorded — the same rule `records_by_dataset` follows above. `None` on
    /// this file therefore also means "a pre-#971 snapshot: never reuse".
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub prepared_identity: Option<String>,
    /// #2421: the sealed blob was deleted once this file's prepared artifact
    /// was durable. Nothing after preparation reads the blob — replay, resume
    /// and metadata updates all stream `prepared.relative_ndjson` — so keeping
    /// a second copy of every indexed source byte made the state dir larger
    /// than the corpus. Only ever true with `prepared: Some`; `open_snapshot`
    /// then verifies the artifact instead of the blob, and a later generation
    /// reuses the artifact by identity or re-copies from the verified live
    /// source. Skip-serialized when false so a manifest sealed before this
    /// field existed keeps hashing to the digest the journal recorded.
    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
    pub blob_released: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct PreparedArtifact {
    pub relative_ndjson: String,
    pub records: u64,
    pub passages: u64,
    pub vectors: u64,
    /// Records the extractor produced that no dataset assignment could accept.
    /// Sealed here because nothing downstream can recover the number: junk is
    /// never published, so no read-back count sees it.
    #[serde(default)]
    pub junk: u64,
    /// The per-file record cap (#381) stopped this file's section stream before
    /// the whole body was emitted. Skip-serialized when false so a non-truncated
    /// artifact hashes exactly as before (a truncating file already differs by
    /// design — it seals fewer records).
    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
    pub truncated: bool,
    /// Per-dataset breakdown of `records`, keyed by the dataset each record was
    /// written under. One file can feed several datasets — a SQL dump is one
    /// file and N tables — so the flat total is not comparable to any single
    /// dataset's read-back. Omitted when empty: `snapshot_digest` covers the
    /// serialized artifact, so an in-flight snapshot sealed before this field
    /// existed must keep hashing to the same value.
    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
    pub records_by_dataset: BTreeMap<String, u64>,
    pub bytes: u64,
    pub digest: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct SourceSnapshot {
    pub version: u32,
    pub tx_id: String,
    /// Publication timestamp sealed once and reused by every replay.
    pub started: String,
    pub preparation_contract_digest: String,
    pub footprint: SnapshotFootprint,
    pub files: Vec<SnapshotFile>,
    pub snapshot_digest: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct SnapshotFootprint {
    pub source_bytes: u64,
    pub prepared_bytes: u64,
    pub total_bytes: u64,
    pub hard_budget_bytes: u64,
}

/// Logical payload accounting for source and prepared-record bytes.
/// This is deliberately not a filesystem-allocation limit: directory,
/// manifest, and block-allocation overhead are excluded.
struct PayloadBudget {
    used: u64,
    limit: u64,
}

impl PayloadBudget {
    fn charge(&mut self, bytes: usize) -> std::io::Result<()> {
        let bytes = u64::try_from(bytes)
            .map_err(|_| std::io::Error::other("snapshot payload write length overflow"))?;
        let next = self
            .used
            .checked_add(bytes)
            .ok_or_else(|| std::io::Error::other("snapshot payload byte count overflow"))?;
        if next > self.limit {
            return Err(std::io::Error::other(format!(
                "snapshot logical payload write of {bytes} bytes would raise staged payload from \
                 {} to {next} bytes, exceeding configured limit of {} bytes; no generation was \
                 committed and the partial snapshot is removed automatically",
                self.used, self.limit
            )));
        }
        self.used = next;
        Ok(())
    }
}

struct BudgetWriter<'a, W> {
    inner: W,
    budget: &'a mut PayloadBudget,
}

impl<W: Write> Write for BudgetWriter<'_, W> {
    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
        self.budget.charge(buf.len())?;
        match self.inner.write(buf) {
            Ok(written) if written == buf.len() => Ok(written),
            Ok(written) => {
                self.budget.used -= (buf.len() - written) as u64;
                Ok(written)
            }
            Err(error) => {
                self.budget.used -= buf.len() as u64;
                Err(error)
            }
        }
    }

    fn flush(&mut self) -> std::io::Result<()> {
        self.inner.flush()
    }
}

struct StagingCleanup {
    path: std::path::PathBuf,
    armed: bool,
}

impl Drop for StagingCleanup {
    fn drop(&mut self) {
        if self.armed {
            let _ = std::fs::remove_dir_all(&self.path);
        }
    }
}

/// One operation this replay attempt still owes, with the name and byte
/// weight the progress surface reports for it. Precomputed once by
/// [`replay_pending_operations`] so the serial loop and the parallel
/// scheduler (#933) describe every operation identically — and so a worker
/// thread never needs the group maps the fields were derived from.
pub struct ReplayItem<'o> {
    pub operation: &'o SyncOperation,
    pub rel: String,
    pub bytes: u64,
}

/// Remote mutations must be convergent: calling `apply` twice for one
/// operation after an accepted-but-unrecorded response must produce exactly
/// the same live state. `validate` is the final generation-wide barrier.
#[cfg_attr(not(test), allow(dead_code))]
pub trait SyncOperationBackend {
    fn provision_generation(&mut self, _desired: &GenerationManifest) -> Result<()> {
        Ok(())
    }

    fn apply(
        &mut self,
        operation: &SyncOperation,
        base: &CommittedManifest,
        desired: &GenerationManifest,
        snapshot: &SourceSnapshot,
    ) -> Result<()>;

    fn validate(
        &mut self,
        base: &CommittedManifest,
        desired: &GenerationManifest,
        snapshot: &SourceSnapshot,
    ) -> Result<()>;

    /// Publish and exactly read back the complete agent-facing catalog for
    /// this generation. This runs before the validation/authority barrier.
    fn publish_generation_catalog(
        &mut self,
        _base: &CommittedManifest,
        _desired: &GenerationManifest,
        _snapshot: &SourceSnapshot,
    ) -> Result<()> {
        Ok(())
    }

    /// Progress hooks for the replay loop (#931). The loop is where a
    /// generated run spends its indexing time, and it used to report nothing:
    /// the stream kept printing the PREVIOUS phase — `scan`, at 100%, with a
    /// `since_progress_s` that only climbed — for as long as documents were
    /// landing, which is exactly what a real hang looks like. A backend with a
    /// progress surface overrides these; the defaults keep test backends silent.
    ///
    /// `items` is the number of operations still to apply and `bytes` the
    /// sealed NDJSON they will send, so the phase has an honest denominator in
    /// the unit the ETA is derived from.
    fn replay_begins(&mut self, _items: u64, _bytes: u64) {}

    /// One operation is about to be applied; `rel` names its source file so the
    /// surface can say what a quiet tail is waiting on.
    fn operation_begins(&mut self, _rel: &str, _bytes: u64) {}

    /// The operation started by [`Self::operation_begins`] has been applied.
    fn operation_applied(&mut self) {}

    /// Apply `items` — the operations this attempt still owes, in operation
    /// order — journaling each one through `journal`.
    ///
    /// The default is the historical serial loop: `Started`, apply,
    /// `Committed`, one operation at a time. [`EsSyncBackend`] overrides it
    /// (#933) with a windowed scheduler when the run asked for more than one
    /// worker: one plan gives every group at most one operation
    /// (`plan_operations` emits at most one per group id), and an operation's
    /// writes are keyed by its group's content id, so two operations' writes
    /// never touch the same documents. The per-operation durable order is
    /// exactly the serial contract, overlapped — `Started` is journaled
    /// before the operation is dispatched, `Committed` after its apply
    /// returned, so a crash mid-window repeats precisely the operations
    /// whose accepted apply was not recorded, and `apply` stays convergent
    /// for exactly that retry.
    fn replay_operations(
        &mut self,
        items: &[ReplayItem<'_>],
        base: &CommittedManifest,
        desired: &GenerationManifest,
        snapshot: &SourceSnapshot,
        journal: &mut Journal,
    ) -> Result<()> {
        replay_serial(self, items, base, desired, snapshot, journal)
    }
}

/// The historical serial replay: one operation at a time, journaling
/// `Started` before its apply and `Committed` after it.
fn replay_serial<B: SyncOperationBackend + ?Sized>(
    backend: &mut B,
    items: &[ReplayItem<'_>],
    base: &CommittedManifest,
    desired: &GenerationManifest,
    snapshot: &SourceSnapshot,
    journal: &mut Journal,
) -> Result<()> {
    for item in items {
        let operation = item.operation;
        let state = journal
            .pending_sync
            .as_ref()
            .and_then(|sync| sync.operation_states.get(&operation.operation_id))
            .cloned();
        if state == Some(SyncOperationState::Committed) {
            continue;
        }
        if state.is_none() {
            journal.sync_operation_state(&operation.operation_id, SyncOperationState::Started)?;
        }
        backend.operation_begins(&item.rel, item.bytes);
        backend.apply(operation, base, desired, snapshot)?;
        replay_fail_after_apply()?;
        journal.sync_operation_state(&operation.operation_id, SyncOperationState::Committed)?;
        backend.operation_applied();
    }
    Ok(())
}

/// The journal-side callbacks of [`replay_windowed`], one object so the
/// caller's mutable state — the journal — is borrowed once, not once per
/// closure.
pub(crate) trait ReplayHooks<T> {
    /// Called on the scheduling thread before the item is dispatched.
    fn begin(&mut self, item: &T) -> Result<()>;
    /// Called on the scheduling thread, in dispatch order, as each item's
    /// apply finishes. Returning an error stops all further dispatch.
    fn applied(&mut self, item: &T, outcome: Result<()>) -> Result<()>;
}

/// #933: apply `items` through a window at most `width` items wide.
///
/// This is the whole concurrency policy of the parallel replay, kept free of
/// ES, journal and progress detail so it can be unit-tested without a
/// server. [`ReplayHooks::begin`] runs on the calling thread before an item
/// is dispatched — where the serial loop journals `Started`; `apply` runs on
/// worker threads and may overlap with other items' `apply`;
/// [`ReplayHooks::applied`] runs on the calling thread, in dispatch order,
/// as each item finishes — where the serial loop journals `Committed`.
/// Completions are joined head-first, so `applied` sees items in exactly the
/// order they were begun, which is what keeps the journal a serial reader
/// can follow.
///
/// Failure semantics are the serial loop's plus one honest addition the
/// overlap forces, and they run through `applied`: the first `applied` that
/// returns an error stops all further dispatch, everything already
/// dispatched is drained to completion, and `applied` is still called for
/// each drained item so the caller can journal the applies the server did
/// accept. On the serial loop nothing else was in flight when an apply
/// failed, so its behaviour is unchanged; overlapped, skipping the
/// `Committed` write for an apply that already landed would only force a
/// convergent redo of it on resume. A caller that deliberately accepts a
/// failed outcome by returning `Ok` keeps the run going — the scheduler
/// trusts `applied`, not the outcome. The first error is the return value;
/// later errors are handed to `applied` and otherwise dropped, because the
/// first failure is the one a retry of the same command will meet again.
pub(crate) fn replay_windowed<T: Sync>(
    items: &[T],
    width: usize,
    hooks: &mut dyn ReplayHooks<T>,
    apply: &(dyn Fn(&T) -> Result<()> + Sync),
) -> Result<()> {
    replay_windowed_with(
        &xerj_common::thread_spawn::StdSpawn,
        xerj_common::thread_spawn::spawn_allowed,
        items,
        width,
        hooks,
        apply,
    )
}

/// How many times a refused worker spawn is retried before the run
/// continues with the workers it already has (#2428).
pub(crate) const WORKER_SPAWN_ATTEMPTS: u32 = 3;
/// First pause after a refused worker spawn; doubled on each retry.
pub(crate) const WORKER_SPAWN_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50);

/// Start `f` on a scoped helper thread, retrying a refused spawn (`EAGAIN`
/// from `pthread_create`, or the spawn gate saying no) up to `attempts`
/// times with a doubling `backoff` (#2428). A refused spawn is a transient
/// thread or memory limit, not a reason to lose the run: `Err` after the
/// last attempt means `f` never ran and the caller carries on with the
/// threads it has, or does the work itself.
pub(crate) fn spawn_scoped_with_backoff<'scope, 'env, S, G, F>(
    scope: &xerj_common::thread_spawn::Scope<'scope, 'env>,
    spawner: &S,
    gate: G,
    name: &str,
    attempts: u32,
    backoff: std::time::Duration,
    f: F,
) -> std::io::Result<()>
where
    S: xerj_common::thread_spawn::Spawn + ?Sized,
    G: Fn() -> bool,
    F: FnOnce() + Send + Copy + 'scope,
{
    let attempts = attempts.max(1);
    let mut delay = backoff;
    let mut attempt = 1;
    loop {
        match scope.spawn_with(spawner, gate(), name, f) {
            Ok(()) => return Ok(()),
            Err(error) if attempt >= attempts => return Err(error),
            Err(_) => {
                std::thread::sleep(delay);
                delay = delay.saturating_mul(2);
                attempt += 1;
            }
        }
    }
}

/// [`replay_windowed`] with an explicit spawner and spawn gate, so the
/// spawn-failure paths are testable without exhausting a real machine.
///
/// #2428: the window used to start one thread per *operation*. A corpus
/// replay is hundreds of thousands of operations, so a long run made
/// hundreds of thousands of `pthread_create` calls, and under memory or
/// thread pressure one of them eventually got `EAGAIN`. Now at most
/// `width` workers are started once per replay and reused for every item.
/// A refused worker spawn is retried with backoff; if it is still refused
/// the replay continues on the workers that did start, and if none did,
/// every item is applied on the calling thread. Ordering and failure
/// semantics are unchanged.
pub(crate) fn replay_windowed_with<T, S, G>(
    spawner: &S,
    gate: G,
    items: &[T],
    width: usize,
    hooks: &mut dyn ReplayHooks<T>,
    apply: &(dyn Fn(&T) -> Result<()> + Sync),
) -> Result<()>
where
    T: Sync,
    S: xerj_common::thread_spawn::Spawn + ?Sized,
    G: Fn() -> bool,
{
    if items.is_empty() {
        return Ok(());
    }
    if width <= 1 || items.len() == 1 {
        // No overlap is possible; same callbacks, same order, no threads.
        for item in items {
            hooks.begin(item)?;
            hooks.applied(item, apply(item))?;
        }
        return Ok(());
    }
    /// One dispatched operation: queued to a worker, or — when no worker
    /// could be started (#1122, #2428) — already applied inline.
    enum Dispatched {
        Worker(std::sync::mpsc::Receiver<Result<()>>),
        Inline(Result<()>),
    }
    type Job<'a, T> = (&'a T, std::sync::mpsc::Sender<Result<()>>);
    let (job_tx, job_rx) = std::sync::mpsc::channel::<Job<'_, T>>();
    let job_rx = std::sync::Mutex::new(job_rx);
    let job_rx = &job_rx;
    // A worker takes queued items until the sender is dropped. A panic in
    // `apply` (unwind builds) becomes that item's error and the worker
    // keeps serving the queue.
    let worker = move || loop {
        let job = {
            let rx = job_rx
                .lock()
                .unwrap_or_else(|poisoned| poisoned.into_inner());
            rx.recv()
        };
        let Ok((item, reply)) = job else { break };
        let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| apply(item)))
            .unwrap_or_else(|panic| {
                Err(anyhow::anyhow!(
                    "replay apply panicked: {}",
                    panic_message(panic)
                ))
            });
        // The scheduler may have stopped listening after a `begin` error.
        let _ = reply.send(outcome);
    };
    // #1122: `Builder::spawn_scoped` can abort the process after returning
    // `Ok`, when the child cannot map its alternate signal stack. This
    // scope uses `pthread_create`.
    let failure = xerj_common::thread_spawn::scope(|scope| -> Result<Option<anyhow::Error>> {
        let wanted = width.min(items.len());
        let mut workers = 0usize;
        while workers < wanted {
            match spawn_scoped_with_backoff(
                scope,
                spawner,
                &gate,
                "xerj-sync-apply",
                WORKER_SPAWN_ATTEMPTS,
                WORKER_SPAWN_BACKOFF,
                worker,
            ) {
                Ok(()) => workers += 1,
                Err(error) => {
                    eprintln!(
                        "autoindex: could not start replay worker {} of {wanted} ({error}); \
                         continuing with {} (#2428)",
                        workers + 1,
                        workers.max(1)
                    );
                    break;
                }
            }
        }
        let job_tx = job_tx;
        let mut next = items.iter();
        let mut in_flight: std::collections::VecDeque<(&T, Dispatched)> =
            std::collections::VecDeque::new();
        let mut failure: Option<anyhow::Error> = None;
        loop {
            while failure.is_none() && in_flight.len() < width {
                // `begin` is the Started write: it happens on THIS thread,
                // before the operation exists remotely, so an operation that
                // was never dispatched is also never recorded as begun.
                let Some(item) = next.next() else { break };
                hooks.begin(item)?;
                let slot = if workers > 0 {
                    let (reply_tx, reply_rx) = std::sync::mpsc::channel();
                    match job_tx.send((item, reply_tx)) {
                        Ok(()) => Dispatched::Worker(reply_rx),
                        // Unreachable while `job_rx` lives; apply inline.
                        Err(_) => Dispatched::Inline(apply(item)),
                    }
                } else {
                    Dispatched::Inline(apply(item))
                };
                in_flight.push_back((item, slot));
            }
            let Some((item, slot)) = in_flight.pop_front() else {
                break;
            };
            let outcome = match slot {
                Dispatched::Worker(reply) => reply.recv().unwrap_or_else(|_| {
                    Err(anyhow::anyhow!("replay worker exited before applying"))
                }),
                Dispatched::Inline(outcome) => outcome,
            };
            let reported = hooks.applied(item, outcome);
            if let (None, Err(error)) = (&failure, reported) {
                // First failure stops dispatch; the drain still reports.
                failure = Some(error);
            }
        }
        // Dropping `job_tx` here ends the workers; the scope joins them.
        Ok(failure)
    })?;
    match failure {
        Some(error) => Err(error),
        None => Ok(()),
    }
}

/// The message of a panicking worker, when it carried one — a panic is not an
/// ES answer and must not look like one.
fn panic_message(panic: Box<dyn std::any::Any + Send>) -> String {
    if let Some(message) = panic.downcast_ref::<String>() {
        message.clone()
    } else if let Some(message) = panic.downcast_ref::<&str>() {
        (*message).to_owned()
    } else {
        "no message".to_owned()
    }
}

/// Per-replay id→position caches for the manifests and snapshot a replay
/// walks (#1224).
///
/// `apply_shared` resolves every operation's group in the base and desired
/// manifests and its content in the snapshot, and the linear `iter().find()`
/// each resolution used is quadratic over a corpus run — the 402,744-operation
/// cve-records build paid milliseconds of scanning per file. Built once per
/// [`EsSyncBackend::replay_operations`] call from that call's arguments, so
/// inside a replay every hit is exact. A lookup outside a replay (or against
/// differently-shaped data than the cache was built from — guarded by the
/// recorded lengths) falls back to the scan the cache replaced, and every hit
/// is verified against the slice it answers from, so a stale cache can never
/// return a group the scan would not have.
#[derive(Debug, Clone, Default)]
struct ReplayLookup {
    base_len: usize,
    base: std::collections::HashMap<String, usize>,
    desired_len: usize,
    desired: std::collections::HashMap<String, usize>,
    files_len: usize,
    files: std::collections::HashMap<String, usize>,
}

impl ReplayLookup {
    fn build(
        base: &crate::sync::CommittedManifest,
        desired: &crate::sync::GenerationManifest,
        snapshot: &SourceSnapshot,
    ) -> Self {
        Self {
            base_len: base.groups.len(),
            base: base
                .groups
                .iter()
                .enumerate()
                .map(|(position, group)| (group.group_id.clone(), position))
                .collect(),
            desired_len: desired.groups.len(),
            desired: desired
                .groups
                .iter()
                .enumerate()
                .map(|(position, group)| (group.group_id.clone(), position))
                .collect(),
            files_len: snapshot.files.len(),
            files: snapshot
                .files
                .iter()
                .enumerate()
                .map(|(position, file)| (file.content_id.clone(), position))
                .collect(),
        }
    }

    /// `groups` by id through a cache side, falling back to the scan when the
    /// cache does not describe this exact slice.
    fn group<'m, G>(
        side: &std::collections::HashMap<String, usize>,
        cached_len: usize,
        groups: &'m [G],
        group_id: &str,
        id_of: impl Fn(&G) -> &str,
    ) -> Option<&'m G> {
        if cached_len != groups.len() {
            return groups.iter().find(|group| id_of(group) == group_id);
        }
        side.get(group_id)
            .copied()
            .and_then(|position| groups.get(position))
            .filter(|group| id_of(group) == group_id)
    }

    fn base_group<'m>(
        &self,
        base: &'m crate::sync::CommittedManifest,
        group_id: &str,
    ) -> Option<&'m crate::sync::ManifestGroup> {
        Self::group(&self.base, self.base_len, &base.groups, group_id, |group| {
            &group.group_id
        })
    }

    fn desired_group<'m>(
        &self,
        desired: &'m crate::sync::GenerationManifest,
        group_id: &str,
    ) -> Option<&'m crate::sync::ManifestGroup> {
        Self::group(
            &self.desired,
            self.desired_len,
            &desired.groups,
            group_id,
            |group| &group.group_id,
        )
    }

    fn file<'m>(&self, snapshot: &'m SourceSnapshot, content_id: &str) -> Option<&'m SnapshotFile> {
        if self.files_len != snapshot.files.len() {
            return snapshot
                .files
                .iter()
                .find(|file| file.content_id == content_id);
        }
        self.files
            .get(content_id)
            .copied()
            .and_then(|position| snapshot.files.get(position))
            .filter(|file| file.content_id == content_id)
    }
}

/// How many per-digest verify windows to issue for one semantic field, and
/// how many to tick without asking (#1267).
///
/// A field whose `exists` total is zero cannot match any digest. Issuing the
/// `term` + `exists` window anyway is the pre-#1260 brute scan: one window
/// per [`EsSyncBackend::VERIFY_WINDOW`] digests, each walking the index, each
/// returning no hits. `window == 0` asks for nothing.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ExistsProbePlan {
    issue_windows: u64,
    skip_windows: u64,
}

fn plan_exists_probes(values: usize, window: usize, exists_total: u64) -> ExistsProbePlan {
    if window == 0 {
        return ExistsProbePlan {
            issue_windows: 0,
            skip_windows: 0,
        };
    }
    let windows = values.div_ceil(window) as u64;
    if exists_total == 0 {
        ExistsProbePlan {
            issue_windows: 0,
            skip_windows: windows,
        }
    } else {
        ExistsProbePlan {
            issue_windows: windows,
            skip_windows: 0,
        }
    }
}

/// Catalog documents whose live `_source` is not already the sealed projection.
///
/// A resume of the same generation re-enters publication with the same
/// `run_id` and the same bytes (#1267). Those ids are not rewritten: the bulk
/// that used to send every projection document is what made the following
/// `run_id` walk match the whole catalog.
fn catalog_docs_to_write(
    expected: &BTreeMap<String, Value>,
    live: &BTreeMap<String, Value>,
) -> BTreeMap<String, Value> {
    expected
        .iter()
        .filter(|(id, document)| live.get(*id) != Some(*document))
        .map(|(id, document)| ((*id).clone(), (*document).clone()))
        .collect()
}

/// How many of the prior generation's published ids this generation kept.
///
/// Kept documents still carry the prior `run_id`. The live count of that
/// `run_id` equals this number when nothing stray remains and every kept
/// document is still there. A smaller count is the documented cross-journal
/// collision (a kept document was overwritten elsewhere). A larger count is a
/// stray. Equal is the fast path that does not walk.
fn prior_run_expected_remainder(prior_written: &BTreeSet<String>, kept: &BTreeSet<String>) -> u64 {
    prior_written.intersection(kept).count() as u64
}

/// Ids carrying one generation's `run_id`, one per line, inside that
/// generation's already-sealed snapshot directory.
///
/// The manifest digest does not cover this file. Missing means "published by
/// an older client": the next generation walks that `run_id` once. Present
/// means the next generation's stray check is a `size: 0` count.
const CATALOG_RUN_IDS_FILE: &str = "catalog-run.ids";

fn catalog_run_ids_path(state_dir: &Path, run_id: &str) -> Result<std::path::PathBuf> {
    validate_tx_id(run_id)?;
    Ok(state_dir
        .join("sync-snapshots")
        .join(run_id)
        .join(CATALOG_RUN_IDS_FILE))
}

fn write_catalog_run_ids<'a>(
    state_dir: &Path,
    run_id: &str,
    ids: impl IntoIterator<Item = &'a String>,
) -> Result<()> {
    let path = catalog_run_ids_path(state_dir, run_id)?;
    let mut lines: Vec<&str> = Vec::new();
    for id in ids {
        anyhow::ensure!(
            !id.contains('\n') && !id.contains('\r'),
            "catalog document id {id:?} contains a newline and cannot be stored one-per-line"
        );
        lines.push(id.as_str());
    }
    lines.sort_unstable();
    let parent = path
        .parent()
        .context("catalog run id file has no parent directory")?;
    let tmp = parent.join(format!("{CATALOG_RUN_IDS_FILE}.tmp"));
    {
        let mut file = OpenOptions::new()
            .create(true)
            .write(true)
            .truncate(true)
            .open(&tmp)
            .with_context(|| format!("create {}", tmp.display()))?;
        for id in &lines {
            writeln!(file, "{id}")?;
        }
        file.sync_all()?;
    }
    std::fs::rename(&tmp, &path)
        .with_context(|| format!("rename {} to {}", tmp.display(), path.display()))?;
    sync_dir(parent)?;
    Ok(())
}

fn read_catalog_run_ids(state_dir: &Path, run_id: &str) -> Result<Option<BTreeSet<String>>> {
    let path = catalog_run_ids_path(state_dir, run_id)?;
    let text = match std::fs::read_to_string(&path) {
        Ok(text) => text,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
        Err(error) => {
            return Err(error).with_context(|| format!("read {}", path.display()));
        }
    };
    let mut ids = BTreeSet::new();
    for line in text.lines() {
        if !line.is_empty() {
            ids.insert(line.to_owned());
        }
    }
    Ok(Some(ids))
}

/// `_id -> _source` for every `found: true` entry of an `_mget` response.
/// `found: false` is an absent id, not an error. A duplicate id is.
fn mget_sources(response: &Value) -> Result<BTreeMap<String, Value>> {
    let docs = response
        .pointer("/docs")
        .and_then(Value::as_array)
        .context("catalog mget response has no docs")?;
    let mut sources = BTreeMap::new();
    for doc in docs {
        let id = doc
            .get("_id")
            .and_then(Value::as_str)
            .context("catalog mget doc has no _id")?;
        if doc.get("found").and_then(Value::as_bool) != Some(true) {
            continue;
        }
        let source = doc
            .get("_source")
            .cloned()
            .with_context(|| format!("catalog mget doc {id} is found but has no _source"))?;
        anyhow::ensure!(
            sources.insert(id.to_owned(), source).is_none(),
            "catalog mget returned duplicate ID {id}"
        );
    }
    Ok(sources)
}

/// Advance the current phase by `n` items. Finalize-verify and the catalog
/// read-back call this once per window (or once per skipped window) so the
/// ticker moves while a request is in flight only for that one window, not
/// for the whole phase (#1267).
fn credit_verify_items(pr: &crate::progress::Progress, n: u64) {
    for _ in 0..n {
        pr.item_done(0);
    }
}

/// Production ES-compatible operation backend for graph-disabled generations.
///
/// Upserts stream sealed index actions after removing both replaced and
/// retry-partial content. Metadata operations read deterministic IDs from the
/// committed snapshot and issue partial updates containing provenance only,
/// so unchanged semantic fields are never re-embedded.
#[cfg_attr(not(test), allow(dead_code))]
pub struct EsSyncBackend<'a> {
    es: &'a crate::esclient::Es,
    state_dir: &'a Path,
    bulk_bytes: usize,
    /// #1224: per-replay lookup caches; empty outside
    /// [`Self::replay_operations`], where every lookup falls back to the
    /// linear scan this field exists to avoid. See [`ReplayLookup`].
    lookup: ReplayLookup,
    /// #755: the run's progress surface, so a legacy-catalog mapping warning
    /// reaches the operator through the surface that owns stderr (#241) rather
    /// than a bare `eprintln!` that `--progress none` cannot silence and
    /// `--progress json` cannot parse.
    pr: &'a crate::progress::Progress,
    /// `index_identity` of a plan whose dataset mappings this same process has
    /// already installed (`preflight_generation_mappings`). Provisioning that
    /// exact generation then skips the per-dataset loop — two round trips per
    /// dataset, ~3,000 requests on the 1,526-dataset corpus that produced
    /// #929 — and installs only the catalog mapping. A replayed generation
    /// (any other identity, or none) is always provisioned in full.
    installed_index_identity: Option<String>,
    /// The operation the replay loop is inside; dropping it counts the
    /// operation done and clears it from the surface's in-flight table.
    in_flight: Option<crate::progress::FileGuard<'a>>,
    /// #933: how many operations [`Self::replay_operations`] may have in
    /// flight at once. `1` — the default — is the historical serial loop,
    /// and what every run without `--workers` beyond it configures.
    replay_workers: usize,
}

#[cfg_attr(not(test), allow(dead_code))]
impl<'a> EsSyncBackend<'a> {
    pub fn new(
        es: &'a crate::esclient::Es,
        state_dir: &'a Path,
        bulk_bytes: usize,
        pr: &'a crate::progress::Progress,
    ) -> Self {
        Self {
            es,
            state_dir,
            bulk_bytes: bulk_bytes.max(64 * 1024),
            lookup: ReplayLookup::default(),
            pr,
            installed_index_identity: None,
            in_flight: None,
            replay_workers: 1,
        }
    }

    /// See `installed_index_identity`.
    pub fn with_installed_mappings(mut self, index_identity: String) -> Self {
        self.installed_index_identity = Some(index_identity);
        self
    }

    /// Set the replay window width (#933). Production passes the run's
    /// `--workers` — the same number that already bounds the run's bulk
    /// admission window, so the operations share one AIMD gate with the
    /// bulks they send and a 429 shrinks what THIS path offers too.
    pub fn with_replay_workers(mut self, workers: usize) -> Self {
        self.replay_workers = workers.max(1);
        self
    }

    fn delete_group(&self, group: &ManifestGroup, plan: &Plan) -> Result<()> {
        for index in group_indices(group, plan)? {
            // The delete only has work when the index already holds documents
            // of this content identity. Every upsert used to fire it
            // unconditionally, and a one-record-per-file corpus pays one
            // upsert per FILE — cvelistV5 is 402,744 groups, each a defensive
            // `?refresh=true` delete_by_query that matched nothing and still
            // cost 5-16 s of server refresh under concurrent write load:
            // ~1 op/s measured, days projected (#1224). A first-time group
            // cannot hold partials, and a retried op replays the SAME
            // prepared artifact over the same deterministic `_id`s, so an
            // unrefreshed partial is overwritten by the replay itself. A
            // size:0 term search answers "is anything visible" in <1 ms; a
            // missing index (404) holds nothing by definition (#1173's
            // rule). A probe the node did not answer is an error, never a
            // skip — the delete path stays exactly as strict as before.
            if !self.index_holds_content(&index, &group.content_id)? {
                continue;
            }
            self.es.delete_by_query(
                &index,
                &serde_json::json!({"term": {
                    "ax_file": &group.content_id
                }}),
            )?;
        }
        Ok(())
    }

    /// Does this index hold any visible document of this content identity?
    /// The [#1224] gate in front of [`Self::delete_group`]'s
    /// `delete_by_query`: `None` (index absent) and 0 hits both mean "the
    /// delete would remove nothing", every other outcome propagates.
    fn index_holds_content(&self, index: &str, content_id: &str) -> Result<bool> {
        let Some(v) = self.es.search_present(
            index,
            &serde_json::json!({
                "size": 0,
                "track_total_hits": true,
                "query": {"term": {"ax_file": content_id}},
            }),
        )?
        else {
            return Ok(false);
        };
        let total = v
            .pointer("/hits/total/value")
            .and_then(Value::as_u64)
            .or_else(|| v.pointer("/hits/total").and_then(Value::as_u64))
            .with_context(|| format!("no total in probe of {index} for {content_id}"))?;
        Ok(total > 0)
    }

    /// The remote work of one operation, through a shared reference: the
    /// windowed replay (#933) runs this from several worker threads at once,
    /// over operations whose groups — and therefore whose documents, selected
    /// by `ax_file` content id — are disjoint. The body is the serial loop's
    /// `apply`, unchanged; only the receiver changed.
    fn apply_shared(
        &self,
        operation: &SyncOperation,
        base: &CommittedManifest,
        desired: &GenerationManifest,
        snapshot: &SourceSnapshot,
    ) -> Result<()> {
        anyhow::ensure!(
            desired
                .execution
                .as_ref()
                .is_some_and(|execution| !execution.graph_enabled),
            "production incremental graph reconciliation is not enabled yet"
        );
        let old = self.lookup.base_group(base, &operation.group_id);
        let new = self.lookup.desired_group(desired, &operation.group_id);
        match operation.kind {
            crate::sync::SyncOperationKind::Delete => self.delete_group(
                old.context("delete operation has no committed group")?,
                &base.plan,
            )?,
            crate::sync::SyncOperationKind::Upsert => {
                if let Some(old) = old {
                    self.delete_group(old, &base.plan)?;
                }
                let new = new.context("upsert operation has no desired group")?;
                // Remove a partial prior retry of the desired identity too.
                self.delete_group(new, &desired.plan)?;
                self.replay_prepared(snapshot, &new.content_id)?;
            }
            crate::sync::SyncOperationKind::Metadata => {
                self.replay_metadata(
                    base,
                    new.context("metadata operation has no desired group")?,
                )?;
            }
        }
        Ok(())
    }

    fn replay_prepared(&self, snapshot: &SourceSnapshot, content_id: &str) -> Result<()> {
        // Cache-first resolution of the sealed artifact (#1224): a linear
        // scan of the snapshot's files per upsert is quadratic over a corpus
        // run. Falls back to `prepared_for`'s scan when no replay cache
        // describes this snapshot.
        let prepared = self
            .lookup
            .file(snapshot, content_id)
            .and_then(|file| file.prepared.as_ref())
            .with_context(|| format!("content {content_id} has no sealed prepared artifact"))?;
        let snapshot_dir = self.state_dir.join("sync-snapshots").join(&snapshot.tx_id);
        let file = File::open(snapshot_dir.join(&prepared.relative_ndjson))?;
        stream_ndjson_pairs(BufReader::new(file), self.bulk_bytes, |body| {
            checked_bulk(self.es, body)
        })
    }

    fn replay_metadata(
        &self,
        base: &CommittedManifest,
        desired_group: &ManifestGroup,
    ) -> Result<()> {
        let base_snapshot = open_committed_snapshot(self.state_dir, base)?;
        let old_group = self
            .lookup
            .base_group(base, &desired_group.group_id)
            .context("metadata operation has no committed group")?;
        let prepared = prepared_for(&base_snapshot, &old_group.content_id)?;
        let snapshot_dir = self
            .state_dir
            .join("sync-snapshots")
            .join(&base_snapshot.tx_id);
        let reader = BufReader::new(File::open(snapshot_dir.join(&prepared.relative_ndjson))?);
        let paths: Vec<Value> = std::iter::once(&desired_group.canonical)
            .chain(desired_group.aliases.iter())
            .map(|path| Value::String(path.rel.clone()))
            .collect();
        stream_metadata_updates(
            reader,
            self.bulk_bytes,
            &desired_group.canonical.rel,
            &paths,
            |body| checked_bulk(self.es, body),
        )
    }

    /// Digests carried per read-back window in the batched finalize-verify.
    /// One window is one `terms` filter plus one `terms` aggregation, so a
    /// corpus with 400k changed groups asks ~400 round trips instead of
    /// ~1.2 M serial searches (#1183's count lane). Window size trades server
    /// work per request against round-trip count; the aggregation must be
    /// allowed as many buckets as the window can hold distinct values, which
    /// `windowed_value_counts` always requests.
    const VERIFY_WINDOW: usize = 1024;

    /// Exact live-row count per value of a keyword content-digest field
    /// (`ax_file` on data indices, `file_key` on the catalog), read windowed:
    /// one `terms` filter + one `terms` aggregation per [`Self::VERIFY_WINDOW`]
    /// values. The engine's terms `doc_count` is exact — no probabilistic
    /// sketch sits in the metric path — so each bucket is the number a
    /// per-value `term` search returned as `hits.total`. A value with no live
    /// row appears in no bucket and keeps the 0 it was seeded with.
    ///
    /// Filter context, `terms` first: the engine's columnar filter executor
    /// checks leaves in order, so the digest set narrows the walk before an
    /// `exists` leaf parses any stored source. In scoring context (`must`)
    /// this shape source-scanned every row — 9.6 s per call on a 91k-doc
    /// segment in the serial era (#1183).
    ///
    /// `extra_filters` are appended verbatim to the window query's
    /// `bool.filter`: the semantic leg's `exists`, the catalog leg's
    /// `run_id` term.
    ///
    /// `credit` ticks the current phase once per window after that window's
    /// answer is in. The capability probe passes `false`: it is one sample,
    /// not one of the phase's windows, and a tick there would walk the bar
    /// past its denominator. A `timed_out` window is re-asked, bounded, and
    /// only its exhaustion bails before the tick, so a partial window is
    /// not counted as zero and is not credited (#1761, #2405).
    fn windowed_value_counts(
        &self,
        index: &str,
        field: &str,
        values: &[String],
        extra_filters: &[Value],
        credit: bool,
    ) -> Result<HashMap<String, u64>> {
        let mut counts: HashMap<String, u64> =
            values.iter().map(|value| (value.clone(), 0u64)).collect();
        for window in values.chunks(Self::VERIFY_WINDOW) {
            let mut filter = Vec::with_capacity(1 + extra_filters.len());
            filter.push(serde_json::json!({"terms": {field: window}}));
            filter.extend(extra_filters.iter().cloned());
            // #2222 + #2405: `search_page_for_verify`, not `search`. A
            // DELETE of this index while the window is in flight used to
            // leave the caller parked on it (`waiting_on` the deleted name)
            // until the transport budget ran out, and the generation never
            // committed; and a `timed_out` answer used to exit 1 on the
            // first partial page. The call re-asks the same window, bounded,
            // and still fails fast on a deleted index. A deadline page is
            // HTTP 200 with `timed_out` and an empty aggregation — counting
            // it as "every digest is absent" would abort a generation whose
            // rows are on the node, so exhaustion stays a loud error and is
            // never counted as zero (#1761).
            let response = self.es.search_page_for_verify(
                index,
                &serde_json::json!({
                    "size": 0,
                    "query": {"bool": {"filter": filter}},
                    // As many buckets as the window can hold distinct values,
                    // so no digest's count is silently dropped by a bucket cap.
                    "aggs": {"values": {"terms": {"field": field, "size": window.len()}}}
                }),
            )?;
            for bucket in response
                .pointer("/aggregations/values/buckets")
                .and_then(Value::as_array)
                .context("batched validation response has no terms aggregation")?
            {
                let key = bucket
                    .get("key")
                    .and_then(Value::as_str)
                    .context("batched validation bucket has no key")?;
                let doc_count = bucket
                    .get("doc_count")
                    .and_then(Value::as_u64)
                    .context("batched validation bucket has no doc_count")?;
                counts.insert(key.to_owned(), doc_count);
            }
            if credit {
                credit_verify_items(self.pr, 1);
            }
        }
        Ok(counts)
    }

    /// The catalog leg of the batched finalize-verify: per-digest counts read
    /// by FETCHING the matched documents, never by aggregation. The catalog
    /// is one global index that holds every corpus's file documents (402,814
    /// on the cve-records node) and has usually taken updates (dataset
    /// documents are rewritten by every finalize-catalog), so its version map
    /// carries delete events — and the engine's columnar agg fast path
    /// refuses exactly that (#1260), falling back to the brute agg corpus that
    /// deep-clones the whole index against `max_query_memory_mb` (measured
    /// 786.7 MB vs the 512 MB default on that catalog → 429, the #1183
    /// breaker shape). A `_source`-restricted fetch of the matched set has no
    /// such gate: 31 ms cold / 14 ms warm for a 1,024-digest window against
    /// that same catalog. Each digest expects `1 + aliases.len()` documents,
    /// so a window's matched set is small; pages advance by `from` until the
    /// exact `total` is collected, a total that moves mid-window fails loud,
    /// and the per-window sum of the counts must equal the query's own total
    /// — the same number the serial per-group count trusted.
    fn windowed_catalog_counts(
        &self,
        values: &[String],
        run_id: &str,
    ) -> Result<HashMap<String, u64>> {
        const PAGE: usize = 8192;
        const MAX_PAGES: usize = 64;
        let mut counts: HashMap<String, u64> =
            values.iter().map(|value| (value.clone(), 0u64)).collect();
        for window in values.chunks(Self::VERIFY_WINDOW) {
            let mut from = 0usize;
            let mut window_total: Option<u64> = None;
            let mut pages = 0usize;
            loop {
                anyhow::ensure!(
                    pages < MAX_PAGES,
                    "catalog read-back window needed more than {MAX_PAGES} pages"
                );
                pages += 1;
                let response = self.es.search_page(
                    crate::catalog::CATALOG_INDEX,
                    &serde_json::json!({
                        "size": PAGE,
                        "from": from,
                        "track_total_hits": true,
                        "_source": ["file_key"],
                        "query": {"bool": {"filter": [
                            {"terms": {"file_key": window}},
                            {"term": {"run_id": run_id}}
                        ]}}
                    }),
                )?;
                let page_total = response
                    .pointer("/hits/total/value")
                    .and_then(Value::as_u64)
                    .context("catalog read-back response has no total hit count")?;
                if let Some(seen) = window_total {
                    anyhow::ensure!(
                        seen == page_total,
                        "catalog read-back total moved from {seen} to {page_total} mid-window"
                    );
                }
                window_total = Some(page_total);
                let hits = response
                    .pointer("/hits/hits")
                    .and_then(Value::as_array)
                    .context("catalog read-back response has no hits array")?;
                if hits.is_empty() {
                    break;
                }
                for hit in hits {
                    if let Some(key) = hit.pointer("/_source/file_key").and_then(Value::as_str) {
                        if let Some(count) = counts.get_mut(key) {
                            *count += 1;
                        }
                    }
                }
                from += hits.len();
                if from as u64 >= page_total {
                    break;
                }
            }
            let counted: u64 = window.iter().filter_map(|value| counts.get(value)).sum();
            anyhow::ensure!(
                counted == window_total.unwrap_or(0),
                "catalog read-back counted {counted} of {} matched documents in a window",
                window_total.unwrap_or(0)
            );
            // One tick per window, after the window's pages have all come
            // back. The phase denominator is windows, not pages (#1267).
            // `search_page` re-asks a `timed_out` page and bails only on
            // exhaustion, so a partial catalog window is not counted as
            // zero and the phase does not sit at 0/N while another corpus
            // publishes to this shared index (#1761, #2405).
            credit_verify_items(self.pr, 1);
        }
        Ok(counts)
    }

    /// How many times [`Self::catalog_generation`] may be re-walked when the
    /// observation falls short of the sealed projection. #1212's second live
    /// shape: under load the engine can answer a short or empty page with
    /// `timed_out` absent, so neither the deadline retry nor the flag can
    /// detect it — only observed-count-versus-expected can. One intermittent
    /// answer must not cost the whole finalize again.
    const READBACK_WALK_ATTEMPTS: usize = 3;

    /// The read-back barrier's walk, retried as a whole while it comes back
    /// short of `expected`. A page-level retry cannot catch an unflagged
    /// truncation (#1212), but a re-walk can: the answer is intermittent, and
    /// a fresh walk after a refresh re-reads the full set. Returns the last
    /// observation when every attempt falls short — the caller's
    /// exact-equality check then produces the loud, precise mismatch error.
    fn catalog_generation_complete(
        &self,
        run_id: &str,
        expected: usize,
    ) -> Result<BTreeMap<String, Value>> {
        Self::rewalk_while_short(
            self.pr,
            expected,
            || self.catalog_generation(run_id),
            || self.es.refresh(crate::catalog::CATALOG_INDEX),
        )
    }

    /// [`Self::catalog_generation_complete`]'s loop, isolated so its contract
    /// is pinnable without a node: walk once, and while the observation is
    /// short of `expected` and attempts remain, refresh and walk again.
    fn rewalk_while_short(
        pr: &crate::progress::Progress,
        expected: usize,
        mut walk: impl FnMut() -> Result<BTreeMap<String, Value>>,
        mut refresh: impl FnMut() -> Result<()>,
    ) -> Result<BTreeMap<String, Value>> {
        let mut observation = walk()?;
        for attempt in 2..=Self::READBACK_WALK_ATTEMPTS {
            if observation.len() >= expected {
                return Ok(observation);
            }
            pr.note(&format!(
                "autoindex: catalog read-back walk {} of {} saw {} of {} expected documents — \
                 the node answered a short page without timed_out (#1212); refreshing and \
                 walking again",
                attempt - 1,
                Self::READBACK_WALK_ATTEMPTS,
                observation.len(),
                expected,
            ));
            refresh()?;
            observation = walk()?;
        }
        Ok(observation)
    }

    /// Page size of the `run_id`-scoped read-back walk. The engine reports the
    /// full match count on every page, so a page shorter than this is the walk
    /// reaching the end of the generation's documents.
    const CATALOG_WALK_PAGE: usize = 1000;

    /// The phase the catalog read-back surfaces itself under.
    ///
    /// Publication used to rewrite every projection document and then walk
    /// `term run_id` + `sort _id` + `search_after`. A resume of the same
    /// generation rewrote the catalog, so that term matched every document
    /// (402,814 on the cve-records node): 403 sorted pages at ~5.2 s each,
    /// behind a finished `finalize-catalog` ticker (#1267). The happy path
    /// now opens this phase on the projection's own length and reads those
    /// ids with `_mget`. [`Self::catalog_generation`] still opens it when a
    /// count does not match and the sorted walk is the way to name the
    /// difference.
    const CATALOG_READBACK_PHASE: &'static str = "finalize-catalog-readback";

    fn catalog_generation(&self, run_id: &str) -> Result<BTreeMap<String, Value>> {
        Self::walk_run_id_pages(
            self.pr,
            Self::CATALOG_READBACK_PHASE,
            Self::CATALOG_WALK_PAGE,
            |search_after| {
                let mut body = serde_json::json!({
                    "size": Self::CATALOG_WALK_PAGE,
                    "sort": [{"_id": "asc"}],
                    // #1267: without this the engine caps `hits.total.value` at
                    // the ES 8.x default 10,000 (relation `gte`), so the walk's
                    // progress denominator froze at 10k on the 402,814-doc
                    // catalog — the exact "stuck ticker" this phase exists to
                    // fix. Ask for the true match count so the bar and ETA are
                    // real. The walk still terminates on a short page, not on
                    // the total, so this only corrects the denominator.
                    "track_total_hits": true,
                    "query": {"term": {"run_id": run_id}}
                });
                if let Some(after) = search_after {
                    body["search_after"] = after.clone();
                }
                // #1212: a `search_page`, not a `search` — the engine answers a
                // missed 30 s default deadline with a partial 200 (`timed_out`),
                // and this walk's short-page break read one partial page as the
                // end of the catalog: 56,441 documents observed against 58,568
                // that were all there, on a node whose sorted pages took 11-40 s
                // while its breaker drained. The page variant retries a timed-out
                // page at the same continuation key instead of returning it.
                self.es.search_page(crate::catalog::CATALOG_INDEX, &body)
            },
        )
    }

    /// Drive a `run_id`-scoped, `_id asc` + `search_after` generation walk to
    /// completion, collecting `_id -> _source` for every matched document and
    /// surfacing the walk as its own progress phase (#1267). `fetch_page` is
    /// handed the previous page's continuation sort key (`None` for the first
    /// page) and returns the raw search response; isolating it this way lets
    /// the paging contract — the duplicate-id guard, the short-page terminator,
    /// the continuation-key check, and the per-page progress ticks — be pinned
    /// without a node, the way [`Self::rewalk_while_short`] is. `page` is the
    /// request `size`: a page shorter than it terminates the walk, so a test
    /// drives real multi-page paging with a handful of documents.
    ///
    /// The phase opens on the first page using the engine's reported match
    /// total as the item denominator, then ticks once per collected document,
    /// so the surface moves and carries an ETA across a walk that can run for
    /// over half an hour. A total the engine does not report leaves the phase
    /// without a denominator (`pct=unknown`) rather than inventing one.
    fn walk_run_id_pages(
        pr: &crate::progress::Progress,
        phase: &'static str,
        page: usize,
        mut fetch_page: impl FnMut(Option<&Value>) -> Result<Value>,
    ) -> Result<BTreeMap<String, Value>> {
        let mut documents = BTreeMap::new();
        let mut search_after: Option<Value> = None;
        let mut phase_opened = false;
        loop {
            let response = fetch_page(search_after.as_ref())?;
            if !phase_opened {
                let total = response
                    .pointer("/hits/total/value")
                    .and_then(Value::as_u64)
                    .unwrap_or(0);
                pr.phase(phase, total, 0);
                phase_opened = true;
            }
            let hits = response
                .pointer("/hits/hits")
                .and_then(Value::as_array)
                .context("catalog generation query has no hits")?;
            for hit in hits {
                let id = hit
                    .get("_id")
                    .and_then(Value::as_str)
                    .context("catalog hit has no _id")?
                    .to_owned();
                let source = hit
                    .get("_source")
                    .cloned()
                    .context("catalog hit has no _source")?;
                anyhow::ensure!(
                    documents.insert(id.clone(), source).is_none(),
                    "catalog generation query returned duplicate ID {id}"
                );
                pr.item_done(0);
            }
            if hits.len() < page {
                break;
            }
            search_after = hits.last().and_then(|hit| hit.get("sort")).cloned();
            anyhow::ensure!(
                search_after.is_some(),
                "full catalog generation page has no continuation sort key"
            );
        }
        Ok(documents)
    }

    /// A warm verify window slower than this means the server is answering
    /// the window shape on its brute path. Measured on the #1267 resume: the
    /// `term ax_file` + `exists` probe took 3.4 s warm on a server without the
    /// #1260 exists arm and 2.7 ms on one with it — three orders of magnitude
    /// either side of this line, so it separates the two generations without
    /// being sensitive to node load.
    const SERVER_PROBE_SLOW: std::time::Duration = std::time::Duration::from_secs(1);

    /// The operational half of #1267: before `finalize-verify` issues its
    /// windows, time ONE representative window and, if it is slow, say so.
    ///
    /// The resume protocol cannot otherwise tell that the server predates the
    /// client's fast-agg fix generation (#1260/#1270). On the cve-records
    /// resume that cost four hours of ~3.4 s probes before anyone looked —
    /// the client was a #1259 binary, the server an older measure build, and
    /// nothing in the handshake differs between them. `GET /` cannot help:
    /// its `version.number` is the ES-compat version (`8.13.0` or the
    /// `--compat-version` override), not XERJ's. `/v1/health` does carry the
    /// XERJ version and is quoted in the warning when it differs from the
    /// client's, but a branch or measure build reports the same string as
    /// the release it forked from — exactly the #1267 shape — so latency is
    /// the signal and the version only context.
    ///
    /// `window_query` runs the window and returns how long it took (the caller
    /// times it, which keeps this pinnable without a node or a clock). A slow
    /// first answer is re-asked once before warning: the first touch after a
    /// server boot pages the index in cold (8.8 s on the FIXED binary in
    /// #1267), and that must not read as an old server. A probe that fails is
    /// not this function's business — the verify window that follows asks the
    /// same thing and fails loud — so it returns `None` without a warning.
    /// It never aborts the run: an old server is slow, not wrong.
    fn probe_server_generation(
        pr: &crate::progress::Progress,
        windows: u64,
        threshold: std::time::Duration,
        mut window_query: impl FnMut() -> Result<std::time::Duration>,
        server_version: impl FnOnce() -> Option<String>,
    ) -> Option<std::time::Duration> {
        let cold = window_query().ok()?;
        if cold <= threshold {
            return Some(cold);
        }
        let warm = window_query().ok()?;
        if warm <= threshold {
            return Some(warm);
        }
        let client = env!("CARGO_PKG_VERSION");
        let version = match server_version() {
            Some(server) if server != client => {
                format!(" The server reports XERJ {server}; this client is {client}.")
            }
            Some(_) => format!(
                " The server reports the same version as this client ({client}), so it is \
                 likely a branch or development build cut before the fix."
            ),
            None => String::new(),
        };
        let projected_s = warm.as_secs_f64() * windows as f64;
        pr.warn(&format!(
            "autoindex: warning: one finalize-verify window took {:.1}s (twice; under {}ms is \
             expected). The server is likely older than this client and lacks the columnar \
             term+exists count lane (#1260/#1270), so every verify window brute-scans its \
             index: {windows} windows at this rate is ~{:.1}h.{version} Continuing — the \
             result will be correct, only slow. To avoid the wait, stop this run, upgrade the \
             server to this client's version, and re-run the same command: the journal resumes \
             where it stopped.",
            warm.as_secs_f64(),
            threshold.as_millis(),
            projected_s / 3600.0,
        ));
        Some(warm)
    }

    /// The XERJ version the server reports on `/v1/health` (`data.version`),
    /// or `None` when the endpoint is absent or unreadable.
    fn server_xerj_version(&self) -> Option<String> {
        self.es
            .get_json("/v1/health")
            .ok()?
            .pointer("/data/version")
            .and_then(Value::as_str)
            .map(str::to_owned)
    }

    /// Ids fetched per `_mget`. Under the engine's default
    /// `limits.max_mget_docs` (10_000), so one window is one request.
    const MGET_WINDOW: usize = 1000;

    /// Read catalog `_source`s by id. `tick` credits one phase item per id
    /// after that id's window returns, so a multi-window read-back moves the
    /// bar between requests instead of once at the end.
    fn mget_catalog(&self, ids: &[String], tick: bool) -> Result<BTreeMap<String, Value>> {
        let mut observed = BTreeMap::new();
        if ids.is_empty() {
            return Ok(observed);
        }
        let _watching = self.pr.watch_file(crate::catalog::CATALOG_INDEX, 0);
        for window in ids.chunks(Self::MGET_WINDOW) {
            let response = self.es.mget(crate::catalog::CATALOG_INDEX, window)?;
            observed.extend(mget_sources(&response)?);
            if tick {
                credit_verify_items(self.pr, window.len() as u64);
            }
        }
        Ok(observed)
    }

    /// `hits.total` of `exists` on `field`. One query per semantic
    /// `(index, field)`, in place of a per-digest `term` + `exists` window
    /// that cannot match when this total is zero (#1267).
    fn field_exists_total(&self, index: &str, field: &str) -> Result<u64> {
        let response = self.es.search_for_verify(
            index,
            &serde_json::json!({
                "size": 0,
                "track_total_hits": true,
                "query": {"bool": {"filter": [{"exists": {"field": field}}]}}
            }),
        )?;
        response
            .pointer("/hits/total/value")
            .and_then(Value::as_u64)
            .context("exists preflight has no total hit count")
    }

    /// How many live catalog documents carry `run_id`. `size: 0`, so the
    /// answer is a count, not a page of stored sources.
    fn catalog_run_count(&self, run_id: &str) -> Result<u64> {
        let response = self.es.search(
            crate::catalog::CATALOG_INDEX,
            &serde_json::json!({
                "size": 0,
                "track_total_hits": true,
                "query": {"bool": {"filter": [{"term": {"run_id": run_id}}]}}
            }),
        )?;
        response
            .pointer("/hits/total/value")
            .and_then(Value::as_u64)
            .context("catalog run_id count has no total hit count")
    }

    /// Live documents still carrying the prior generation's `run_id` must be
    /// a subset of the ids this generation kept.
    ///
    /// When this process published that generation, `catalog-run.ids` is the
    /// exact id set that carried the `run_id`. The live count equals the
    /// intersection of that set with the ids kept now, or the sorted walk
    /// names the strays. A missing file (a generation published before this
    /// file existed) takes the walk once.
    fn assert_no_prior_strays(
        &self,
        prior_run_id: &str,
        projection: &crate::generation_catalog::CatalogProjection,
    ) -> Result<()> {
        let written: BTreeSet<String> = projection.documents.keys().cloned().collect();
        let kept: BTreeSet<String> = projection
            .managed_ids
            .difference(&written)
            .cloned()
            .collect();
        match read_catalog_run_ids(self.state_dir, prior_run_id)? {
            Some(prior_written) => {
                let expected = prior_run_expected_remainder(&prior_written, &kept);
                let live = self.catalog_run_count(prior_run_id)?;
                if live == expected {
                    return Ok(());
                }
                self.pr.note(&format!(
                    "autoindex: prior catalog generation {prior_run_id} still counts {live} \
                     document(s) under its run_id against {expected} kept; walking them to name \
                     any stray (#1267)"
                ));
            }
            None => {
                self.pr.note(&format!(
                    "autoindex: prior catalog generation {prior_run_id} has no saved id list; \
                     walking documents that still carry its run_id (#1267)"
                ));
            }
        }
        let remaining = self.catalog_generation(prior_run_id)?;
        let strays: Vec<&String> = remaining.keys().filter(|id| !kept.contains(*id)).collect();
        anyhow::ensure!(
            strays.is_empty(),
            "prior catalog generation {prior_run_id} still has {} document(s) this generation \
             neither rewrote, deleted nor kept: {:?}",
            strays.len(),
            strays
        );
        Ok(())
    }

    /// `exists` total for one semantic field.
    ///
    /// The first field of the phase is the capability probe (#1267): a slow
    /// answer is re-asked once, and a slow warm answer warns, but only when
    /// the field actually matches documents — those are the windows about to
    /// brute-scan. A zero total skips the windows, so a slow empty answer
    /// must not project that one query across the whole phase. Later fields
    /// ask once and do not warn again.
    fn semantic_exists_total(
        &self,
        index: &str,
        field: &str,
        probe: bool,
        windows_if_matched: u64,
    ) -> Result<u64> {
        let started = std::time::Instant::now();
        let mut total = self.field_exists_total(index, field)?;
        let first = started.elapsed();
        if !probe || total == 0 || first <= Self::SERVER_PROBE_SLOW {
            return Ok(total);
        }
        let started = std::time::Instant::now();
        total = self.field_exists_total(index, field)?;
        let warm = started.elapsed();
        if total > 0 && warm > Self::SERVER_PROBE_SLOW {
            let samples = std::cell::RefCell::new(vec![first, warm]);
            Self::probe_server_generation(
                self.pr,
                windows_if_matched,
                Self::SERVER_PROBE_SLOW,
                || {
                    let mut samples = samples.borrow_mut();
                    anyhow::ensure!(!samples.is_empty(), "probe timing exhausted");
                    Ok(samples.remove(0))
                },
                || self.server_xerj_version(),
            );
        }
        Ok(total)
    }

    fn exact_dataset_catalog_stats(
        &self,
        desired: &GenerationManifest,
    ) -> Result<BTreeMap<String, crate::generation_catalog::DatasetCatalogStats>> {
        let mut out = BTreeMap::new();
        for dataset in &desired.plan.datasets {
            let groups: Vec<&ManifestGroup> = desired
                .groups
                .iter()
                .filter(|group| group.dataset_slugs.contains(&dataset.slug))
                .collect();
            let content_ids: Vec<&str> = groups
                .iter()
                .map(|group| group.content_id.as_str())
                .collect();
            // Filter on `ax_dataset` as well as `ax_file`: the read-back is then
            // per-dataset by construction, matching the identity the records
            // were sealed under, and stays exact even if two datasets ever share
            // one index. `ax_dataset` is written at every sink site and is a
            // mapped `PROVENANCE_FIELDS` keyword.
            //
            // #1183: this read-back must never carry `aggs`. A `size:0 + aggs`
            // query whose agg has no columnar fast path makes the server
            // deep-clone every matching document into owned Values before the
            // agg runs (the "aggregation corpus materialisation" charge in the
            // engine's search path, ~2 KB a doc against
            // `limits.max_query_memory_mb`). On the xerj-search rebuild one
            // ~570 k-record dataset estimated 1.1 GB against the 512 MB
            // default, the server answered 429 circuit_breaking_exception, the
            // client retried for its full 600 s budget, and the run aborted —
            // the "finalize-catalog deadlock". The count needs no corpus; the
            // time bounds come from two size-1 searches sorted on the time
            // field (`extreme_time` below), which sort from doc values and
            // clone nothing.
            let filter = serde_json::json!([
                {"terms": {"ax_file": content_ids}},
                {"term": {"ax_dataset": dataset.slug}}
            ]);
            let body = serde_json::json!({
                "size": 0,
                "track_total_hits": true,
                "query": {"bool": {"filter": filter}}
            });
            let response = self.es.search(&dataset.index, &body)?;
            let record_count = response
                .pointer("/hits/total/value")
                .and_then(Value::as_u64)
                .context("dataset exact read-back has no total")?;
            // A group's sealed record count is a property of the *content*, not
            // of a dataset: one file fans out over N datasets (a SQL dump is N
            // tables), so its flat total is comparable to no single dataset's
            // read-back. Fold the per-dataset ledger instead — the counts keyed
            // by the identity they were written under. `None` is a group sealed
            // before that ledger existed which fanned out anyway: genuinely
            // unattributable, so the equality is skipped for this dataset rather
            // than guessed at, and the exact read-back is still published as the
            // statistic. The check itself stays fatal — a read-back that
            // disagrees with a seal it *can* be compared against is a corruption
            // signal, not junk.
            let expected = groups
                .iter()
                .map(|group| group.expected_records_for(&dataset.slug))
                .try_fold(Some(0u64), |sum, count| match (sum, count) {
                    (Some(sum), Some(count)) => sum
                        .checked_add(count)
                        .context("dataset expected record count overflow")
                        .map(Some),
                    _ => Ok(None),
                })?;
            if let Some(expected) = expected {
                anyhow::ensure!(
                    record_count == expected,
                    "dataset {} exact read-back count {record_count} disagrees with sealed count \
                     {expected} across groups {}",
                    dataset.slug,
                    groups
                        .iter()
                        .map(|group| group.group_id.as_str())
                        .collect::<Vec<_>>()
                        .join(",")
                );
            }
            let mut formats: Vec<String> = desired
                .plan
                .files
                .values()
                .filter(|assignment| {
                    assignment
                        .assignments
                        .iter()
                        .any(|(_, slug)| slug == &dataset.slug)
                })
                .map(|assignment| {
                    if assignment.gzip {
                        format!("{}(gzip)", assignment.family)
                    } else {
                        assignment.family.clone()
                    }
                })
                .collect();
            formats.sort();
            formats.dedup();
            let bytes = groups.iter().try_fold(0u64, |sum, group| {
                sum.checked_add(group.content_size)
                    .context("dataset source byte count overflow")
            })?;
            // Junk is a property of the *file*, not of a dataset: a record that
            // no assignment accepted belongs to no dataset by definition. A
            // group that feeds several datasets therefore charges its junk to
            // exactly one of them — the first of its sorted slugs — so the
            // run-level `junk_records_total` (a sum over datasets) stays exact
            // instead of multiplying by fan-out. The legacy path attributes it
            // the same way, to `fa.assignments.first()` (lib.rs).
            let junk_records = groups
                .iter()
                .filter(|group| group.dataset_slugs.first() == Some(&dataset.slug))
                .try_fold(0u64, |sum, group| {
                    sum.checked_add(group.expected_junk_records)
                        .context("dataset junk-record count overflow")
                })?;
            let time_bound = |order: &'static str| -> Result<Option<String>> {
                match &dataset.time_field {
                    Some(field) => self.extreme_time(&dataset.index, &filter, field, order),
                    None => Ok(None),
                }
            };
            out.insert(
                dataset.slug.clone(),
                crate::generation_catalog::DatasetCatalogStats {
                    record_count,
                    junk_records,
                    bytes,
                    formats,
                    time_min: time_bound("asc")?,
                    time_max: time_bound("desc")?,
                    sample_queries: crate::catalog::build_sample_queries(dataset, &[]),
                    notes: dataset
                        .group
                        .iter()
                        .map(|group| format!("source table: {group}"))
                        .chain(dataset.specs.iter().flat_map(|spec| {
                            spec.notes
                                .iter()
                                .map(move |note| format!("{}: {note}", spec.name))
                        }))
                        .collect(),
                },
            );
            self.pr.item_done(0);
        }
        Ok(out)
    }

    /// One end of a dataset's time range, via a size-1 search sorted on the
    /// time field under the dataset's own filter (#1183). This replaced a
    /// `min`/`max` aggregation on the same field: the agg made the server
    /// materialise the aggregation corpus for the whole dataset and refuse at
    /// the query-memory breaker on large ones, while a sort reads doc values
    /// and clones nothing. Exact for the same reason the agg was: the filter
    /// is the dataset's own identity, and the sort's first hit IS the extreme.
    ///
    /// The returned string matches the agg's `value_as_string`: the stored
    /// `_source` value verbatim when the field holds strings (autoindex date
    /// fields are ISO strings), and epoch-milliseconds rendered by the same
    /// ISO helper the engine's `min`/`max` aggs use when it holds numbers —
    /// so the catalog document does not change shape between corpus builds.
    fn extreme_time(
        &self,
        index: &str,
        filter: &Value,
        field: &str,
        order: &str,
    ) -> Result<Option<String>> {
        let body = serde_json::json!({
            "size": 1,
            "query": {"bool": {"filter": filter}},
            "sort": [{field: {"order": order}}],
            "_source": [field]
        });
        let response = self.es.search(index, &body)?;
        Ok(response
            .pointer("/hits/hits/0/_source")
            .and_then(|src| src.get(field))
            .and_then(|v| match v {
                Value::String(s) => Some(s.clone()),
                Value::Number(n) => n.as_i64().map(xerj_common::schema::epoch_ms_to_iso8601_utc),
                _ => None,
            }))
    }
}

impl SyncOperationBackend for EsSyncBackend<'_> {
    fn provision_generation(&mut self, desired: &GenerationManifest) -> Result<()> {
        let execution = desired
            .execution
            .as_ref()
            .context("desired generation has no execution identity")?;
        let (_, index_identity) = crate::generation_contract_identities(&desired.plan)?;
        anyhow::ensure!(
            execution.index_identity == index_identity,
            "desired generation index identity disagrees with its frozen mappings"
        );
        if self.installed_index_identity.as_deref() == Some(index_identity.as_str()) {
            return crate::ensure_generation_catalog_mapping(self.es, self.pr);
        }
        crate::ensure_generation_mappings(self.es, &desired.plan, self.pr)
    }

    fn replay_begins(&mut self, items: u64, bytes: u64) {
        self.pr.phase("index", items, bytes);
    }

    fn operation_begins(&mut self, rel: &str, bytes: u64) {
        let pr = self.pr;
        self.in_flight = Some(pr.file(rel, bytes));
    }

    fn operation_applied(&mut self) {
        self.in_flight = None;
    }

    fn apply(
        &mut self,
        operation: &SyncOperation,
        base: &CommittedManifest,
        desired: &GenerationManifest,
        snapshot: &SourceSnapshot,
    ) -> Result<()> {
        self.apply_shared(operation, base, desired, snapshot)
    }

    fn replay_operations(
        &mut self,
        items: &[ReplayItem<'_>],
        base: &CommittedManifest,
        desired: &GenerationManifest,
        snapshot: &SourceSnapshot,
        journal: &mut Journal,
    ) -> Result<()> {
        // #1224: resolve per-operation groups and prepared artifacts through
        // id indexes built once here, not a linear scan per operation.
        self.lookup = ReplayLookup::build(base, desired, snapshot);
        let width = self.replay_workers;
        if width <= 1 || items.len() <= 1 {
            return replay_serial(self, items, base, desired, snapshot, journal);
        }
        // The durability argument of the overlap rests on disjoint groups:
        // one plan gives a group at most one operation (`plan_operations`
        // walks base and desired by group id), so no two in-flight
        // operations ever write the same documents. Refuse here rather
        // than silently corrupt if a future planner breaks that.
        let mut groups = std::collections::BTreeSet::new();
        anyhow::ensure!(
            items
                .iter()
                .all(|item| groups.insert(item.operation.group_id.as_str())),
            "parallel replay requires at most one operation per group; this plan needs the \
             serial loop"
        );
        let this: &EsSyncBackend<'_> = self;
        let pr = this.pr;
        let apply = |item: &ReplayItem<'_>| {
            // The in-flight entry IS this worker's file: the guard counts the
            // item done on every exit path, exactly as the serial loop's
            // `operation_begins`/`operation_applied` pair does (#931).
            let _guard = pr.file(&item.rel, item.bytes);
            this.apply_shared(item.operation, base, desired, snapshot)
        };
        /// The serial loop's two journal writes, verbatim: Started before
        /// dispatch, Committed after the apply returned — the durable order
        /// #933 must not change, only overlap it.
        struct JournalHooks<'j> {
            journal: &'j mut Journal,
        }
        impl ReplayHooks<ReplayItem<'_>> for JournalHooks<'_> {
            fn begin(&mut self, item: &ReplayItem<'_>) -> Result<()> {
                let state = self
                    .journal
                    .pending_sync
                    .as_ref()
                    .and_then(|sync| sync.operation_states.get(&item.operation.operation_id))
                    .cloned();
                if state.is_none() {
                    self.journal.sync_operation_state(
                        &item.operation.operation_id,
                        SyncOperationState::Started,
                    )?;
                }
                Ok(())
            }

            fn applied(&mut self, item: &ReplayItem<'_>, outcome: Result<()>) -> Result<()> {
                let () = outcome?;
                replay_fail_after_apply()?;
                self.journal.sync_operation_state(
                    &item.operation.operation_id,
                    SyncOperationState::Committed,
                )
            }
        }
        let mut hooks = JournalHooks { journal };
        replay_windowed(items, width, &mut hooks, &apply)
    }

    fn publish_generation_catalog(
        &mut self,
        base: &CommittedManifest,
        desired: &GenerationManifest,
        snapshot: &SourceSnapshot,
    ) -> Result<()> {
        // #931: one exact read-back query per dataset. Named like the legacy
        // path's phase so a reader of either route sees the same vocabulary.
        self.pr
            .phase("finalize-catalog", desired.plan.datasets.len() as u64, 0);
        // #1183: the refreshes used to run one index per request, serially —
        // 1,479 round-trips on the xerj-search corpus, each taking seconds
        // against a node whose memory breaker was draining between them, so
        // the phase read as `stalled` for the better part of an hour before
        // the read-back behind it even started. The refresh path takes a
        // comma-list (the server resolves it through the same selector as a
        // wildcard), so one request per window of indexes, with one progress
        // tick per index so the phase denominator keeps its meaning.
        let indexes: Vec<&str> = desired
            .plan
            .datasets
            .iter()
            .map(|dataset| dataset.index.as_str())
            .collect::<std::collections::BTreeSet<_>>()
            .into_iter()
            .collect();
        const REFRESH_WINDOW: usize = 50;
        for batch in indexes.chunks(REFRESH_WINDOW) {
            // #2234: one attempt. `Es::refresh` would retry a timed-out window
            // five more times (300s each) with no new information.
            self.es.refresh_once(&batch.join(","))?;
            for _ in batch {
                self.pr.item_done(0);
            }
        }
        self.es.refresh_once(crate::catalog::CATALOG_INDEX)?;
        let prior_run_id =
            base.execution
                .as_ref()
                .and_then(|execution| match &execution.source_policy {
                    SourceExecutionPolicy::DurableSnapshot { reference, .. } => {
                        reference.strip_prefix("sync-snapshots/")
                    }
                    SourceExecutionPolicy::AbortOnSourceChange { .. } => None,
                });
        let stats = self.exact_dataset_catalog_stats(desired)?;
        let projection = crate::generation_catalog::project_generation(
            base,
            desired,
            &crate::generation_catalog::GenerationCatalogMetadata {
                generation_id: snapshot.tx_id.clone(),
                started: snapshot.started.clone(),
            },
            &stats,
            &BTreeMap::new(),
            &BTreeSet::new(),
        )?;
        // #1267: read the projection's ids before writing. A resume of this
        // same generation already stored these bytes under this `run_id`;
        // sending them again is what made the old `run_id` walk match the
        // whole catalog. The phase denominator is the projection, which is
        // the changed set (#971) plus the documents a same-generation resume
        // still has to confirm.
        let ids: Vec<String> = projection.documents.keys().cloned().collect();
        self.pr
            .phase(Self::CATALOG_READBACK_PHASE, ids.len() as u64, 0);
        let mut observed = self.mget_catalog(&ids, true)?;
        let to_write = catalog_docs_to_write(&projection.documents, &observed);
        let mut body = Vec::new();
        for id in &projection.stale_ids {
            let action =
                serde_json::json!({"delete": {"_index": crate::catalog::CATALOG_INDEX, "_id": id}});
            body.extend_from_slice(serde_json::to_string(&action)?.as_bytes());
            body.push(b'\n');
        }
        for (id, document) in &to_write {
            append_index_action(crate::catalog::CATALOG_INDEX, id, document, &mut body)?;
        }
        // #955: one document per file — this body grows with the corpus. On
        // the 48,533-file reference corpus it was 51,129 actions in 31.9 MB,
        // sent as ONE request under an 8 MB `--bulk-mb`, and the engine's
        // 50,000-action limit ended the run here after every operation had
        // been applied. Windowed like every other bulk; deletes stay first.
        // #971: the projection is now incremental — a one-file change sends
        // that file's document (plus datasets and the run document), not one
        // document per file in the corpus. #1267: a document whose live
        // source already equals that projection is not in `to_write`.
        if !body.is_empty() {
            checked_bulk_windowed(self.es, body, self.bulk_bytes)?;
            // #2234: one attempt. `Es::refresh` would retry a timed-out
            // catalog refresh five more times (300s each).
            self.es.refresh_once(crate::catalog::CATALOG_INDEX)?;
        }
        if !to_write.is_empty() {
            let written_ids: Vec<String> = to_write.keys().cloned().collect();
            for (id, source) in self.mget_catalog(&written_ids, false)? {
                observed.insert(id, source);
            }
        }
        projection.validate_observed(&observed)?;
        // The mget above sees only the projection's ids. A document this
        // generation did not project that still carries its `run_id` would
        // pass that check and inflate the next generation's kept set. The
        // count is that extra. When it disagrees, the sorted walk — retried
        // while short (#1212) — is what names the missing and unexpected ids.
        // A walk that then matches the projection is the publish: the count
        // was the optimization, and a capped total must not fail a complete
        // read-back.
        let live = self.catalog_run_count(&snapshot.tx_id)?;
        if live != projection.documents.len() as u64 {
            self.pr.note(&format!(
                "autoindex: catalog run_id {} counts {live} live documents against {} in the \
                 projection; reading them back by the sorted walk (#1267)",
                snapshot.tx_id,
                projection.documents.len()
            ));
            projection.validate_observed(
                &self.catalog_generation_complete(&snapshot.tx_id, projection.documents.len())?,
            )?;
        }
        write_catalog_run_ids(self.state_dir, &snapshot.tx_id, projection.documents.keys())?;
        if let Some(prior_run_id) = prior_run_id {
            // #971 sweep. Documents still carrying the prior generation's
            // run_id are legitimate — they are the intentionally kept
            // (unchanged) ones — but NOTHING ELSE may. The check itself is
            // unchanged (subset, not equality; a kept document may have been
            // overwritten by another journal). What changed (#1267) is that
            // the common case is a count against the ids the prior generation
            // saved, not another sorted walk of the catalog.
            self.assert_no_prior_strays(prior_run_id, &projection)?;
        }
        Ok(())
    }

    fn validate(
        &mut self,
        base: &CommittedManifest,
        desired: &GenerationManifest,
        snapshot: &SourceSnapshot,
    ) -> Result<()> {
        // #931: the read-back barrier is only exact against refreshed indices,
        // so every dataset is refreshed first — one request each, serially. On
        // the 1,526-dataset corpus that produced #931 that measured ~106 ms a
        // request, i.e. close to three minutes. Folded into `finalize-verify`
        // it would hold that phase at `0/N` with `since_progress_s` climbing
        // until it read `stalled`: the same lie this change removes, at a
        // smaller scale. So it is a phase of its own, named like the legacy
        // path's, counted in indices, with the index it is waiting on named.
        self.pr.phase(
            "finalize-refresh",
            desired.plan.datasets.len() as u64 + 1,
            0,
        );
        let mut refresh_failures: Vec<String> = Vec::new();
        for dataset in &desired.plan.datasets {
            let _refreshing = self.pr.file(&dataset.index, 0);
            // #2234: one attempt per index. The file guard names the index
            // on the progress line for as long as this request is in flight.
            if let Err(error) = self.es.refresh_once(&dataset.index) {
                refresh_failures.push(format!("{}: {error:#}", dataset.index));
            }
        }
        {
            let _refreshing = self.pr.file(crate::catalog::CATALOG_INDEX, 0);
            if let Err(error) = self.es.refresh_once(crate::catalog::CATALOG_INDEX) {
                refresh_failures.push(format!("{}: {error:#}", crate::catalog::CATALOG_INDEX));
            }
        }
        if !refresh_failures.is_empty() {
            anyhow::bail!(
                "finalize-refresh failed for {} index(es): {}",
                refresh_failures.len(),
                refresh_failures.join(" | ")
            );
        }
        // The generation-wide barrier reads changed groups back. Every
        // question it asks has the same shape — "how many rows in this index
        // carry one of these content digests" — so it is asked windowed
        // instead of the three serial searches per file this barrier used to
        // issue. That serial shape is #1183's count lane: measured 2.2
        // groups/s on cve-records (402,695 changed groups, ~50 h projected)
        // against an index phase that itself takes hours. Measured on the
        // live crawl node (2026-10-08): the record windows answer in
        // 6–63 ms per 1,024 digests while the index is append-only (the
        // engine's columnar agg fast path), the semantic windows pay the
        // brute path — the fast path does not yet columnarize `exists`
        // (#1260) — at ~16 s per window on a 53k-doc index, and the catalog leg
        // cannot aggregate at all: its version map carries delete events
        // (dataset documents are rewritten by every finalize-catalog),
        // which the fast path refuses, and the brute agg corpus trips the
        // query-memory breaker (786.7 MB vs 512 MB → 429). The catalog
        // therefore fetch-counts its small matched sets instead
        // (`windowed_catalog_counts`, 14–31 ms a window). The phase counts
        // read-back windows and ticks as each one resolves, so the line
        // never sits at `0/N` for the length of the walk (#931's rule).
        //
        // #971: a group wholly equal to its committed self is skipped. That
        // equality is exactly the condition under which `plan_operations`
        // planned no operation — nothing this generation wrote can have moved
        // its live counts, and its read-back was exact at the barrier of the
        // generation that last touched it. Verifying it again every run is
        // what made a one-file change cost O(corpus) queries.
        let base_group_by_id: BTreeMap<&str, &ManifestGroup> = base
            .groups
            .iter()
            .map(|group| (group.group_id.as_str(), group))
            .collect();
        let changed_groups: Vec<&ManifestGroup> = desired
            .groups
            .iter()
            .filter(|group| base_group_by_id.get(group.group_id.as_str()).copied() != Some(*group))
            .collect();
        let desired_ids: std::collections::HashSet<&str> = desired
            .groups
            .iter()
            .map(|group| group.content_id.as_str())
            .collect();
        let deleted_groups: Vec<&ManifestGroup> = base
            .groups
            .iter()
            .filter(|old| !desired_ids.contains(old.content_id.as_str()))
            .collect();

        // What each leg needs, gathered before anything is asked so the phase
        // opens with an honest denominator. Record counts are keyed by index:
        // a changed group asks in every index its datasets touch, a deleted
        // group asks for zero in every index the BASE plan touched — a renamed
        // dataset yields two entries, exactly the two names that must be read.
        // Duplicate digests inside one index's list are harmless: the count map
        // is keyed by digest and each group sums its own lookups.
        let mut record_wanted: BTreeMap<String, Vec<String>> = BTreeMap::new();
        for group in &changed_groups {
            for index in group_indices(group, &desired.plan)? {
                record_wanted
                    .entry(index)
                    .or_default()
                    .push(group.content_id.clone());
            }
        }
        for old in &deleted_groups {
            for index in group_indices(old, &base.plan)? {
                record_wanted
                    .entry(index)
                    .or_default()
                    .push(old.content_id.clone());
            }
        }
        // Semantic counts are keyed by (index, semantic field): a changed
        // group asks once per dataset that carries one.
        let by_slug: HashMap<&str, &crate::state::PlanDataset> = desired
            .plan
            .datasets
            .iter()
            .map(|dataset| (dataset.slug.as_str(), dataset))
            .collect();
        let mut semantic_wanted: BTreeMap<(String, String), Vec<String>> = BTreeMap::new();
        for group in &changed_groups {
            for slug in &group.dataset_slugs {
                let dataset = by_slug
                    .get(slug.as_str())
                    .copied()
                    .with_context(|| format!("group references absent dataset {slug}"))?;
                if let Some(field) = &dataset.semantic_field {
                    semantic_wanted
                        .entry((dataset.index.clone(), field.clone()))
                        .or_default()
                        .push(group.content_id.clone());
                }
            }
        }
        let windows = |values: usize| values.div_ceil(Self::VERIFY_WINDOW) as u64;
        let mut readbacks = 0u64;
        for values in record_wanted.values() {
            readbacks += windows(values.len());
        }
        for values in semantic_wanted.values() {
            // One exists preflight per field, then its digest windows — issued
            // when the field matches anything, ticked and skipped when it
            // matches nothing (#1267).
            readbacks += 1 + windows(values.len());
        }
        let catalog_wanted: Vec<String> = changed_groups
            .iter()
            .map(|group| group.content_id.clone())
            .collect();
        if !catalog_wanted.is_empty() {
            readbacks += windows(catalog_wanted.len());
        }
        self.pr.phase("finalize-verify", readbacks, 0);

        // #1267 operational: a generation with no semantic field still times
        // one record window, so an old server is named before the walk. The
        // semantic leg's probe is the exists preflight below — a per-digest
        // `term` + `exists` sample is the query this change exists to stop
        // issuing when the field matches nothing. `watch_file` names the
        // index and does not credit an item: the probe is not one of
        // `readbacks`.
        if semantic_wanted.is_empty() {
            if let Some((index, value)) = record_wanted
                .iter()
                .find_map(|(index, values)| values.first().map(|value| (index.as_str(), value)))
            {
                let _probing = self.pr.watch_file(index, 0);
                Self::probe_server_generation(
                    self.pr,
                    readbacks,
                    Self::SERVER_PROBE_SLOW,
                    || {
                        let started = std::time::Instant::now();
                        self.windowed_value_counts(
                            index,
                            "ax_file",
                            std::slice::from_ref(value),
                            &[],
                            false,
                        )?;
                        Ok(started.elapsed())
                    },
                    || self.server_xerj_version(),
                );
            }
        }

        let mut record_counts: HashMap<String, HashMap<String, u64>> = HashMap::new();
        for (index, values) in &record_wanted {
            let _reading = self.pr.watch_file(index, 0);
            let counts = self.windowed_value_counts(index, "ax_file", values, &[], true)?;
            record_counts.insert(index.clone(), counts);
        }
        let mut semantic_counts: HashMap<(String, String), HashMap<String, u64>> = HashMap::new();
        let mut first_semantic = true;
        for ((index, field), values) in &semantic_wanted {
            let _reading = self.pr.watch_file(index, 0);
            let probe = first_semantic;
            first_semantic = false;
            let total = self.semantic_exists_total(index, field, probe, windows(values.len()))?;
            credit_verify_items(self.pr, 1);
            let plan = plan_exists_probes(values.len(), Self::VERIFY_WINDOW, total);
            if plan.skip_windows > 0 {
                self.pr.note(&format!(
                    "autoindex: semantic field {field} on {index} matches no documents; skipped \
                     {} verify window(s) (#1267)",
                    plan.skip_windows
                ));
                credit_verify_items(self.pr, plan.skip_windows);
                semantic_counts.insert(
                    (index.clone(), field.clone()),
                    values.iter().cloned().map(|value| (value, 0)).collect(),
                );
            } else {
                let exists = serde_json::json!({"exists": {"field": field}});
                let counts =
                    self.windowed_value_counts(index, "ax_file", values, &[exists], true)?;
                semantic_counts.insert((index.clone(), field.clone()), counts);
            }
        }
        // Scoped to this generation's `run_id`, the same way the run-summary
        // read-back is (`lib.rs`). `file_key` is derived from CONTENT alone
        // (`content::full_digest`) and the catalog is one global index that no
        // `--prefix` namespaces, so an unscoped count also sees the
        // canonical and alias documents that ANOTHER corpus on this node
        // published for byte-identical content — two Apache-2.0 checkouts
        // sharing a LICENSE is enough. Those documents are that run's, not
        // this one's: counting them aborted a generation whose own
        // publication was exactly right (#360). What this barrier is for is
        // "this generation published one canonical document and its aliases",
        // and that is what it now asks. For the same #971 reason as the
        // record counts above, only changed groups ask — a kept group's
        // canonical/alias documents intentionally keep the run_id of the
        // generation that last wrote them.
        let mut catalog_counts = if catalog_wanted.is_empty() {
            HashMap::new()
        } else {
            let _reading = self.pr.watch_file(crate::catalog::CATALOG_INDEX, 0);
            self.windowed_catalog_counts(&catalog_wanted, &snapshot.tx_id)?
        };

        // The checks are in-memory lookups in the same per-group order the
        // serial walk used, so the first disagreement still names the same
        // group it always did. A key missing from a map here is a bookkeeping
        // bug in the gathering above, not a count of zero — the catalog check
        // makes it fail loud either way.
        for group in changed_groups {
            let mut records = 0u64;
            let indices = group_indices(group, &desired.plan)?;
            for index in &indices {
                records += record_counts[index][&group.content_id];
            }
            anyhow::ensure!(
                records == group.expected_records,
                "live record count disagrees with desired group {}: live_records={records} \
                 expected_records={} indices=[{}]; a resume checks the same live indices, \
                 so it stops here again until they are rebuilt",
                group.group_id,
                group.expected_records,
                indices.join(", ")
            );
            let mut semantic = 0u64;
            for slug in &group.dataset_slugs {
                let dataset = by_slug
                    .get(slug.as_str())
                    .copied()
                    .with_context(|| format!("group references absent dataset {slug}"))?;
                if let Some(field) = &dataset.semantic_field {
                    semantic +=
                        semantic_counts[&(dataset.index.clone(), field.clone())][&group.content_id];
                }
            }
            anyhow::ensure!(
                semantic == group.expected_passages && semantic == group.expected_vectors,
                "live semantic count disagrees with desired group {}: \
                 live_semantic_records={}, expected_passages={}, expected_vectors={}, datasets={:?}",
                group.group_id,
                semantic,
                group.expected_passages,
                group.expected_vectors,
                group.dataset_slugs
            );
            // #2314: the equality used to be one shot. A folder of identical
            // empty files published the canonical doc and its aliases, then
            // lost the race with catalog visibility and aborted; the same
            // command retried clean. Recount after a refresh, the same bound
            // as the short read-back walk, before calling it a disagreement.
            let expected = 1 + group.aliases.len() as u64;
            let mut observed = catalog_counts.get(&group.content_id).copied().unwrap_or(0);
            let mut attempt = 1usize;
            while observed != expected && attempt < Self::READBACK_WALK_ATTEMPTS {
                self.pr.note(&format!(
                    "autoindex: catalog canonical/alias count for {} saw {observed} of \
                     {expected} document(s) — refreshing and recounting (attempt {attempt} of {})",
                    group.group_id,
                    Self::READBACK_WALK_ATTEMPTS
                ));
                self.es.refresh_once(crate::catalog::CATALOG_INDEX)?;
                catalog_counts = self.windowed_catalog_counts(&catalog_wanted, &snapshot.tx_id)?;
                observed = catalog_counts.get(&group.content_id).copied().unwrap_or(0);
                attempt += 1;
            }
            anyhow::ensure!(
                observed == expected,
                "catalog canonical/alias count disagrees with desired group {}: \
                 observed {observed}, expected {expected}",
                group.group_id
            );
        }
        for old in &deleted_groups {
            let mut live = 0u64;
            for index in group_indices(old, &base.plan)? {
                live += record_counts[&index][&old.content_id];
            }
            anyhow::ensure!(
                live == 0,
                "replaced or deleted content {} remains live",
                old.content_id
            );
        }
        Ok(())
    }
}

/// Resume a journaled generation strictly from its verified snapshot.
///
/// The durable order is Started -> convergent remote apply -> Committed.
/// Therefore a crash after an accepted response repeats the same operation;
/// a crash after Committed skips it. Only an exact backend validation allows
/// `sync_validated` and the minimal authority switch in `sync_commit`.
#[cfg_attr(not(test), allow(dead_code))]
pub fn replay_pending_operations(
    state_dir: &Path,
    journal: &mut Journal,
    backend: &mut impl SyncOperationBackend,
) -> Result<()> {
    let pending = journal
        .pending_sync
        .clone()
        .context("cannot replay without a pending corpus generation")?;
    let base = journal
        .committed_manifest
        .clone()
        .context("pending corpus generation has no committed base")?;
    // Same contract as the journal-replay wrap in `state.rs` (#283): a pending
    // generation that fails its own re-validation is unrepairable by
    // re-running, and the internal invariant alone is not actionable.
    pending.validate_against(&base).with_context(|| {
        format!(
            "the pending corpus generation {} no longer re-validates against committed \
             generation {}; the generation journal is not internally consistent, and re-running \
             will not repair it. No remote data was changed. Rebuild with a new --state-dir and \
             a new --prefix",
            pending.desired.generation, base.generation
        )
    })?;
    let snapshot = open_snapshot(state_dir, &pending.tx_id)?;
    verify_snapshot_binding(&pending, &snapshot)?;
    backend.provision_generation(&pending.desired)?;

    // #931: what is left to apply, in the units the progress surface reports.
    // An operation already `Committed` by an earlier attempt is not work this
    // run will do, so it is in neither the numerator nor the denominator — a
    // resumed replay starts at 0% of what REMAINS, not at a percentage that
    // credits this invocation with a previous one's writes.
    let committed = |journal: &Journal, operation: &SyncOperation| {
        journal
            .pending_sync
            .as_ref()
            .and_then(|sync| sync.operation_states.get(&operation.operation_id))
            == Some(&SyncOperationState::Committed)
    };
    let desired_by_group: HashMap<&str, &ManifestGroup> = pending
        .desired
        .groups
        .iter()
        .map(|group| (group.group_id.as_str(), group))
        .collect();
    let base_by_group: HashMap<&str, &ManifestGroup> = base
        .groups
        .iter()
        .map(|group| (group.group_id.as_str(), group))
        .collect();
    let prepared_bytes: HashMap<&str, u64> = snapshot
        .files
        .iter()
        .filter_map(|file| {
            file.prepared
                .as_ref()
                .map(|artifact| (file.content_id.as_str(), artifact.bytes))
        })
        .collect();
    // Only an upsert sends the sealed NDJSON; a delete or a metadata rewrite
    // moves no prepared bytes, so it counts as an item and as zero bytes.
    let operation_bytes = |operation: &SyncOperation| -> u64 {
        if operation.kind != crate::sync::SyncOperationKind::Upsert {
            return 0;
        }
        desired_by_group
            .get(operation.group_id.as_str())
            .and_then(|group| prepared_bytes.get(group.content_id.as_str()))
            .copied()
            .unwrap_or(0)
    };
    let remaining: Vec<&SyncOperation> = pending
        .operations
        .iter()
        .filter(|operation| !committed(journal, operation))
        .collect();
    // #931: what the loop reports per operation — its source path and its
    // sealed bytes — resolved once here, so the serial loop and the windowed
    // scheduler (#933) name and measure every operation identically.
    let items: Vec<ReplayItem> = remaining
        .iter()
        .map(|operation| ReplayItem {
            operation,
            rel: desired_by_group
                .get(operation.group_id.as_str())
                .or_else(|| base_by_group.get(operation.group_id.as_str()))
                .map_or(operation.group_id.as_str(), |group| {
                    group.canonical.rel.as_str()
                })
                .to_owned(),
            bytes: operation_bytes(operation),
        })
        .collect();
    backend.replay_begins(
        items.len() as u64,
        items.iter().map(|item| item.bytes).sum(),
    );
    backend.replay_operations(&items, &base, &pending.desired, &snapshot, journal)?;
    backend.publish_generation_catalog(&base, &pending.desired, &snapshot)?;
    backend.validate(&base, &pending.desired, &snapshot)?;
    journal.sync_validated()?;
    journal.sync_commit()?;
    gc_snapshots(state_dir, journal).context(
        "generation committed durably, but snapshot cleanup failed; inspect the attached cause \
         (remove a refused symlink or repair the reported filesystem permission/I/O problem), \
         then retry the same command to continue bounded cleanup without republishing data",
    )
}

/// [`replay_pending_operations`] for a run with a progress surface: when the
/// server's back-pressure outlasts the client's patience, say THAT on the way
/// out instead of `reason=aborted`.
///
/// #944 made a throttled server slow the run down rather than end it; this is
/// the one case left where it still ends — the node accepted nothing for the
/// whole patience (the node behind #950 never accepts again until it is
/// restarted). Such a stop is not a crash and not a bad corpus: every applied
/// operation is journaled, the same command resumes, and the only thing the
/// reader needs is how much is left and what it was doing. So the terminal
/// line names the cause and the two counts, and a note names the operation in
/// flight. The exit code stays 1: `3` is published as "a finished run, retry
/// nothing", and this run did not finish — an agent told to retry nothing
/// would report a half-applied generation as searchable.
pub fn replay_pending_operations_reporting(
    state_dir: &Path,
    journal: &mut Journal,
    backend: &mut impl SyncOperationBackend,
    pr: &crate::progress::Progress,
) -> Result<()> {
    let result = replay_pending_operations(state_dir, journal, backend);
    let Err(error) = &result else {
        return result;
    };
    if crate::esclient::BackpressureExhausted::in_chain(error).is_none() {
        return result;
    }
    let Some(pending) = journal.pending_sync.as_ref() else {
        return result;
    };
    let committed = |operation: &SyncOperation| {
        pending.operation_states.get(&operation.operation_id)
            == Some(&SyncOperationState::Committed)
    };
    let applied = pending.operations.iter().filter(|op| committed(op)).count() as u64;
    let remaining = pending.operations.len() as u64 - applied;
    // The operation in flight is the one journaled Started and not Committed.
    let in_flight = pending
        .operations
        .iter()
        .find(|operation| {
            pending.operation_states.get(&operation.operation_id)
                == Some(&SyncOperationState::Started)
        })
        .and_then(|operation| {
            pending
                .desired
                .groups
                .iter()
                .find(|group| group.group_id == operation.group_id)
                .map(|group| group.canonical.rel.clone())
        });
    pr.note(&match (remaining, in_flight) {
        (0, _) => format!(
            "autoindex: stopped by server back-pressure after all {applied} operation(s) were \
             applied; the catalog write and the read-back barrier are what remain — the same \
             command resumes there"
        ),
        (_, Some(rel)) => format!(
            "autoindex: stopped by server back-pressure while applying {rel}: {applied} \
             operation(s) are journaled applied, {remaining} are not (this one first) — the \
             same command resumes from here once the node accepts writes again"
        ),
        (_, None) => format!(
            "autoindex: stopped by server back-pressure: {applied} operation(s) are journaled \
             applied, {remaining} are not — the same command resumes from here once the node \
             accepts writes again"
        ),
    });
    pr.finish(
        false,
        1,
        "server-backpressure",
        &[("ops_applied", applied), ("ops_remaining", remaining)],
    );
    result
}

const SNAPSHOT_GC_BATCH_SIZE: usize = 4096;

fn protected_snapshot(
    state_dir: &Path,
    execution: &crate::sync::ExecutionIdentity,
) -> Result<String> {
    let SourceExecutionPolicy::DurableSnapshot {
        reference,
        snapshot_digest,
    } = &execution.source_policy
    else {
        anyhow::bail!("generated authority does not reference a durable snapshot");
    };
    let tx_id = reference
        .strip_prefix("sync-snapshots/")
        .context("protected snapshot reference is not state-relative")?;
    validate_tx_id(tx_id)?;
    let snapshot = open_snapshot(state_dir, tx_id)?;
    anyhow::ensure!(
        snapshot.snapshot_digest == *snapshot_digest,
        "protected snapshot digest disagrees with journal authority"
    );
    Ok(tx_id.to_owned())
}

/// Reclaim only snapshots proven unreferenced by replayed journal authority.
/// The caller owns the journal lock for the complete validation/rename/fsync
/// sequence.
pub fn gc_snapshots(state_dir: &Path, journal: &Journal) -> Result<()> {
    let root = state_dir.join("sync-snapshots");
    let mut protected = std::collections::HashSet::new();
    if let Some(execution) = journal
        .committed_manifest
        .as_ref()
        .and_then(|manifest| manifest.execution.as_ref())
    {
        protected.insert(protected_snapshot(state_dir, execution)?);
    }
    if let Some(execution) = journal
        .pending_sync
        .as_ref()
        .and_then(|pending| pending.desired.execution.as_ref())
    {
        protected.insert(protected_snapshot(state_dir, execution)?);
    }
    let entries = match std::fs::read_dir(&root) {
        Ok(entries) => entries,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
        Err(error) => {
            return Err(anyhow::Error::from(error)
                .context(format!("list snapshot directory {}", root.display())));
        }
    };
    let mut entries = entries.collect::<std::io::Result<Vec<_>>>()?;
    entries.sort_by_key(|entry| entry.file_name());
    // Validate the complete directory before the first mutation. In
    // particular, a late symlink must not be discovered after earlier
    // snapshots have already been deleted.
    for entry in &entries {
        anyhow::ensure!(
            !entry.file_type()?.is_symlink(),
            "snapshot cleanup refuses symlink {}",
            entry.path().display()
        );
    }
    for entry in entries.into_iter().take(SNAPSHOT_GC_BATCH_SIZE) {
        let metadata = entry.file_type()?;
        if !metadata.is_dir() {
            continue;
        }
        let name = entry.file_name().to_string_lossy().into_owned();
        if protected.contains(&name) {
            continue;
        }
        let tombstone = if name.ends_with(".gc") {
            entry.path()
        } else {
            let tombstone = root.join(format!(".{name}.gc"));
            anyhow::ensure!(
                !tombstone.exists(),
                "snapshot tombstone already exists: {}",
                tombstone.display()
            );
            std::fs::rename(entry.path(), &tombstone).with_context(|| {
                format!(
                    "rename snapshot {} to tombstone {}",
                    entry.path().display(),
                    tombstone.display()
                )
            })?;
            sync_dir(&root)?;
            #[cfg(test)]
            if GC_FAIL_AFTER_RENAME.swap(false, std::sync::atomic::Ordering::SeqCst) {
                anyhow::bail!("injected snapshot GC crash after durable tombstone rename");
            }
            tombstone
        };
        std::fs::remove_dir_all(&tombstone)
            .with_context(|| format!("remove snapshot tombstone {}", tombstone.display()))?;
        sync_dir(&root)?;
    }
    Ok(())
}

/// Test helper proving a pending generation binds before mutable discovery.
/// Production calls `replay_pending_operations` at this boundary.
#[cfg(test)]
pub fn require_resumable_pending_source(
    state_dir: &Path,
    pending: Option<&PendingSync>,
) -> Result<()> {
    let Some(pending) = pending else {
        return Ok(());
    };
    let snapshot = open_snapshot(state_dir, &pending.tx_id).with_context(|| {
        format!(
            "pending corpus generation {} must resume before source discovery",
            pending.tx_id
        )
    })?;
    verify_snapshot_binding(pending, &snapshot)?;
    anyhow::bail!(
        "corpus generation {} is pending with verified durable source snapshot {} ({} files); \
         incremental operation replay is not enabled by this executor slice, so no source \
         discovery or remote mutation was attempted",
        pending.tx_id,
        snapshot.snapshot_digest,
        snapshot.files.len()
    )
}

fn verify_snapshot_binding(pending: &PendingSync, snapshot: &SourceSnapshot) -> Result<()> {
    let execution = pending
        .desired
        .execution
        .as_ref()
        .context("pending generation has no execution identity")?;
    let SourceExecutionPolicy::DurableSnapshot {
        reference,
        snapshot_digest,
    } = &execution.source_policy
    else {
        anyhow::bail!("pending generation is not bound to a durable source snapshot");
    };
    anyhow::ensure!(
        reference == &format!("sync-snapshots/{}", pending.tx_id)
            && snapshot_digest == &snapshot.snapshot_digest,
        "pending generation source snapshot binding does not match verified snapshot"
    );
    Ok(())
}

pub(crate) fn open_committed_snapshot(
    state_dir: &Path,
    committed: &CommittedManifest,
) -> Result<SourceSnapshot> {
    let execution = committed
        .execution
        .as_ref()
        .context("committed generation has no execution identity")?;
    let SourceExecutionPolicy::DurableSnapshot {
        reference,
        snapshot_digest,
    } = &execution.source_policy
    else {
        anyhow::bail!("committed generation is not bound to a durable snapshot");
    };
    let tx_id = reference
        .strip_prefix("sync-snapshots/")
        .context("committed snapshot reference is not state-relative")?;
    let snapshot = open_snapshot(state_dir, tx_id)?;
    anyhow::ensure!(
        &snapshot.snapshot_digest == snapshot_digest,
        "committed snapshot digest mismatch"
    );
    Ok(snapshot)
}

fn prepared_for<'a>(
    snapshot: &'a SourceSnapshot,
    content_id: &str,
) -> Result<&'a PreparedArtifact> {
    snapshot
        .files
        .iter()
        .find(|file| file.content_id == content_id)
        .and_then(|file| file.prepared.as_ref())
        .with_context(|| format!("content {content_id} has no sealed prepared artifact"))
}

fn group_indices(group: &ManifestGroup, plan: &Plan) -> Result<Vec<String>> {
    let by_slug: HashMap<&str, &str> = plan
        .datasets
        .iter()
        .map(|dataset| (dataset.slug.as_str(), dataset.index.as_str()))
        .collect();
    let mut indices = group
        .dataset_slugs
        .iter()
        .map(|slug| {
            by_slug
                .get(slug.as_str())
                .map(|index| (*index).to_string())
                .with_context(|| format!("group references absent dataset {slug}"))
        })
        .collect::<Result<Vec<_>>>()?;
    indices.sort();
    indices.dedup();
    Ok(indices)
}

fn checked_bulk(es: &crate::esclient::Es, body: Vec<u8>) -> Result<()> {
    if body.is_empty() {
        return Ok(());
    }
    let outcome = es.bulk(body.clone())?;
    check_bulk_outcome(es, outcome, &body)
}

/// [`checked_bulk`] for a body nobody windowed — the catalog projection, which
/// holds one document per file and so grows with the corpus (#955). Sent as
/// requests of at most `bulk_bytes` bytes, in order.
fn checked_bulk_windowed(es: &crate::esclient::Es, body: Vec<u8>, bulk_bytes: usize) -> Result<()> {
    if body.is_empty() {
        return Ok(());
    }
    let outcome = es.bulk_windowed(body.clone(), bulk_bytes)?;
    check_bulk_outcome(es, outcome, &body)
}

/// The `ax_path` of each refused record, read back from the body that was
/// sent: a refusal reports only the record's `_id`, which is a hash.
fn refused_paths(body: &[u8], outcome: &crate::esclient::BulkOutcome) -> Vec<String> {
    let ids: HashSet<&str> = outcome
        .refused
        .iter()
        .filter_map(|item| item.id.as_deref())
        .collect();
    let mut paths = Vec::new();
    let mut lines = body.split(|byte| *byte == b'\n');
    while let (Some(action), Some(doc)) = (lines.next(), lines.next()) {
        let Ok(action) = serde_json::from_slice::<Value>(action) else {
            continue;
        };
        let id = action
            .as_object()
            .and_then(|op| op.values().next())
            .and_then(|op| op.get("_id"))
            .and_then(Value::as_str);
        if !id.is_some_and(|id| ids.contains(id)) {
            continue;
        }
        if let Some(path) = serde_json::from_slice::<Value>(doc)
            .ok()
            .and_then(|doc| doc.get("ax_path")?.as_str().map(str::to_owned))
        {
            if !paths.contains(&path) {
                paths.push(path);
            }
        }
    }
    paths
}

fn check_bulk_outcome(
    es: &crate::esclient::Es,
    outcome: crate::esclient::BulkOutcome,
    body: &[u8],
) -> Result<()> {
    // `Es::bulk` has already re-sent per-item 429s, and only those, for the
    // whole of its patience (`THROTTLE_PATIENCE`, #944/#949, shortened to
    // `ZERO_PROGRESS_PATIENCE` when a parent memory breaker accepts nothing,
    // #2238). What is left is either a server condition that did not clear —
    // fatal, and resumable, because a sealed operation is journaled applied
    // only after this returns — or a record the server refused outright.
    if outcome.throttled_out > 0 {
        let patience = outcome
            .exhausted_after
            .unwrap_or_else(|| es.throttle_patience());
        let reason = outcome
            .first_server_error
            .unwrap_or_else(|| "unknown server error".into());
        return Err(anyhow::Error::new(crate::esclient::backpressure_exhausted(
            outcome.server_errors,
            patience,
            &reason,
        )));
    }
    if outcome.server_errors > 0 {
        let reason = outcome
            .first_server_error
            .as_deref()
            .map(crate::esclient::memory_cap_hint)
            .unwrap_or_else(|| "unknown server error".into());
        anyhow::bail!(
            "the server failed {} of a prepared bulk's items: {reason}. Nothing from this bulk was \
             journaled applied; fix the reported server condition and rerun the same command — \
             the run resumes from its last committed operation",
            outcome.server_errors,
        );
    }
    if outcome.item_errors == 0 {
        return Ok(());
    }
    // This route seals the generation before it sends, so a file can no
    // longer leave the plan here, and a rerun replays the same refusal. Name
    // the file and the ways out instead (#2807).
    let paths = refused_paths(body, &outcome);
    let source = if paths.is_empty() {
        String::new()
    } else {
        format!(" from {}", paths.join(", "))
    };
    let too_large = outcome.refused.iter().any(|item| item.too_large);
    let remedy = if too_large {
        " The server adds vectors to a `semantic_text` field, and they count toward its \
         per-document limit. A sealed generation cannot set a file aside, so rerunning \
         repeats this refusal: index the folder under a new --prefix and --state-dir with \
         the file left out (`--exclude`), or with `--no-semantic`"
    } else {
        ""
    };
    anyhow::bail!(
        "the server refused {} record(s){source}: {}.{remedy}",
        outcome.item_errors,
        outcome
            .first_error
            .unwrap_or_else(|| "unknown error".into())
    )
}

fn append_index_action(index: &str, id: &str, doc: &Value, body: &mut Vec<u8>) -> Result<()> {
    serde_json::to_writer(
        &mut *body,
        &serde_json::json!({"index": {"_index": index, "_id": id}}),
    )?;
    body.push(b'\n');
    serde_json::to_writer(&mut *body, doc)?;
    body.push(b'\n');
    Ok(())
}

fn stream_ndjson_pairs(
    mut reader: impl std::io::BufRead,
    bulk_bytes: usize,
    mut send: impl FnMut(Vec<u8>) -> Result<()>,
) -> Result<()> {
    let mut body = Vec::with_capacity(bulk_bytes);
    loop {
        let mut action = Vec::new();
        if reader.read_until(b'\n', &mut action)? == 0 {
            break;
        }
        let mut document = Vec::new();
        anyhow::ensure!(
            reader.read_until(b'\n', &mut document)? > 0,
            "prepared NDJSON ended without a document"
        );
        if !body.is_empty() && body.len() + action.len() + document.len() > bulk_bytes {
            send(std::mem::take(&mut body))?;
            body.reserve(bulk_bytes);
        }
        body.extend_from_slice(&action);
        body.extend_from_slice(&document);
    }
    send(body)
}

fn stream_metadata_updates(
    mut reader: impl std::io::BufRead,
    bulk_bytes: usize,
    canonical: &str,
    paths: &[Value],
    mut send: impl FnMut(Vec<u8>) -> Result<()>,
) -> Result<()> {
    let mut body = Vec::with_capacity(bulk_bytes);
    loop {
        let mut action_line = String::new();
        if reader.read_line(&mut action_line)? == 0 {
            break;
        }
        let mut ignored_document = String::new();
        anyhow::ensure!(
            reader.read_line(&mut ignored_document)? > 0,
            "prepared NDJSON ended without a document"
        );
        let action: Value = serde_json::from_str(&action_line)?;
        let index = action
            .pointer("/index/_index")
            .and_then(Value::as_str)
            .context("prepared action has no index")?;
        let id = action
            .pointer("/index/_id")
            .and_then(Value::as_str)
            .context("prepared action has no ID")?;
        let update =
            serde_json::to_vec(&serde_json::json!({"update": {"_index": index, "_id": id}}))?;
        let patch = serde_json::to_vec(&serde_json::json!({"doc": {
            "ax_path": canonical,
            "ax_paths": paths
        }}))?;
        let required = update.len() + patch.len() + 2;
        if !body.is_empty() && body.len() + required > bulk_bytes {
            send(std::mem::take(&mut body))?;
            body.reserve(bulk_bytes);
        }
        body.extend_from_slice(&update);
        body.push(b'\n');
        body.extend_from_slice(&patch);
        body.push(b'\n');
    }
    send(body)
}

/// Build desired groups without guessing assignments. Output counts survive
/// only when the committed content identity is byte-identical.
#[cfg_attr(not(test), allow(dead_code))]
pub fn groups_from_inventory(
    inventory: &Inventory,
    plan: &Plan,
    previous: &[ManifestGroup],
) -> Result<Vec<ManifestGroup>> {
    ensure_inventory_lengths(inventory)?;
    let previous_by_content: HashMap<&str, &ManifestGroup> = previous
        .iter()
        .map(|group| (group.content_id.as_str(), group))
        .collect();
    let mut aliases_by_key: HashMap<&str, Vec<ManifestPath>> = HashMap::new();
    for alias in &inventory.duplicates {
        aliases_by_key
            .entry(alias.file_key.as_str())
            .or_default()
            .push(ManifestPath {
                path_id: alias.path_id.clone(),
                rel: alias.rel.clone(),
                is_symlink: alias.is_symlink.with_context(|| {
                    format!("alias {} has no persisted symlink rank", alias.rel)
                })?,
            });
    }
    let desired = inventory
        .files
        .iter()
        .zip(&inventory.keys)
        .zip(&inventory.digests)
        .map(|((file, content_id), content_digest)| {
            let assignment = plan.files.get(content_id).with_context(|| {
                format!("typed plan has no assignment for content {content_id}")
            })?;
            anyhow::ensure!(
                assignment.rel == file.rel
                    && assignment.path_id == file.rel_id
                    && assignment.content_digest.as_deref() == Some(content_digest.as_str()),
                "typed plan canonical projection disagrees with inventory for {}",
                file.rel
            );
            let mut dataset_slugs: Vec<String> = assignment
                .assignments
                .iter()
                .map(|(_, slug)| slug.clone())
                .collect();
            dataset_slugs.sort();
            dataset_slugs.dedup();
            let prior = previous_by_content.get(content_id.as_str()).copied();
            Ok(DesiredContentGroup {
                content_id: content_id.clone(),
                content_digest: content_digest.clone(),
                content_size: file.size,
                paths: std::iter::once(ManifestPath {
                    path_id: file.rel_id.clone(),
                    rel: file.rel.clone(),
                    is_symlink: file.is_symlink,
                })
                .chain(
                    aliases_by_key
                        .get(content_id.as_str())
                        .into_iter()
                        .flatten()
                        .cloned(),
                )
                .collect(),
                dataset_slugs,
                expected_records: prior.map_or(0, |group| group.expected_records),
                expected_passages: prior.map_or(0, |group| group.expected_passages),
                expected_vectors: prior.map_or(0, |group| group.expected_vectors),
                expected_junk_records: prior.map_or(0, |group| group.expected_junk_records),
                expected_records_by_dataset: prior
                    .map(|group| group.expected_records_by_dataset.clone())
                    .unwrap_or_default(),
            })
        })
        .collect::<Result<Vec<_>>>()?;
    sync::reconcile_groups(previous, desired)
}

/// Bind exact prepared output cardinalities into the desired manifest before
/// `sync_begin`. Content without a prepared artifact is rejected; unchanged
/// groups should retain their already committed counts instead.
#[cfg_attr(not(test), allow(dead_code))]
pub fn bind_prepared_counts(
    groups: &mut [ManifestGroup],
    snapshot: &SourceSnapshot,
    prepared_content: &[String],
) -> Result<()> {
    let prepared_content: std::collections::HashSet<&str> =
        prepared_content.iter().map(String::as_str).collect();
    let by_content: HashMap<&str, &PreparedArtifact> = snapshot
        .files
        .iter()
        .filter_map(|file| {
            file.prepared
                .as_ref()
                .map(|prepared| (file.content_id.as_str(), prepared))
        })
        .collect();
    for group in groups {
        if !prepared_content.contains(group.content_id.as_str()) {
            continue;
        }
        let prepared = by_content
            .get(group.content_id.as_str())
            .context("desired content has no prepared artifact")?;
        group.expected_records = prepared.records;
        group.expected_passages = prepared.passages;
        group.expected_vectors = prepared.vectors;
        group.expected_junk_records = prepared.junk;
        group.expected_records_by_dataset = prepared.records_by_dataset.clone();
    }
    Ok(())
}

/// Fsync blobs and manifest in a private directory, then atomically publish
/// the transaction snapshot. Retries reuse a verified final snapshot or
/// replace only that transaction's incomplete staging directory.
#[cfg_attr(not(test), allow(dead_code))]
pub fn create_snapshot(
    state_dir: &Path,
    tx_id: &str,
    inventory: &Inventory,
) -> Result<SourceSnapshot> {
    create_snapshot_inner(
        state_dir,
        tx_id,
        inventory,
        None,
        "source-snapshot-v1",
        u64::MAX,
        &crate::progress::Progress::silent(),
        None,
        "",
        // The source-snapshot wrapper has no plan and therefore never runs
        // `prepare_artifact`; a labeler would have nothing to vote on.
        None,
        None,
    )
}

/// Seal deterministic bulk actions together with source bytes. Extraction and
/// coercion happen before `sync_begin`; retries stream this artifact and never
/// re-extract or re-embed unchanged content.
#[cfg_attr(not(test), allow(dead_code))]
pub fn create_prepared_snapshot(
    state_dir: &Path,
    tx_id: &str,
    inventory: &Inventory,
    plan: &Plan,
    preparation_contract_digest: &str,
    hard_budget_bytes: u64,
) -> Result<SourceSnapshot> {
    create_prepared_snapshot_reporting(
        state_dir,
        tx_id,
        inventory,
        plan,
        preparation_contract_digest,
        hard_budget_bytes,
        &crate::progress::Progress::silent(),
        // Test-only wrapper: these snapshots are never handed back as a reuse
        // source, so the chunker identity they seal is a fixed label.
        None,
        "prepared-records-v1",
        // Test-only wrapper: no `--label` in these snapshots' contracts.
        None,
        // Test-only wrapper: no server vectors to size records against.
        None,
    )
}

/// [`create_prepared_snapshot`], reporting through the run's progress surface.
///
/// Sealing is the generated path's extraction pass — every file is verified,
/// copied, verified again and parsed into sealed NDJSON, one at a time — and on
/// the corpus that produced #931 it is minutes of work. It used to run with no
/// phase of its own, so the stream kept describing the `scan` that had already
/// finished. It is now the `snapshot` phase, with a byte denominator.
///
/// #971: `reuse` is the prior *committed* generation's snapshot. A file whose
/// content digest and preparation identity both match its entry there is
/// hardlinked from it instead of being re-copied and re-extracted, so a
/// one-file change seals O(changed) new bytes. `chunker_identity` is the run's
/// `prepared_records_identity` and becomes part of each file's sealed
/// preparation identity.
///
/// #2807: `vector_dims` is the width of the vectors the server adds to each
/// record's semantic field, from its embedding identity. A record that would
/// pass [`xerj_common::document_limit::MAX_INDEXED_DOCUMENT_BYTES`] once they
/// are added is set aside as a junk record instead of sealed. `None` sizes
/// records without vectors. The width is pinned in the generation's execution
/// identity, so a reused artifact was sized against the same width.
#[allow(clippy::too_many_arguments)]
pub fn create_prepared_snapshot_reporting(
    state_dir: &Path,
    tx_id: &str,
    inventory: &Inventory,
    plan: &Plan,
    preparation_contract_digest: &str,
    hard_budget_bytes: u64,
    pr: &crate::progress::Progress,
    reuse: Option<&SourceSnapshot>,
    chunker_identity: &str,
    labeler: Option<&crate::label::Labeler>,
    vector_dims: Option<usize>,
) -> Result<SourceSnapshot> {
    create_snapshot_inner(
        state_dir,
        tx_id,
        inventory,
        Some(plan),
        preparation_contract_digest,
        hard_budget_bytes,
        pr,
        reuse,
        chunker_identity,
        labeler,
        vector_dims,
    )
}

#[allow(clippy::too_many_arguments)]
fn create_snapshot_inner(
    state_dir: &Path,
    tx_id: &str,
    inventory: &Inventory,
    plan: Option<&Plan>,
    preparation_contract_digest: &str,
    hard_budget_bytes: u64,
    pr: &crate::progress::Progress,
    reuse: Option<&SourceSnapshot>,
    chunker_identity: &str,
    labeler: Option<&crate::label::Labeler>,
    vector_dims: Option<usize>,
) -> Result<SourceSnapshot> {
    validate_tx_id(tx_id)?;
    ensure_inventory_lengths(inventory)?;
    let root = state_dir.join("sync-snapshots");
    std::fs::create_dir_all(&root)?;
    let final_dir = root.join(tx_id);
    if final_dir.exists() {
        let existing = open_snapshot(state_dir, tx_id)?;
        anyhow::ensure!(
            existing.preparation_contract_digest == preparation_contract_digest,
            "existing final snapshot was prepared under a different contract"
        );
        let requested = inventory
            .keys
            .iter()
            .zip(&inventory.digests)
            .zip(&inventory.files)
            .map(|((content_id, digest), file)| (content_id.as_str(), digest.as_str(), file.size))
            .collect::<Vec<_>>();
        let sealed = existing
            .files
            .iter()
            .map(|file| {
                (
                    file.content_id.as_str(),
                    file.content_digest.as_str(),
                    file.content_size,
                )
            })
            .collect::<Vec<_>>();
        anyhow::ensure!(
            sealed == requested,
            "existing final snapshot inventory differs from this preparation attempt"
        );
        anyhow::ensure!(
            existing.footprint.total_bytes <= hard_budget_bytes,
            "existing final snapshot exceeds the current logical payload limit"
        );
        return Ok(existing);
    }
    let staging = root.join(format!(".{tx_id}.partial"));
    if staging.exists() {
        std::fs::remove_dir_all(&staging)?;
    }
    std::fs::create_dir(&staging)?;
    let mut cleanup = StagingCleanup {
        path: staging.clone(),
        armed: true,
    };
    let blobs = staging.join("blobs");
    std::fs::create_dir(&blobs)?;
    let prepared_dir = staging.join("prepared");
    if plan.is_some() {
        std::fs::create_dir(&prepared_dir)?;
    }
    let source_bytes = inventory.files.iter().try_fold(0u64, |total, file| {
        total
            .checked_add(file.size)
            .context("snapshot source byte overflow")
    })?;
    if source_bytes > hard_budget_bytes {
        anyhow::bail!(
            "snapshot source footprint {source_bytes} bytes exceeds logical payload limit \
             {hard_budget_bytes} bytes before preparation"
        );
    }
    let mut budget = PayloadBudget {
        used: 0,
        limit: hard_budget_bytes,
    };
    // Built once per snapshot: per-file preparation used to rebuild the
    // dataset/coercion maps, rescan every duplicate alias, and re-serialize
    // each assigned dataset for every file — O(files × (aliases + datasets)).
    let lookup = plan.map(PreparationLookup::new).transpose()?;
    // #971: the reuse index. Sealing is O(corpus) because every file is
    // re-copied, re-verified and re-extracted per generation — a one-file
    // change in a 10,000-file corpus sealed 10,000 blobs twice over (verify,
    // copy+fsync, verify, extract). A file whose content digest and size match
    // the prior committed snapshot's entry is hardlinked from it instead. A
    // hardlink, not a symlink and not a copy: `gc_snapshots` removes the prior
    // snapshot's directory entries after this generation commits, and a
    // hardlinked inode survives that while a symlink would dangle — the same
    // manifest-referenced-GC property tantivy's `list_segment_files` relies on
    // to share unchanged segment files across commits. The blob needs no
    // re-verify either: `open_snapshot` (which the caller ran to hand us a
    // digest-checked `reuse`) proved the prior blob matches its manifest, and
    // this inventory's digest for the file is the same value — so the sealed
    // bytes equal the scan-time bytes by construction.
    let reuse_root = reuse.map(|prior| state_dir.join("sync-snapshots").join(&prior.tx_id));
    let reuse_by_content: HashMap<&str, &SnapshotFile> = reuse
        .map(|prior| {
            prior
                .files
                .iter()
                .map(|file| (file.content_id.as_str(), file))
                .collect()
        })
        .unwrap_or_default();
    let mut files = Vec::with_capacity(inventory.files.len());
    // Entered only when there is work to report: a retry that reuses a verified
    // final snapshot returned above and never claims a phase it did not run.
    pr.phase("snapshot", inventory.files.len() as u64, source_bytes);
    // #2314: a test grows one source in the window after the hash and before
    // this seal. Production runs leave the hook empty.
    snapshot_source_hook(state_dir);
    for (ordinal, ((source, content_id), content_digest)) in inventory
        .files
        .iter()
        .zip(&inventory.keys)
        .zip(&inventory.digests)
        .enumerate()
    {
        // Counted done on every exit from this iteration, including the `?`s:
        // the phase measures files drained, and the guard names the file a
        // quiet tail is inside.
        let _sealing = pr.file(&source.rel, source.size);
        let relative_blob = format!("blobs/{ordinal:08}");
        let destination = staging.join(&relative_blob);
        // A junk/skipped file has no `plan.files` entry *by construction* — it
        // lives in `plan.junk_files` and is never indexed — so demanding one
        // here made `--no-graph` fail outright on any folder holding a single
        // unreadable, empty or unrecognised file. Its bytes still belong in the
        // sealed snapshot (a resume replays from the snapshot, not from the
        // mutable tree, and the inventory it is verified against lists the
        // file), but there is nothing to prepare for it: `prepared: None`.
        let assignment = plan.and_then(|plan| plan.files.get(content_id.as_str()));
        let prepared_identity = match (lookup.as_ref(), assignment) {
            (Some(lookup), Some(assignment)) => Some(prepared_artifact_identity(
                chunker_identity,
                lookup,
                content_id.as_str(),
                assignment,
            )?),
            _ => None,
        };
        let reusable = reuse_by_content
            .get(content_id.as_str())
            .copied()
            .filter(|prior| {
                prior.content_digest == *content_digest && prior.content_size == source.size
            })
            // #2421: a prior file whose blob was released can only be reused
            // through its prepared artifact. If the preparation identity moved,
            // there are no sealed bytes to re-extract from, so it re-seals from
            // the live source (verified against this run's digest) below.
            .filter(|prior| {
                !prior.blob_released
                    || (prior.prepared.is_some()
                        && prior.prepared_identity.is_some()
                        && prior.prepared_identity == prepared_identity)
            });
        match reusable {
            Some(prior) => {
                let prior_blob = reuse_root
                    .as_ref()
                    .expect("reuse root exists whenever reuse_by_content is non-empty")
                    .join(&prior.relative_blob);
                if !prior.blob_released {
                    std::fs::hard_link(&prior_blob, &destination).with_context(|| {
                        format!(
                            "hardlink unchanged snapshot blob {} -> {}",
                            prior_blob.display(),
                            destination.display()
                        )
                    })?;
                }
                budget.charge(source.size as usize)?;
                let prepared = match &prior.prepared {
                    Some(prior_artifact)
                        if prior.prepared_identity.as_deref().is_some_and(|identity| {
                            Some(identity) == prepared_identity.as_deref()
                        }) =>
                    {
                        // Extraction inputs are byte-identical to the prior
                        // generation's, so the sealed NDJSON is too (the only
                        // run-varying field it carries is `ax_run`, which the
                        // prior generation stamped — and the live records this
                        // artifact replays still carry that stamp, so reusing
                        // it is consistent with the ax_run provenance rule:
                        // only an Upsert, which always re-prepares, may move
                        // it). Link it under this snapshot's ordinal and keep
                        // the verified artifact metadata verbatim.
                        let relative_ndjson = format!("prepared/{ordinal:08}.ndjson");
                        let prior_path = reuse_root
                            .as_ref()
                            .expect("reuse root exists whenever reuse_by_content is non-empty")
                            .join(&prior_artifact.relative_ndjson);
                        std::fs::hard_link(&prior_path, staging.join(&relative_ndjson))
                            .with_context(|| {
                                format!(
                                    "hardlink unchanged prepared artifact {}",
                                    prior_path.display()
                                )
                            })?;
                        budget.charge(prior_artifact.bytes as usize)?;
                        Some(PreparedArtifact {
                            relative_ndjson,
                            ..prior_artifact.clone()
                        })
                    }
                    _ => match assignment {
                        Some(assignment) => Some(prepare_artifact(
                            &staging,
                            ordinal,
                            tx_id,
                            source,
                            content_id,
                            &destination,
                            assignment,
                            lookup.as_ref().expect("assignment implies a plan"),
                            &mut budget,
                            labeler,
                            vector_dims,
                            pr,
                        )?),
                        None => None,
                    },
                };
                let blob_released = release_prepared_blob(&destination, prepared.as_ref())?;
                files.push(SnapshotFile {
                    content_id: content_id.clone(),
                    content_digest: content_digest.clone(),
                    content_size: source.size,
                    relative_blob,
                    prepared,
                    prepared_identity,
                    blob_released,
                });
            }
            None => {
                // #2314: a file that changes size (or disappears) between the
                // hash and this copy used to abort the generation on the `?`.
                // The error names the live source so the caller can drop that
                // one file and seal the rest.
                verify_live_source(source, content_digest)?;
                if let Err(error) = copy_synced(&source.path, &destination, &mut budget) {
                    if verify_live_source(source, content_digest).is_err() {
                        return Err(source_changed(
                            &source.path,
                            format!(
                                "{} changed while it was being copied into the snapshot ({error:#})",
                                source.path.display()
                            ),
                        ));
                    }
                    return Err(error);
                }
                verify_sealed_blob(source, &destination, content_digest)?;
                #[cfg(test)]
                apply_post_seal_source_replacement(&source.path)?;
                let prepared = match assignment {
                    Some(assignment) => Some(prepare_artifact(
                        &staging,
                        ordinal,
                        tx_id,
                        source,
                        content_id,
                        &destination,
                        assignment,
                        lookup.as_ref().expect("assignment implies a plan"),
                        &mut budget,
                        labeler,
                        vector_dims,
                        pr,
                    )?),
                    None => None,
                };
                let blob_released = release_prepared_blob(&destination, prepared.as_ref())?;
                files.push(SnapshotFile {
                    content_id: content_id.clone(),
                    content_digest: content_digest.clone(),
                    content_size: source.size,
                    relative_blob,
                    prepared,
                    prepared_identity,
                    blob_released,
                });
            }
        }
    }
    let prepared_bytes = files.iter().try_fold(0u64, |total, file| {
        total
            .checked_add(file.prepared.as_ref().map_or(0, |artifact| artifact.bytes))
            .context("prepared snapshot byte overflow")
    })?;
    let total_bytes = source_bytes
        .checked_add(prepared_bytes)
        .context("total snapshot byte overflow")?;
    anyhow::ensure!(
        total_bytes == budget.used && total_bytes <= hard_budget_bytes,
        "snapshot logical payload accounting mismatch"
    );
    let footprint = SnapshotFootprint {
        source_bytes,
        prepared_bytes,
        total_bytes,
        hard_budget_bytes,
    };
    snapshot_failpoint(1)?;
    let snapshot = SourceSnapshot {
        version: SNAPSHOT_VERSION,
        tx_id: tx_id.to_string(),
        preparation_contract_digest: preparation_contract_digest.to_owned(),
        footprint: footprint.clone(),
        started: chrono::Utc::now().to_rfc3339(),
        snapshot_digest: String::new(),
        files,
    };
    let mut snapshot = snapshot;
    snapshot.snapshot_digest = snapshot_digest(
        tx_id,
        &snapshot.started,
        preparation_contract_digest,
        &footprint,
        &snapshot.files,
    )?;
    write_synced_json(&staging.join("manifest.json"), &snapshot)?;
    sync_dir(&blobs)?;
    if plan.is_some() {
        sync_dir(&prepared_dir)?;
    }
    sync_dir(&staging)?;
    snapshot_failpoint(2)?;
    std::fs::rename(&staging, &final_dir)?;
    cleanup.armed = false;
    sync_dir(&root)?;
    snapshot_failpoint(3)?;
    open_snapshot(state_dir, tx_id)
}

/// #2421: once a file's prepared artifact is durable (`prepare_artifact`
/// fsyncs it before returning), its sealed blob has served its only purpose —
/// a stable input for extraction — so it is unlinked from the staging dir
/// before the manifest is written. The blob is a hardlink when reused, so this
/// never touches the prior generation's copy. Files with nothing prepared
/// (junk/skipped, or a plan-less snapshot) keep their blob.
fn release_prepared_blob(destination: &Path, prepared: Option<&PreparedArtifact>) -> Result<bool> {
    if prepared.is_none() {
        return Ok(false);
    }
    match std::fs::remove_file(destination) {
        Ok(()) => {}
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
        Err(error) => {
            return Err(error).with_context(|| {
                format!("release prepared snapshot blob {}", destination.display())
            })
        }
    }
    Ok(true)
}

pub fn open_snapshot(state_dir: &Path, tx_id: &str) -> Result<SourceSnapshot> {
    validate_tx_id(tx_id)?;
    let dir = state_dir.join("sync-snapshots").join(tx_id);
    let snapshot: SourceSnapshot =
        serde_json::from_slice(&std::fs::read(dir.join("manifest.json"))?)?;
    anyhow::ensure!(
        snapshot.version == SNAPSHOT_VERSION && snapshot.tx_id == tx_id,
        "source snapshot identity mismatch"
    );
    let source_bytes = snapshot.files.iter().try_fold(0u64, |sum, file| {
        sum.checked_add(file.content_size)
            .context("snapshot source footprint overflow")
    })?;
    let prepared_bytes = snapshot.files.iter().try_fold(0u64, |sum, file| {
        sum.checked_add(file.prepared.as_ref().map_or(0, |artifact| artifact.bytes))
            .context("snapshot prepared footprint overflow")
    })?;
    let total_bytes = source_bytes
        .checked_add(prepared_bytes)
        .context("snapshot total footprint overflow")?;
    anyhow::ensure!(
        snapshot.footprint.source_bytes == source_bytes
            && snapshot.footprint.prepared_bytes == prepared_bytes
            && snapshot.footprint.total_bytes == total_bytes
            && total_bytes <= snapshot.footprint.hard_budget_bytes,
        "source snapshot footprint does not match its sealed artifacts"
    );
    anyhow::ensure!(
        snapshot.snapshot_digest
            == snapshot_digest(
                tx_id,
                &snapshot.started,
                &snapshot.preparation_contract_digest,
                &snapshot.footprint,
                &snapshot.files,
            )?,
        "source snapshot manifest digest mismatch"
    );
    for file in &snapshot.files {
        anyhow::ensure!(
            file.relative_blob.starts_with("blobs/")
                && !file.relative_blob.contains("..")
                && !Path::new(&file.relative_blob).is_absolute(),
            "invalid source snapshot blob path"
        );
        if file.blob_released {
            // #2421: the bytes live only in the prepared artifact, which is
            // size- and digest-verified below.
            anyhow::ensure!(
                file.prepared.is_some(),
                "source snapshot released a blob that has no prepared artifact"
            );
        } else {
            crate::content::verify(
                &dir.join(&file.relative_blob),
                file.content_size,
                &file.content_digest,
            )?;
        }
        if let Some(prepared) = &file.prepared {
            validate_relative_path(&prepared.relative_ndjson, "prepared/")?;
            let path = dir.join(&prepared.relative_ndjson);
            let metadata = std::fs::metadata(&path)?;
            anyhow::ensure!(
                metadata.len() == prepared.bytes
                    && stream_digest(&path, "axp1")? == prepared.digest,
                "prepared artifact digest or size mismatch"
            );
        }
    }
    Ok(snapshot)
}

/// Per-snapshot index over the plan, built once by `create_snapshot_inner`
/// so preparing F files costs O(F + aliases + datasets) instead of
/// O(F × (aliases + datasets)). Duplicate slugs resolve last-wins, exactly
/// as the per-file `collect()`/`insert` loops it replaces did, so
/// `prepared_artifact_identity` stays byte-identical (it is a persisted
/// reuse key).
struct PreparationLookup<'a> {
    datasets: HashMap<&'a str, &'a crate::state::PlanDataset>,
    dataset_values: HashMap<&'a str, Value>,
    coercions: HashMap<&'a str, HashMap<String, crate::coerce::Coerce>>,
    /// `file_key` -> sorted `(rel, path_id)` of its duplicate aliases.
    aliases_by_key: HashMap<&'a str, Vec<(&'a str, &'a str)>>,
}

impl<'a> PreparationLookup<'a> {
    fn new(plan: &'a Plan) -> Result<Self> {
        let mut datasets = HashMap::with_capacity(plan.datasets.len());
        let mut dataset_values = HashMap::with_capacity(plan.datasets.len());
        let mut coercions = HashMap::with_capacity(plan.datasets.len());
        for dataset in &plan.datasets {
            let slug = dataset.slug.as_str();
            datasets.insert(slug, dataset);
            dataset_values.insert(slug, serde_json::to_value(dataset)?);
            coercions.insert(slug, crate::coerce::plan_from_specs(&dataset.specs));
        }
        let mut aliases_by_key: HashMap<&'a str, Vec<(&'a str, &'a str)>> = HashMap::new();
        for alias in &plan.duplicate_files {
            aliases_by_key
                .entry(alias.file_key.as_str())
                .or_default()
                .push((alias.rel.as_str(), alias.path_id.as_str()));
        }
        for aliases in aliases_by_key.values_mut() {
            aliases.sort_unstable();
        }
        Ok(Self {
            datasets,
            dataset_values,
            coercions,
            aliases_by_key,
        })
    }
}

#[allow(clippy::too_many_arguments)]
fn prepare_artifact(
    staging: &Path,
    ordinal: usize,
    generation_identity: &str,
    source: &crate::walk::FileEntry,
    content_id: &str,
    snapshot_blob: &Path,
    assignment: &crate::state::FileAssignment,
    lookup: &PreparationLookup<'_>,
    budget: &mut PayloadBudget,
    // #1062: `--label` runs each record's payload through the node's
    // `/_decide` HERE, before the record is sealed, so the labels become part
    // of the durable artifact — replays and hardlink reuse carry them without
    // a second vote. The question-set bytes are in the run's
    // `chunker_identity` (`lib.rs::prepared_records_identity`), which is what
    // makes a changed set re-prepare instead of hardlinking unlabelled (or
    // differently-labelled) NDJSON. `None` — the default, and every caller
    // without `--label` — seals unlabelled exactly as before.
    labeler: Option<&crate::label::Labeler>,
    vector_dims: Option<usize>,
    pr: &crate::progress::Progress,
) -> Result<PreparedArtifact> {
    let datasets = &lookup.datasets;
    let coercions = &lookup.coercions;
    let mut paths = vec![assignment.rel.clone()];
    if let Some(aliases) = lookup.aliases_by_key.get(content_id) {
        paths.extend(aliases.iter().map(|(rel, _)| (*rel).to_owned()));
    }
    paths.sort();
    paths.dedup();
    let assignments: HashMap<Option<String>, &str> = assignment
        .assignments
        .iter()
        .map(|(group, slug)| (group.clone(), slug.as_str()))
        .collect();
    let sniffed = crate::sniff::sniff_with_name(snapshot_blob, &source.path)
        .with_context(|| format!("sniff {} for durable preparation", source.rel))?;
    let relative_ndjson = format!("prepared/{ordinal:08}.ndjson");
    let path = staging.join(&relative_ndjson);
    let file = OpenOptions::new()
        .create_new(true)
        .write(true)
        .open(&path)?;
    let mut writer = BudgetWriter {
        inner: BufWriter::new(file),
        budget,
    };
    let mut records = 0u64;
    let mut records_by_dataset: BTreeMap<String, u64> = BTreeMap::new();
    let mut passages = 0u64;
    let mut vectors = 0u64;
    // Records set aside for size, and the first one's locator and size.
    let mut oversized = 0u64;
    let mut first_oversized: Option<(String, usize, bool)> = None;
    let mut sink_error: Option<anyhow::Error> = None;
    let mut sink = |record: crate::extract::RawRecord| -> bool {
        let Some(slug) = assignments
            .get(&record.group)
            .or_else(|| assignments.get(&None))
            .copied()
        else {
            sink_error = Some(anyhow::anyhow!(
                "record {} has no stable dataset assignment",
                record.locator
            ));
            return false;
        };
        let Some(dataset) = datasets.get(slug).copied() else {
            sink_error = Some(anyhow::anyhow!(
                "dataset {slug} is absent from prepared plan"
            ));
            return false;
        };
        let mut fields: Map<String, Value> = record.fields;
        let Some(coercion) = coercions.get(slug) else {
            sink_error = Some(anyhow::anyhow!("dataset {slug} has no coercion plan"));
            return false;
        };
        crate::coerce::coerce_record(&mut fields, coercion);
        fields.insert("ax_path".into(), Value::String(assignment.rel.clone()));
        fields.insert(
            "ax_paths".into(),
            Value::Array(paths.iter().cloned().map(Value::String).collect()),
        );
        fields.insert("ax_file".into(), Value::String(content_id.to_string()));
        fields.insert("ax_locator".into(), Value::String(record.locator.clone()));
        fields.insert("ax_dataset".into(), Value::String(slug.to_string()));
        fields.insert(
            "ax_run".into(),
            Value::String(generation_identity.to_owned()),
        );
        fields.insert(
            "ax_format".into(),
            Value::String(source.path.extension().map_or_else(
                || "unknown".into(),
                |extension| extension.to_string_lossy().to_ascii_lowercase(),
            )),
        );
        // #1062: the decide vote runs after coercion and provenance stamping
        // and before the record is written, on the same `fields` map that is
        // about to be sealed — the question templates see the record exactly
        // as the index will. A FAILED vote fails the seal (sink_error), which
        // fails the run: sealing the record unlabelled would hand the index a
        // document the operator believes was judged. An abstention is an
        // answer and stamps `label: null` with the raw p (see `label.rs`).
        if let Some(labeler) = labeler {
            match labeler.label_fields(&fields) {
                Ok(pairs) => {
                    for (name, value) in pairs {
                        fields.insert(name, value);
                    }
                }
                Err(error) => {
                    sink_error = Some(error.context(format!(
                        "--label {} (record {})",
                        source.rel, record.locator
                    )));
                    return false;
                }
            }
        }
        let semantic = dataset
            .semantic_field
            .as_ref()
            .is_some_and(|field| fields.get(field).is_some());
        // #2807: the server adds the semantic vectors before it checks the
        // per-document limit, so a record well under it can be refused, and a
        // refusal after sealing stops every rerun of this generation. Size the
        // record as the server will see it, and set it aside as a junk record
        // when it would not fit. The file's other records are still indexed.
        let vector_bytes = match (vector_dims, dataset.semantic_field.as_deref()) {
            (Some(dims), Some(field)) => {
                fields.get(field).and_then(Value::as_str).map_or(0, |text| {
                    xerj_common::document_limit::semantic_vector_bytes(
                        field,
                        &format!("{field}_vector"),
                        dims,
                        &xerj_ai::semantic_lexical::semantic_passage_offsets(text),
                    )
                })
            }
            _ => 0,
        };
        let document = Value::Object(fields).to_string();
        let indexed_bytes = document.len().saturating_add(vector_bytes);
        if xerj_common::document_limit::indexed_document_too_large(indexed_bytes) {
            oversized += 1;
            first_oversized.get_or_insert((record.locator, indexed_bytes, vector_bytes > 0));
            return true;
        }
        let id = crate::ids::doc_id(slug, content_id, &record.locator);
        let action = serde_json::json!({"index": {"_index": dataset.index, "_id": id}});
        if let Err(error) =
            writeln!(writer, "{action}").and_then(|_| writeln!(writer, "{document}"))
        {
            sink_error = Some(anyhow::Error::new(error).context("write prepared NDJSON"));
            return false;
        }
        records += 1;
        // Seal the count under the same dataset identity the record was written
        // under, so a file that fans out over several datasets stays reconcilable
        // against each one separately.
        *records_by_dataset.entry(slug.to_string()).or_default() += 1;
        if semantic {
            passages += 1;
            vectors += 1;
        }
        true
    };
    // Junk records are recorded, never fatal — that is the documented contract
    // (`cli.rs` EXIT CODES: "3 completed-with-junk (junk recorded, never
    // fatal)") and what the legacy path has always done. Aborting the whole
    // generation because one line of one log file did not parse would make a
    // realistic corpus unindexable. The count is sealed into the artifact so
    // the generation, and then the catalog, can report it.
    //
    // #722: `assignment.as_document` (a demoted one-off config file, #173)
    // decides the record *shape*, the same precedence the legacy path's own
    // phase-B dispatch gives it (`lib.rs`, `if fa.as_document { … }`) — the
    // frozen dataset's mapping was built by re-sampling through
    // `extract_as_document`, so that is the only extractor whose output may
    // be compared against it. Before this, the durable/generated pipeline
    // never checked the flag at all: a demoted file was correctly ROUTED
    // into the docs dataset (`reconcile_plan`/`dataset::cluster` agree it
    // belongs there) but its published fields still came from its raw
    // family extractor, silently disagreeing with the mapping. `sniffed` is
    // already sniffed from `snapshot_blob`, a real path on the sealed
    // snapshot (not a byte blob), so the branch is exactly the legacy
    // path's — no format change needed.
    let stats = if assignment.as_document {
        // `extract_as_document_with_name`, not `extract_as_document`:
        // `snapshot_blob` is a real path, but to the SEALED SNAPSHOT under
        // its own ordinal name, not the source file's — the title must come
        // from `source.path` (the same split `sniff_with_name` above makes
        // for the same reason).
        crate::extract::extract_as_document_with_name(
            snapshot_blob,
            &source.path,
            sniffed.gzip,
            &mut sink,
        )
    } else {
        crate::extract::extract(snapshot_blob, &sniffed, None, &mut sink)
    }
    .with_context(|| format!("extract {} into durable preparation", source.rel))?;
    if let Some(error) = sink_error {
        return Err(error);
    }
    if let Some((locator, indexed_bytes, with_vectors)) = first_oversized {
        let (sized, remedy) = if with_vectors {
            (
                "with the vectors the server adds, ",
                "; --no-semantic indexes it without vectors",
            )
        } else {
            ("", "")
        };
        pr.note(&format!(
            "warning: {} had {oversized} record(s) set aside as junk: {sized}record \
             {locator} would be {indexed_bytes} bytes, over the {} byte limit for one \
             document. The file's other records are indexed{remedy}",
            source.rel,
            xerj_common::document_limit::MAX_INDEXED_DOCUMENT_BYTES,
        ));
    }
    writer.flush()?;
    writer.inner.get_ref().sync_all()?;
    drop(writer);
    let bytes = std::fs::metadata(&path)?.len();
    Ok(PreparedArtifact {
        relative_ndjson,
        records,
        passages,
        vectors,
        junk: stats.junk + oversized,
        truncated: stats.truncated,
        records_by_dataset,
        bytes,
        digest: stream_digest(&path, "axp1")?,
    })
}

/// #971: the identity of every input `prepare_artifact` consumes for one file.
///
/// Equal identity + equal content digest (checked separately against the
/// inventory) means the sealed NDJSON this run would produce is byte-identical
/// to the prior generation's, so the prior artifact is hardlinked instead of
/// re-extracted. The enumeration deliberately mirrors what `prepare_artifact`
/// reads: the chunker-level settings and their version label, the document-id
/// scheme, the file's whole assignment (rel drives `ax_path`, the sniff name
/// and `ax_format`; `assignments` drives record routing; `as_document` switches
/// the extractor), the alias paths folded into `ax_paths`, and the complete
/// `PlanDataset` records for every dataset a record can land in (index name,
/// specs/coercions, semantic field). **If `prepare_artifact` ever grows an
/// input, it must be added here** — the chunker and document-id labels are the
/// existing contract-digest versioning points and change with any extractor
/// behaviour change, which is what makes this safe across builds.
fn prepared_artifact_identity(
    chunker_identity: &str,
    lookup: &PreparationLookup<'_>,
    content_id: &str,
    assignment: &crate::state::FileAssignment,
) -> Result<String> {
    let aliases: &[(&str, &str)] = lookup
        .aliases_by_key
        .get(content_id)
        .map_or(&[], Vec::as_slice);
    let mut datasets: BTreeMap<&str, &Value> = BTreeMap::new();
    for (_, slug) in &assignment.assignments {
        if let Some(value) = lookup.dataset_values.get(slug.as_str()) {
            datasets.insert(slug.as_str(), value);
        }
    }
    let encoded = serde_json::to_vec(&serde_json::json!({
        "chunker_identity": chunker_identity,
        "document_ids": crate::DOCUMENT_IDS_IDENTITY,
        "assignment": serde_json::to_value(assignment)?,
        "aliases": aliases,
        "datasets": datasets,
    }))?;
    Ok(format!(
        "axfi1-{:032x}",
        xxhash_rust::xxh3::xxh3_128(&encoded)
    ))
}

fn validate_relative_path(path: &str, prefix: &str) -> Result<()> {
    anyhow::ensure!(
        path.starts_with(prefix) && !path.contains("..") && !Path::new(path).is_absolute(),
        "invalid snapshot artifact path"
    );
    Ok(())
}

fn stream_digest(path: &Path, prefix: &str) -> Result<String> {
    let mut reader = BufReader::new(File::open(path)?);
    let mut hash = xxhash_rust::xxh3::Xxh3::new();
    let mut buffer = [0u8; 64 * 1024];
    loop {
        let read = reader.read(&mut buffer)?;
        if read == 0 {
            break;
        }
        hash.update(&buffer[..read]);
    }
    Ok(format!("{prefix}-{:016x}", hash.digest()))
}

fn ensure_inventory_lengths(inventory: &Inventory) -> Result<()> {
    anyhow::ensure!(
        inventory.files.len() == inventory.keys.len()
            && inventory.keys.len() == inventory.digests.len(),
        "inventory vectors have inconsistent lengths"
    );
    Ok(())
}

fn snapshot_digest(
    tx_id: &str,
    started: &str,
    preparation_contract_digest: &str,
    footprint: &SnapshotFootprint,
    files: &[SnapshotFile],
) -> Result<String> {
    let mut files = files.to_vec();
    files.sort_by(|left, right| {
        left.content_id
            .cmp(&right.content_id)
            .then_with(|| left.relative_blob.cmp(&right.relative_blob))
    });
    let encoded = serde_json::to_vec(&(
        SNAPSHOT_VERSION,
        tx_id,
        started,
        preparation_contract_digest,
        footprint,
        files,
    ))?;
    Ok(format!(
        "axs1-{:032x}",
        xxhash_rust::xxh3::xxh3_128(&encoded)
    ))
}

fn validate_tx_id(tx_id: &str) -> Result<()> {
    anyhow::ensure!(
        !tx_id.is_empty()
            && tx_id.len() <= 128
            && tx_id
                .bytes()
                .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')),
        "invalid source snapshot transaction ID"
    );
    Ok(())
}

fn copy_synced(source: &Path, destination: &Path, budget: &mut PayloadBudget) -> Result<()> {
    let mut input = File::open(source)?;
    let output = OpenOptions::new()
        .create_new(true)
        .write(true)
        .open(destination)?;
    let mut output = BudgetWriter {
        inner: output,
        budget,
    };
    std::io::copy(&mut input, &mut output)?;
    output.inner.sync_all()?;
    Ok(())
}

fn write_synced_json(path: &Path, value: &impl Serialize) -> Result<()> {
    let mut file = OpenOptions::new().create_new(true).write(true).open(path)?;
    serde_json::to_writer(&mut file, value)?;
    file.write_all(b"\n")?;
    file.sync_all()?;
    Ok(())
}

/// fsync a directory so the entries created or renamed inside it survive
/// power loss.
///
/// #482: this used to be `File::open(path)?.sync_all()?` — a Unix-only idiom.
/// On Windows `File::open` on a *directory* fails with `ERROR_ACCESS_DENIED`
/// (os error 5) on **every** call, because `std` cannot pass
/// `FILE_FLAG_BACKUP_SEMANTICS`, so sealing a source snapshot aborted the
/// whole `--no-graph` run before a single document was indexed. Exactly the
/// same mistake had already been found and fixed once in
/// [`xerj_common::fsio::fsync_dir`]; route through it instead of re-deriving
/// it here. Its Windows body is a documented no-op that makes no durability
/// claim — which costs Windows nothing, because the code it replaces flushed
/// nothing either: it returned `Err` and killed the run.
fn sync_dir(path: &Path) -> Result<()> {
    xerj_common::fsio::fsync_dir(path)
        .with_context(|| format!("fsync snapshot directory {}", path.display()))
}

#[cfg(test)]
fn arm_source_mutation_after_seal(path: &Path, replacement: &[u8]) {
    *POST_SEAL_SOURCE_REPLACEMENT.lock().unwrap() =
        Some((path.to_path_buf(), replacement.to_vec()));
}

#[cfg(test)]
fn apply_post_seal_source_replacement(path: &Path) -> Result<()> {
    let replacement = {
        let mut armed = POST_SEAL_SOURCE_REPLACEMENT.lock().unwrap();
        if armed.as_ref().is_some_and(|(expected, _)| expected == path) {
            armed.take().map(|(_, bytes)| bytes)
        } else {
            None
        }
    };
    if let Some(replacement) = replacement {
        std::fs::write(path, replacement)
            .with_context(|| format!("inject post-seal source replacement {}", path.display()))?;
    }
    Ok(())
}

/// A live source no longer matches the bytes this generation hashed.
///
/// Stat failures are included: a file deleted between the hash and the seal
/// is the same skip, not a reason to abort every other file.
fn verify_live_source(source: &crate::walk::FileEntry, digest: &str) -> Result<()> {
    match crate::content::verify(&source.path, source.size, digest) {
        Ok(()) => Ok(()),
        Err(error) => Err(source_changed(&source.path, format!("{error:#}"))),
    }
}

/// The staged blob does not match the hashed source. A size or content miss
/// is reported against the live path, which is the file the caller can drop.
fn verify_sealed_blob(source: &crate::walk::FileEntry, sealed: &Path, digest: &str) -> Result<()> {
    match crate::content::verify(sealed, source.size, digest) {
        Ok(()) => Ok(()),
        Err(error)
            if error
                .downcast_ref::<crate::content::SourceChanged>()
                .is_some() =>
        {
            Err(source_changed(&source.path, format!("{error:#}")))
        }
        Err(error) => Err(error),
    }
}

fn source_changed(path: &Path, detail: String) -> anyhow::Error {
    crate::content::SourceChanged {
        path: path.to_path_buf(),
        detail,
        locked: false,
    }
    .into()
}

/// Test-only: run once at the start of a snapshot seal, keyed by state dir
/// so a parallel test cannot grow another run's files (#2314).
#[cfg(test)]
type SnapshotSourceHook = (std::path::PathBuf, Box<dyn FnOnce() + Send>);
#[cfg(test)]
static SNAPSHOT_SOURCE_HOOK: std::sync::Mutex<Option<SnapshotSourceHook>> =
    std::sync::Mutex::new(None);

#[cfg(test)]
pub(crate) fn arm_snapshot_source_hook(state_dir: &Path, action: impl FnOnce() + Send + 'static) {
    *SNAPSHOT_SOURCE_HOOK
        .lock()
        .unwrap_or_else(|poison| poison.into_inner()) =
        Some((state_dir.to_path_buf(), Box::new(action)));
}

#[cfg(test)]
fn snapshot_source_hook(state_dir: &Path) {
    let action = {
        let mut armed = SNAPSHOT_SOURCE_HOOK
            .lock()
            .unwrap_or_else(|poison| poison.into_inner());
        match armed.as_ref() {
            Some((dir, _)) if dir == state_dir => armed.take().map(|(_, action)| action),
            _ => None,
        }
    };
    if let Some(action) = action {
        action();
    }
}

#[cfg(not(test))]
fn snapshot_source_hook(_state_dir: &Path) {}

#[cfg(test)]
fn fail_next_snapshot(boundary: u8) {
    SNAPSHOT_FAILPOINT.with(|failpoint| failpoint.set(boundary));
}

#[cfg(test)]
fn snapshot_failpoint(boundary: u8) -> Result<()> {
    let armed = SNAPSHOT_FAILPOINT.with(|failpoint| {
        if failpoint.get() == boundary {
            failpoint.set(0);
            true
        } else {
            false
        }
    });
    if armed {
        anyhow::bail!("injected snapshot failure at boundary {boundary}");
    }
    Ok(())
}

#[cfg(not(test))]
fn snapshot_failpoint(_boundary: u8) -> Result<()> {
    Ok(())
}

#[cfg(test)]
fn fail_replay_after_next_apply() {
    REPLAY_FAIL_AFTER_APPLY.with(|failpoint| failpoint.set(true));
}

#[cfg(test)]
fn replay_fail_after_apply() -> Result<()> {
    if REPLAY_FAIL_AFTER_APPLY.with(|failpoint| failpoint.replace(false)) {
        anyhow::bail!("injected crash after accepted operation");
    }
    Ok(())
}

#[cfg(not(test))]
fn replay_fail_after_apply() -> Result<()> {
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::state::{FileAssignment, PlanDataset};
    use crate::sync::{
        ExecutionIdentity, GenerationManifest, SourceExecutionPolicy, SyncOperationKind,
        EXECUTION_IDENTITY_VERSION,
    };
    use crate::walk;
    use std::collections::BTreeMap;
    #[cfg(unix)]
    use std::os::unix::fs::MetadataExt;

    fn inventory(root: &Path) -> Inventory {
        crate::content::resolve_reporting(walk::walk(root, false).unwrap(), &|_| {}).unwrap()
    }

    fn plan_for(inventory: &Inventory) -> Plan {
        let key = inventory.keys[0].clone();
        let file = &inventory.files[0];
        Plan {
            datasets: vec![PlanDataset {
                slug: "docs".into(),
                index: "ax-docs".into(),
                family: "text".into(),
                group: None,
                specs: vec![],
                time_field: None,
                semantic_field: None,
                text_analyzer: None,
                scope: None,
                sampled_records: 1,
                file_count: 1,
            }],
            files: HashMap::from([(
                key,
                FileAssignment {
                    rel: file.rel.clone(),
                    path_id: file.rel_id.clone(),
                    is_symlink: Some(file.is_symlink),
                    family: "text".into(),
                    gzip: false,
                    content_digest: Some(inventory.digests[0].clone()),
                    assignments: vec![(None, "docs".into())],
                    as_document: false,
                },
            )]),
            alias_paths_indexed: false,
            ..Plan::default()
        }
    }

    /// `prepared_artifact_identity` is a persisted reuse key: the
    /// once-per-snapshot `PreparationLookup` must hash exactly what the old
    /// per-file plan scan hashed, including unordered aliases, aliases of other
    /// files, a duplicate slug (last wins) and an assigned slug with no dataset.
    #[test]
    fn preparation_lookup_identity_matches_the_per_file_plan_scan() {
        fn legacy_identity(
            chunker_identity: &str,
            plan: &Plan,
            content_id: &str,
            assignment: &FileAssignment,
        ) -> String {
            let mut aliases: Vec<(&str, &str)> = plan
                .duplicate_files
                .iter()
                .filter(|alias| alias.file_key == content_id)
                .map(|alias| (alias.rel.as_str(), alias.path_id.as_str()))
                .collect();
            aliases.sort_unstable();
            let slugs: BTreeSet<&str> = assignment
                .assignments
                .iter()
                .map(|(_, slug)| slug.as_str())
                .collect();
            let mut datasets: BTreeMap<&str, Value> = BTreeMap::new();
            for dataset in plan
                .datasets
                .iter()
                .filter(|dataset| slugs.contains(&dataset.slug.as_str()))
            {
                datasets.insert(
                    dataset.slug.as_str(),
                    serde_json::to_value(dataset).unwrap(),
                );
            }
            let encoded = serde_json::to_vec(&serde_json::json!({
                "chunker_identity": chunker_identity,
                "document_ids": crate::DOCUMENT_IDS_IDENTITY,
                "assignment": serde_json::to_value(assignment).unwrap(),
                "aliases": aliases,
                "datasets": datasets,
            }))
            .unwrap();
            format!("axfi1-{:032x}", xxhash_rust::xxh3::xxh3_128(&encoded))
        }
        let dataset = |slug: &str, index: &str| PlanDataset {
            slug: slug.into(),
            index: index.into(),
            family: "text".into(),
            group: None,
            specs: vec![],
            time_field: None,
            semantic_field: None,
            text_analyzer: None,
            scope: None,
            sampled_records: 1,
            file_count: 1,
        };
        let alias = |file_key: &str, rel: &str| crate::state::DuplicateFile {
            file_key: file_key.into(),
            rel: rel.into(),
            path_id: format!("id-{rel}"),
            is_symlink: Some(false),
            duplicate_of: "canon".into(),
            bytes: 1,
        };
        let assignment = FileAssignment {
            rel: "canon".into(),
            path_id: "id-canon".into(),
            is_symlink: Some(false),
            family: "text".into(),
            gzip: false,
            content_digest: None,
            assignments: vec![
                (None, "docs".into()),
                (Some("g".into()), "logs".into()),
                (Some("h".into()), "missing".into()),
            ],
            as_document: false,
        };
        let plan = Plan {
            datasets: vec![
                dataset("docs", "ax-docs"),
                dataset("logs", "ax-logs-old"),
                dataset("other", "ax-other"),
                dataset("logs", "ax-logs"),
            ],
            files: HashMap::from([("k1".to_string(), assignment.clone())]),
            duplicate_files: vec![
                alias("k1", "z/copy"),
                alias("k2", "elsewhere"),
                alias("k1", "a/copy"),
                alias("k1", "m/copy"),
            ],
            ..Plan::default()
        };
        let lookup = PreparationLookup::new(&plan).unwrap();
        for content_id in ["k1", "k2", "k3"] {
            assert_eq!(
                prepared_artifact_identity("chunker-v1", &lookup, content_id, &assignment).unwrap(),
                legacy_identity("chunker-v1", &plan, content_id, &assignment),
                "identity drifted for {content_id}"
            );
        }
    }

    /// #971: like `plan_for` but with an assignment for EVERY file in the
    /// inventory — the shape a real corpus plan has.
    fn plan_for_all(inventory: &Inventory) -> Plan {
        let files: HashMap<String, FileAssignment> = inventory
            .files
            .iter()
            .enumerate()
            .map(|(index, file)| {
                (
                    inventory.keys[index].clone(),
                    FileAssignment {
                        rel: file.rel.clone(),
                        path_id: file.rel_id.clone(),
                        is_symlink: Some(file.is_symlink),
                        family: "text".into(),
                        gzip: false,
                        content_digest: Some(inventory.digests[index].clone()),
                        assignments: vec![(None, "docs".into())],
                        as_document: false,
                    },
                )
            })
            .collect();
        Plan {
            datasets: vec![PlanDataset {
                slug: "docs".into(),
                index: "ax-docs".into(),
                family: "text".into(),
                group: None,
                specs: vec![],
                time_field: None,
                semantic_field: None,
                text_analyzer: None,
                scope: None,
                sampled_records: 1,
                file_count: files.len(),
            }],
            files,
            alias_paths_indexed: false,
            ..Plan::default()
        }
    }

    /// #971 fail-before shape: a one-file change in a three-file corpus must
    /// seal new bytes ONLY for the changed file. Unchanged files' blobs and
    /// prepared artifacts are hardlinked from the prior committed snapshot —
    /// same inode, no copy, no re-extraction, no per-file fsync.
    #[cfg(unix)]
    #[test]
    fn one_changed_file_hardlinks_unchanged_blobs_and_prepared_artifacts() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        for (name, body) in [
            ("a.md", "alpha contents one\n"),
            ("b.md", "bravo contents two\n"),
            ("c.md", "charlie contents three\n"),
        ] {
            std::fs::write(corpus.path().join(name), body).unwrap();
        }
        let first_inventory = inventory(corpus.path());
        let first_plan = plan_for_all(&first_inventory);
        let first = create_prepared_snapshot_reporting(
            state.path(),
            "tx-g1",
            &first_inventory,
            &first_plan,
            "test-preparation-v1",
            u64::MAX,
            &crate::progress::Progress::silent(),
            None,
            "chunker-v1",
            None,
            None,
        )
        .unwrap();
        assert_eq!(first.files.len(), 3);
        assert!(first
            .files
            .iter()
            .all(|file| file.prepared.is_some() && file.prepared_identity.is_some()));

        // One file changes; the tree is re-scanned; generation two seals.
        std::fs::write(corpus.path().join("b.md"), "bravo contents CHANGED\n").unwrap();
        let second_inventory = inventory(corpus.path());
        let second_plan = plan_for_all(&second_inventory);
        let changed_key = second_inventory
            .keys
            .iter()
            .zip(&second_inventory.digests)
            .find(|(key, digest)| {
                first
                    .files
                    .iter()
                    .find(|file| file.content_id == **key)
                    .is_none_or(|prior| &prior.content_digest != *digest)
            })
            .map(|(key, _)| key.clone())
            .unwrap();
        let second = create_prepared_snapshot_reporting(
            state.path(),
            "tx-g2",
            &second_inventory,
            &second_plan,
            "test-preparation-v1",
            u64::MAX,
            &crate::progress::Progress::silent(),
            Some(&first),
            "chunker-v1",
            None,
            None,
        )
        .unwrap();

        // Content ids are content-derived, so the changed file's id is new in
        // generation two — gen1 holds it under the OLD id. Classify by content
        // identity instead: a gen2 entry whose (content_id, digest) has no
        // gen1 match is the changed file.
        let prior_by_content: HashMap<&str, &SnapshotFile> = first
            .files
            .iter()
            .map(|file| (file.content_id.as_str(), file))
            .collect();
        let fresh: Vec<&SnapshotFile> = second
            .files
            .iter()
            .filter(|file| {
                prior_by_content
                    .get(file.content_id.as_str())
                    .is_none_or(|prior| prior.content_digest != file.content_digest)
            })
            .collect();
        assert_eq!(
            fresh.len(),
            1,
            "exactly the changed file seals fresh bytes: {:?}",
            fresh
                .iter()
                .map(|file| file.content_id.clone())
                .collect::<Vec<_>>()
        );
        assert_eq!(fresh[0].content_id, changed_key);

        let artifact_inode = |snapshot: &SourceSnapshot, relative: &str| {
            std::fs::metadata(
                state
                    .path()
                    .join("sync-snapshots")
                    .join(&snapshot.tx_id)
                    .join(relative),
            )
            .unwrap()
            .ino()
        };
        // #2421: every prepared file's blob is released at seal, in both
        // generations — reuse carries the prepared artifact, not the bytes.
        for (snapshot, file) in first
            .files
            .iter()
            .map(|file| (&first, file))
            .chain(second.files.iter().map(|file| (&second, file)))
        {
            assert!(file.blob_released, "{}", file.content_id);
            assert!(!state
                .path()
                .join("sync-snapshots")
                .join(&snapshot.tx_id)
                .join(&file.relative_blob)
                .exists());
        }

        for file in &second.files {
            let new_prepared = file.prepared.as_ref().unwrap();
            match prior_by_content
                .get(file.content_id.as_str())
                .filter(|prior| prior.content_digest == file.content_digest)
            {
                None => {
                    // The changed file: everything sealed fresh, under the new
                    // generation's identity.
                    let ndjson = std::fs::read_to_string(
                        state
                            .path()
                            .join("sync-snapshots/tx-g2")
                            .join(&new_prepared.relative_ndjson),
                    )
                    .unwrap();
                    assert!(
                        ndjson.contains("\"ax_run\":\"tx-g2\""),
                        "the changed file's artifact must carry the new generation: {ndjson}"
                    );
                }
                Some(prior) => {
                    let prior_prepared = prior.prepared.as_ref().unwrap();
                    assert_eq!(
                        artifact_inode(&first, &prior_prepared.relative_ndjson),
                        artifact_inode(&second, &new_prepared.relative_ndjson),
                        "an unchanged file's prepared artifact must be the prior generation's \
                         inode"
                    );
                    let ndjson = std::fs::read_to_string(
                        state
                            .path()
                            .join("sync-snapshots/tx-g2")
                            .join(&new_prepared.relative_ndjson),
                    )
                    .unwrap();
                    assert!(
                        !ndjson.contains("tx-g2"),
                        "a reused artifact keeps the generation that extracted it: {ndjson}"
                    );
                    assert_eq!(new_prepared.digest, prior_prepared.digest);
                }
            }
        }
        // The sealed snapshot still opens and re-verifies every artifact,
        // hardlinked or not.
        let reopened = open_snapshot(state.path(), "tx-g2").unwrap();
        assert_eq!(reopened, second);
    }

    /// #2421: a committed generation's snapshot used to hold a full copy of
    /// every source byte (`blobs/`) next to the prepared NDJSON, so the state
    /// dir was larger than the corpus it indexed. With prepared blobs released
    /// at seal, nothing under `blobs/` survives, and what the protected
    /// snapshots hold beyond the prepared records a resume replays from —
    /// counted per inode, so hardlinked reuse is not double-counted — stays
    /// under 1x the source after a commit and after the next generation's gc.
    /// (The prepared records themselves are the replay authority; their size
    /// depends on the corpus: overlapping text chunks run above the source,
    /// compacted JSON below it.)
    #[cfg(unix)]
    #[test]
    fn committed_snapshot_keeps_no_source_copy_beside_its_prepared_records() {
        use std::os::unix::fs::MetadataExt;
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        for file in 0..8 {
            let body: String = (0..400)
                .map(|line| format!("file {file} line {line}: the quick brown fox jumps over it\n"))
                .collect();
            std::fs::write(corpus.path().join(format!("f{file}.md")), body).unwrap();
        }
        // (bytes beyond the prepared records, prepared bytes)
        let footprint = |state: &Path| -> (u64, u64) {
            let mut seen = std::collections::HashSet::new();
            let mut other = 0;
            let mut prepared = 0;
            for entry in walkdir::WalkDir::new(state.join("sync-snapshots")) {
                let entry = entry.unwrap();
                let metadata = entry.metadata().unwrap();
                if metadata.is_file() && seen.insert(metadata.ino()) {
                    let parts: Vec<_> = entry.path().components().collect();
                    let under = |dir: &str| parts.iter().any(|part| part.as_os_str() == dir);
                    assert!(
                        !under("blobs"),
                        "a released snapshot keeps no source copy: {}",
                        entry.path().display()
                    );
                    if under("prepared") {
                        prepared += metadata.len();
                    } else {
                        other += metadata.len();
                    }
                }
            }
            (other, prepared)
        };
        let seal = |tx: &str, prior: Option<&SourceSnapshot>| {
            let inventory = inventory(corpus.path());
            let plan = plan_for_all(&inventory);
            let source: u64 = inventory.files.iter().map(|file| file.size).sum();
            let snapshot = create_prepared_snapshot_reporting(
                state.path(),
                tx,
                &inventory,
                &plan,
                "test-preparation-v1",
                u64::MAX,
                &crate::progress::Progress::silent(),
                prior,
                "chunker-v1",
                None,
                None,
            )
            .unwrap();
            (snapshot, source)
        };
        let (first, source) = seal("tx-g1", None);
        let (used, prepared) = footprint(state.path());
        assert!(prepared > 0);
        assert!(
            used < source,
            "snapshot bytes beyond the prepared records ({used}) must stay below the \
             {source}-byte source"
        );
        assert!(first.files.iter().all(|file| file.blob_released));

        std::fs::write(corpus.path().join("f3.md"), "changed\n").unwrap();
        let (second, source) = seal("tx-g2", Some(&first));
        // What gc does once g2 is the committed authority.
        std::fs::remove_dir_all(state.path().join("sync-snapshots/tx-g1")).unwrap();
        let (used, _) = footprint(state.path());
        assert!(
            used < source,
            "after gc the bytes beyond the prepared records ({used}) must stay below the \
             {source}-byte source"
        );
        assert_eq!(open_snapshot(state.path(), "tx-g2").unwrap(), second);
    }

    /// #2421: a released blob is never silently trusted. The prepared artifact
    /// that replaces it is still size- and digest-verified on every open.
    #[test]
    fn released_blob_snapshot_still_verifies_its_prepared_artifact() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.md"), "alpha contents one\n").unwrap();
        let inventory = inventory(corpus.path());
        let plan = plan_for_all(&inventory);
        let snapshot = create_prepared_snapshot(
            state.path(),
            "tx-released",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
        )
        .unwrap();
        assert!(snapshot.files[0].blob_released);
        let artifact = state
            .path()
            .join("sync-snapshots/tx-released")
            .join(&snapshot.files[0].prepared.as_ref().unwrap().relative_ndjson);
        let original = std::fs::read(&artifact).unwrap();
        let mut corrupt = original.clone();
        corrupt[0] ^= 0x20;
        std::fs::write(&artifact, &corrupt).unwrap();
        assert!(open_snapshot(state.path(), "tx-released").is_err());
        std::fs::write(&artifact, &original).unwrap();
        assert_eq!(
            open_snapshot(state.path(), "tx-released").unwrap(),
            snapshot
        );
    }

    /// #971: gc removes the prior snapshot's directory entries after a new
    /// generation commits (`gc_snapshots`); the new generation's hardlinked
    /// inodes must survive that, and the sealed snapshot must still verify.
    /// This is the property that made a hardlink the right sharing primitive —
    /// a symlink would dangle here.
    #[test]
    fn reused_blobs_survive_removal_of_the_prior_snapshot_directory() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        for (name, body) in [
            ("a.md", "alpha contents one\n"),
            ("b.md", "bravo contents two\n"),
        ] {
            std::fs::write(corpus.path().join(name), body).unwrap();
        }
        let first_inventory = inventory(corpus.path());
        let first_plan = plan_for_all(&first_inventory);
        let first = create_prepared_snapshot_reporting(
            state.path(),
            "tx-g1",
            &first_inventory,
            &first_plan,
            "test-preparation-v1",
            u64::MAX,
            &crate::progress::Progress::silent(),
            None,
            "chunker-v1",
            None,
            None,
        )
        .unwrap();
        std::fs::write(corpus.path().join("b.md"), "bravo contents CHANGED\n").unwrap();
        let second_inventory = inventory(corpus.path());
        let second_plan = plan_for_all(&second_inventory);
        let second = create_prepared_snapshot_reporting(
            state.path(),
            "tx-g2",
            &second_inventory,
            &second_plan,
            "test-preparation-v1",
            u64::MAX,
            &crate::progress::Progress::silent(),
            Some(&first),
            "chunker-v1",
            None,
            None,
        )
        .unwrap();

        // What gc does to the prior generation once g2 is authority.
        std::fs::remove_dir_all(state.path().join("sync-snapshots/tx-g1")).unwrap();
        // Every file re-verifies: blobs by content digest, prepared artifacts
        // by size and digest — none of that consults the removed directory.
        let reopened = open_snapshot(state.path(), "tx-g2").unwrap();
        assert_eq!(reopened, second);
        assert_eq!(reopened.files.len(), 2);
    }

    /// #971: a preparation-input change (chunker identity) busts prepared
    /// reuse even when the bytes are identical — the artifact is re-extracted
    /// under the new contract. #2421: the prior blob was released at seal, so
    /// the bytes are re-sealed from the verified live source and released again.
    #[cfg(unix)]
    #[test]
    fn changed_preparation_identity_reextracts_from_the_live_source() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.md"), "alpha contents one\n").unwrap();
        let inventory = inventory(corpus.path());
        let plan = plan_for_all(&inventory);
        let first = create_prepared_snapshot_reporting(
            state.path(),
            "tx-g1",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
            &crate::progress::Progress::silent(),
            None,
            "chunker-v1",
            None,
            None,
        )
        .unwrap();
        let second = create_prepared_snapshot_reporting(
            state.path(),
            "tx-g2",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
            &crate::progress::Progress::silent(),
            Some(&first),
            "chunker-v2",
            None,
            None,
        )
        .unwrap();
        let key = &inventory.keys[0];
        let blob = |snapshot: &SourceSnapshot| {
            state
                .path()
                .join("sync-snapshots")
                .join(&snapshot.tx_id)
                .join(
                    &snapshot
                        .files
                        .iter()
                        .find(|file| file.content_id == *key)
                        .unwrap()
                        .relative_blob,
                )
        };
        let inode = |path: std::path::PathBuf| std::fs::metadata(path).unwrap().ino();
        assert!(!blob(&first).exists() && !blob(&second).exists());
        assert!(first.files[0].blob_released && second.files[0].blob_released);
        // The artifacts live in different snapshot directories under the same
        // ordinal name, so compare inodes, not relative paths.
        let prepared = |snapshot: &SourceSnapshot| {
            state
                .path()
                .join("sync-snapshots")
                .join(&snapshot.tx_id)
                .join(
                    &snapshot
                        .files
                        .iter()
                        .find(|file| file.content_id == *key)
                        .unwrap()
                        .prepared
                        .as_ref()
                        .unwrap()
                        .relative_ndjson,
                )
        };
        assert_ne!(
            inode(prepared(&first)),
            inode(prepared(&second)),
            "a changed preparation identity must re-extract, not hardlink"
        );
        let ndjson = std::fs::read_to_string(
            state
                .path()
                .join("sync-snapshots/tx-g2")
                .join(&second.files[0].prepared.as_ref().unwrap().relative_ndjson),
        )
        .unwrap();
        assert!(
            ndjson.contains("\"ax_run\":\"tx-g2\""),
            "a re-extracted artifact carries the new generation: {ndjson}"
        );
    }

    /// One file, two tables, two datasets — the ordinary shape of a SQL dump.
    fn sql_plan_for(inventory: &Inventory) -> Plan {
        let key = inventory.keys[0].clone();
        let file = &inventory.files[0];
        let dataset = |slug: &str| PlanDataset {
            slug: slug.into(),
            index: format!("ax-{slug}"),
            family: "sqldump".into(),
            group: Some(slug.into()),
            specs: vec![],
            time_field: None,
            semantic_field: None,
            text_analyzer: None,
            scope: None,
            sampled_records: 2,
            file_count: 1,
        };
        Plan {
            datasets: vec![dataset("orders"), dataset("users")],
            files: HashMap::from([(
                key,
                FileAssignment {
                    rel: file.rel.clone(),
                    path_id: file.rel_id.clone(),
                    is_symlink: Some(file.is_symlink),
                    family: "sqldump".into(),
                    gzip: false,
                    content_digest: Some(inventory.digests[0].clone()),
                    assignments: vec![
                        (Some("orders".into()), "orders".into()),
                        (Some("users".into()), "users".into()),
                    ],
                    as_document: false,
                },
            )]),
            alias_paths_indexed: false,
            ..Plan::default()
        }
    }

    fn two_table_dump() -> &'static str {
        "CREATE TABLE `users` (`id` int, `name` varchar(64));\n\
         INSERT INTO `users` VALUES (1,'ann'),(2,'bob');\n\
         CREATE TABLE `orders` (`id` int, `total` int);\n\
         INSERT INTO `orders` VALUES (10,100),(11,200);\n"
    }

    /// #360: the seal has to say which dataset each record went to. A flat
    /// per-file total is comparable to no single dataset's read-back once the
    /// file fans out, and the executor reconciled it against every one of them.
    #[test]
    fn preparation_seals_records_under_each_dataset_a_file_feeds() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("dump.sql"), two_table_dump()).unwrap();
        let inventory = inventory(corpus.path());
        let plan = sql_plan_for(&inventory);
        let snapshot = create_prepared_snapshot(
            state.path(),
            "tx-fan-out",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
        )
        .unwrap();

        let prepared = snapshot.files[0].prepared.as_ref().unwrap();
        assert_eq!(prepared.records, 4);
        assert_eq!(
            prepared.records_by_dataset,
            BTreeMap::from([("orders".to_string(), 2), ("users".to_string(), 2)])
        );

        let mut groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
        bind_prepared_counts(&mut groups, &snapshot, &inventory.keys).unwrap();
        assert_eq!(groups[0].expected_records, 4);
        assert_eq!(groups[0].expected_records_for("users"), Some(2));
        assert_eq!(groups[0].expected_records_for("orders"), Some(2));
        assert_eq!(groups[0].expected_records_for("absent"), Some(0));
    }

    /// #722: `assignment.as_document` was never read here at all — a demoted
    /// one-off config file (#173) was correctly ROUTED into the docs dataset
    /// (`reconcile_plan`/`dataset::cluster` agree) but published through its
    /// raw family extractor, not `extract_as_document`, silently disagreeing
    /// with the docs mapping the run itself built. A first, file-path-only
    /// fix used `extract_as_document(snapshot_blob, …)` directly and derived
    /// the title from `snapshot_blob`'s own name — the SEALED SNAPSHOT's
    /// ordinal filename (`prepared/00000000`), not the source file's, so
    /// every demoted document titled itself `"00000000"`. Pins both halves:
    /// the record is document-shaped (`title`/`body`, not the source's raw
    /// JSON keys) AND the title is the real source name.
    #[test]
    fn preparation_of_a_demoted_file_publishes_a_real_document_with_the_source_title() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(
            corpus.path().join("config.json"),
            br#"{"host": "example.com", "port": 8080}"#,
        )
        .unwrap();
        let inventory = inventory(corpus.path());
        let mut plan = plan_for(&inventory);
        let key = inventory.keys[0].clone();
        plan.files.get_mut(&key).unwrap().as_document = true;

        let snapshot = create_prepared_snapshot(
            state.path(),
            "tx-demoted-config",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
        )
        .unwrap();

        let prepared = snapshot.files[0].prepared.as_ref().unwrap();
        assert_eq!(prepared.records, 1);
        let root = state.path().join("sync-snapshots/tx-demoted-config");
        let ndjson = std::fs::read_to_string(root.join(&prepared.relative_ndjson)).unwrap();
        let document: Value = serde_json::from_str(ndjson.lines().nth(1).unwrap()).unwrap();
        assert_eq!(
            document["title"], "config",
            "title must be the source file's own stem, not the sealed snapshot's ordinal \
             filename: {document}"
        );
        assert!(
            document["body"]
                .as_str()
                .unwrap()
                .contains(r#""host": "example.com""#),
            "body must be the decoded document text, not raw extracted JSON fields: {document}"
        );
        assert!(
            document.get("host").is_none() && document.get("port").is_none(),
            "a demoted file must not publish its raw family-extractor fields: {document}"
        );
    }

    /// `snapshot_digest` covers the serialized `PreparedArtifact` and
    /// `open_snapshot` re-verifies it on every resume, so a snapshot sealed
    /// before the per-dataset ledger existed has to keep hashing to the value
    /// it was sealed with. Otherwise this fix turns a bug affecting SQL corpora
    /// into "protected snapshot digest disagrees" for everyone mid-generation.
    #[test]
    fn a_snapshot_sealed_without_the_per_dataset_ledger_still_resumes() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("dump.sql"), two_table_dump()).unwrap();
        let inventory = inventory(corpus.path());
        let plan = sql_plan_for(&inventory);
        let snapshot = create_prepared_snapshot(
            state.path(),
            "tx-legacy-ledger",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
        )
        .unwrap();
        assert!(!snapshot.files[0]
            .prepared
            .as_ref()
            .unwrap()
            .records_by_dataset
            .is_empty());

        // Rewrite the manifest exactly as a pre-upgrade binary wrote it: no
        // `records_by_dataset` key anywhere, and a digest computed over that
        // form. `skip_serializing_if` is what makes the two byte-identical.
        let manifest = state
            .path()
            .join("sync-snapshots/tx-legacy-ledger/manifest.json");
        let raw = std::fs::read_to_string(&manifest).unwrap();
        assert!(raw.contains("records_by_dataset"));
        let mut legacy: SourceSnapshot =
            serde_json::from_str(&raw.replace("\"records_by_dataset\"", "\"ignored_by_serde\""))
                .unwrap();
        assert!(legacy.files[0]
            .prepared
            .as_ref()
            .unwrap()
            .records_by_dataset
            .is_empty());
        assert!(!serde_json::to_string(&legacy)
            .unwrap()
            .contains("records_by_dataset"));
        legacy.snapshot_digest = snapshot_digest(
            &legacy.tx_id,
            &legacy.started,
            &legacy.preparation_contract_digest,
            &legacy.footprint,
            &legacy.files,
        )
        .unwrap();
        std::fs::write(&manifest, serde_json::to_vec(&legacy).unwrap()).unwrap();

        let resumed = open_snapshot(state.path(), "tx-legacy-ledger").unwrap();
        assert_eq!(resumed, legacy);
        assert_eq!(resumed.files[0].prepared.as_ref().unwrap().records, 4);
    }

    fn execution(tx_id: &str, digest: &str) -> ExecutionIdentity {
        ExecutionIdentity {
            version: EXECUTION_IDENTITY_VERSION,
            root_identity: "root".into(),
            url: "http://engine".into(),
            prefix: "ax".into(),
            follow_symlinks: false,
            chunker_identity: "chunker-v1".into(),
            embedding_identity_sha256: "a".repeat(64),
            embedding_backend: "lexical".into(),
            embedding_dimension: Some(384),
            embedding_semantic_contract: "semantic_text-derived-vector.v1".into(),
            embedding_resumable: true,
            graph_enabled: false,
            brain: "disabled".into(),
            detector_identity: "disabled".into(),
            schema_identity: "schema-v1".into(),
            index_identity: "index-v1".into(),
            source_policy: SourceExecutionPolicy::DurableSnapshot {
                reference: format!("sync-snapshots/{tx_id}"),
                snapshot_digest: digest.into(),
            },
        }
    }

    fn desired(
        generation: u64,
        tx_id: &str,
        snapshot: &SourceSnapshot,
        plan: Plan,
        groups: Vec<ManifestGroup>,
    ) -> GenerationManifest {
        GenerationManifest {
            generation,
            execution: Some(execution(tx_id, &snapshot.snapshot_digest)),
            plan,
            groups,
        }
    }

    #[derive(Debug, Clone, PartialEq, Eq)]
    struct LiveGroup {
        content_id: String,
        canonical: String,
        aliases: Vec<String>,
    }

    #[derive(Default)]
    struct FakeBackend {
        live: BTreeMap<String, LiveGroup>,
        applications: Vec<String>,
        provisions: usize,
        validations: usize,
    }

    impl SyncOperationBackend for FakeBackend {
        fn provision_generation(&mut self, _desired: &GenerationManifest) -> Result<()> {
            self.provisions += 1;
            Ok(())
        }

        fn apply(
            &mut self,
            operation: &SyncOperation,
            _base: &CommittedManifest,
            desired: &GenerationManifest,
            _snapshot: &SourceSnapshot,
        ) -> Result<()> {
            self.applications.push(operation.operation_id.clone());
            match operation.kind {
                SyncOperationKind::Delete => {
                    self.live.remove(&operation.group_id);
                }
                SyncOperationKind::Upsert | SyncOperationKind::Metadata => {
                    let group = desired
                        .groups
                        .iter()
                        .find(|group| group.group_id == operation.group_id)
                        .context("desired operation group is absent")?;
                    self.live.insert(
                        group.group_id.clone(),
                        LiveGroup {
                            content_id: group.content_id.clone(),
                            canonical: group.canonical.path_id.clone(),
                            aliases: group
                                .aliases
                                .iter()
                                .map(|alias| alias.path_id.clone())
                                .collect(),
                        },
                    );
                }
            }
            Ok(())
        }

        fn validate(
            &mut self,
            _base: &CommittedManifest,
            desired: &GenerationManifest,
            _snapshot: &SourceSnapshot,
        ) -> Result<()> {
            self.validations += 1;
            let expected: BTreeMap<String, LiveGroup> = desired
                .groups
                .iter()
                .map(|group| {
                    (
                        group.group_id.clone(),
                        LiveGroup {
                            content_id: group.content_id.clone(),
                            canonical: group.canonical.path_id.clone(),
                            aliases: group
                                .aliases
                                .iter()
                                .map(|alias| alias.path_id.clone())
                                .collect(),
                        },
                    )
                })
                .collect();
            anyhow::ensure!(self.live == expected, "fake live generation mismatch");
            Ok(())
        }
    }

    fn begin(
        journal: &mut Journal,
        tx_id: &str,
        snapshot: &SourceSnapshot,
        plan: Plan,
        groups: Vec<ManifestGroup>,
    ) {
        let base = journal.committed_manifest.as_ref().unwrap();
        let pending = PendingSync::new(
            tx_id.into(),
            base,
            desired(base.generation + 1, tx_id, snapshot, plan, groups),
        )
        .unwrap();
        journal.sync_begin(&pending).unwrap();
    }

    /// The `windows:` native identity `stable_path_id` writes for `rel` on a
    /// Windows walk: each UTF-16 unit of the NATIVE path (separator `\`) as
    /// four hex digits. `native_path_id_supported` accepts it on every
    /// platform, so a Windows walk can be reproduced from a macOS/Linux one.
    fn windows_path_id(rel: &str) -> String {
        let mut out = String::from("windows:");
        for unit in rel.replace('/', "\\").encode_utf16() {
            use std::fmt::Write;
            write!(out, "{unit:04x}").unwrap();
        }
        out
    }

    /// #1723: `corpus index` on Windows aborted every generation — even with
    /// `--fresh` — with "plan canonical projection disagrees with manifest
    /// group" once the corpus held byte-identical files at `Pill/x.scss` and
    /// `PillS/x.scss` (any pair whose paths first differ at a separator on
    /// one side and a digit, `A-Z`, or `:;<=>?@[` on the other).
    ///
    /// The inventory picked the canonical copy by `rel`, where the separator
    /// is `/` (0x2F) and sorts first; the manifest ranks by `path_id`, the
    /// native identity, where the separator is `\` (0x5C) and sorts after
    /// those characters. The plan projected one path, the manifest group
    /// another, and `PendingSync::new` refused the generation.
    #[test]
    fn windows_duplicate_canonical_agrees_between_plan_and_manifest() {
        let corpus = tempfile::tempdir().unwrap();
        for dir in ["Pill", "PillS", "Pill1"] {
            std::fs::create_dir_all(corpus.path().join(dir)).unwrap();
            std::fs::write(corpus.path().join(dir).join("x.scss"), ".pill{}\n").unwrap();
        }
        std::fs::write(corpus.path().join("other.txt"), "unrelated").unwrap();
        let mut files = walk::walk(corpus.path(), false).unwrap();
        for file in &mut files {
            file.rel_id = windows_path_id(&file.rel);
        }
        let inventory = crate::content::resolve_reporting(files, &|_| {}).unwrap();
        assert_eq!(inventory.files.len(), 2, "three copies, one content file");
        let canonical = inventory
            .files
            .iter()
            .find(|file| file.rel.ends_with("x.scss"))
            .unwrap();
        assert_eq!(
            canonical.rel, "Pill1/x.scss",
            "the canonical copy is the lowest native identity: `1` (0x31) and \
             `S` (0x53) both sort below `\\` (0x5C)"
        );

        let mut plan = plan_for_all(&inventory);
        plan.duplicate_files = inventory.duplicates.clone();
        let groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
        let base = CommittedManifest::genesis().unwrap();
        PendingSync::new(
            "tx-1723".into(),
            &base,
            GenerationManifest {
                generation: 1,
                execution: Some(execution("tx-1723", "digest")),
                plan,
                groups,
            },
        )
        .expect("#1723: plan and manifest must agree on the canonical duplicate");
    }

    #[test]
    fn inventory_bridge_retains_counts_only_for_identical_content() {
        let corpus = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
        let inventory = inventory(corpus.path());
        let plan = plan_for(&inventory);
        let first = groups_from_inventory(&inventory, &plan, &[]).unwrap();
        assert_eq!(first[0].expected_records, 0);
        let mut committed = first;
        committed[0].expected_records = 7;
        committed[0].expected_passages = 3;
        committed[0].expected_vectors = 3;
        let resumed = groups_from_inventory(&inventory, &plan, &committed).unwrap();
        assert_eq!(
            (
                resumed[0].expected_records,
                resumed[0].expected_passages,
                resumed[0].expected_vectors
            ),
            (7, 3, 3)
        );
    }

    #[test]
    fn snapshot_is_immutable_and_detects_blob_corruption() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
        let inventory = inventory(corpus.path());
        let snapshot = create_snapshot(state.path(), "tx-1", &inventory).unwrap();
        std::fs::write(corpus.path().join("a.txt"), "changed source").unwrap();
        assert_eq!(open_snapshot(state.path(), "tx-1").unwrap(), snapshot);
        let blob = state
            .path()
            .join("sync-snapshots/tx-1")
            .join(&snapshot.files[0].relative_blob);
        std::fs::write(blob, "corrupt").unwrap();
        assert!(open_snapshot(state.path(), "tx-1").is_err());
    }

    #[test]
    fn prepared_snapshot_streams_exact_actions_counts_and_digest() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(
            corpus.path().join("a.jsonl"),
            "{\"message\":\"alpha\"}\n{\"message\":\"beta\"}\n",
        )
        .unwrap();
        let inventory = inventory(corpus.path());
        let plan = plan_for(&inventory);
        let snapshot = create_prepared_snapshot(
            state.path(),
            "tx-prepared",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
        )
        .unwrap();
        let prepared = snapshot.files[0].prepared.as_ref().unwrap();
        assert_eq!(prepared.records, 2);
        assert!(prepared.bytes > 0);
        let ndjson = std::fs::read_to_string(
            state
                .path()
                .join("sync-snapshots/tx-prepared")
                .join(&prepared.relative_ndjson),
        )
        .unwrap();
        assert_eq!(ndjson.lines().count(), 4);
        assert!(ndjson.contains("\"ax_file\""));
        assert!(ndjson.contains("\"_index\":\"ax-docs\""));

        let mut groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
        bind_prepared_counts(&mut groups, &snapshot, &inventory.keys).unwrap();
        assert_eq!(groups[0].expected_records, 2);
        let artifact_path = state
            .path()
            .join("sync-snapshots/tx-prepared")
            .join(&prepared.relative_ndjson);
        std::fs::write(artifact_path, "corrupt").unwrap();
        assert!(open_snapshot(state.path(), "tx-prepared").is_err());
    }

    #[test]
    fn preparation_sniffs_and_extracts_only_the_sealed_blob_after_live_mutation() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        let source = corpus.path().join("records.jsonl");
        let sealed = b"{\"message\":\"sealed alpha\"}\n{\"message\":\"sealed beta\"}\n";
        let replacement = b"id,value\n9,live mutation\n";
        std::fs::write(&source, sealed).unwrap();
        let inventory = inventory(corpus.path());
        let plan = plan_for(&inventory);
        arm_source_mutation_after_seal(&source, replacement);

        let snapshot = create_prepared_snapshot(
            state.path(),
            "tx-post-seal-mutation",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
        )
        .unwrap();

        assert_eq!(std::fs::read(&source).unwrap(), replacement);
        let prepared = snapshot.files[0].prepared.as_ref().unwrap();
        assert_eq!(prepared.records, 2);
        let root = state.path().join("sync-snapshots/tx-post-seal-mutation");
        let ndjson = std::fs::read_to_string(root.join(&prepared.relative_ndjson)).unwrap();
        assert!(ndjson.contains("sealed alpha"), "{ndjson}");
        assert!(ndjson.contains("sealed beta"), "{ndjson}");
        assert!(!ndjson.contains("live mutation"), "{ndjson}");
        for document in ndjson
            .lines()
            .skip(1)
            .step_by(2)
            .map(|line| serde_json::from_str::<Value>(line).unwrap())
        {
            assert_eq!(document["ax_format"], "jsonl");
            assert!(document.get("message").is_some(), "{document}");
            assert!(document.get("id").is_none(), "{document}");
            assert!(document.get("value").is_none(), "{document}");
        }
        // #2421: the sealed bytes served extraction and were then released;
        // the artifact above is the durable copy.
        assert!(snapshot.files[0].blob_released);
        assert!(!root.join(&snapshot.files[0].relative_blob).exists());
    }

    /// #294: snapshot blobs are content-addressed and extensionless
    /// (`blobs/00000000`), and the code extractor keyed its grammar lookup on
    /// the CONTENT path instead of the logical one carried by `Sniffed`. Every
    /// source file on the durable path therefore prepared as junk — zero
    /// documents — while the generation still committed and reported success.
    #[test]
    fn preparation_extracts_code_from_extensionless_snapshot_blobs() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(
            corpus.path().join("app.py"),
            "def alpha_helper():\n    return 1\n",
        )
        .unwrap();
        let inventory = inventory(corpus.path());
        let plan = plan_for(&inventory);
        let snapshot = create_prepared_snapshot(
            state.path(),
            "tx-code",
            &inventory,
            &plan,
            "test-preparation-v1",
            u64::MAX,
        )
        .unwrap();
        let prepared = snapshot.files[0].prepared.as_ref().unwrap();
        // #500: a code file prepares the file-level document PLUS one document
        // per declaration — so records is now ≥1 (was exactly 1). The invariant
        // the test guards is "documents, not silent junk": junk stays 0.
        assert!(
            prepared.records >= 1 && prepared.junk == 0,
            "a code file must prepare documents, not silent junk: records={} junk={}",
            prepared.records,
            prepared.junk
        );
        let ndjson = std::fs::read_to_string(
            state
                .path()
                .join("sync-snapshots/tx-code")
                .join(&prepared.relative_ndjson),
        )
        .unwrap();
        let document: Value = serde_json::from_str(ndjson.lines().nth(1).unwrap()).unwrap();
        assert_eq!(document["language"], "python", "{document}");
        // The title must be the logical file name, not the blob ordinal.
        assert_eq!(document["title"], "app.py", "{document}");
        assert!(
            document["defs"]
                .as_str()
                .unwrap_or("")
                .contains("function alpha_helper"),
            "{document}"
        );
    }

    #[test]
    fn prepared_snapshot_budget_aborts_and_removes_staging() {
        let corpus = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.txt"), "budgeted source bytes").unwrap();
        let inventory = crate::content::resolve_reporting(
            crate::walk::walk(corpus.path(), false).unwrap(),
            &|_| {},
        )
        .unwrap();
        let plan = plan_for(&inventory);
        let state = tempfile::tempdir().unwrap();
        let error = create_prepared_snapshot(
            state.path(),
            "tx-budget",
            &inventory,
            &plan,
            "test-preparation-v1",
            inventory.files[0].size + 1,
        )
        .unwrap_err();
        assert!(format!("{error:#}").contains("logical payload"));
        assert!(!state
            .path()
            .join("sync-snapshots/.tx-budget.partial")
            .exists());
        assert!(!state.path().join("sync-snapshots/tx-budget").exists());
    }

    #[test]
    fn payload_writer_refuses_bytes_before_they_reach_disk() {
        let state = tempfile::tempdir().unwrap();
        let path = state.path().join("payload");
        let file = File::create(&path).unwrap();
        let mut budget = PayloadBudget { used: 0, limit: 3 };
        let mut writer = BudgetWriter {
            inner: file,
            budget: &mut budget,
        };
        writer.write_all(b"abc").unwrap();
        assert!(writer.write_all(b"d").is_err());
        writer.flush().unwrap();
        drop(writer);
        assert_eq!(std::fs::metadata(path).unwrap().len(), 3);
        assert_eq!(budget.used, 3);
    }

    #[test]
    fn gc_tombstone_crash_is_retryable_and_symlinks_are_refused() {
        let state = tempfile::tempdir().unwrap();
        let mut journal = Journal::open(state.path(), "root", "url", "prefix", 300, false).unwrap();
        journal.sync_bootstrap_genesis().unwrap();
        let snapshots = state.path().join("sync-snapshots");
        std::fs::create_dir_all(snapshots.join("orphan")).unwrap();
        std::fs::write(snapshots.join("orphan/blob"), b"x").unwrap();
        GC_FAIL_AFTER_RENAME.store(true, std::sync::atomic::Ordering::SeqCst);
        assert!(gc_snapshots(state.path(), &journal).is_err());
        assert!(!snapshots.join("orphan").exists());
        assert!(snapshots.join(".orphan.gc").exists());
        gc_snapshots(state.path(), &journal).unwrap();
        assert!(!snapshots.join(".orphan.gc").exists());

        #[cfg(unix)]
        {
            std::fs::create_dir(snapshots.join("a-orphan")).unwrap();
            std::os::unix::fs::symlink(state.path(), snapshots.join("hostile")).unwrap();
            assert!(gc_snapshots(state.path(), &journal).is_err());
            assert!(snapshots.join("a-orphan").exists());
            assert!(snapshots.join("hostile").exists());
        }
    }

    #[test]
    fn gc_validates_every_protected_snapshot_before_deleting_orphans() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.txt"), "protected bytes").unwrap();
        let inventory = crate::content::resolve_reporting(
            crate::walk::walk(corpus.path(), false).unwrap(),
            &|_| {},
        )
        .unwrap();
        let plan = plan_for(&inventory);
        let state = tempfile::tempdir().unwrap();
        let snapshot = create_snapshot(state.path(), "tx-protected", &inventory).unwrap();
        let mut journal = Journal::open(state.path(), "root", "url", "prefix", 300, false).unwrap();
        journal.sync_bootstrap_genesis().unwrap();
        let groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
        begin(&mut journal, "tx-protected", &snapshot, plan, groups);
        let orphan = state.path().join("sync-snapshots/orphan");
        std::fs::create_dir(&orphan).unwrap();
        std::fs::write(orphan.join("blob"), b"orphan").unwrap();
        let protected_blob = state
            .path()
            .join("sync-snapshots/tx-protected")
            .join(&snapshot.files[0].relative_blob);
        std::fs::write(protected_blob, b"corrupt").unwrap();
        assert!(gc_snapshots(state.path(), &journal).is_err());
        assert!(orphan.exists(), "validation must precede every deletion");
        assert!(state.path().join("sync-snapshots/tx-protected").exists());
    }

    #[test]
    fn prepared_and_metadata_streams_preserve_pairs_under_tiny_budgets() {
        let input = concat!(
            "{\"index\":{\"_index\":\"docs\",\"_id\":\"a\"}}\n",
            "{\"body\":\"alpha\"}\n",
            "{\"index\":{\"_index\":\"docs\",\"_id\":\"b\"}}\n",
            "{\"body\":\"beta\"}\n"
        );
        let mut index_chunks = Vec::new();
        stream_ndjson_pairs(input.as_bytes(), 1, |body| {
            index_chunks.push(String::from_utf8(body).unwrap());
            Ok(())
        })
        .unwrap();
        assert_eq!(index_chunks.len(), 2);
        assert!(index_chunks.iter().all(|chunk| chunk.lines().count() == 2));

        let mut update_chunks = Vec::new();
        stream_metadata_updates(
            input.as_bytes(),
            1,
            "renamed.txt",
            &[Value::String("renamed.txt".into())],
            |body| {
                update_chunks.push(String::from_utf8(body).unwrap());
                Ok(())
            },
        )
        .unwrap();
        assert_eq!(update_chunks.len(), 2);
        assert!(update_chunks.iter().all(|chunk| {
            chunk.lines().count() == 2
                && chunk.contains("\"update\"")
                && chunk.contains("\"ax_path\":\"renamed.txt\"")
                && !chunk.contains("\"body\"")
        }));

        let es = crate::esclient::Es::new("http://127.0.0.1:1", None).unwrap();
        let state = tempfile::tempdir().unwrap();
        let (pr, _sink) = crate::progress::Progress::capture(
            crate::progress::Surface::Silent,
            std::time::Duration::from_secs(3600),
        );
        let _backend = EsSyncBackend::new(&es, state.path(), 1, &pr);
    }

    #[test]
    fn every_snapshot_crash_boundary_restarts_deterministically() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        for boundary in 1..=3 {
            let corpus = tempfile::tempdir().unwrap();
            let state = tempfile::tempdir().unwrap();
            std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
            let inventory = inventory(corpus.path());
            fail_next_snapshot(boundary);
            assert!(create_snapshot(state.path(), "tx-restart", &inventory).is_err());
            let recovered = create_snapshot(state.path(), "tx-restart", &inventory).unwrap();
            assert_eq!(
                open_snapshot(state.path(), "tx-restart").unwrap(),
                recovered
            );
        }
    }

    #[test]
    fn snapshot_failpoint_cannot_be_stolen_by_an_unrelated_thread() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let (armed_tx, armed_rx) = std::sync::mpsc::channel();
        let (thief_done_tx, thief_done_rx) = std::sync::mpsc::channel();

        let owner = std::thread::spawn(move || {
            fail_next_snapshot(2);
            armed_tx.send(()).unwrap();
            thief_done_rx.recv().unwrap();
            (
                snapshot_failpoint(2).is_err(),
                snapshot_failpoint(2).is_ok(),
            )
        });

        armed_rx.recv().unwrap();
        let thief_result = snapshot_failpoint(2);
        thief_done_tx.send(()).unwrap();
        let (owner_consumed_once, owner_second_probe_succeeded) = owner.join().unwrap();

        assert!(
            thief_result.is_ok(),
            "a thread that did not arm the snapshot failpoint consumed it"
        );
        assert!(
            owner_consumed_once,
            "the thread that armed the snapshot failpoint did not consume it"
        );
        assert!(
            owner_second_probe_succeeded,
            "the snapshot failpoint must retain one-shot semantics"
        );
    }

    #[test]
    fn replay_failpoint_cannot_be_stolen_by_an_unrelated_thread() {
        let _guard = REPLAY_FAILPOINT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let (armed_tx, armed_rx) = std::sync::mpsc::channel();
        let (thief_done_tx, thief_done_rx) = std::sync::mpsc::channel();

        let owner = std::thread::spawn(move || {
            fail_replay_after_next_apply();
            armed_tx.send(()).unwrap();
            thief_done_rx.recv().unwrap();
            (
                replay_fail_after_apply().is_err(),
                replay_fail_after_apply().is_ok(),
            )
        });

        armed_rx.recv().unwrap();
        let thief_result = replay_fail_after_apply();
        thief_done_tx.send(()).unwrap();
        let (owner_consumed_once, owner_second_probe_succeeded) = owner.join().unwrap();

        assert!(
            thief_result.is_ok(),
            "a thread that did not arm the replay failpoint consumed it"
        );
        assert!(
            owner_consumed_once,
            "the thread that armed the replay failpoint did not consume it"
        );
        assert!(
            owner_second_probe_succeeded,
            "the replay failpoint must retain one-shot semantics"
        );
    }

    #[test]
    fn pending_generation_is_bound_before_mutable_source_replanning() {
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let state = tempfile::tempdir().unwrap();
        let corpus = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
        let inventory = inventory(corpus.path());
        let snapshot = create_snapshot(state.path(), "tx-pending", &inventory).unwrap();
        let pending: PendingSync = serde_json::from_value(serde_json::json!({
            "tx_id": "tx-pending",
            "base_generation": 0,
            "base_manifest_digest": "base",
            "desired_manifest_digest": "desired",
            "operation_hash": "operations",
            "desired": {
                "generation": 1,
                "execution": {
                    "version": 1,
                    "root_identity": "root",
                    "url": "http://engine",
                    "prefix": "ax",
                    "follow_symlinks": false,
                    "chunker_identity": "chunker",
                    "embedding_identity_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
                    "embedding_backend": "lexical",
                    "embedding_dimension": 384,
                    "embedding_semantic_contract": "semantic_text-derived-vector.v1",
                    "embedding_resumable": true,
                    "graph_enabled": false,
                    "brain": "disabled",
                    "detector_identity": "disabled",
                    "schema_identity": "schema",
                    "index_identity": "index",
                    "source_policy": {
                        "policy": "durable_snapshot",
                        "reference": "sync-snapshots/tx-pending",
                        "snapshot_digest": snapshot.snapshot_digest
                    }
                },
                "plan": {
                    "datasets": [],
                    "files": {},
                    "junk_files": [],
                    "duplicate_files": [],
                    "alias_paths_indexed": false
                },
                "groups": []
            },
            "operations": []
        }))
        .unwrap();
        let error = require_resumable_pending_source(state.path(), Some(&pending))
            .unwrap_err()
            .to_string();
        std::fs::write(corpus.path().join("a.txt"), "mutated after sync_begin").unwrap();
        let repeated = require_resumable_pending_source(state.path(), Some(&pending))
            .unwrap_err()
            .to_string();
        assert_eq!(error, repeated);
        assert!(error.contains("verified durable source snapshot"));
    }

    #[test]
    fn accepted_upsert_replays_without_duplicate_live_state() {
        let _journal_guard = crate::state::SYNC_IO_FAILPOINT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let corpus = tempfile::tempdir().unwrap();
        let state = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
        let inventory = inventory(corpus.path());
        let plan = plan_for(&inventory);
        let groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
        let snapshot = create_snapshot(state.path(), "tx-add", &inventory).unwrap();
        let mut journal =
            Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
        journal.sync_bootstrap_genesis().unwrap();
        begin(&mut journal, "tx-add", &snapshot, plan, groups);
        let mut backend = FakeBackend::default();

        fail_replay_after_next_apply();
        assert!(replay_pending_operations(state.path(), &mut journal, &mut backend).is_err());
        assert_eq!(backend.live.len(), 1);
        drop(journal);

        let mut journal =
            Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
        replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
        assert_eq!(backend.live.len(), 1);
        assert_eq!(backend.applications.len(), 2);
        assert!(journal.pending_sync.is_none());
        assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 1);
    }

    /// The concurrency a window of applies actually reached, counted from
    /// inside `apply` — shared with the worker threads through an `Arc`.
    #[derive(Default)]
    struct WindowProbe {
        in_flight: std::sync::atomic::AtomicUsize,
        max_in_flight: std::sync::atomic::AtomicUsize,
    }

    impl WindowProbe {
        fn enter(&self) {
            let entered = self
                .in_flight
                .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
                + 1;
            self.max_in_flight
                .fetch_max(entered, std::sync::atomic::Ordering::SeqCst);
        }

        fn leave(&self) {
            self.in_flight
                .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
        }

        fn max_seen(&self) -> usize {
            self.max_in_flight.load(std::sync::atomic::Ordering::SeqCst)
        }
    }

    /// What the scheduling thread saw, in the order it saw it.
    #[derive(Default)]
    struct WindowHooks {
        begun: Vec<usize>,
        applied: Vec<(usize, bool)>,
    }

    impl ReplayHooks<usize> for WindowHooks {
        fn begin(&mut self, item: &usize) -> Result<()> {
            self.begun.push(*item);
            Ok(())
        }

        fn applied(&mut self, item: &usize, outcome: Result<()>) -> Result<()> {
            self.applied.push((*item, outcome.is_ok()));
            outcome
        }
    }

    #[test]
    fn replay_windowed_overlaps_up_to_the_width_and_reports_in_dispatch_order() {
        let items: Vec<usize> = (0..12).collect();
        let probe = std::sync::Arc::new(WindowProbe::default());
        let apply_probe = std::sync::Arc::clone(&probe);
        let apply = move |_item: &usize| {
            apply_probe.enter();
            // Long enough that the whole first window is inside `apply` at
            // once; short enough that the test stays instant.
            std::thread::sleep(std::time::Duration::from_millis(25));
            apply_probe.leave();
            Ok(())
        };
        let mut hooks = WindowHooks::default();
        replay_windowed(&items, 4, &mut hooks, &apply).unwrap();
        assert_eq!(hooks.begun, items, "every item begun, in dispatch order");
        assert_eq!(
            hooks
                .applied
                .iter()
                .map(|(item, _)| *item)
                .collect::<Vec<_>>(),
            items,
            "completions reported in dispatch order"
        );
        assert!(hooks.applied.iter().all(|(_, ok)| *ok));
        let max = probe.max_seen();
        assert!(
            (2..=4).contains(&max),
            "a 4-wide window over 25 ms applies must overlap; max concurrent was {max}"
        );
    }

    #[test]
    fn replay_windowed_stops_dispatch_on_failure_and_drains_what_was_dispatched() {
        let items: Vec<usize> = (0..6).collect();
        let mut hooks = WindowHooks::default();
        let apply = |item: &usize| {
            if *item == 1 {
                // Inside the first window, slow enough that items 0, 2 and
                // the next dispatch (3) are already in flight when it lands.
                std::thread::sleep(std::time::Duration::from_millis(30));
                Err(anyhow::anyhow!("injected apply failure"))
            } else {
                std::thread::sleep(std::time::Duration::from_millis(5));
                Ok(())
            }
        };
        let error = replay_windowed(&items, 3, &mut hooks, &apply)
            .unwrap_err()
            .to_string();
        assert_eq!(error, "injected apply failure");
        assert_eq!(
            hooks.begun,
            vec![0, 1, 2, 3],
            "dispatch stops at the failure"
        );
        assert_eq!(
            hooks.applied,
            vec![(0, true), (1, false), (2, true), (3, true)],
            "in-flight successes after the failure are still reported, so their \
             Committed writes are not lost; undispatched items are untouched"
        );
    }

    #[test]
    fn replay_windowed_width_one_never_spawns_and_keeps_order() {
        let items: Vec<usize> = (0..3).collect();
        let mut hooks = WindowHooks::default();
        let seen_threads =
            std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashSet::new()));
        let apply_of = std::sync::Arc::clone(&seen_threads);
        let apply = move |_item: &usize| {
            apply_of.lock().unwrap().insert(std::thread::current().id());
            Ok(())
        };
        replay_windowed(&items, 1, &mut hooks, &apply).unwrap();
        assert_eq!(hooks.begun, items);
        assert_eq!(hooks.applied, vec![(0, true), (1, true), (2, true)]);
        assert_eq!(
            seen_threads.lock().unwrap().len(),
            1,
            "width 1 runs every apply on the calling thread"
        );
    }

    #[test]
    fn replay_windowed_converts_a_worker_panic_into_the_run_error() {
        let items: Vec<usize> = (0..4).collect();
        let mut hooks = WindowHooks::default();
        let apply = |item: &usize| {
            if *item == 0 {
                std::panic::panic_any("worker exploded");
            }
            Ok(())
        };
        // Keep the injected panic out of the test output; it is the input,
        // not a failure to report.
        let previous_hook = std::panic::take_hook();
        std::panic::set_hook(Box::new(|_| {}));
        let error = replay_windowed(&items, 4, &mut hooks, &apply).unwrap_err();
        std::panic::set_hook(previous_hook);
        assert_eq!(error.to_string(), "replay apply panicked: worker exploded");
        assert_eq!(hooks.begun, vec![0, 1, 2, 3], "the window was dispatched");
        assert_eq!(
            hooks.applied,
            vec![(0, false), (1, true), (2, true), (3, true)],
            "the drain still reports the applies that did land"
        );
    }

    /// A spawner that refuses its first `refuse` calls with `EAGAIN`, as
    /// `pthread_create` does under thread or memory pressure, then starts
    /// real threads. Counts every call (#2428).
    struct FlakySpawn {
        refuse: usize,
        calls: std::sync::atomic::AtomicUsize,
        started: std::sync::atomic::AtomicUsize,
    }

    impl FlakySpawn {
        fn refusing(refuse: usize) -> Self {
            FlakySpawn {
                refuse,
                calls: std::sync::atomic::AtomicUsize::new(0),
                started: std::sync::atomic::AtomicUsize::new(0),
            }
        }
    }

    impl xerj_common::thread_spawn::Spawn for FlakySpawn {
        fn spawn(
            &self,
            name: &str,
            job: xerj_common::thread_spawn::BoxedJob,
        ) -> std::io::Result<xerj_common::thread_spawn::SpawnedThread> {
            let call = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            if call < self.refuse {
                return Err(std::io::Error::from_raw_os_error(11));
            }
            self.started
                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            xerj_common::thread_spawn::StdSpawn.spawn(name, job)
        }
    }

    #[test]
    fn replay_windowed_reuses_at_most_width_workers_for_a_long_replay() {
        // #2428: one thread per operation was hundreds of thousands of
        // `pthread_create` calls over a corpus replay. The window must start
        // at most `width` workers once and reuse them.
        let items: Vec<usize> = (0..500).collect();
        let spawner = FlakySpawn::refusing(0);
        let threads = std::sync::Mutex::new(std::collections::HashSet::new());
        let apply = |_item: &usize| {
            threads.lock().unwrap().insert(std::thread::current().id());
            Ok(())
        };
        let mut hooks = WindowHooks::default();
        replay_windowed_with(&spawner, || true, &items, 4, &mut hooks, &apply).unwrap();
        assert_eq!(hooks.begun, items);
        assert_eq!(hooks.applied.len(), items.len());
        assert!(hooks.applied.iter().all(|(_, ok)| *ok));
        let spawned = spawner.calls.load(std::sync::atomic::Ordering::SeqCst);
        assert!(
            spawned <= 4,
            "500 operations at width 4 must start at most 4 threads, started {spawned}"
        );
        assert!(threads.lock().unwrap().len() <= 4);
    }

    #[test]
    fn replay_windowed_completes_inline_when_every_worker_spawn_is_refused() {
        // #2428: EAGAIN on every spawn is a slower replay, not a lost run.
        let items: Vec<usize> = (0..20).collect();
        let spawner = FlakySpawn::refusing(usize::MAX);
        let caller = std::thread::current().id();
        let threads = std::sync::Mutex::new(std::collections::HashSet::new());
        let apply = |_item: &usize| {
            threads.lock().unwrap().insert(std::thread::current().id());
            Ok(())
        };
        let mut hooks = WindowHooks::default();
        replay_windowed_with(&spawner, || true, &items, 4, &mut hooks, &apply).unwrap();
        assert_eq!(hooks.begun, items);
        assert_eq!(
            hooks
                .applied
                .iter()
                .map(|(item, _)| *item)
                .collect::<Vec<_>>(),
            items,
            "every item applied and reported in dispatch order"
        );
        assert!(hooks.applied.iter().all(|(_, ok)| *ok));
        assert_eq!(
            *threads.lock().unwrap(),
            std::collections::HashSet::from([caller]),
            "with no worker, every apply runs on the calling thread"
        );
        assert_eq!(spawner.started.load(std::sync::atomic::Ordering::SeqCst), 0);
        assert_eq!(
            spawner.calls.load(std::sync::atomic::Ordering::SeqCst),
            WORKER_SPAWN_ATTEMPTS as usize,
            "the first worker is retried with backoff, then the replay goes inline"
        );
    }

    #[test]
    fn replay_windowed_retries_a_transient_spawn_refusal() {
        // Two EAGAINs, then the OS has threads again: the retry gets the
        // worker instead of giving up on it.
        let items: Vec<usize> = (0..50).collect();
        let spawner = FlakySpawn::refusing(2);
        let apply = |_item: &usize| Ok(());
        let mut hooks = WindowHooks::default();
        replay_windowed_with(&spawner, || true, &items, 3, &mut hooks, &apply).unwrap();
        assert_eq!(hooks.applied.len(), items.len());
        assert!(hooks.applied.iter().all(|(_, ok)| *ok));
        assert_eq!(
            spawner.started.load(std::sync::atomic::Ordering::SeqCst),
            3,
            "all three workers started after the transient refusals"
        );
    }

    #[test]
    fn replay_windowed_continues_on_the_workers_that_started() {
        // Spawn succeeds once, then EAGAIN forever: the replay finishes on
        // that one worker.
        struct OneThenRefuse(std::sync::atomic::AtomicUsize);
        impl xerj_common::thread_spawn::Spawn for OneThenRefuse {
            fn spawn(
                &self,
                name: &str,
                job: xerj_common::thread_spawn::BoxedJob,
            ) -> std::io::Result<xerj_common::thread_spawn::SpawnedThread> {
                if self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
                    xerj_common::thread_spawn::StdSpawn.spawn(name, job)
                } else {
                    Err(std::io::Error::from_raw_os_error(11))
                }
            }
        }
        let items: Vec<usize> = (0..40).collect();
        let spawner = OneThenRefuse(std::sync::atomic::AtomicUsize::new(0));
        let apply = |_item: &usize| Ok(());
        let mut hooks = WindowHooks::default();
        replay_windowed_with(&spawner, || true, &items, 4, &mut hooks, &apply).unwrap();
        assert_eq!(hooks.begun, items);
        assert_eq!(
            hooks
                .applied
                .iter()
                .map(|(item, _)| *item)
                .collect::<Vec<_>>(),
            items
        );
    }

    #[test]
    fn spawn_scoped_with_backoff_gives_up_after_the_attempts_without_running_the_job() {
        let spawner = FlakySpawn::refusing(usize::MAX);
        let ran = std::sync::atomic::AtomicBool::new(false);
        let ran_ref = &ran;
        let result = xerj_common::thread_spawn::scope(|scope| {
            spawn_scoped_with_backoff(
                scope,
                &spawner,
                || true,
                "test-worker",
                3,
                std::time::Duration::from_millis(1),
                move || ran_ref.store(true, std::sync::atomic::Ordering::SeqCst),
            )
        });
        assert!(result.is_err());
        assert_eq!(spawner.calls.load(std::sync::atomic::Ordering::SeqCst), 3);
        assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));
    }

    /// A backend whose Nth `apply` is the server still answering 429 after
    /// the client's patience — the one way a throttled node still ends a run.
    struct ThrottledOutBackend {
        inner: FakeBackend,
        fail_apply_number: usize,
        applies: usize,
    }

    impl SyncOperationBackend for ThrottledOutBackend {
        fn apply(
            &mut self,
            operation: &SyncOperation,
            base: &CommittedManifest,
            desired: &GenerationManifest,
            snapshot: &SourceSnapshot,
        ) -> Result<()> {
            self.applies += 1;
            if self.applies == self.fail_apply_number {
                return Err(anyhow::Error::new(crate::esclient::BackpressureExhausted {
                    items: 7,
                    patience: std::time::Duration::from_secs(120),
                    reason: "breaker".into(),
                }))
                .context("replay sealed bulk");
            }
            self.inner.apply(operation, base, desired, snapshot)
        }

        fn validate(
            &mut self,
            base: &CommittedManifest,
            desired: &GenerationManifest,
            snapshot: &SourceSnapshot,
        ) -> Result<()> {
            self.inner.validate(base, desired, snapshot)
        }
    }

    /// #944, the terminal case: the node accepted nothing for the client's
    /// whole patience. The run still ends — nothing on this side can make a
    /// pinned node (#950) accept — but it ends NAMED: the terminal line says
    /// `reason=server-backpressure` with what is applied and what is left, a
    /// note names the file it was on, and the same journal resumes to a commit.
    /// Any other failure keeps `reason=aborted`.
    #[test]
    fn a_backpressure_stop_names_itself_and_what_is_left_then_resumes() {
        let _journal_guard = crate::state::SYNC_IO_FAILPOINT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let state = tempfile::tempdir().unwrap();
        let corpus = tempfile::tempdir().unwrap();
        for name in ["a.txt", "b.txt", "c.txt"] {
            std::fs::write(corpus.path().join(name), format!("content of {name}")).unwrap();
        }
        let inventory = inventory(corpus.path());
        // `plan_for` assigns the first file only; this test needs three
        // operations, so every file gets the same one-dataset assignment.
        let mut plan = plan_for(&inventory);
        for ((key, file), digest) in inventory
            .keys
            .iter()
            .zip(&inventory.files)
            .zip(&inventory.digests)
        {
            plan.files.insert(
                key.clone(),
                FileAssignment {
                    rel: file.rel.clone(),
                    path_id: file.rel_id.clone(),
                    is_symlink: Some(file.is_symlink),
                    family: "text".into(),
                    gzip: false,
                    content_digest: Some(digest.clone()),
                    assignments: vec![(None, "docs".into())],
                    as_document: false,
                },
            );
        }
        let groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
        let snapshot = create_snapshot(state.path(), "tx-throttled", &inventory).unwrap();
        let mut journal =
            Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
        journal.sync_bootstrap_genesis().unwrap();
        begin(&mut journal, "tx-throttled", &snapshot, plan, groups);
        let second_rel = {
            let pending = journal.pending_sync.as_ref().unwrap();
            let operation = &pending.operations[1];
            pending
                .desired
                .groups
                .iter()
                .find(|group| group.group_id == operation.group_id)
                .unwrap()
                .canonical
                .rel
                .clone()
        };

        let mut backend = ThrottledOutBackend {
            inner: FakeBackend::default(),
            fail_apply_number: 2,
            applies: 0,
        };
        let (pr, sink) = crate::progress::Progress::capture(
            crate::progress::Surface::Plain,
            std::time::Duration::from_secs(3600),
        );
        let error =
            replay_pending_operations_reporting(state.path(), &mut journal, &mut backend, &pr)
                .unwrap_err();
        assert!(
            format!("{error:#}").contains("the server kept rejecting 7 of a prepared bulk's items"),
            "{error:#}"
        );
        let stream = String::from_utf8(sink.lock().unwrap().clone()).unwrap();
        let done = stream
            .lines()
            .find(|line| line.starts_with("xerj-done "))
            .unwrap_or_else(|| panic!("{stream}"));
        assert!(
            done.starts_with("xerj-done ok=false exit=1 reason=server-backpressure ")
                && done.ends_with(" ops_applied=1 ops_remaining=2"),
            "{done}"
        );
        assert!(
            stream.lines().any(|line| line.contains(&format!(
                "stopped by server back-pressure while applying {second_rel}: 1 operation(s) \
                 are journaled applied, 2 are not"
            ))),
            "{stream}"
        );

        // The same journal resumes: two operations left, then a commit.
        backend.fail_apply_number = usize::MAX;
        let (pr, sink) = crate::progress::Progress::capture(
            crate::progress::Surface::Plain,
            std::time::Duration::from_secs(3600),
        );
        replay_pending_operations_reporting(state.path(), &mut journal, &mut backend, &pr).unwrap();
        assert_eq!(backend.inner.applications.len(), 3);
        assert!(journal.pending_sync.is_none(), "the generation committed");
        let stream = String::from_utf8(sink.lock().unwrap().clone()).unwrap();
        assert!(
            !stream.contains("xerj-done"),
            "a successful replay leaves the terminal line to its caller: {stream}"
        );
    }

    /// Only back-pressure earns the named reason. Anything else — here an
    /// injected failure after an apply — leaves the stream to close itself
    /// `reason=aborted`, exactly as before.
    #[test]
    fn a_failure_that_is_not_backpressure_is_not_renamed() {
        let _journal_guard = crate::state::SYNC_IO_FAILPOINT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let state = tempfile::tempdir().unwrap();
        let corpus = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
        let inventory = inventory(corpus.path());
        let plan = plan_for(&inventory);
        let groups = groups_from_inventory(&inventory, &plan, &[]).unwrap();
        let snapshot = create_snapshot(state.path(), "tx-other", &inventory).unwrap();
        let mut journal =
            Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
        journal.sync_bootstrap_genesis().unwrap();
        begin(&mut journal, "tx-other", &snapshot, plan, groups);
        let mut backend = FakeBackend::default();
        let (pr, sink) = crate::progress::Progress::capture(
            crate::progress::Surface::Plain,
            std::time::Duration::from_secs(3600),
        );
        fail_replay_after_next_apply();
        assert!(
            replay_pending_operations_reporting(state.path(), &mut journal, &mut backend, &pr)
                .is_err()
        );
        let stream = String::from_utf8(sink.lock().unwrap().clone()).unwrap();
        assert!(!stream.contains("xerj-done"), "{stream}");
        assert!(!stream.contains("back-pressure"), "{stream}");
    }

    #[test]
    fn replay_converges_change_metadata_and_delete_without_resurrection() {
        let _journal_guard = crate::state::SYNC_IO_FAILPOINT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let _guard = SNAPSHOT_TEST_LOCK
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());
        let state = tempfile::tempdir().unwrap();
        let corpus = tempfile::tempdir().unwrap();
        std::fs::write(corpus.path().join("a.txt"), "alpha").unwrap();
        let first_inventory = inventory(corpus.path());
        let first_plan = plan_for(&first_inventory);
        let first_groups = groups_from_inventory(&first_inventory, &first_plan, &[]).unwrap();
        let first_snapshot = create_snapshot(state.path(), "tx-first", &first_inventory).unwrap();
        let mut journal =
            Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
        journal.sync_bootstrap_genesis().unwrap();
        begin(
            &mut journal,
            "tx-first",
            &first_snapshot,
            first_plan,
            first_groups,
        );
        let mut backend = FakeBackend::default();
        replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
        let stable_group_id = journal.committed_manifest.as_ref().unwrap().groups[0]
            .group_id
            .clone();

        std::fs::write(corpus.path().join("a.txt"), "beta").unwrap();
        let changed_inventory = inventory(corpus.path());
        let changed_plan = plan_for(&changed_inventory);
        let changed_groups = groups_from_inventory(
            &changed_inventory,
            &changed_plan,
            &journal.committed_manifest.as_ref().unwrap().groups,
        )
        .unwrap();
        assert_eq!(changed_groups[0].group_id, stable_group_id);
        let changed_snapshot =
            create_snapshot(state.path(), "tx-change", &changed_inventory).unwrap();
        begin(
            &mut journal,
            "tx-change",
            &changed_snapshot,
            changed_plan,
            changed_groups,
        );
        fail_replay_after_next_apply();
        assert!(replay_pending_operations(state.path(), &mut journal, &mut backend).is_err());
        replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
        let changed_content = backend.live[&stable_group_id].content_id.clone();

        let mut renamed_plan = journal.committed_manifest.as_ref().unwrap().plan.clone();
        let content_id = renamed_plan.files.keys().next().unwrap().clone();
        let assignment = renamed_plan.files.get_mut(&content_id).unwrap();
        assignment.rel = "renamed.txt".into();
        assignment.path_id = "unix:72656e616d65642e747874".into();
        let mut renamed_groups = journal.committed_manifest.as_ref().unwrap().groups.clone();
        renamed_groups[0].canonical.rel = "renamed.txt".into();
        renamed_groups[0].canonical.path_id = "unix:72656e616d65642e747874".into();
        let renamed_snapshot =
            create_snapshot(state.path(), "tx-metadata", &changed_inventory).unwrap();
        begin(
            &mut journal,
            "tx-metadata",
            &renamed_snapshot,
            renamed_plan,
            renamed_groups,
        );
        replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
        assert_eq!(backend.live[&stable_group_id].content_id, changed_content);
        assert_eq!(
            backend.live[&stable_group_id].canonical,
            "unix:72656e616d65642e747874"
        );

        let empty = Inventory {
            files: vec![],
            keys: vec![],
            digests: vec![],
            duplicates: vec![],
            unreadable: vec![],
        };
        let empty_snapshot = create_snapshot(state.path(), "tx-delete", &empty).unwrap();
        let mut empty_plan = journal.committed_manifest.as_ref().unwrap().plan.clone();
        empty_plan.files.clear();
        empty_plan.datasets[0].file_count = 0;
        begin(
            &mut journal,
            "tx-delete",
            &empty_snapshot,
            empty_plan,
            vec![],
        );
        fail_replay_after_next_apply();
        assert!(replay_pending_operations(state.path(), &mut journal, &mut backend).is_err());
        assert!(backend.live.is_empty());
        replay_pending_operations(state.path(), &mut journal, &mut backend).unwrap();
        assert!(backend.live.is_empty());
        assert!(journal
            .committed_manifest
            .as_ref()
            .unwrap()
            .groups
            .is_empty());
    }

    #[test]
    fn serialized_graph_identity_with_zero_operations_fails_before_backend_or_commit() {
        let state = tempfile::tempdir().unwrap();
        let inventory = Inventory {
            files: vec![],
            keys: vec![],
            digests: vec![],
            duplicates: vec![],
            unreadable: vec![],
        };
        let snapshot = create_snapshot(state.path(), "tx-hostile", &inventory).unwrap();
        let mut journal =
            Journal::open(state.path(), "root", "http://engine", "ax", 300, false).unwrap();
        journal.sync_bootstrap_genesis().unwrap();
        let base = journal.committed_manifest.as_ref().unwrap().clone();
        let desired = desired(1, "tx-hostile", &snapshot, Plan::default(), vec![]);
        let valid = PendingSync::new("tx-hostile".into(), &base, desired).unwrap();
        assert!(valid.operations.is_empty());
        journal.sync_begin(&valid).unwrap();

        let mut encoded = serde_json::to_value(journal.pending_sync.as_ref().unwrap()).unwrap();
        encoded["desired"]["execution"]["graph_enabled"] = Value::Bool(true);
        encoded["desired"]["execution"]["brain"] = Value::String("hostile".into());
        encoded["desired"]["execution"]["detector_identity"] =
            Value::String("hostile-detectors".into());
        journal.pending_sync = Some(serde_json::from_value(encoded).unwrap());

        let mut backend = FakeBackend::default();
        assert!(replay_pending_operations(state.path(), &mut journal, &mut backend).is_err());
        assert_eq!(backend.provisions, 0);
        assert!(backend.applications.is_empty());
        assert_eq!(backend.validations, 0);
        assert_eq!(journal.committed_manifest.as_ref().unwrap().generation, 0);
        assert!(journal.pending_sync.is_some());
    }

    // ── #1212 shape 2: the read-back barrier re-walks an unflagged short page ──
    //
    // The live failure: finalize's run_id-scoped sorted walk read 9 full
    // 1,000-hit pages, then an EMPTY page with `timed_out` absent — 9,000 of
    // 58,568, no flag, so neither the transport retry nor #1213's page-level
    // deadline check could see it. Only observed-count-versus-expected can,
    // and the answer was intermittent: the same walk on an idle node read all
    // 59 pages clean. These pin the re-walk loop's contract without a node.

    fn doc_map(ids: &[&str]) -> BTreeMap<String, Value> {
        ids.iter()
            .map(|id| ((*id).to_string(), Value::String((*id).into())))
            .collect()
    }

    #[test]
    fn a_complete_first_read_back_walk_is_returned_without_refreshing() {
        let pr = crate::progress::Progress::silent();
        let walks = std::cell::Cell::new(0u32);
        let refreshes = std::cell::Cell::new(0u32);
        let observed = EsSyncBackend::rewalk_while_short(
            &pr,
            2,
            || {
                walks.set(walks.get() + 1);
                Ok(doc_map(&["a", "b"]))
            },
            || {
                refreshes.set(refreshes.get() + 1);
                Ok(())
            },
        )
        .unwrap();
        assert_eq!(observed.len(), 2);
        assert_eq!(walks.get(), 1, "the happy path is exactly one walk");
        assert_eq!(refreshes.get(), 0, "a complete walk never refreshes");
    }

    #[test]
    fn a_short_walk_is_refreshed_and_rewalked_until_complete() {
        let pr = crate::progress::Progress::silent();
        let walks = std::cell::Cell::new(0u32);
        let refreshes = std::cell::Cell::new(0u32);
        let observed = EsSyncBackend::rewalk_while_short(
            &pr,
            2,
            || {
                walks.set(walks.get() + 1);
                // walk 1 is the #1212 shape: an unflagged truncated answer;
                // walk 2 sees the whole set, the way the idle-node probe did
                Ok(if walks.get() == 1 {
                    doc_map(&["a"])
                } else {
                    doc_map(&["a", "b"])
                })
            },
            || {
                refreshes.set(refreshes.get() + 1);
                Ok(())
            },
        )
        .unwrap();
        assert_eq!(observed.len(), 2, "the complete observation is returned");
        assert_eq!(walks.get(), 2);
        assert_eq!(refreshes.get(), 1, "one refresh between the two walks");
    }

    #[test]
    fn a_walk_short_every_time_costs_all_attempts_and_returns_the_last_observation() {
        let pr = crate::progress::Progress::silent();
        let walks = std::cell::Cell::new(0u32);
        let observed = EsSyncBackend::rewalk_while_short(
            &pr,
            58_568,
            || {
                walks.set(walks.get() + 1);
                // every walk truncates differently short — never complete
                let id = format!("only-{0}", walks.get());
                Ok(BTreeMap::from([(id.clone(), Value::String(id))]))
            },
            || Ok(()),
        )
        .unwrap();
        assert_eq!(
            walks.get(),
            EsSyncBackend::READBACK_WALK_ATTEMPTS as u32,
            "bounded: no infinite re-walk over a persistently truncating node"
        );
        assert_eq!(observed.len(), 1);
        assert_eq!(
            observed.keys().next().map(String::as_str),
            Some(format!("only-{0}", EsSyncBackend::READBACK_WALK_ATTEMPTS).as_str()),
            "the LAST observation is what validate_observed judges — the error \
             names the final walk's counts, not a stale one"
        );
    }

    #[test]
    fn a_transport_error_in_the_walk_propagates_without_a_retry() {
        let pr = crate::progress::Progress::silent();
        let refreshes = std::cell::Cell::new(0u32);
        let verdict = EsSyncBackend::rewalk_while_short(
            &pr,
            2,
            || Err(anyhow::anyhow!("connection reset")),
            || {
                refreshes.set(refreshes.get() + 1);
                Ok(())
            },
        );
        assert!(verdict.is_err(), "an error is not a short walk");
        assert_eq!(refreshes.get(), 0, "no refresh over a dead transport");
    }

    // #1267: the `run_id`-scoped catalog read-back walk. On a resume that
    // rewrote the whole catalog its `run_id` matches every document, so it
    // runs for ~35 min — and used to do so invisibly, behind a finished
    // `finalize-catalog` ticker that read as a hang. These pin the walk as its
    // own phase with the engine's match total as the denominator, plus the
    // paging contract, without a node.

    /// One page of a `run_id` walk: every hit carries a `sort` key (the
    /// continuation the next page is fetched from) and the engine reports the
    /// full match `total` on every page, not just the first.
    fn walk_page(total: u64, ids: &[&str]) -> Value {
        let hits: Vec<Value> = ids
            .iter()
            .map(|id| serde_json::json!({"_id": id, "_source": {"run": id}, "sort": [id]}))
            .collect();
        serde_json::json!({"hits": {"total": {"value": total}, "hits": hits}})
    }

    #[test]
    fn a_run_id_walk_collects_every_page_and_opens_an_honest_phase() {
        let (pr, buffer) = crate::progress::Progress::capture(
            crate::progress::Surface::Plain,
            std::time::Duration::from_secs(3600),
        );
        // Page size 2: two full pages, then a short page ends the walk. The
        // engine reports total=5 on every page.
        let pages = std::cell::RefCell::new(vec![
            walk_page(5, &["a", "b"]),
            walk_page(5, &["c", "d"]),
            walk_page(5, &["e"]),
        ]);
        let calls = std::cell::Cell::new(0usize);
        let observed = EsSyncBackend::walk_run_id_pages(
            &pr,
            "finalize-catalog-readback",
            2,
            |_search_after| {
                calls.set(calls.get() + 1);
                Ok(pages.borrow_mut().remove(0))
            },
        )
        .unwrap();
        assert_eq!(
            calls.get(),
            3,
            "two full pages then one short page ends the walk"
        );
        assert_eq!(
            observed.len(),
            5,
            "every document across every page is collected"
        );
        assert!(
            observed.contains_key("e"),
            "the short final page is not dropped"
        );

        let out = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
        assert!(
            out.contains("phase=finalize-catalog-readback"),
            "the walk is its own phase, not a frozen finalize-catalog ticker: {out}"
        );
        // The denominator is the engine's own match total from page one — not
        // the page size, not the running count — so the bar is honest from the
        // first page instead of after 35 min of silence (#1267).
        assert!(
            out.contains("items=0/5"),
            "the phase opens with the full match total as its denominator: {out}"
        );
    }

    #[test]
    fn the_read_back_walk_advances_its_surface_per_document() {
        // The whole point of #1267: the walk must TICK, so a long read-back is
        // never a stalled line. A ticker on a short cadence proves the surface
        // reaches `items=3/3` without any item needing to "complete" a file.
        let (pr, buffer) = crate::progress::Progress::capture(
            crate::progress::Surface::Plain,
            std::time::Duration::from_millis(40),
        );
        let ticker = pr.spawn_ticker();
        // One short page (size 1000 default; 3 < 1000 ends the walk).
        let observed = EsSyncBackend::walk_run_id_pages(
            &pr,
            "finalize-catalog-readback",
            1000,
            |_search_after| Ok(walk_page(3, &["a", "b", "c"])),
        )
        .unwrap();
        assert_eq!(observed.len(), 3);

        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
        loop {
            let out = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
            if out.contains("phase=finalize-catalog-readback") && out.contains("items=3/3") {
                break;
            }
            assert!(
                std::time::Instant::now() < deadline,
                "the ticker never showed the walk advance to items=3/3: {out}"
            );
            std::thread::yield_now();
        }
        drop(ticker);
    }

    // #1267 operational: the server-generation latency probe that runs before
    // `finalize-verify`. Timings are injected, so these pin the decision
    // without a node or a clock.

    fn run_probe(
        surface: crate::progress::Surface,
        timings: &[std::time::Duration],
        version: Option<&str>,
    ) -> (Option<std::time::Duration>, usize, String) {
        let (pr, buffer) =
            crate::progress::Progress::capture(surface, std::time::Duration::from_secs(3600));
        let queue = std::cell::RefCell::new(timings.to_vec());
        let calls = std::cell::Cell::new(0usize);
        let verdict = EsSyncBackend::probe_server_generation(
            &pr,
            400,
            EsSyncBackend::SERVER_PROBE_SLOW,
            || {
                calls.set(calls.get() + 1);
                let mut queue = queue.borrow_mut();
                anyhow::ensure!(!queue.is_empty(), "transport down");
                Ok(queue.remove(0))
            },
            || version.map(str::to_owned),
        );
        let out = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
        (verdict, calls.get(), out)
    }

    #[test]
    fn a_fast_verify_window_probes_once_and_stays_silent() {
        let ms = std::time::Duration::from_millis;
        let (verdict, calls, out) =
            run_probe(crate::progress::Surface::Plain, &[ms(3)], Some("0.0.0-old"));
        assert_eq!(verdict, Some(ms(3)));
        assert_eq!(calls, 1, "a fast first answer is not re-asked");
        assert!(
            !out.contains("warning"),
            "a fixed server is not warned about: {out}"
        );
    }

    #[test]
    fn a_cold_first_window_is_reasked_and_not_mistaken_for_an_old_server() {
        // #1267's own numbers: 8.8 s cold page-in on the FIXED binary, then 2.7 ms.
        let ms = std::time::Duration::from_millis;
        let (verdict, calls, out) =
            run_probe(crate::progress::Surface::Plain, &[ms(8800), ms(3)], None);
        assert_eq!(verdict, Some(ms(3)));
        assert_eq!(calls, 2);
        assert!(
            !out.contains("warning"),
            "cold page-in is not an old server: {out}"
        );
    }

    #[test]
    fn a_slow_warm_window_warns_on_the_plain_surface_and_does_not_abort() {
        let ms = std::time::Duration::from_millis;
        let (verdict, calls, out) = run_probe(
            crate::progress::Surface::Plain,
            &[ms(8800), ms(3400)],
            Some("1.0.0-rc.80"),
        );
        assert_eq!(
            verdict,
            Some(ms(3400)),
            "the probe reports and the run continues"
        );
        assert_eq!(calls, 2);
        for needle in [
            "warning",
            "3.4s",
            "older than this client",
            "#1260",
            "400 windows",
            "~0.4h",
            "XERJ 1.0.0-rc.80",
            env!("CARGO_PKG_VERSION"),
            "upgrade the server",
            "Continuing",
        ] {
            assert!(out.contains(needle), "warning lacks {needle:?}: {out}");
        }
    }

    #[test]
    fn a_slow_server_with_the_same_version_is_named_a_development_build() {
        let ms = std::time::Duration::from_millis;
        let (_, _, out) = run_probe(
            crate::progress::Surface::Plain,
            &[ms(3400), ms(3400)],
            Some(env!("CARGO_PKG_VERSION")),
        );
        assert!(out.contains("same version as this client"), "{out}");
        assert!(out.contains("development build"), "{out}");
    }

    #[test]
    fn the_slow_server_warning_is_one_json_event_on_the_machine_surface() {
        let ms = std::time::Duration::from_millis;
        let (_, _, out) = run_probe(crate::progress::Surface::Json, &[ms(3400), ms(3400)], None);
        let events: Vec<Value> = out
            .lines()
            .map(|line| serde_json::from_str(line).expect("one JSON object per line"))
            .collect();
        assert_eq!(events.len(), 1, "{out}");
        assert_eq!(events[0]["event"], "warning");
        assert!(events[0]["message"]
            .as_str()
            .unwrap()
            .contains("older than this client"));
    }

    #[test]
    fn the_slow_server_warning_survives_the_silent_surface() {
        // `--quiet` drops routine notes; a four-hour wait is not routine.
        let ms = std::time::Duration::from_millis;
        let (_, _, out) = run_probe(
            crate::progress::Surface::Silent,
            &[ms(3400), ms(3400)],
            None,
        );
        assert!(out.contains("older than this client"), "{out}");
    }

    #[test]
    fn a_failing_probe_neither_warns_nor_propagates() {
        let (verdict, calls, out) = run_probe(crate::progress::Surface::Plain, &[], None);
        assert_eq!(verdict, None);
        assert_eq!(calls, 1);
        assert!(out.is_empty(), "{out}");
    }

    #[test]
    fn a_duplicate_id_across_pages_fails_the_walk_loudly() {
        let pr = crate::progress::Progress::silent();
        let pages =
            std::cell::RefCell::new(vec![walk_page(4, &["a", "b"]), walk_page(4, &["b", "c"])]);
        let verdict = EsSyncBackend::walk_run_id_pages(
            &pr,
            "finalize-catalog-readback",
            2,
            |_search_after| Ok(pages.borrow_mut().remove(0)),
        );
        let err = verdict.unwrap_err().to_string();
        assert!(
            err.contains("duplicate ID b"),
            "a document seen twice is a loud failure, never a silent overwrite: {err}"
        );
    }

    #[test]
    fn a_full_page_without_a_continuation_key_cannot_advance() {
        let pr = crate::progress::Progress::silent();
        // A full page (size 2) whose hits carry no `sort`: the walk cannot form
        // the next `search_after` and must refuse rather than loop or truncate.
        let page = serde_json::json!({
            "hits": {"total": {"value": 9}, "hits": [
                {"_id": "a", "_source": {}},
                {"_id": "b", "_source": {}}
            ]}
        });
        let verdict = EsSyncBackend::walk_run_id_pages(
            &pr,
            "finalize-catalog-readback",
            2,
            |_search_after| Ok(page.clone()),
        );
        let err = verdict.unwrap_err().to_string();
        assert!(
            err.contains("continuation sort key"),
            "a full page with no sort key is a loud failure: {err}"
        );
    }

    /// #2222: deleting a finalize-verify target while its window search is in
    /// flight must end the wait. The search connection is held open on purpose
    /// — that is the in-flight request whose handle the node already resolved,
    /// so it does not become a 404 — and `HEAD` starts answering 404 once the
    /// index is "deleted". Verify has to return the deleted-index error while
    /// that search is still unanswered.
    #[test]
    fn finalize_verify_stops_when_its_target_index_is_deleted() {
        use std::io::{Read, Write};
        use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
        use std::sync::mpsc;
        use std::time::Duration;

        const INDEX: &str = "xc-elastic-charts-cpu-profile-tree-mock";

        fn read_http(stream: &mut std::net::TcpStream) -> String {
            stream
                .set_read_timeout(Some(Duration::from_secs(2)))
                .unwrap();
            let mut buf = Vec::new();
            let mut tmp = [0u8; 2048];
            loop {
                match stream.read(&mut tmp) {
                    Ok(0) => break,
                    Ok(n) => {
                        buf.extend_from_slice(&tmp[..n]);
                        if buf.windows(4).any(|w| w == b"\r\n\r\n") {
                            break;
                        }
                    }
                    Err(_) => break,
                }
            }
            String::from_utf8_lossy(&buf).into_owned()
        }

        fn write_status(stream: &mut std::net::TcpStream, status: &str) {
            write!(
                stream,
                "HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
            )
            .unwrap();
        }

        let listener = crate::test_loopback::bind();
        let address = listener.local_addr().unwrap();
        // Accepted sockets inherit non-blocking from the listener. Put each
        // one back into blocking mode inside its handler. The accept loop
        // itself stays non-blocking so a parked `_search` — handled on its
        // own thread — cannot stop a later `HEAD` from being answered.
        listener.set_nonblocking(true).unwrap();
        let deleted = std::sync::Arc::new(AtomicBool::new(false));
        let release = std::sync::Arc::new(AtomicBool::new(false));
        let heads = std::sync::Arc::new(AtomicUsize::new(0));
        let deleted_srv = std::sync::Arc::clone(&deleted);
        let release_srv = std::sync::Arc::clone(&release);
        let heads_srv = std::sync::Arc::clone(&heads);
        let server = std::thread::spawn(move || {
            // Hold the search well past the test's own deadline so a client
            // that waits for the response fails the test instead of passing
            // by accident. `release` lets the passing path close the socket.
            let hold_until = std::time::Instant::now() + Duration::from_secs(45);
            let started = std::time::Instant::now();
            let mut handlers = Vec::new();
            while started.elapsed() < Duration::from_secs(60) {
                if release_srv.load(Ordering::SeqCst) {
                    break;
                }
                match listener.accept() {
                    Ok((stream, _)) => {
                        let deleted_one = std::sync::Arc::clone(&deleted_srv);
                        let release_one = std::sync::Arc::clone(&release_srv);
                        let heads_one = std::sync::Arc::clone(&heads_srv);
                        handlers.push(std::thread::spawn(move || {
                            let mut stream = stream;
                            stream.set_nonblocking(false).unwrap();
                            let request = read_http(&mut stream);
                            if request.starts_with("HEAD ") {
                                heads_one.fetch_add(1, Ordering::SeqCst);
                                let status = if deleted_one.load(Ordering::SeqCst) {
                                    "404 Not Found"
                                } else {
                                    "200 OK"
                                };
                                write_status(&mut stream, status);
                            } else if request.contains("_search") {
                                while !release_one.load(Ordering::SeqCst)
                                    && std::time::Instant::now() < hold_until
                                {
                                    std::thread::sleep(Duration::from_millis(20));
                                }
                                write_status(&mut stream, "200 OK");
                            }
                        }));
                    }
                    Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
                        std::thread::sleep(Duration::from_millis(10));
                    }
                    Err(_) => break,
                }
            }
            for handler in handlers {
                let _ = handler.join();
            }
        });

        let deleted_watch = std::sync::Arc::clone(&deleted);
        let heads_watch = std::sync::Arc::clone(&heads);
        std::thread::spawn(move || {
            let deadline = std::time::Instant::now() + Duration::from_secs(10);
            while heads_watch.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < deadline {
                std::thread::sleep(Duration::from_millis(20));
            }
            deleted_watch.store(true, Ordering::SeqCst);
        });

        let url = format!("http://{address}");
        let (tx, rx) = mpsc::channel();
        // Release the parked search even if an assertion panics, so the
        // server thread does not sit out the rest of its hold.
        struct ReleaseOnDrop(std::sync::Arc<AtomicBool>);
        impl Drop for ReleaseOnDrop {
            fn drop(&mut self) {
                self.0.store(true, Ordering::SeqCst);
            }
        }
        let _release_on_drop = ReleaseOnDrop(std::sync::Arc::clone(&release));
        std::thread::spawn(move || {
            let es = crate::esclient::Es::new(&url, None).unwrap();
            let dir = tempfile::tempdir().unwrap();
            let pr = crate::progress::Progress::silent();
            let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);
            let result =
                backend.windowed_value_counts(INDEX, "ax_file", &["digest".into()], &[], false);
            let _ = tx.send(result.map_err(|error| format!("{error:#}")));
        });

        let message = rx
            .recv_timeout(Duration::from_secs(20))
            .expect("finalize-verify hung on an index that was deleted during verify")
            .expect_err("a deleted verify target must fail that item");
        assert!(
            !release.load(Ordering::SeqCst),
            "verify returned only after the hung search was released"
        );
        assert!(message.contains("was deleted during verify"), "{message}");
        assert!(message.contains(INDEX), "{message}");

        release.store(true, Ordering::SeqCst);
        // Unblock `accept` if it is parked, then let the held search finish.
        let _ = std::net::TcpStream::connect(address);
        server.join().unwrap();
    }

    /// The same window, when the index stays and the search answers, still
    /// returns the bucket counts. The deletion check must not fire on a live
    /// target.
    #[test]
    fn finalize_verify_window_counts_a_live_index() {
        use std::io::{Read, Write};

        let listener = crate::test_loopback::bind();
        let address = listener.local_addr().unwrap();
        let server = std::thread::spawn(move || {
            let (mut stream, _) = listener.accept().unwrap();
            stream
                .set_read_timeout(Some(std::time::Duration::from_secs(2)))
                .unwrap();
            let mut buf = Vec::new();
            let mut tmp = [0u8; 2048];
            loop {
                match stream.read(&mut tmp) {
                    Ok(0) => break,
                    Ok(n) => {
                        buf.extend_from_slice(&tmp[..n]);
                        if buf.windows(4).any(|w| w == b"\r\n\r\n") {
                            break;
                        }
                    }
                    Err(_) => break,
                }
            }
            let body =
                br#"{"aggregations":{"values":{"buckets":[{"key":"digest-a","doc_count":3}]}}}"#;
            write!(
                stream,
                "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
                body.len()
            )
            .unwrap();
            stream.write_all(body).unwrap();
        });

        let es = crate::esclient::Es::new(&format!("http://{address}"), None).unwrap();
        let dir = tempfile::tempdir().unwrap();
        let pr = crate::progress::Progress::silent();
        let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);
        let counts = backend
            .windowed_value_counts(
                "xc-live",
                "ax_file",
                &["digest-a".into(), "digest-b".into()],
                &[],
                false,
            )
            .unwrap();
        assert_eq!(counts.get("digest-a"), Some(&3));
        assert_eq!(counts.get("digest-b"), Some(&0));
        server.join().unwrap();
    }

    /// A scripted node for the finalize-verify retry tests (#2405). Every
    /// connection gets its own handler thread so a `_search` in flight
    /// cannot starve the `HEAD` probe `search_for_verify` may send while a
    /// search is unanswered. `HEAD` always answers 200 (the index lives);
    /// each `_search` records its request body and answers with the next
    /// scripted response — the last one repeats when the script runs out,
    /// so a caller that keeps re-asking past the script is still answered.
    struct ScriptedSearch {
        address: std::net::SocketAddr,
        search_bodies: std::sync::Arc<std::sync::Mutex<Vec<Value>>>,
        stop: std::sync::Arc<std::sync::atomic::AtomicBool>,
        server: Option<std::thread::JoinHandle<()>>,
    }

    impl ScriptedSearch {
        fn start(script: &[&str]) -> Self {
            use std::io::{Read, Write};
            use std::sync::atomic::{AtomicBool, Ordering};
            use std::sync::{Arc, Mutex};
            use std::time::Duration;

            fn read_request(stream: &mut std::net::TcpStream) -> (String, Vec<u8>) {
                stream
                    .set_read_timeout(Some(Duration::from_secs(5)))
                    .unwrap();
                let mut buf = Vec::new();
                let mut tmp = [0u8; 4096];
                loop {
                    match stream.read(&mut tmp) {
                        Ok(0) => break,
                        Ok(n) => {
                            buf.extend_from_slice(&tmp[..n]);
                            if let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
                                let head = String::from_utf8_lossy(&buf[..end]).into_owned();
                                let length = head
                                    .lines()
                                    .find_map(|line| {
                                        line.to_ascii_lowercase()
                                            .strip_prefix("content-length:")
                                            .and_then(|v| v.trim().parse::<usize>().ok())
                                    })
                                    .unwrap_or(0);
                                let have = buf.len() - (end + 4);
                                if have >= length {
                                    return (head, buf[end + 4..end + 4 + length].to_vec());
                                }
                            }
                        }
                        Err(_) => break,
                    }
                }
                let head = String::from_utf8_lossy(&buf).into_owned();
                (head, Vec::new())
            }

            fn write_json(stream: &mut std::net::TcpStream, body: &str) {
                write!(
                    stream,
                    "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
                    body.len()
                )
                .unwrap();
                stream.write_all(body.as_bytes()).unwrap();
            }

            let listener = crate::test_loopback::bind();
            let address = listener.local_addr().unwrap();
            listener.set_nonblocking(true).unwrap();
            let search_bodies = Arc::new(Mutex::new(Vec::new()));
            let stop = Arc::new(AtomicBool::new(false));
            let script: Arc<Mutex<std::collections::VecDeque<String>>> =
                Arc::new(Mutex::new(script.iter().map(|s| s.to_string()).collect()));
            let bodies_srv = Arc::clone(&search_bodies);
            let stop_srv = Arc::clone(&stop);
            let script_srv = Arc::clone(&script);
            let server = std::thread::spawn(move || {
                let mut handlers = Vec::new();
                while !stop_srv.load(Ordering::SeqCst) {
                    match listener.accept() {
                        Ok((stream, _)) => {
                            let bodies_one = Arc::clone(&bodies_srv);
                            let script_one = Arc::clone(&script_srv);
                            handlers.push(std::thread::spawn(move || {
                                let mut stream = stream;
                                stream.set_nonblocking(false).unwrap();
                                let (head, body) = read_request(&mut stream);
                                if head.starts_with("HEAD ") {
                                    write_json(&mut stream, "{}");
                                } else if head.contains("_search") {
                                    let parsed =
                                        serde_json::from_slice(&body).unwrap_or(Value::Null);
                                    bodies_one.lock().unwrap().push(parsed);
                                    let answer = {
                                        let mut scripted = script_one.lock().unwrap();
                                        if scripted.len() > 1 {
                                            scripted.pop_front().unwrap()
                                        } else {
                                            scripted
                                                .front()
                                                .cloned()
                                                .unwrap_or_else(|| "{}".to_string())
                                        }
                                    };
                                    write_json(&mut stream, &answer);
                                } else {
                                    write_json(&mut stream, "{}");
                                }
                            }));
                        }
                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
                            std::thread::sleep(Duration::from_millis(10));
                        }
                        Err(_) => break,
                    }
                }
                for handler in handlers {
                    let _ = handler.join();
                }
            });
            ScriptedSearch {
                address,
                search_bodies,
                stop,
                server: Some(server),
            }
        }

        fn searches(&self) -> Vec<Value> {
            self.search_bodies.lock().unwrap().clone()
        }

        fn es(&self) -> crate::esclient::Es {
            crate::esclient::Es::new(&format!("http://{}", self.address), None).unwrap()
        }

        fn shutdown(&mut self) {
            self.stop.store(true, std::sync::atomic::Ordering::SeqCst);
            // Unblock `accept` if it is parked.
            let _ = std::net::TcpStream::connect(self.address);
            if let Some(server) = self.server.take() {
                server.join().unwrap();
            }
        }
    }

    impl Drop for ScriptedSearch {
        fn drop(&mut self) {
            self.shutdown();
        }
    }

    /// #2405: a `timed_out` window used to exit 1 on the spot. The window
    /// is re-asked — with the page timeout the body lacked — and the second
    /// answer counts.
    #[test]
    fn finalize_verify_window_retries_a_timed_out_answer() {
        let mut server = ScriptedSearch::start(&[
            r#"{"timed_out":true,"hits":{"hits":[]}}"#,
            r#"{"aggregations":{"values":{"buckets":[{"key":"digest-a","doc_count":3}]}}}"#,
        ]);
        let es = server.es();
        let dir = tempfile::tempdir().unwrap();
        let pr = crate::progress::Progress::silent();
        let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);
        let counts = backend
            .windowed_value_counts(
                "xc-retry",
                "ax_file",
                &["digest-a".into(), "digest-b".into()],
                &[],
                false,
            )
            .unwrap();
        assert_eq!(counts.get("digest-a"), Some(&3));
        assert_eq!(counts.get("digest-b"), Some(&0));
        let searches = server.searches();
        assert_eq!(
            searches.len(),
            2,
            "one retry, not a page storm: {searches:?}"
        );
        for body in &searches {
            assert_eq!(
                body.get("timeout").and_then(Value::as_str),
                Some("120s"),
                "the retried window must carry the page timeout: {body}"
            );
        }
        server.shutdown();
    }

    /// The retry is bounded: a node that answers every attempt partial
    /// gets the loud error, not an infinite wait, and the window was never
    /// counted.
    #[test]
    fn finalize_verify_window_fails_loud_when_every_attempt_times_out() {
        let mut server = ScriptedSearch::start(&[r#"{"timed_out":true,"hits":{"hits":[]}}"#]);
        let es = server.es();
        let dir = tempfile::tempdir().unwrap();
        let pr = crate::progress::Progress::silent();
        let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);
        let error = backend
            .windowed_value_counts("xc-wedged", "ax_file", &["digest-a".into()], &[], false)
            .unwrap_err()
            .to_string();
        assert!(error.contains("xc-wedged"), "{error}");
        assert!(error.contains("timed_out"), "{error}");
        assert!(error.contains("consecutive attempts"), "{error}");
        assert!(error.contains("rerun the same command"), "{error}");
        assert_eq!(
            server.searches().len(),
            crate::esclient::PARTIAL_PAGE_ATTEMPTS,
        );
        server.shutdown();
    }

    /// The catalog read-back's `search_page` leg retries the same way.
    #[test]
    fn catalog_read_back_window_retries_a_timed_out_page() {
        let mut server = ScriptedSearch::start(&[
            r#"{"timed_out":true,"hits":{"hits":[]}}"#,
            r#"{"timed_out":false,"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_id":"x","_source":{"file_key":"k1"}}]}}"#,
        ]);
        let es = server.es();
        let dir = tempfile::tempdir().unwrap();
        let pr = crate::progress::Progress::silent();
        let backend = EsSyncBackend::new(&es, dir.path(), 8 * 1024 * 1024, &pr);
        let counts = backend
            .windowed_catalog_counts(&["k1".into()], "run-1")
            .unwrap();
        assert_eq!(counts.get("k1"), Some(&1));
        assert_eq!(server.searches().len(), 2);
        server.shutdown();
    }

    #[test]
    fn a_zero_exists_total_skips_the_per_digest_windows() {
        // #1267 measured 2,619 term+exists probes, each a brute scan with zero
        // hits. After the windowed verify those are three windows of 1,024.
        // A field that matches nothing issues none of them.
        let skipped = plan_exists_probes(2619, EsSyncBackend::VERIFY_WINDOW, 0);
        assert_eq!(skipped.issue_windows, 0);
        assert_eq!(skipped.skip_windows, 3);
        let issued = plan_exists_probes(2619, EsSyncBackend::VERIFY_WINDOW, 1);
        assert_eq!(issued.issue_windows, 3);
        assert_eq!(issued.skip_windows, 0);
        assert_eq!(
            plan_exists_probes(0, EsSyncBackend::VERIFY_WINDOW, 0),
            ExistsProbePlan {
                issue_windows: 0,
                skip_windows: 0
            }
        );
        assert_eq!(plan_exists_probes(5, 0, 0).issue_windows, 0);
    }

    #[test]
    fn skipped_exists_windows_advance_the_verify_phase() {
        let (pr, buffer) = crate::progress::Progress::capture(
            crate::progress::Surface::Plain,
            std::time::Duration::from_millis(40),
        );
        let ticker = pr.spawn_ticker();
        let plan = plan_exists_probes(2619, EsSyncBackend::VERIFY_WINDOW, 0);
        pr.phase("finalize-verify", plan.skip_windows, 0);
        credit_verify_items(&pr, plan.skip_windows);

        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
        loop {
            let out = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
            if out.contains("phase=finalize-verify") && out.contains("items=3/3") {
                break;
            }
            assert!(
                std::time::Instant::now() < deadline,
                "the ticker never showed the skipped windows advance to items=3/3: {out}"
            );
            std::thread::yield_now();
        }
        drop(ticker);
    }

    #[test]
    fn an_unchanged_catalog_source_is_not_rewritten() {
        let same = serde_json::json!({"run_id": "g1", "path": "a"});
        let expected = BTreeMap::from([
            ("file:same".to_owned(), same.clone()),
            (
                "file:changed".to_owned(),
                serde_json::json!({"run_id": "g2", "path": "b"}),
            ),
            (
                "file:missing".to_owned(),
                serde_json::json!({"run_id": "g2", "path": "c"}),
            ),
        ]);
        let live = BTreeMap::from([
            ("file:same".to_owned(), same),
            (
                "file:changed".to_owned(),
                serde_json::json!({"run_id": "g1", "path": "b"}),
            ),
        ]);
        let write = catalog_docs_to_write(&expected, &live);
        assert!(
            !write.contains_key("file:same"),
            "a resume that already stored this projection must not bulk it again"
        );
        assert_eq!(write["file:changed"]["run_id"], "g2");
        assert!(write.contains_key("file:missing"));
        assert_eq!(write.len(), 2);
    }

    #[test]
    fn the_prior_run_remainder_is_the_kept_intersection() {
        let prior = BTreeSet::from([
            "kept".to_owned(),
            "rewritten".to_owned(),
            "not-kept".to_owned(),
        ]);
        let kept = BTreeSet::from(["kept".to_owned(), "born-kept".to_owned()]);
        assert_eq!(prior_run_expected_remainder(&prior, &kept), 1);
    }

    #[test]
    fn catalog_run_ids_round_trip_and_a_missing_file_is_an_upgrade() {
        let dir = tempfile::tempdir().unwrap();
        let snap = dir.path().join("sync-snapshots").join("tx-1267");
        std::fs::create_dir_all(&snap).unwrap();
        assert!(
            read_catalog_run_ids(dir.path(), "tx-1267")
                .unwrap()
                .is_none(),
            "a generation published before this file existed walks once"
        );
        let ids = ["b".to_owned(), "a".to_owned()];
        write_catalog_run_ids(dir.path(), "tx-1267", [&ids[0], &ids[1]]).unwrap();
        assert_eq!(
            read_catalog_run_ids(dir.path(), "tx-1267")
                .unwrap()
                .unwrap(),
            BTreeSet::from(["a".to_owned(), "b".to_owned()])
        );
        let bad = "a\nb".to_owned();
        let err = write_catalog_run_ids(dir.path(), "tx-1267", [&bad]).unwrap_err();
        assert!(
            err.to_string().contains("newline"),
            "a newline id must not split into two lines: {err}"
        );
        assert!(read_catalog_run_ids(dir.path(), "../tx").is_err());
    }

    #[test]
    fn mget_sources_keep_found_documents_and_reject_a_duplicate_id() {
        let body = serde_json::json!({"docs": [
            {"_id": "a", "found": true, "_source": {"k": 1}},
            {"_id": "b", "found": false}
        ]});
        let got = mget_sources(&body).unwrap();
        assert_eq!(got.len(), 1);
        assert_eq!(got["a"]["k"], 1);
        let dup = serde_json::json!({"docs": [
            {"_id": "a", "found": true, "_source": {}},
            {"_id": "a", "found": true, "_source": {}}
        ]});
        let err = mget_sources(&dup).unwrap_err().to_string();
        assert!(err.contains("duplicate ID a"), "{err}");
    }
}

/// Issue #482 regression guard.
///
/// The durable-snapshot path (`--no-graph`) fsyncs four directories per seal
/// plus two per GC step. `File::open` applied to a *directory* is a Unix-only
/// idiom: on Windows it returns `ERROR_ACCESS_DENIED` (os error 5) for every
/// call, because `std` cannot pass `FILE_FLAG_BACKUP_SEMANTICS`. Each of
/// those sites was therefore an unconditional abort of the whole run on that
/// platform. The engine had already learned this once —
/// `xerj_common::fsio::fsync_dir` carries the `#[cfg(windows)]` shim and the
/// write-up — and this module re-derived the broken form anyway.
///
/// The fix is `#[cfg(windows)]`-shaped, so no behavioural test compiled for
/// Unix can tell fixed code from unfixed. The behavioural coverage is the
/// windows-latest `autoindex-fd-smoke` job, which since this change runs the
/// reported `--no-graph` flow. This guard is what keeps the idiom from coming
/// back on the platform the unit suite is never compiled for.
#[cfg(test)]
mod windows_directory_open_guard {
    /// The body of the `fn` whose signature line starts with `header`,
    /// brace-matched out of this file's own source at compile time.
    fn function_body(source: &str, header: &str) -> String {
        let start = source
            .find(header)
            .unwrap_or_else(|| panic!("signature moved: {header}"))
            + header.len();
        let mut depth = 1usize;
        for (offset, ch) in source[start..].char_indices() {
            match ch {
                '{' => depth += 1,
                '}' => {
                    depth -= 1;
                    if depth == 0 {
                        return source[start..start + offset].to_string();
                    }
                }
                _ => {}
            }
        }
        panic!("unbalanced braces after {header}");
    }

    /// Every directory this module makes durable, named by the binding the
    /// code passes around. None of them may reach `File::open`.
    const DIRECTORY_BINDINGS: [&str; 6] = [
        "&root",
        "&staging",
        "&blobs",
        "&prepared_dir",
        "&final_dir",
        "&snapshot_dir",
    ];

    #[test]
    fn no_directory_is_opened_as_a_file() {
        const SRC: &str = include_str!("sync_executor.rs");
        for binding in DIRECTORY_BINDINGS {
            // Assembled, never written out literally, so this test's own
            // source can never satisfy the search it performs.
            let forbidden = format!("File::open({binding})");
            assert!(
                !SRC.contains(&forbidden),
                "#482: `{forbidden}` opens a directory as a file. That call returns \
                 ERROR_ACCESS_DENIED (os error 5) on Windows every single time and \
                 aborts the run. Use `sync_dir`, which routes through \
                 `xerj_common::fsio::fsync_dir`."
            );
        }
    }

    #[test]
    fn sync_dir_delegates_to_the_platform_aware_shim() {
        const SRC: &str = include_str!("sync_executor.rs");
        let body = function_body(SRC, "fn sync_dir(path: &Path) -> Result<()> {");
        assert!(
            body.contains("xerj_common::fsio::fsync_dir(path)"),
            "#482: `sync_dir` must delegate to `xerj_common::fsio::fsync_dir`, whose \
             Windows body is a documented no-op; body was:\n{body}"
        );
        assert!(
            !body.contains("File::open"),
            "#482: `sync_dir` re-derived the Unix-only `File::open(dir).sync_all()` \
             idiom, which fails with os error 5 on every Windows call; body was:\n{body}"
        );
    }
}