Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

131 KB
Raw Blame
//! The cache engine: ties documents, normalization, the hot tier, storage
//! and dependency tracking together behind the API the hosts (wasm worker /
//! Tauri) expose over RPC.

mod calendar;
#[cfg(test)]
mod test;

pub mod live_query;
pub mod watch_query;
mod window_observation;

use crate::calendar::{CalendarError, EVENT_TYPENAME};
use crate::denormalize::{DenormalizeError, ReadOutcome, ReadPlans, ReadSession, RecordSource};
use crate::deps::{
    DepIndex, OpId, QueryDependencies, ViewerFieldUpdate, ViewerFields, changed_viewer_fields,
};
use crate::document::{Document, DocumentError, OperationKind};
use crate::entity_resolver::{EntityResolver, EntityResolverError, EntityResolverLookup};
use crate::identity::{self, IdentityBinding, IdentityMap};
use crate::link_patch::{
    LinkPatchError, OptimisticLinkPatch, QueryRevalidation, apply_link_patches,
    deduplicate_patches, missing_patch_records,
};
use crate::normalize::{
    DependencyCompleteness, NormalizeError, RecordUpdates, normalize, normalize_with_dependencies,
    project_hydration_response,
};
use crate::predicate::reconciliation::{
    MAX_RECONCILIATION_BASELINE, PredicateBaselineEntry, PredicateReconciliation,
};
use crate::predicate::{
    OptimisticShadowReconciliation, OptimisticUpsertReconciliation, PredicateIndexStorage,
    PredicateQueryResult, ProjectionMutation, ProjectionMutationLayer, ProjectionState,
    StagedOptimisticProjection, StagedOptimisticProjectionOwner, apply_authoritative_exact_members,
    apply_authoritative_projection_mutations, apply_authoritative_projection_patch,
    compose_effective_optimistic_projection,
};
use crate::query_inspection::{
    CachedQueryInstance, CachedQueryVariant, OwnerResolution, QueryInspection,
    QueryInspectionError, matches_variable_filters, prepare, recover_variants, resolve_owner,
    selected_result_value,
};
use crate::queue::{
    ClaimedMutation, MutationClaimRequest, MutationClaimToken, MutationId, MutationQueueSnapshot,
    MutationRequest, MutationUpsertKind, NewQueuedMutation, OptimisticSource,
    PersistedOptimisticLayer, QueuedMutation, StoredMutation, decode_optimistic_source,
    encode_optimistic_source,
};
use crate::record_selection::{
    MAX_RECORD_SELECTION_KEYS, RecordSelection, RecordSelectionError, SelectedRecord,
};
use crate::revision::{CacheRevision, Revisioned};
use crate::search::{
    SearchCatalogs, SearchCursor, SearchDocument, SearchError, SearchPage, SearchProfile,
    SearchRequest, collect_search_changes, project_search_documents, rank_documents,
    snapshot_search_fields, validate_search_request,
};
use crate::store::{QueueDiagnostics, Storage};
use crate::value::{EntityKey, Record, canonical_json};
use lru::LruCache;
use predicate_index::{
    OptimisticProjectionMutation, RecordKey as PredicateRecordKey, ValidatedIndexQuery,
};
use serde_json::Value as Json;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::num::NonZeroUsize;
use thiserror::Error;
use uuid::Uuid;

#[derive(Debug, Error)]
pub enum EngineError<S: std::error::Error + 'static> {
    #[error(transparent)]
    Document(#[from] DocumentError),
    #[error(transparent)]
    Normalize(#[from] NormalizeError),
    #[error(transparent)]
    Denormalize(#[from] DenormalizeError),
    #[error(transparent)]
    EntityResolver(#[from] EntityResolverError),
    #[error(transparent)]
    LinkPatch(#[from] LinkPatchError),
    #[error(transparent)]
    QueryInspection(#[from] QueryInspectionError),
    #[error(transparent)]
    RecordSelection(#[from] RecordSelectionError),
    #[error(transparent)]
    Search(#[from] SearchError),
    #[error(transparent)]
    Calendar(#[from] CalendarError),
    #[error("unknown or already-settled optimistic transaction {0}")]
    UnknownTransaction(OptimisticTransactionId),
    #[error("stale claim for optimistic transaction {0}")]
    StaleMutationClaim(OptimisticTransactionId),
    #[error("invalid queued mutation {id}: {detail}")]
    InvalidQueuedMutation {
        id: OptimisticTransactionId,
        detail: String,
    },
    #[error("invalid optimistic projection: {0}")]
    InvalidOptimisticProjection(String),
    /// Identity resolution and normalization failed; the mutation was durably discarded.
    #[error("invalid optimistic identity: {}", .0.error)]
    IdentityResolutionFailed(Box<FailedMutationResult>),
    #[error("invalid optimistic mutation UUID `{0}`")]
    InvalidMutationUuid(String),
    #[error("durable optimistic queue changed while staging UUID upsert")]
    StaleOptimisticUpsert,
    #[error("storage: {0}")]
    Storage(#[source] S),
    #[error("cache revision overflow")]
    RevisionOverflow,
}

/// Result of a cache read.
#[derive(Debug)]
pub enum ReadResult {
    /// Fully answerable from cache.
    Hit { data: Json },
    /// Not answerable; forward to the network.
    Miss,
}

/// Borrowed inputs for one network response write.
#[derive(Debug, Clone, Copy)]
pub struct NetworkWrite<'a> {
    /// GraphQL operation document.
    pub query: &'a str,
    /// Selected operation name.
    pub operation_name: Option<&'a str>,
    /// Resolved operation variables.
    pub variables: &'a serde_json::Map<String, Json>,
    /// GraphQL response data.
    pub data: &'a Json,
    /// Optional opaque session identity witness.
    pub identity: Option<&'a str>,
}

/// An active query whose dependencies should be installed by a network write.
#[derive(Debug, Clone, Copy)]
pub struct QueryRegistration<'a> {
    /// Host-scoped active operation id.
    pub op_id: OpId,
    /// Read-only relations that change which normalized entities the query uses.
    pub entity_resolvers: &'a [EntityResolver],
}

/// Result of writing a network response.
#[derive(Debug)]
pub struct WriteResult {
    /// Effective scalar changes; absent when consumers must re-read queries.
    pub field_changes: Option<Vec<crate::field_changes::RecordFieldChange>>,
    /// Identity bindings omitted while committing otherwise normalizable server data.
    pub identity_errors: Vec<String>,
    /// Revision installed after this logical cache mutation.
    pub revision: CacheRevision,
    /// Whether this write advanced [`Self::revision`].
    pub revision_advanced: bool,
    /// Search buckets changed by an ordinary query response. `None` keeps
    /// mutation/reset notifications conservative; an empty set proves no change.
    pub search_changed_buckets: Option<BTreeSet<String>>,
    /// Records whose contents changed.
    pub changed: BTreeSet<EntityKey<'static>>,
    /// Active operations depending on changed records (host re-executes
    /// these). Excludes the operation that performed the write. After a
    /// `reset` this is *every* active operation except the origin.
    pub affected_ops: BTreeSet<OpId>,
    /// True when the identity witness observed a different user than the one
    /// bound to this cache: all previous state was wiped (silent restart)
    /// before this response was written. Hosts must broadcast this to other
    /// engine instances sharing the same storage.
    pub reset: bool,
    /// Queries that should be revalidated after terminal settlement.
    pub revalidations: Vec<QueryRevalidation>,
    /// Stable caller identity of a settled mutation.
    pub mutation_uuid: Option<String>,
}

/// Internal durable deltas; viewer field proof is never sent across host boundaries.
struct PersistedChanges {
    field_changes: Vec<crate::field_changes::RecordFieldChange>,
    changed: BTreeSet<EntityKey<'static>>,
    revision: CacheRevision,
    revision_advanced: bool,
    search_changed_buckets: BTreeSet<String>,
    viewer_fields: ViewerFields,
}

/// Result of hydrating a query while returning only non-`@cacheOnly` fields.
#[derive(Debug)]
pub struct HydrationWriteResult {
    /// Quick Access buckets whose searchable or materialized fields changed.
    /// An empty set proves that search-backed consumers need no refresh.
    pub search_changed_buckets: BTreeSet<String>,
    /// Cache changes used by hosts for invalidation fan-out.
    pub write_result: WriteResult,
    /// Small caller-visible projection, or `None` when every field is cache-only.
    pub data: Option<Json>,
}

/// Outcome of the initial strict-head claim attempted after enqueue.
#[derive(Debug)]
pub enum InitialClaimOutcome<E> {
    /// The strict queue head was runnable and is now durably leased.
    Claimed(Box<ClaimedMutation>),
    /// The strict queue head is leased, deferred, or the queue is empty.
    NotRunnable,
    /// Enqueue succeeded, but attempting to claim the strict head failed.
    Failed(E),
}

/// Result of durably enqueueing an optimistic mutation and attempting its
/// initial strict-head claim.
#[derive(Debug)]
pub struct EnqueueOptimisticMutationResult<E> {
    /// Engine-assigned id of the newly enqueued optimistic mutation.
    pub transaction_id: OptimisticTransactionId,
    /// How the caller UUID changed the queue.
    pub upsert_kind: MutationUpsertKind,
    /// Visible cache changes caused by the newly published optimistic layer.
    pub write_result: WriteResult,
    /// Outcome of the claim attempt made before hosts publish cache changes.
    pub initial_claim: InitialClaimOutcome<E>,
}

/// Cache change and replacement identity produced by settling superseded work.
#[derive(Debug)]
pub struct SupersededMutationResult {
    /// Cache changes caused by settling the superseded layer.
    pub write_result: WriteResult,
    /// Current transaction carrying the caller's newer intent.
    pub replacement_transaction_id: OptimisticTransactionId,
}

/// Tagged commit result used by transport adapters for settlement fanout.
#[derive(Debug)]
pub enum CommitOptimisticWriteResult {
    /// A current mutation committed normally.
    Committed(WriteResult),
    /// The response committed beneath a newer optimistic replacement.
    CommittedSuperseded(SupersededMutationResult),
    /// Invalid identity data and an unnormalizable response permanently failed this attempt.
    Failed(FailedMutationResult),
}

/// Cache changes and diagnostics from a permanently failed commit attempt.
#[derive(Debug)]
pub struct FailedMutationResult {
    /// Changes caused by removing the failed optimistic layer.
    pub write_result: WriteResult,
    /// Identity-resolution diagnostic for the host's cache error handler.
    pub error: String,
    /// Newer intent preserved when the failed attempt was superseded.
    pub replacement_transaction_id: Option<OptimisticTransactionId>,
}

/// Result of attempting to defer a failed queue attempt.
#[derive(Debug)]
pub enum DeferOptimisticWriteResult {
    /// The current mutation retained its optimistic layer and retry state.
    Deferred,
    /// The attempt was superseded, so it was discarded instead of retried.
    DiscardedSuperseded(SupersededMutationResult),
}

/// Tagged rollback result used by transport adapters for settlement fanout.
#[derive(Debug)]
pub enum RollbackOptimisticWriteResult {
    /// A current mutation permanently failed normally.
    RolledBack(WriteResult),
    /// A superseded attempt was accepted and discarded.
    DiscardedSuperseded(SupersededMutationResult),
}

/// Borrowed inputs for atomically beginning one optimistic mutation.
pub struct BeginOptimisticWrite<'a> {
    /// Opaque durable client correlation, never sent to the server.
    pub client_metadata: Option<&'a Json>,
    /// Local entities whose IDs must be resolved from the committed response.
    pub identity_bindings: &'a [IdentityBinding],
    /// Caller-supplied RFC UUID used only for safe coalescing.
    pub uuid: &'a str,
    /// GraphQL mutation document.
    pub query: &'a str,
    /// Selected operation name.
    pub operation_name: Option<&'a str>,
    /// Mutation variables.
    pub variables: &'a serde_json::Map<String, Json>,
    /// Optimistic mutation response.
    pub data: &'a Json,
    /// Ordered constrained relation recipes.
    pub link_patches: &'a [OptimisticLinkPatch],
    /// Revalidations for relevant fields that could not be patched.
    pub revalidations: &'a [QueryRevalidation],
    /// Wall-clock enqueue timestamp.
    pub created_at_ms: i64,
}

/// Engine-assigned id of one optimistic mutation transaction. Never reuse
/// host operation keys: identical concurrent mutations share an urql key,
/// but each needs its own layer.
pub type OptimisticTransactionId = MutationId;

/// One queued optimistic mutation's contribution to the cache view. Layers
/// are persisted and ordered by their mutation ids; only the strict queue
/// head can be claimed and settled.
#[derive(Clone)]
struct OptimisticLayer {
    identity_bindings: Vec<IdentityBinding>,
    identity_keys: BTreeSet<EntityKey<'static>>,
    id: OptimisticTransactionId,
    uuid: Uuid,
    superseded: bool,
    updates: RecordUpdates,
    link_patches: Vec<OptimisticLinkPatch>,
    revalidations: Vec<QueryRevalidation>,
    projection_mutations: Vec<OptimisticProjectionMutation>,
    uncertain_calendar_event_keys: Vec<EntityKey<'static>>,
}

struct BegunOptimisticWrite {
    transaction_id: OptimisticTransactionId,
    upsert_kind: MutationUpsertKind,
    write_result: WriteResult,
}

/// Reserved storage key holding the identity bound to this cache. The
/// `__meta:` prefix can never collide with entity keys (typenames can't
/// contain `:`).
const IDENTITY_META_KEY: &str = "__meta:identity";
const IDENTITY_VALUE_FIELD: &str = "identity";
const STORAGE_GENERATION_META_KEY: &str = "__meta:storage-generation";
const STORAGE_GENERATION_VALUE_FIELD: &str = "generation";

/// Hydration/binding state of the session identity tag for this cache.
#[derive(Debug, Clone, PartialEq, Eq)]
enum IdentityState {
    /// Not yet loaded from storage.
    NotHydrated,
    /// Hydrated: no identity has been bound to this cache yet.
    Missing,
    /// Hydrated: bound to this identity. (Named `Bound` rather than `Some`
    /// to avoid shadowing/confusion with `Option::Some` in matches.)
    Bound(String),
}

/// Default hot-tier capacity (records, not bytes — byte budgets are a
/// hardening-phase refinement).
pub const DEFAULT_HOT_CAPACITY: usize = 10_000;

/// Bound on the uncertain calendar events one optimistic layer may declare.
const MAX_UNCERTAIN_CALENDAR_EVENTS: usize = 256;

// Parsed plans contain no cached user data. Bound dynamic document churn while
// leaving room for the production operation catalog and fragment variants.
const DOCUMENT_CACHE_CAPACITY: usize = 128;

pub struct Engine<S: Storage> {
    schema: std::sync::Arc<crate::meta::Schema>,
    live_queries: live_query::LiveQueries,
    window_observations: window_observation::WindowObservations,
    query_watches: watch_query::QueryWatches,
    storage: S,
    revision: CacheRevision,
    hot: LruCache<EntityKey<'static>, Record>,
    docs: LruCache<String, Document>,
    deps: DepIndex,
    identity: IdentityState,
    /// Ordered optimistic mutation layers hydrated from durable storage.
    optimistic: Vec<OptimisticLayer>,
    optimistic_hydrated: bool,
    /// Compact durable catalogs are loaded lazily for text search. Empty
    /// queries use the storage index directly and do not populate this map.
    search_catalogs: SearchCatalogs,
}

impl<S: Storage> Engine<S> {
    pub fn new(storage: S) -> Self {
        Self::with_capacity(storage, DEFAULT_HOT_CAPACITY)
    }

    pub fn with_capacity(storage: S, hot_capacity: usize) -> Self {
        Self::with_schema(storage, hot_capacity, crate::meta::bundled_schema())
    }

    /// Creates an engine with immutable metadata supplied by its host.
    pub fn with_schema(
        storage: S,
        hot_capacity: usize,
        schema: std::sync::Arc<crate::meta::Schema>,
    ) -> Self {
        Engine {
            schema,
            live_queries: live_query::LiveQueries::default(),
            window_observations: window_observation::WindowObservations::default(),
            query_watches: watch_query::QueryWatches::default(),
            storage,
            revision: CacheRevision::ZERO,
            hot: LruCache::new(NonZeroUsize::new(hot_capacity).expect("capacity > 0")),
            docs: LruCache::new(NonZeroUsize::new(DOCUMENT_CACHE_CAPACITY).unwrap()),
            deps: DepIndex::new(),
            identity: IdentityState::NotHydrated,
            optimistic: Vec::new(),
            optimistic_hydrated: false,
            search_catalogs: SearchCatalogs::default(),
        }
    }

    /// Returns the current effective-view revision of this engine generation.
    pub fn current_revision(&self) -> CacheRevision {
        self.revision
    }

    /// Installs a compatible superset without discarding records or queued mutations.
    pub fn install_schema(
        &mut self,
        schema: &crate::meta::Schema,
    ) -> Result<(), crate::meta::SchemaError> {
        self.schema = self.schema.merge(schema)?;
        Ok(())
    }

    /// Shared immutable snapshot for host plan validation.
    pub fn schema_snapshot(&self) -> std::sync::Arc<crate::meta::Schema> {
        std::sync::Arc::clone(&self.schema)
    }

    /// Metadata used for every operation in this engine instance.
    pub fn schema(&self) -> &crate::meta::Schema {
        &self.schema
    }

    /// Returns the durable identity of the currently stored cache data.
    ///
    /// A fresh generation is installed when the marker is absent or malformed.
    /// Existing records are preserved when upgrading a cache without this marker.
    /// Logical clears, identity changes, and physical resets
    /// remove the marker with the data, so old hydration checkpoints cannot resume.
    /// Read storage each time: engine replacement and external resets must not
    /// leave an in-memory generation that outlives its records.
    pub async fn current_storage_generation(&mut self) -> Result<Uuid, EngineError<S::Error>> {
        let key = EntityKey(STORAGE_GENERATION_META_KEY.into());
        let fetched = self
            .storage
            .get_batch(std::slice::from_ref(&key))
            .await
            .map_err(EngineError::Storage)?;
        if let Some(record) = fetched.into_iter().next().flatten()
            && let Some(crate::value::CacheValue::String(value)) =
                record.fields.get(STORAGE_GENERATION_VALUE_FIELD)
            && let Ok(generation) = Uuid::parse_str(value)
        {
            return Ok(generation);
        }

        let generation = Uuid::now_v7();
        let mut record = Record::default();
        record.fields.insert(
            STORAGE_GENERATION_VALUE_FIELD.to_string(),
            crate::value::CacheValue::String(generation.to_string()),
        );
        self.storage
            .put_batch(vec![(key, record)])
            .await
            .map_err(EngineError::Storage)?;
        Ok(generation)
    }

    fn ensure_revision_can_advance(&self) -> Result<(), EngineError<S::Error>> {
        self.revision
            .checked_successor()
            .map(|_| ())
            .ok_or(EngineError::RevisionOverflow)
    }

    fn advance_revision(&mut self) -> Result<CacheRevision, EngineError<S::Error>> {
        let next = self
            .revision
            .checked_successor()
            .ok_or(EngineError::RevisionOverflow)?;
        self.revision = next;
        self.live_queries.advance(next);
        Ok(next)
    }

    fn revisioned<T>(&self, value: T) -> Revisioned<T> {
        Revisioned {
            revision: self.revision,
            value,
        }
    }

    /// Hydrates durable optimistic layers before the first operation. Queue
    /// order is the optimistic composition order. Relation recipes are
    /// reconstructed against the durable base and preceding layers.
    async fn hydrate_optimistic(&mut self) -> Result<(), EngineError<S::Error>> {
        if self.optimistic_hydrated {
            return Ok(());
        }
        let queued = self
            .storage
            .load_mutation_queue()
            .await
            .map_err(EngineError::Storage)?;
        self.optimistic = self.rebuild_queued_layers(queued).await?;
        self.optimistic_hydrated = true;
        Ok(())
    }

    /// Reloads durable optimistic layers after another engine sharing this
    /// storage changes the queue. Returns operations whose effective view
    /// changed.
    pub async fn refresh_optimistic_queue(&mut self) -> Result<WriteResult, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        let mut live_projection_keys = live_query::projection_keys(&self.optimistic);
        let queued = self
            .storage
            .load_mutation_queue()
            .await
            .map_err(EngineError::Storage)?;
        let old_candidates = layer_keys(&self.optimistic);
        let old_bases = self.load_bases(&old_candidates).await?;
        let before = effective_records(&old_bases, &self.optimistic, &old_candidates);
        let replacement = self.rebuild_queued_layers(queued).await?;
        let mut candidates = old_candidates;
        candidates.extend(layer_keys(&replacement));
        let bases = self.load_bases(&candidates).await?;
        let mut before_all = effective_records(&bases, &self.optimistic, &candidates);
        before_all.extend(before);
        let after = effective_records(&bases, &replacement, &candidates);
        self.optimistic = replacement;
        live_projection_keys.extend(live_query::projection_keys(&self.optimistic));
        let live_records = candidates.clone();
        self.optimistic_hydrated = true;
        let changed: BTreeSet<EntityKey<'static>> = candidates
            .into_iter()
            .filter(|key| before_all.get(key) != after.get(key))
            .collect();
        let affected_ops = self.deps.ops_for_keys(changed.iter());
        let revision = self.advance_revision()?;
        self.live_queries.record(
            revision,
            live_query::Changes {
                records: live_records,
                projections: live_projection_keys,
            },
        );
        Ok(WriteResult {
            field_changes: Some(crate::field_changes::between(&before_all, &after)),
            identity_errors: Vec::new(),
            revision,
            revision_advanced: true,
            search_changed_buckets: None,
            changed,
            affected_ops,
            reset: false,
            revalidations: Vec::new(),
            mutation_uuid: None,
        })
    }

    async fn rebuild_queued_layers(
        &mut self,
        queued: Vec<QueuedMutation>,
    ) -> Result<Vec<OptimisticLayer>, EngineError<S::Error>> {
        self.rebuild_queued_layers_with_strict_tail(queued, None)
            .await
    }

    async fn rebuild_queued_layers_with_strict_tail(
        &mut self,
        queued: Vec<QueuedMutation>,
        strict_tail: Option<MutationId>,
    ) -> Result<Vec<OptimisticLayer>, EngineError<S::Error>> {
        self.rebuild_queued_layers_with_staged(queued, strict_tail, &RecordUpdates::new())
            .await
    }

    /// Prepare the visible layers against a proposed durable write. All reads
    /// and recipe validation finish before that write can commit.
    async fn rebuild_queued_layers_with_staged(
        &mut self,
        queued: Vec<QueuedMutation>,
        strict_tail: Option<MutationId>,
        staged: &RecordUpdates,
    ) -> Result<Vec<OptimisticLayer>, EngineError<S::Error>> {
        let mut layers = Vec::with_capacity(queued.len());
        for queued in queued {
            let variables: Json = serde_json::from_str(&queued.mutation.request.variables_json)
                .map_err(|error| EngineError::InvalidQueuedMutation {
                    id: queued.id,
                    detail: format!("invalid variables: {error}"),
                })?;
            let Json::Object(variables) = variables else {
                return Err(EngineError::InvalidQueuedMutation {
                    id: queued.id,
                    detail: "variables are not an object".to_string(),
                });
            };
            let mut source = decode_optimistic_source(&queued.optimistic.optimistic_data_json)
                .map_err(|detail| EngineError::InvalidQueuedMutation {
                    id: queued.id,
                    detail: format!("invalid optimistic response: {detail}"),
                })?;
            let identities = self
                .load_identity_bindings(&source.identity_bindings, staged)
                .await?;
            for patch in &mut source.link_patches {
                identity::remap_patch(patch, &source.identity_bindings, &identities);
            }
            for revalidation in &mut source.revalidations {
                identity::remap_variables(
                    &mut revalidation.variables_json,
                    &source.identity_bindings,
                    &identities,
                );
            }
            for projection in &mut source.projection_mutations {
                identity::remap_projection(projection, &identities);
            }
            let document = Self::document(&mut self.docs, &queued.mutation.request.query)?;
            let operation =
                document.operation(queued.mutation.request.operation_name.as_deref())?;
            let mut updates = identity::remap_updates(
                normalize(&self.schema, operation, &variables, &source.mutation_data)?,
                &source.identity_bindings,
                &identities,
            );
            identity::apply_record_lifecycle(&mut updates, &source.identity_bindings, &identities);
            let patches =
                deduplicate_patches(&self.schema, &source.link_patches).map_err(|error| {
                    EngineError::InvalidQueuedMutation {
                        id: queued.id,
                        detail: error.to_string(),
                    }
                })?;
            let candidates: BTreeSet<EntityKey<'static>> = updates.keys().cloned().collect();
            let (candidates, bases) = self
                .load_link_patch_bases(candidates, &layers, &updates, &patches, staged)
                .await?;
            let composed = effective_records(&bases, &layers, &candidates);
            let mut effective = present_records(composed);
            merge_updates_into_effective(&mut effective, &updates);
            // Missing query fields after a format wipe are intentionally not
            // recreated from stale recipes during hydration. A newly submitted
            // tail remains strict so invalid caller patches cannot be persisted.
            apply_link_patches(
                &self.schema,
                &mut effective,
                &mut updates,
                &patches,
                strict_tail != Some(queued.id),
            )?;
            let revalidations = deduplicate_revalidations(
                source.revalidations.iter().cloned().chain(
                    source
                        .link_patches
                        .iter()
                        .filter_map(OptimisticLinkPatch::revalidation),
                ),
            );
            layers.push(OptimisticLayer {
                identity_keys: source
                    .identity_bindings
                    .iter()
                    .map(|binding| {
                        identities
                            .get(&binding.local_key)
                            .unwrap_or(&binding.local_key)
                            .clone()
                    })
                    .collect(),
                identity_bindings: source.identity_bindings,
                id: queued.id,
                uuid: queued.uuid,
                superseded: queued.superseded,
                updates,
                link_patches: patches,
                revalidations,
                projection_mutations: source.projection_mutations,
                uncertain_calendar_event_keys: source
                    .uncertain_calendar_event_keys
                    .into_iter()
                    .map(|key| identities.get(&key).cloned().unwrap_or(key))
                    .collect(),
            });
        }
        Ok(layers)
    }

    async fn load_identity_bindings(
        &self,
        bindings: &[IdentityBinding],
        staged: &RecordUpdates,
    ) -> Result<IdentityMap, EngineError<S::Error>> {
        let keys = bindings
            .iter()
            .map(|binding| binding.local_key.clone())
            .collect::<Vec<_>>();
        if keys.is_empty() {
            return Ok(IdentityMap::new());
        }
        let mut targets = keys.clone();
        let mut visited = keys
            .iter()
            .map(|key| BTreeSet::from([key.clone()]))
            .collect::<Vec<_>>();
        for _ in 0..identity::MAX_ALIAS_CHAIN_DEPTH {
            let records = self
                .storage
                .get_batch(&targets)
                .await
                .map_err(EngineError::Storage)?;
            let mut advanced = false;
            for ((target, record), visited) in targets.iter_mut().zip(records).zip(&mut visited) {
                let record = staged.get(target).or(record.as_ref());
                if let Some(next) = record.and_then(identity::alias_target) {
                    if !visited.insert(next.clone()) {
                        return Err(EngineError::InvalidOptimisticProjection(
                            "cyclic identity binding".into(),
                        ));
                    }
                    *target = next.clone();
                    advanced = true;
                }
            }
            if !advanced {
                let mut identities = keys
                    .into_iter()
                    .zip(targets)
                    .filter(|(key, target)| key != target)
                    .collect();
                identity::expand_reference_identities(bindings, &mut identities);
                return Ok(identities);
            }
        }
        Err(EngineError::InvalidOptimisticProjection(
            "identity binding chain is too deep".into(),
        ))
    }

    /// The identity binding of this cache, hydrating it from storage on
    /// first use. Never returns [`IdentityState::NotHydrated`].
    async fn bound_identity(&mut self) -> Result<IdentityState, EngineError<S::Error>> {
        if self.identity == IdentityState::NotHydrated {
            let key = EntityKey(IDENTITY_META_KEY.into());
            let fetched = self
                .storage
                .get_batch(std::slice::from_ref(&key))
                .await
                .map_err(EngineError::Storage)?;
            let stored = fetched.into_iter().next().flatten().and_then(|record| {
                match record.fields.get(IDENTITY_VALUE_FIELD) {
                    Some(crate::value::CacheValue::String(s)) => Some(s.clone()),
                    _ => None,
                }
            });
            self.identity = match stored {
                Some(identity) => IdentityState::Bound(identity),
                None => IdentityState::Missing,
            };
        }
        Ok(self.identity.clone())
    }

    /// Returns the opaque identity currently bound to this cache, hydrating
    /// it from persistent storage when necessary.
    pub async fn current_identity(&mut self) -> Result<Option<String>, EngineError<S::Error>> {
        match self.bound_identity().await? {
            IdentityState::NotHydrated => unreachable!("bound_identity hydrates"),
            IdentityState::Missing => Ok(None),
            IdentityState::Bound(identity) => Ok(Some(identity)),
        }
    }

    async fn bind_identity(&mut self, identity: &str) -> Result<(), EngineError<S::Error>> {
        let mut record = Record::default();
        record.fields.insert(
            IDENTITY_VALUE_FIELD.to_string(),
            crate::value::CacheValue::String(identity.to_string()),
        );
        self.storage
            .put_batch(vec![(EntityKey(IDENTITY_META_KEY.into()), record)])
            .await
            .map_err(EngineError::Storage)?;
        self.identity = IdentityState::Bound(identity.to_string());
        Ok(())
    }

    /// Attempts to answer a query from cache. When `op_id` is given the
    /// operation is registered as active with the dependencies it touched
    /// (hit *or* miss — a miss still re-executes when its records change).
    pub async fn read_query(
        &mut self,
        op_id: Option<OpId>,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
    ) -> Result<ReadResult, EngineError<S::Error>> {
        self.read_query_with_entity_resolvers(op_id, query, operation_name, variables, &[])
            .await
    }

    /// Attempts to answer a query while applying validated read-only entity
    /// relations. Resolver descriptors are request policy and never persisted.
    pub async fn read_query_with_entity_resolvers(
        &mut self,
        op_id: Option<OpId>,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        entity_resolvers: &[EntityResolver],
    ) -> Result<ReadResult, EngineError<S::Error>> {
        self.read_query_tracked(
            op_id,
            query,
            operation_name,
            variables,
            entity_resolvers,
            false,
        )
        .await
        .map(|(result, _)| result)
    }

    async fn read_query_tracked(
        &mut self,
        op_id: Option<OpId>,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        entity_resolvers: &[EntityResolver],
        track_projection: bool,
    ) -> Result<(ReadResult, Option<crate::denormalize::QueryProjection>), EngineError<S::Error>>
    {
        let entity_resolvers = EntityResolverLookup::compile(&self.schema, entity_resolvers)?;
        self.hydrate_optimistic().await?;
        let doc = Self::document(&mut self.docs, query)?;
        let op = doc.operation(operation_name)?.prepare(variables)?;
        if op.kind != OperationKind::Query {
            return Err(EngineError::Document(
                DocumentError::UnsupportedOperationType(format!(
                    "{:?} (cache reads are query-only)",
                    op.kind
                )),
            ));
        }

        // Effective read view = durable base + optimistic layers, composed
        // per key. `composed` holds the merged records for optimistically
        // touched keys and is never promoted into the hot tier — the durable
        // LRU base must stay free of optimistic values.
        let optimistic = merged_optimistic(&self.optimistic);
        let mut composed: HashMap<EntityKey<'static>, Record> = HashMap::new();
        for (key, update) in &optimistic {
            if let Some(base) = self.hot.peek(key) {
                let mut merged = base.clone();
                merged.merge(update.clone());
                composed.insert(key.clone(), merged);
            }
        }

        // Durable records fetched from storage this read (pre-merge — safe
        // to promote into the hot tier afterwards).
        let mut fetched_base: HashMap<EntityKey<'static>, Record> = HashMap::new();
        let mut known_absent: BTreeSet<EntityKey<'static>> = BTreeSet::new();
        let mut deps = QueryDependencies::default();

        let mut plans = ReadPlans::default();
        let mut session = ReadSession::new(
            &self.schema,
            &EntityKey::root(),
            self.schema.query_root(),
            &op.selection_set,
        );
        if track_projection {
            session.projection = Some(Default::default());
        }
        let outcome = loop {
            let source = EngineSource {
                hot: &self.hot,
                fetched: &fetched_base,
                composed: &composed,
            };
            match session.resume(variables, &source, &mut deps, &entity_resolvers, &mut plans)? {
                ReadOutcome::Complete(data) => break ReadResult::Hit { data },
                ReadOutcome::Miss { .. } => break ReadResult::Miss,
                ReadOutcome::NeedRecords(missing) => {
                    let to_fetch: Vec<EntityKey<'static>> = missing
                        .into_iter()
                        .filter(|k| {
                            !known_absent.contains(k)
                                && !fetched_base.contains_key(k)
                                && !composed.contains_key(k)
                        })
                        .collect();
                    if to_fetch.is_empty() {
                        // Everything missing is genuinely absent → miss.
                        break ReadResult::Miss;
                    }
                    let fetched = self
                        .storage
                        .get_batch(&to_fetch)
                        .await
                        .map_err(EngineError::Storage)?;
                    for (key, record) in to_fetch.into_iter().zip(fetched) {
                        match record {
                            Some(r) => {
                                if let Some(update) = optimistic.get(&key) {
                                    let mut merged = r.clone();
                                    merged.merge(update.clone());
                                    composed.insert(key.clone(), merged);
                                }
                                fetched_base.insert(key, r);
                            }
                            None => {
                                if let Some(update) = optimistic.get(&key) {
                                    // Entity exists only optimistically.
                                    composed.insert(key, update.clone());
                                } else {
                                    known_absent.insert(key);
                                }
                            }
                        }
                    }
                }
            }
        };

        // Refresh borrowed hot records before cold promotions can evict them.
        for key in &deps.records {
            let _ = self.hot.get(key);
        }
        for (key, record) in fetched_base {
            self.hot.put(key, record);
        }
        if let Some(op_id) = op_id {
            self.deps.set_query_deps(op_id, deps);
        }
        Ok((outcome, session.projection))
    }

    /// Projects a bounded explicit set of normalized entity keys through a
    /// named fragment without scanning storage. Missing, wrong-type, and
    /// incomplete records are omitted; output preserves first-occurrence key
    /// order.
    pub async fn read_records_by_keys(
        &mut self,
        selection: &RecordSelection,
        keys: &[EntityKey<'static>],
    ) -> Result<Revisioned<Vec<SelectedRecord>>, EngineError<S::Error>> {
        let (records, dependencies) = self.read_records_tracked(selection, keys).await?;
        self.bind_window_dependencies(keys, &dependencies);
        Ok(self.revisioned(records))
    }

    async fn read_records_tracked(
        &mut self,
        selection: &RecordSelection,
        keys: &[EntityKey<'static>],
    ) -> Result<(Vec<SelectedRecord>, live_query::RowDependencies), EngineError<S::Error>> {
        if keys.len() > MAX_RECORD_SELECTION_KEYS {
            return Err(RecordSelectionError::TooManyKeys {
                count: keys.len(),
                max: MAX_RECORD_SELECTION_KEYS,
            }
            .into());
        }
        if keys.iter().any(|key| {
            key.as_ref().len() > 1024
                || key.as_ref().split_once(':').is_none_or(|(typename, _)| {
                    typename.is_empty()
                        || !typename.bytes().enumerate().all(|(index, byte)| {
                            byte == b'_'
                                || byte.is_ascii_alphabetic()
                                || (index > 0 && byte.is_ascii_digit())
                        })
                })
        }) {
            return Err(RecordSelectionError::InvalidKey.into());
        }
        if keys.is_empty() {
            return Ok((Vec::new(), BTreeMap::new()));
        }
        self.hydrate_optimistic().await?;

        let selected_types: BTreeSet<_> =
            selection.type_names().iter().map(String::as_str).collect();
        let mut seen = BTreeSet::new();
        let ordered_keys: Vec<_> = keys
            .iter()
            .filter(|key| {
                key.typename()
                    .is_some_and(|name| selected_types.contains(name))
                    && seen.insert((*key).clone())
            })
            .cloned()
            .collect();
        let optimistic = merged_optimistic(&self.optimistic);
        let (projected, dependencies) = self
            .project_record_batch(selection, &ordered_keys, &optimistic)
            .await?;
        let records = projected;
        let canonical_keys = records
            .iter()
            .map(|(key, record)| {
                record
                    .get("id")
                    .and_then(Json::as_str)
                    .and_then(|id| {
                        key.typename()
                            .map(|typename| EntityKey::entity(typename, &[id]))
                    })
                    .unwrap_or_else(|| key.clone())
            })
            .collect::<Vec<_>>();
        let mutation_uuid = |record: &Record| match record.fields.get(identity::MUTATION_UUID_FIELD)
        {
            Some(crate::value::CacheValue::String(uuid)) => Some(uuid.clone()),
            _ => None,
        };
        // Projection already warmed these records. Read only identity metadata
        // from hot rows, without fetching or cloning their unselected bodies.
        let mut durable_uuids = vec![None; canonical_keys.len()];
        let mut missing = Vec::new();
        for (index, key) in canonical_keys.iter().enumerate() {
            if let Some(record) = self.hot.get(key) {
                durable_uuids[index] = mutation_uuid(record);
            } else {
                missing.push((index, key.clone()));
            }
        }
        if !missing.is_empty() {
            let keys: Vec<_> = missing.iter().map(|(_, key)| key.clone()).collect();
            let records = self
                .storage
                .get_batch(&keys)
                .await
                .map_err(EngineError::Storage)?;
            for ((index, _), record) in missing.into_iter().zip(records) {
                durable_uuids[index] = record.as_ref().and_then(mutation_uuid);
            }
        }
        let records = records
            .into_iter()
            .zip(canonical_keys)
            .zip(durable_uuids)
            .map(|(((record_key, record), canonical), durable)| {
                let pending = self
                    .optimistic
                    .iter()
                    .rev()
                    .find(|layer| layer.identity_keys.contains(&canonical));
                let mutation_uuid = pending.map(|layer| layer.uuid.to_string()).or(durable);
                SelectedRecord {
                    record_key,
                    record,
                    identity: identity::IdentityStatus {
                        mutation_uuid,
                        pending: pending.is_some(),
                    },
                }
            })
            .collect();
        Ok((records, dependencies))
    }

    async fn project_record_batch(
        &mut self,
        selection: &RecordSelection,
        candidate_keys: &[EntityKey<'static>],
        optimistic: &BTreeMap<EntityKey<'static>, Record>,
    ) -> Result<(Vec<(EntityKey<'static>, Json)>, live_query::RowDependencies), EngineError<S::Error>>
    {
        // Borrow hot bases through EngineSource, just like ordinary query reads.
        // Only cold records and optimistic compositions need owned snapshots.
        let mut fetched_base = HashMap::new();
        let mut composed = HashMap::new();
        for (key, update) in optimistic {
            if let Some(base) = self.hot.peek(key) {
                let mut effective = base.clone();
                effective.merge(update.clone());
                composed.insert(key.clone(), effective);
            }
        }

        let variables = serde_json::Map::new();
        let mut pending: BTreeMap<_, _> = candidate_keys
            .iter()
            .map(|key| {
                (
                    key.clone(),
                    ReadSession::new(
                        &self.schema,
                        key,
                        key.typename().unwrap_or_default(),
                        selection.selection_set(),
                    ),
                )
            })
            .collect();
        let mut plans = ReadPlans::default();
        let mut completed = BTreeMap::new();
        let mut known_absent = BTreeSet::new();
        let mut dependencies = live_query::RowDependencies::new();
        while !pending.is_empty() {
            let mut missing = BTreeSet::new();
            let source = EngineSource {
                hot: &self.hot,
                fetched: &fetched_base,
                composed: &composed,
            };
            let current: Vec<_> = pending.keys().cloned().collect();
            for key in current {
                match pending.get_mut(&key).expect("pending record").resume(
                    &variables,
                    &source,
                    dependencies.entry(key.clone()).or_default(),
                    &EntityResolverLookup::default(),
                    &mut plans,
                )? {
                    ReadOutcome::Complete(record) => {
                        pending.remove(&key);
                        completed.insert(key, record);
                    }
                    ReadOutcome::Miss { .. } => {
                        pending.remove(&key);
                    }
                    ReadOutcome::NeedRecords(keys) => missing.extend(keys),
                }
            }
            if pending.is_empty() {
                break;
            }

            let to_fetch: Vec<_> = missing
                .into_iter()
                .filter(|key| {
                    !known_absent.contains(key)
                        && !fetched_base.contains_key(key)
                        && !composed.contains_key(key)
                })
                .collect();
            if to_fetch.is_empty() {
                break;
            }
            let fetched = self
                .storage
                .get_batch(&to_fetch)
                .await
                .map_err(EngineError::Storage)?;
            for (key, record) in to_fetch.into_iter().zip(fetched) {
                match record {
                    Some(record) => {
                        if let Some(update) = optimistic.get(&key) {
                            let mut effective = record.clone();
                            effective.merge(update.clone());
                            composed.insert(key.clone(), effective);
                        }
                        fetched_base.insert(key, record);
                    }
                    None => {
                        if let Some(update) = optimistic.get(&key) {
                            composed.insert(key, update.clone());
                        } else {
                            known_absent.insert(key);
                        }
                    }
                }
            }
        }

        // Keep this read's working set ahead of unrelated hot records.
        for key in dependencies.values().flatten() {
            let _ = self.hot.get(key);
        }
        for (key, record) in fetched_base {
            self.hot.put(key, record);
        }
        Ok((
            candidate_keys
                .iter()
                .filter_map(|key| completed.remove(key).map(|record| (key.clone(), record)))
                .collect(),
            dependencies,
        ))
    }

    /// Searches the compact materialized projection without scanning or
    /// decoding normalized-record payloads.
    ///
    /// Empty queries fan out over the per-profile/per-bucket timestamp index.
    /// Text queries lazily load only requested buckets and rank borrowed entries.
    /// Active optimistic layers are projected from their fully composed record
    /// values and overlaid explicitly on either durable path.
    pub async fn search(
        &mut self,
        request: &SearchRequest,
    ) -> Result<SearchPage, EngineError<S::Error>> {
        let requested_buckets = validate_search_request(request)?;
        self.hydrate_optimistic().await?;
        let buckets: Vec<String> = if requested_buckets.is_empty() {
            request
                .profile
                .buckets()
                .iter()
                .map(|bucket| (*bucket).to_owned())
                .collect()
        } else {
            requested_buckets
        };
        let bucket_set: BTreeSet<_> = buckets.iter().map(String::as_str).collect();
        let overlay = self.optimistic_search_overlay(request.profile).await?;
        let trimmed_query = request.query.trim();

        let browse_candidates: HashMap<EntityKey<'static>, SearchDocument> =
            if trimmed_query.is_empty() {
                // Fetch enough extra durable rows to compensate for optimistic
                // replacements/removals without turning this into a record scan.
                let per_bucket_limit = request
                    .limit
                    .saturating_add(overlay.len())
                    .saturating_add(1);
                let mut candidates = HashMap::new();
                for bucket in &buckets {
                    let rows = self
                        .storage
                        .browse_search_documents(
                            request.profile,
                            bucket,
                            request.cursor.as_ref(),
                            per_bucket_limit,
                        )
                        .await
                        .map_err(EngineError::Storage)?;
                    for document in rows {
                        candidates.insert(document.record_key.clone(), document);
                    }
                }
                candidates
            } else {
                for bucket in &buckets {
                    // Unknown (but syntactically valid) buckets have no projection.
                    // Do not grow the catalog map with arbitrary empty names.
                    if !request.profile.buckets().contains(&bucket.as_str()) {
                        continue;
                    }
                    if self.search_catalogs.get(request.profile, bucket).is_none() {
                        let documents = self
                            .storage
                            .load_search_documents(request.profile, bucket)
                            .await
                            .map_err(EngineError::Storage)?;
                        self.search_catalogs
                            .insert(request.profile, bucket.clone(), documents);
                    }
                }
                HashMap::new()
            };

        let cached = buckets
            .iter()
            .filter(|_| !trimmed_query.is_empty())
            .filter_map(|bucket| self.search_catalogs.get(request.profile, bucket))
            .flat_map(|catalog| catalog.values());
        // A shadow removes its durable counterpart even when the optimistic
        // value is deleted, moved to another bucket, or excluded by the cursor.
        let candidates = browse_candidates
            .values()
            .chain(cached)
            .filter(|document| !overlay.contains_key(&document.record_key))
            .chain(overlay.values().filter_map(Option::as_ref))
            .filter(|document| bucket_set.contains(document.bucket.as_str()))
            .filter(|document| cursor_allows(request.cursor.as_ref(), document));
        Ok(rank_documents(request, candidates))
    }

    async fn optimistic_search_overlay(
        &mut self,
        profile: SearchProfile,
    ) -> Result<HashMap<EntityKey<'static>, Option<SearchDocument>>, EngineError<S::Error>> {
        let keys = layer_keys(&self.optimistic);
        if keys.is_empty() {
            return Ok(HashMap::new());
        }
        let bases = self.load_bases(&keys).await?;
        Ok(effective_records(&bases, &self.optimistic, &keys)
            .into_iter()
            .map(|(key, record)| {
                let document = record.and_then(|record| {
                    project_search_documents(&key, &record)
                        .into_iter()
                        .find(|document| document.profile == profile)
                });
                (key, document)
            })
            .collect())
    }

    fn update_loaded_search_catalogs(&mut self, entries: &[(EntityKey<'static>, Record)]) {
        self.search_catalogs.update(entries);
    }

    /// Normalizes and stores a network response. Returns changed records and
    /// the affected active operations (excluding `origin_op`).
    ///
    /// `identity` is an opaque session tag extracted by the host (e.g. the
    /// viewer id from the response). The engine knows nothing about its
    /// meaning — only that a write tagged with a different identity than the
    /// one bound to this cache wipes everything before the write proceeds
    /// (silent restart), atomically with this write.
    pub async fn write_query(
        &mut self,
        origin_op: Option<OpId>,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        data: &Json,
        identity: Option<&str>,
    ) -> Result<WriteResult, EngineError<S::Error>> {
        self.write_query_with_registration(
            origin_op,
            None,
            NetworkWrite {
                query,
                operation_name,
                variables,
                data,
                identity,
            },
        )
        .await
    }

    /// Normalizes and stores a network response, installing the active query's
    /// dependencies from the same response without denormalizing it again.
    pub async fn write_query_with_registration(
        &mut self,
        origin_op: Option<OpId>,
        registration: Option<QueryRegistration<'_>>,
        input: NetworkWrite<'_>,
    ) -> Result<WriteResult, EngineError<S::Error>> {
        self.write_query_with_registration_and_projections(
            origin_op,
            registration,
            input,
            Vec::new(),
        )
        .await
    }

    /// Normalizes records and atomically applies caller-composed generic projections.
    pub async fn write_query_with_registration_and_projections(
        &mut self,
        origin_op: Option<OpId>,
        registration: Option<QueryRegistration<'_>>,
        input: NetworkWrite<'_>,
        projections: Vec<ProjectionMutation>,
    ) -> Result<WriteResult, EngineError<S::Error>> {
        self.write_network(origin_op, registration, input, projections, true)
            .await
            .map(|(result, _)| result)
    }

    async fn write_network(
        &mut self,
        origin_op: Option<OpId>,
        registration: Option<QueryRegistration<'_>>,
        input: NetworkWrite<'_>,
        projections: Vec<ProjectionMutation>,
        retain_pages: bool,
    ) -> Result<(WriteResult, BTreeSet<String>), EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        let NetworkWrite {
            query,
            operation_name,
            variables,
            data,
            identity,
        } = input;
        self.hydrate_optimistic().await?;
        let entity_resolvers = EntityResolverLookup::compile(
            &self.schema,
            registration.map_or(&[][..], |registration| registration.entity_resolvers),
        )?;
        let doc = Self::document(&mut self.docs, query)?;
        let op = doc.operation(operation_name)?;
        if registration.is_some() && op.kind != OperationKind::Query {
            return Err(EngineError::Document(
                DocumentError::UnsupportedOperationType(format!(
                    "{:?} (dependency registration is query-only)",
                    op.kind
                )),
            ));
        }
        let is_query = op.kind == OperationKind::Query;
        let normalized =
            normalize_with_dependencies(&self.schema, op, variables, data, &entity_resolvers)?;
        let mut updates = normalized.updates;
        if !retain_pages {
            crate::page_retention::omit_hydration_pages(&mut updates);
        }

        if let Some(observed) = identity {
            match self.bound_identity().await? {
                IdentityState::NotHydrated => unreachable!("bound_identity hydrates"),
                IdentityState::Missing => self.bind_identity(observed).await?,
                IdentityState::Bound(bound) if bound == observed => {}
                IdentityState::Bound(_) => {
                    let changed = updates.keys().cloned().collect();
                    let mut witness = Record::default();
                    witness.fields.insert(
                        IDENTITY_VALUE_FIELD.into(),
                        crate::value::CacheValue::String(observed.into()),
                    );
                    updates.insert(EntityKey(IDENTITY_META_KEY.into()), witness);
                    let entries: Vec<_> = updates.into_iter().collect();
                    // Identity, records, and old queue removal are one transaction.
                    // There is no reset to publish if any part of it fails.
                    self.storage
                        .reset_with_records(entries.clone(), projections)
                        .await
                        .map_err(EngineError::Storage)?;
                    self.external_reset()?;
                    self.identity = IdentityState::Bound(observed.into());
                    self.optimistic_hydrated = true;
                    for (key, record) in entries {
                        self.hot.put(key, record);
                    }
                    let mut affected_ops = self.deps.all_ops();
                    if let Some(origin) = origin_op {
                        affected_ops.remove(&origin);
                    }
                    if let Some(registration) = registration {
                        if normalized.completeness == DependencyCompleteness::Exact {
                            self.deps
                                .set_query_deps(registration.op_id, normalized.dependencies);
                        } else {
                            self.deps.set_op_broad(registration.op_id);
                        }
                    }
                    return Ok((
                        WriteResult {
                            field_changes: None,
                            identity_errors: Vec::new(),
                            mutation_uuid: None,
                            revision: self.revision,
                            revision_advanced: true,
                            search_changed_buckets: None,
                            changed,
                            affected_ops,
                            reset: true,
                            revalidations: Vec::new(),
                        },
                        BTreeSet::new(),
                    ));
                }
            }
        }

        // Without optimistic layers, durable changes are exactly the visible
        // changes. Avoid loading/cloning the whole batch again on both sides
        // of an ordinary network refresh.
        let optimistic_update = if self.optimistic.is_empty() {
            None
        } else {
            let mut candidates = layer_keys(&self.optimistic);
            candidates.extend(updates.keys().cloned());
            let bases = self.load_bases(&candidates).await?;
            let before = effective_records(&bases, &self.optimistic, &candidates);
            let staged: RecordUpdates = stage_updates(&bases, updates.clone())
                .1
                .into_iter()
                .collect();
            let queued = self
                .storage
                .load_mutation_queue()
                .await
                .map_err(EngineError::Storage)?;
            let replacement = self
                .rebuild_queued_layers_with_staged(queued, None, &staged)
                .await?;
            candidates.extend(layer_keys(&replacement));
            let mut staged_bases = self.load_bases(&candidates).await?;
            staged_bases.extend(staged);
            let after = effective_records(&staged_bases, &replacement, &candidates);
            Some((candidates, before, replacement, after))
        };
        let PersistedChanges {
            mut field_changes,
            changed,
            revision,
            revision_advanced,
            mut search_changed_buckets,
            viewer_fields: mut viewer_changes,
        } = self.persist_updates(updates, projections).await?;

        let visible_changed =
            if let Some((candidates, before, replacement, after)) = optimistic_update {
                self.optimistic = replacement;
                field_changes = crate::field_changes::between(&before, &after);
                if revision_advanced {
                    self.live_queries.extend(
                        revision,
                        candidates.iter().cloned(),
                        live_query::projection_keys(&self.optimistic),
                    );
                }
                // Compare the composed view: a hydration hidden beneath a pending
                // edit must not invalidate the search projection it did not change.
                search_changed_buckets.clear();
                viewer_changes.clear();
                for key in &candidates {
                    if let Some(fields) = changed_viewer_fields(
                        before.get(key).and_then(Option::as_ref),
                        after.get(key).and_then(Option::as_ref),
                    ) {
                        viewer_changes.insert(key.clone(), fields);
                    }
                    collect_search_changes(
                        key,
                        before.get(key).and_then(Option::as_ref),
                        after.get(key).and_then(Option::as_ref),
                        &mut search_changed_buckets,
                    );
                }
                candidates
                    .into_iter()
                    .filter(|key| before.get(key) != after.get(key))
                    .collect()
            } else {
                changed.clone()
            };

        let mut affected_ops = if is_query {
            self.deps.ops_for_changes(&visible_changed, &viewer_changes)
        } else {
            self.deps.ops_for_keys(visible_changed.iter())
        };
        if let Some(origin) = origin_op {
            affected_ops.remove(&origin);
        }
        if let Some(registration) = registration {
            if normalized.completeness == DependencyCompleteness::Exact
                && self.optimistic.is_empty()
            {
                self.deps
                    .set_query_deps(registration.op_id, normalized.dependencies);
            } else {
                self.deps.set_op_broad(registration.op_id);
            }
        }
        Ok((
            WriteResult {
                field_changes: Some(field_changes),
                identity_errors: Vec::new(),
                mutation_uuid: None,
                revision,
                revision_advanced,
                search_changed_buckets: is_query.then(|| search_changed_buckets.clone()),
                changed,
                affected_ops,
                reset: false,
                revalidations: Vec::new(),
            },
            search_changed_buckets,
        ))
    }

    /// Stores a network response and returns only fields not marked
    /// `@cacheOnly`. Projection is taken directly from the validated network
    /// payload, so hydration never denormalizes the response back out of
    /// storage. Soup page wrappers are transient; their normalized descendants
    /// and projections are persisted without retaining cursor-qualified pages.
    pub async fn hydrate_query(
        &mut self,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        data: &Json,
        identity: Option<&str>,
    ) -> Result<HydrationWriteResult, EngineError<S::Error>> {
        self.hydrate_query_with_projections(
            query,
            operation_name,
            variables,
            data,
            identity,
            Vec::new(),
        )
        .await
    }

    /// Hydrates a response while atomically maintaining generic projections.
    pub async fn hydrate_query_with_projections(
        &mut self,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        data: &Json,
        identity: Option<&str>,
        projections: Vec<ProjectionMutation>,
    ) -> Result<HydrationWriteResult, EngineError<S::Error>> {
        let projected = {
            let doc = Self::document(&mut self.docs, query)?;
            let op = doc.operation(operation_name)?;
            if op.kind != OperationKind::Query {
                return Err(EngineError::Document(
                    DocumentError::UnsupportedOperationType(format!(
                        "{:?} (cache hydration is query-only)",
                        op.kind
                    )),
                ));
            }
            project_hydration_response(&self.schema, &op.prepare(variables)?, data)?
        };
        let (write_result, search_changed_buckets) = self
            .write_network(
                None,
                None,
                NetworkWrite {
                    query,
                    operation_name,
                    variables,
                    data,
                    identity,
                },
                projections,
                false,
            )
            .await?;
        Ok(HydrationWriteResult {
            write_result,
            search_changed_buckets,
            data: projected,
        })
    }

    /// Merges normalized updates into the hot tier and storage. Returns the
    /// keys whose durable contents actually changed.
    async fn persist_updates(
        &mut self,
        updates: RecordUpdates,
        projections: Vec<ProjectionMutation>,
    ) -> Result<PersistedChanges, EngineError<S::Error>> {
        let projection_keys = projections
            .iter()
            .map(|mutation| mutation.record_key().clone())
            .collect();
        // Load current values (hot tier, then storage) so merges detect real
        // changes. Merges are staged in a plain map, NOT the LRU: a batch
        // larger than the hot capacity would otherwise evict its own
        // records mid-merge and overwrite storage with partial updates.
        let mut staging: HashMap<EntityKey<'static>, Record> = HashMap::new();
        let mut missing: Vec<EntityKey<'static>> = Vec::new();
        for key in updates.keys() {
            match self.hot.peek(key) {
                Some(record) => {
                    staging.insert(key.clone(), record.clone());
                }
                None => missing.push(key.clone()),
            }
        }
        if !missing.is_empty() {
            let fetched = self
                .storage
                .get_batch(&missing)
                .await
                .map_err(EngineError::Storage)?;
            for (key, record) in missing.into_iter().zip(fetched) {
                if let Some(r) = record {
                    staging.insert(key, r);
                }
            }
        }

        let mut changed = BTreeSet::new();
        let mut field_changes = Vec::new();
        let mut search_changed_buckets = BTreeSet::new();
        let mut viewer_fields = ViewerFields::new();
        let mut to_persist: Vec<(EntityKey<'static>, Record)> = Vec::new();
        let mut touched = Vec::with_capacity(updates.len());
        for (key, update) in updates {
            let (merged, did_change) = match staging.remove(&key) {
                Some(mut existing) => {
                    // Compare only fields supplied by this partial response,
                    // overlaid onto the existing row, without cloning its body.
                    let before = snapshot_search_fields(&key, &existing);
                    let viewer_update = ViewerFieldUpdate::capture(&existing, &update);
                    field_changes
                        .extend(crate::field_changes::from_update(&key, &existing, &update));
                    let did_change = existing.merge(update);
                    if let Some(fields) = viewer_update.and_then(|update| update.finish(&existing))
                    {
                        viewer_fields.insert(key.clone(), fields);
                    }
                    if did_change {
                        collect_search_changes(
                            &key,
                            before.as_ref(),
                            Some(&existing),
                            &mut search_changed_buckets,
                        );
                    }
                    (existing, did_change)
                }
                None => {
                    field_changes.push(crate::field_changes::RecordFieldChange::Invalidate {
                        key: key.clone(),
                    });
                    collect_search_changes(&key, None, Some(&update), &mut search_changed_buckets);
                    (update, true)
                }
            };
            if did_change {
                changed.insert(key.clone());
                to_persist.push((key.clone(), merged.clone()));
            }
            touched.push((key, merged));
        }

        // Unchanged bases are already durable, including those fetched after
        // hot-tier eviction. Still apply every projection mutation: a partial
        // response can change projection authority without changing records.
        let revision_advanced = if changed.is_empty() {
            self.projection_mutations_change(&projections).await?
        } else {
            true
        };
        self.storage
            .put_batch_with_projections(to_persist, projections)
            .await
            .map_err(EngineError::Storage)?;
        let revision = if revision_advanced {
            self.advance_revision()?
        } else {
            self.revision
        };
        if revision_advanced {
            self.live_queries.record(
                revision,
                live_query::Changes {
                    records: changed.clone(),
                    projections: projection_keys,
                },
            );
        }
        // Publish only after the atomic write succeeds. Otherwise a failed
        // write could poison the hot tier and make its retry look unchanged.
        self.update_loaded_search_catalogs(&touched);
        for (key, record) in touched {
            self.hot.put(key, record);
        }
        Ok(PersistedChanges {
            field_changes,
            changed,
            revision,
            revision_advanced,
            search_changed_buckets,
            viewer_fields,
        })
    }

    async fn projection_mutations_change(
        &self,
        mutations: &[ProjectionMutation],
    ) -> Result<bool, EngineError<S::Error>> {
        if mutations.is_empty() {
            return Ok(false);
        }
        let keys = mutations
            .iter()
            .map(ProjectionMutation::record_key)
            .cloned()
            .collect::<BTreeSet<_>>()
            .into_iter()
            .collect::<Vec<_>>();
        let states = self
            .storage
            .load_projection_states(&keys)
            .await
            .map_err(EngineError::Storage)?;
        let mut projected = keys
            .into_iter()
            .zip(states)
            .filter_map(|(key, state)| state.map(|state| (key, state)))
            .collect::<HashMap<_, _>>();
        let before = projected.clone();
        apply_authoritative_projection_mutations(&mut projected, mutations);
        Ok(projected != before)
    }

    /// Atomically enqueues a mutation together with its optimistic layer.
    /// The layer is durable before it becomes visible or the caller is
    /// allowed to forward the mutation to the network.
    pub async fn begin_optimistic_write(
        &mut self,
        origin_op: Option<OpId>,
        input: BeginOptimisticWrite<'_>,
    ) -> Result<(OptimisticTransactionId, WriteResult), EngineError<S::Error>> {
        self.begin_optimistic_write_with_projections(origin_op, input, Vec::new())
            .await
    }

    /// Atomically enqueue an optimistic normalized layer and generic projection overlay.
    pub async fn begin_optimistic_write_with_projections(
        &mut self,
        origin_op: Option<OpId>,
        input: BeginOptimisticWrite<'_>,
        projection_mutations: Vec<OptimisticProjectionMutation>,
    ) -> Result<(OptimisticTransactionId, WriteResult), EngineError<S::Error>> {
        let begun = self
            .upsert_optimistic_write(origin_op, input, projection_mutations, Vec::new())
            .await?;
        Ok((begun.transaction_id, begun.write_result))
    }

    async fn upsert_optimistic_write(
        &mut self,
        origin_op: Option<OpId>,
        input: BeginOptimisticWrite<'_>,
        projection_mutations: Vec<OptimisticProjectionMutation>,
        uncertain_calendar_event_keys: Vec<EntityKey<'static>>,
    ) -> Result<BegunOptimisticWrite, EngineError<S::Error>> {
        let uuid = Uuid::parse_str(input.uuid)
            .map_err(|_| EngineError::InvalidMutationUuid(input.uuid.to_owned()))?;
        self.ensure_revision_can_advance()?;
        let mut live_projection_keys = live_query::projection_keys(&self.optimistic);
        let BeginOptimisticWrite {
            client_metadata,
            uuid: _,
            query,
            operation_name,
            variables,
            data,
            link_patches,
            revalidations,
            created_at_ms,
            identity_bindings,
        } = input;
        identity::validate(identity_bindings).map_err(EngineError::InvalidOptimisticProjection)?;
        if uncertain_calendar_event_keys.len() > MAX_UNCERTAIN_CALENDAR_EVENTS
            || uncertain_calendar_event_keys
                .iter()
                .any(|key| key.typename() != Some(EVENT_TYPENAME))
        {
            return Err(CalendarError::InvalidKey.into());
        }
        self.hydrate_optimistic().await?;

        let queued = self
            .storage
            .load_mutation_queue()
            .await
            .map_err(EngineError::Storage)?;
        let expected_queue = queued
            .iter()
            .map(MutationQueueSnapshot::from)
            .collect::<Vec<_>>();
        let old_layers = self.rebuild_queued_layers(queued.clone()).await?;
        let collision = queued
            .iter()
            .find(|queued| queued.uuid == uuid && !queued.superseded)
            .map(|queued| {
                (
                    queued.id,
                    crate::queue::collision_stays_active(
                        queued.mutation.lease_expires_at_ms,
                        created_at_ms,
                        queued.mutation.attempt_count > 0,
                        &queued.optimistic.optimistic_data_json,
                    ),
                    queued.mutation.created_at_ms,
                )
            });
        let expected_kind = match collision {
            None => MutationUpsertKind::Inserted,
            Some((active_id, true, _)) => MutationUpsertKind::AppendedAfterActive { active_id },
            Some((removed_id, false, _)) => MutationUpsertKind::ReplacedPending { removed_id },
        };
        // The removed row never reached the server, so its wait carries over;
        // an active row stays queued and keeps reporting its own.
        let enqueued_at_ms = match collision {
            Some((_, false, replaced_at_ms)) => replaced_at_ms.min(created_at_ms),
            _ => created_at_ms,
        };

        let patches = deduplicate_patches(&self.schema, link_patches)?;
        let revalidations = deduplicate_revalidations(
            revalidations
                .iter()
                .cloned()
                .chain(patches.iter().filter_map(OptimisticLinkPatch::revalidation)),
        );
        for mutation in &projection_mutations {
            mutation
                .validate()
                .map_err(|error| EngineError::InvalidOptimisticProjection(error.to_string()))?;
        }
        let identity = match self.bound_identity().await? {
            IdentityState::Bound(identity) => Some(identity),
            IdentityState::NotHydrated | IdentityState::Missing => None,
        };
        let source = OptimisticSource {
            client_metadata: client_metadata.cloned(),
            identity_bindings: identity_bindings.to_vec(),
            mutation_data: data.clone(),
            link_patches: patches,
            revalidations,
            projection_mutations,
            uncertain_calendar_event_keys,
        };
        let mut entry = NewQueuedMutation {
            uuid,
            mutation: StoredMutation::new(
                MutationRequest {
                    query: query.to_string(),
                    operation_name: operation_name.map(str::to_string),
                    variables_json: canonical_json(&Json::Object(variables.clone())),
                    identity,
                },
                enqueued_at_ms,
            ),
            optimistic: PersistedOptimisticLayer {
                optimistic_data_json: encode_optimistic_source(&source),
                normalized_updates: RecordUpdates::default(),
            },
        };

        let mut proposed_queue = queued.clone();
        match expected_kind {
            MutationUpsertKind::Inserted => {}
            MutationUpsertKind::ReplacedPending { removed_id } => {
                proposed_queue.retain(|queued| queued.id != removed_id);
            }
            MutationUpsertKind::AppendedAfterActive { active_id } => {
                proposed_queue
                    .iter_mut()
                    .find(|queued| queued.id == active_id)
                    .expect("collision was loaded")
                    .superseded = true;
            }
        }
        proposed_queue.push(QueuedMutation {
            id: MutationId::MAX,
            uuid,
            superseded: false,
            mutation: entry.mutation.clone(),
            optimistic: entry.optimistic.clone(),
        });
        let mut proposed_layers = self
            .rebuild_queued_layers_with_strict_tail(proposed_queue, Some(MutationId::MAX))
            .await?;
        entry.optimistic.normalized_updates = proposed_layers
            .last()
            .expect("proposed queue contains the new tail")
            .updates
            .clone();

        let mut candidates = layer_keys(&old_layers);
        candidates.extend(layer_keys(&proposed_layers));
        let bases = self.load_bases(&candidates).await?;
        let before = effective_records(&bases, &old_layers, &candidates);
        let after = effective_records(&bases, &proposed_layers, &candidates);

        let affected_projection_keys = old_layers
            .iter()
            .chain(&proposed_layers)
            .flat_map(|layer| &layer.projection_mutations)
            .map(|mutation| mutation.record_key().clone())
            .collect::<BTreeSet<_>>()
            .into_iter()
            .collect::<Vec<_>>();
        let authoritative = self
            .storage
            .load_projection_states(&affected_projection_keys)
            .await
            .map_err(EngineError::Storage)?;
        if authoritative.len() != affected_projection_keys.len() {
            return Err(EngineError::InvalidOptimisticProjection(
                "storage returned misaligned authoritative projection states".to_owned(),
            ));
        }
        let projection_layers = proposed_layers
            .iter()
            .map(|layer| ProjectionMutationLayer {
                owner: layer.id,
                mutations: &layer.projection_mutations,
            })
            .collect::<Vec<_>>();
        let replacements = affected_projection_keys
            .iter()
            .zip(authoritative.iter())
            .filter_map(|(key, authoritative)| {
                compose_effective_optimistic_projection(
                    key,
                    authoritative.as_ref(),
                    &projection_layers,
                )
                .transpose()
            })
            .collect::<Result<Vec<_>, _>>()
            .map_err(|error| EngineError::InvalidOptimisticProjection(error.to_string()))?
            .into_iter()
            .map(|projection| StagedOptimisticProjection {
                owner: if projection.owner == MutationId::MAX {
                    StagedOptimisticProjectionOwner::Enqueued
                } else {
                    StagedOptimisticProjectionOwner::Existing(projection.owner)
                },
                state: projection.state,
                uncertainty: projection.uncertainty,
            })
            .collect();
        let upsert = self
            .storage
            .upsert_mutation_with_shadow(
                entry,
                created_at_ms,
                OptimisticUpsertReconciliation {
                    expected_queue: Some(expected_queue),
                    affected_keys: affected_projection_keys,
                    replacements,
                },
            )
            .await
            .map_err(EngineError::Storage)?;
        if upsert.kind != expected_kind {
            return Err(EngineError::StaleOptimisticUpsert);
        }
        proposed_layers
            .last_mut()
            .expect("proposed queue contains the new tail")
            .id = upsert.id;
        self.optimistic = proposed_layers;
        live_projection_keys.extend(live_query::projection_keys(&self.optimistic));
        let live_records = candidates.clone();

        let changed = candidates
            .into_iter()
            .filter(|key| before.get(key) != after.get(key))
            .collect::<BTreeSet<_>>();
        let mut affected_ops = self.deps.ops_for_keys(changed.iter());
        if let Some(origin) = origin_op {
            affected_ops.remove(&origin);
        }
        let revision = self.advance_revision()?;
        self.live_queries.record(
            revision,
            live_query::Changes {
                records: live_records,
                projections: live_projection_keys,
            },
        );
        Ok(BegunOptimisticWrite {
            transaction_id: upsert.id,
            upsert_kind: upsert.kind,
            write_result: WriteResult {
                field_changes: Some(crate::field_changes::between(&before, &after)),
                identity_errors: Vec::new(),
                revision,
                revision_advanced: true,
                search_changed_buckets: None,
                changed,
                affected_ops,
                reset: false,
                revalidations: Vec::new(),
                mutation_uuid: None,
            },
        })
    }

    /// Durably enqueues a mutation and publishes its optimistic layer, then
    /// attempts to claim the strict queue head before returning. A claim
    /// failure is nested in the successful enqueue result so callers never
    /// bypass or duplicate an already durable mutation.
    pub async fn enqueue_optimistic_mutation(
        &mut self,
        origin_op: Option<OpId>,
        input: BeginOptimisticWrite<'_>,
        claim: MutationClaimRequest,
    ) -> Result<EnqueueOptimisticMutationResult<EngineError<S::Error>>, EngineError<S::Error>> {
        self.enqueue_optimistic_mutation_with_projections(origin_op, input, claim, Vec::new())
            .await
    }

    /// Durably enqueue normalized optimism and a queryable projection overlay.
    pub async fn enqueue_optimistic_mutation_with_projections(
        &mut self,
        origin_op: Option<OpId>,
        input: BeginOptimisticWrite<'_>,
        claim: MutationClaimRequest,
        projection_mutations: Vec<OptimisticProjectionMutation>,
    ) -> Result<EnqueueOptimisticMutationResult<EngineError<S::Error>>, EngineError<S::Error>> {
        self.enqueue_optimistic_mutation_with_calendar(
            origin_op,
            input,
            claim,
            projection_mutations,
            Vec::new(),
        )
        .await
    }

    /// Durably enqueue optimism, projection overlays, and the calendar events
    /// whose occurrence sets stay uncertain until the mutation settles.
    pub async fn enqueue_optimistic_mutation_with_calendar(
        &mut self,
        origin_op: Option<OpId>,
        input: BeginOptimisticWrite<'_>,
        claim: MutationClaimRequest,
        projection_mutations: Vec<OptimisticProjectionMutation>,
        uncertain_calendar_event_keys: Vec<EntityKey<'static>>,
    ) -> Result<EnqueueOptimisticMutationResult<EngineError<S::Error>>, EngineError<S::Error>> {
        let begun = self
            .upsert_optimistic_write(
                origin_op,
                input,
                projection_mutations,
                uncertain_calendar_event_keys,
            )
            .await?;
        let initial_claim = match self.claim_next_mutation(claim).await {
            Ok(Some(claimed)) => InitialClaimOutcome::Claimed(Box::new(claimed)),
            Ok(None) => InitialClaimOutcome::NotRunnable,
            Err(error) => InitialClaimOutcome::Failed(error),
        };
        Ok(EnqueueOptimisticMutationResult {
            transaction_id: begun.transaction_id,
            upsert_kind: begun.upsert_kind,
            write_result: begun.write_result,
            initial_claim,
        })
    }

    /// Reads durable queue entries without acquiring or changing their leases.
    pub async fn inspect_mutations(
        &self,
    ) -> Result<Vec<crate::queue::MutationInspection>, EngineError<S::Error>> {
        self.storage
            .load_mutation_queue()
            .await
            .map_err(EngineError::Storage)?
            .into_iter()
            .map(crate::queue::MutationInspection::try_from)
            .collect::<Result<_, _>>()
            .map_err(EngineError::InvalidOptimisticProjection)
    }

    /// Claims the oldest runnable mutation. A leased or backed-off head
    /// blocks every later mutation.
    pub async fn claim_next_mutation(
        &mut self,
        request: MutationClaimRequest,
    ) -> Result<Option<ClaimedMutation>, EngineError<S::Error>> {
        self.hydrate_optimistic().await?;
        self.storage
            .claim_next_mutation(request)
            .await
            .map_err(EngineError::Storage)
    }

    /// Releases a retryable mutation, or discards it when a newer UUID match exists.
    pub async fn defer_optimistic_write(
        &mut self,
        transaction: OptimisticTransactionId,
        claim: MutationClaimToken,
        next_attempt_at_ms: i64,
        error: String,
        server_failure: bool,
    ) -> Result<DeferOptimisticWriteResult, EngineError<S::Error>> {
        self.hydrate_optimistic().await?;
        let layer = self
            .optimistic
            .iter()
            .find(|layer| layer.id == transaction)
            .ok_or(EngineError::UnknownTransaction(transaction))?;
        if layer.superseded
            && !layer
                .identity_bindings
                .iter()
                .any(|binding| !binding.response_path.is_empty())
        {
            let replacement_transaction_id = self
                .optimistic
                .iter()
                .find(|candidate| candidate.uuid == layer.uuid && !candidate.superseded)
                .map(|candidate| candidate.id)
                .ok_or(EngineError::UnknownTransaction(transaction))?;
            let write_result = self.rollback_optimistic_write(transaction, claim).await?;
            return Ok(DeferOptimisticWriteResult::DiscardedSuperseded(
                SupersededMutationResult {
                    write_result,
                    replacement_transaction_id,
                },
            ));
        }
        if !self
            .storage
            .defer_mutation(
                transaction,
                claim,
                next_attempt_at_ms,
                error,
                server_failure,
            )
            .await
            .map_err(EngineError::Storage)?
        {
            return Err(EngineError::StaleMutationClaim(transaction));
        }
        Ok(DeferOptimisticWriteResult::Deferred)
    }

    async fn stage_shadow_reconciliation(
        &self,
        transaction: OptimisticTransactionId,
        authoritative_mutations: &[ProjectionMutation],
        identities: &IdentityMap,
    ) -> Result<OptimisticShadowReconciliation, EngineError<S::Error>> {
        let expected_queue = self
            .optimistic
            .iter()
            .map(|layer| layer.id)
            .collect::<Vec<_>>();
        let settled = self
            .optimistic
            .iter()
            .find(|layer| layer.id == transaction)
            .ok_or(EngineError::UnknownTransaction(transaction))?;
        let mut rebased_layers = self.optimistic.clone();
        for layer in &mut rebased_layers {
            for projection in &mut layer.projection_mutations {
                identity::remap_projection(projection, identities);
            }
        }
        let affected_keys = settled
            .projection_mutations
            .iter()
            .map(|mutation| mutation.record_key().clone())
            .chain(
                authoritative_mutations
                    .iter()
                    .map(|mutation| mutation.record_key().clone()),
            )
            .chain(
                self.optimistic
                    .iter()
                    .chain(&rebased_layers)
                    .flat_map(|layer| {
                        layer
                            .projection_mutations
                            .iter()
                            .map(|mutation| mutation.record_key().clone())
                    }),
            )
            .collect::<BTreeSet<_>>()
            .into_iter()
            .collect::<Vec<_>>();
        let loaded = self
            .storage
            .load_projection_states(&affected_keys)
            .await
            .map_err(EngineError::Storage)?;
        if loaded.len() != affected_keys.len() {
            return Err(EngineError::InvalidOptimisticProjection(
                "storage returned misaligned authoritative projection states".to_owned(),
            ));
        }
        let mut authoritative = affected_keys
            .iter()
            .cloned()
            .zip(loaded)
            .collect::<BTreeMap<_, _>>();
        for mutation in authoritative_mutations {
            match mutation {
                ProjectionMutation::Replace(document) => {
                    let mut document = document.clone();
                    document.canonicalize();
                    document.validate().map_err(|error| {
                        EngineError::InvalidOptimisticProjection(error.to_string())
                    })?;
                    authoritative.insert(
                        document.record_key.clone(),
                        Some(ProjectionState::Complete(document)),
                    );
                }
                ProjectionMutation::Patch {
                    record_key,
                    profile,
                    partition,
                    exact,
                    integers,
                    sorts,
                } => {
                    let state = apply_authoritative_projection_patch(
                        authoritative.get(record_key).and_then(Option::as_ref),
                        record_key,
                        profile,
                        partition,
                        exact,
                        integers,
                        sorts,
                    );
                    authoritative.insert(record_key.clone(), Some(state));
                }
                ProjectionMutation::MarkIncomplete {
                    record_key,
                    profile,
                    partition,
                    kind,
                } => {
                    authoritative.insert(
                        record_key.clone(),
                        Some(ProjectionState::Incomplete {
                            record_key: record_key.clone(),
                            profile: profile.clone(),
                            partition: partition.clone(),
                            kind: *kind,
                        }),
                    );
                }
                ProjectionMutation::Delete(record_key) => {
                    authoritative.insert(record_key.clone(), None);
                }
                ProjectionMutation::PatchExact {
                    record_key,
                    profile,
                    partition,
                    remove,
                    insert,
                } => {
                    let state = apply_authoritative_exact_members(
                        authoritative.get(record_key).and_then(Option::as_ref),
                        record_key,
                        profile,
                        partition,
                        remove,
                        insert,
                    );
                    authoritative.insert(record_key.clone(), Some(state));
                }
            }
        }
        let remaining_layers = rebased_layers
            .iter()
            .filter(|layer| layer.id != transaction)
            .map(|layer| ProjectionMutationLayer {
                owner: layer.id,
                mutations: &layer.projection_mutations,
            })
            .collect::<Vec<_>>();
        let replacements = affected_keys
            .iter()
            .filter_map(|key| {
                compose_effective_optimistic_projection(
                    key,
                    authoritative.get(key).and_then(Option::as_ref),
                    &remaining_layers,
                )
                .transpose()
            })
            .collect::<Result<Vec<_>, _>>()
            .map_err(|error| EngineError::InvalidOptimisticProjection(error.to_string()))?;
        let reconciliation = OptimisticShadowReconciliation {
            expected_queue,
            affected_keys,
            replacements,
        };
        if reconciliation.expected_queue.first().copied() != Some(transaction) {
            return Err(EngineError::StaleMutationClaim(transaction));
        }
        reconciliation
            .validate(transaction)
            .map_err(|error| EngineError::InvalidOptimisticProjection(error.to_string()))?;
        Ok(reconciliation)
    }

    /// Replaces the claimed head's optimistic contribution with the real
    /// network response without flickering through the pre-mutation value.
    /// Real records and queue deletion commit in one storage transaction.
    /// Invalid response identities are reported in [`WriteResult::identity_errors`].
    /// If that response also cannot normalize, the attempt is discarded and
    /// [`EngineError::IdentityResolutionFailed`] carries the rollback changes.
    pub async fn commit_optimistic_write(
        &mut self,
        transaction: OptimisticTransactionId,
        claim: MutationClaimToken,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        data: &Json,
    ) -> Result<WriteResult, EngineError<S::Error>> {
        self.commit_optimistic_write_with_projections(
            transaction,
            claim,
            query,
            operation_name,
            variables,
            data,
            Vec::new(),
        )
        .await
    }

    /// Commits an optimistic write and reports whether a newer UUID match superseded it.
    #[allow(clippy::too_many_arguments)]
    pub async fn commit_optimistic_write_with_outcome(
        &mut self,
        transaction: OptimisticTransactionId,
        claim: MutationClaimToken,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        data: &Json,
    ) -> Result<CommitOptimisticWriteResult, EngineError<S::Error>> {
        self.commit_optimistic_write_with_projections_outcome(
            transaction,
            claim,
            query,
            operation_name,
            variables,
            data,
            Vec::new(),
        )
        .await
    }

    /// Commits an optimistic write with projections and reports supersession.
    #[allow(clippy::too_many_arguments)]
    pub async fn commit_optimistic_write_with_projections_outcome(
        &mut self,
        transaction: OptimisticTransactionId,
        claim: MutationClaimToken,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        data: &Json,
        projections: Vec<ProjectionMutation>,
    ) -> Result<CommitOptimisticWriteResult, EngineError<S::Error>> {
        self.hydrate_optimistic().await?;
        let layer = self
            .optimistic
            .iter()
            .find(|layer| layer.id == transaction)
            .ok_or(EngineError::UnknownTransaction(transaction))?;
        let replacement_transaction_id = if layer.superseded {
            Some(
                self.optimistic
                    .iter()
                    .find(|candidate| candidate.uuid == layer.uuid && !candidate.superseded)
                    .map(|candidate| candidate.id)
                    .ok_or(EngineError::UnknownTransaction(transaction))?,
            )
        } else {
            None
        };
        let write_result = match self
            .commit_optimistic_write_with_projections(
                transaction,
                claim,
                query,
                operation_name,
                variables,
                data,
                projections,
            )
            .await
        {
            Ok(result) => result,
            Err(EngineError::IdentityResolutionFailed(result)) => {
                return Ok(CommitOptimisticWriteResult::Failed(*result));
            }
            Err(error) => return Err(error),
        };
        match replacement_transaction_id {
            Some(replacement_transaction_id) => Ok(
                CommitOptimisticWriteResult::CommittedSuperseded(SupersededMutationResult {
                    write_result,
                    replacement_transaction_id,
                }),
            ),
            None => Ok(CommitOptimisticWriteResult::Committed(write_result)),
        }
    }

    /// Settles an optimistic write with atomic generic projection replacement.
    #[allow(clippy::too_many_arguments)]
    pub async fn commit_optimistic_write_with_projections(
        &mut self,
        transaction: OptimisticTransactionId,
        claim: MutationClaimToken,
        query: &str,
        operation_name: Option<&str>,
        variables: &serde_json::Map<String, Json>,
        data: &Json,
        projections: Vec<ProjectionMutation>,
    ) -> Result<WriteResult, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        let mut live_projection_keys = live_query::projection_keys(&self.optimistic);
        live_projection_keys.extend(
            projections
                .iter()
                .map(|projection| projection.record_key().clone()),
        );
        self.hydrate_optimistic().await?;
        let index = self
            .optimistic
            .iter()
            .position(|layer| layer.id == transaction)
            .ok_or(EngineError::UnknownTransaction(transaction))?;
        let mutation_uuid = Some(self.optimistic[index].uuid.to_string());
        let mut bindings = self.optimistic[index].identity_bindings.clone();
        let mut identity_errors = Vec::new();
        let identities = match identity::resolve(&bindings, data) {
            Ok(identities) => identities,
            Err(_) => {
                // Salvage valid bindings without guessing targets for failed ones.
                let mut identities = IdentityMap::new();
                bindings.retain(|binding| {
                    match identity::resolve(std::slice::from_ref(binding), data) {
                        Ok(resolved) => {
                            identities.extend(resolved);
                            true
                        }
                        Err(error) => {
                            identity_errors.push(error);
                            false
                        }
                    }
                });
                identities
            }
        };
        let doc = Self::document(&mut self.docs, query)?;
        let op = doc.operation(operation_name)?;
        let mut updates = match normalize(&self.schema, op, variables, data) {
            Ok(updates) => updates,
            Err(error) if identity_errors.is_empty() => return Err(error.into()),
            Err(error) => {
                // No safe authoritative write is possible. Retire this attempt
                // rather than replaying an already successful server mutation.
                let error = format!("{}; {error}", identity_errors.join("; "));
                let rollback = self
                    .rollback_optimistic_write_with_outcome(transaction, claim)
                    .await?;
                let (write_result, replacement_transaction_id) = match rollback {
                    RollbackOptimisticWriteResult::RolledBack(result) => (result, None),
                    RollbackOptimisticWriteResult::DiscardedSuperseded(result) => {
                        (result.write_result, Some(result.replacement_transaction_id))
                    }
                };
                return Err(EngineError::IdentityResolutionFailed(Box::new(
                    FailedMutationResult {
                        write_result,
                        error,
                        replacement_transaction_id,
                    },
                )));
            }
        };
        // Recipes and revalidation variables may contain unresolved local IDs.
        // On a binding error, commit only authoritative data and valid aliases.
        let (mut recipes, mut revalidations) = if identity_errors.is_empty() {
            (
                self.optimistic[index].link_patches.clone(),
                self.optimistic[index].revalidations.clone(),
            )
        } else {
            (Vec::new(), Vec::new())
        };
        for patch in &mut recipes {
            identity::remap_patch(patch, &bindings, &identities);
        }
        for revalidation in &mut revalidations {
            identity::remap_variables(&mut revalidation.variables_json, &bindings, &identities);
        }
        for (local, target) in &identities {
            if !updates.contains_key(target) {
                if bindings.iter().any(|binding| &binding.local_key == local) {
                    return Err(EngineError::InvalidOptimisticProjection(
                        "identity target is absent from normalized response".into(),
                    ));
                }
                continue;
            }
            updates.insert(local.clone(), identity::alias_record(target));
            updates.entry(target.clone()).or_default().fields.insert(
                identity::MUTATION_UUID_FIELD.into(),
                crate::value::CacheValue::String(self.optimistic[index].uuid.to_string()),
            );
        }

        let stored_identities = self
            .load_identity_bindings(&bindings, &RecordUpdates::new())
            .await?;
        let mut settled_identities = stored_identities;
        settled_identities.extend(identities.clone());
        identity::apply_record_lifecycle(&mut updates, &bindings, &settled_identities);
        let mut candidates = layer_keys(&self.optimistic);
        candidates.extend(updates.keys().cloned());
        let (mut candidates, bases) = self
            .load_link_patch_bases(candidates, &[], &updates, &recipes, &RecordUpdates::new())
            .await?;
        let before = effective_records(&bases, &self.optimistic, &candidates);

        // Reapply the idempotent recipes to the latest durable base, never a
        // captured optimistic query snapshot. Stale/missing fields are skipped
        // and recovered by the returned revalidations.
        let mut effective = bases.clone();
        merge_updates_into_effective(&mut effective, &updates);
        let links_applied =
            apply_link_patches(&self.schema, &mut effective, &mut updates, &recipes, true)?;
        revalidations
            .retain(|query| !query.only_on_link_failure || recipes.is_empty() || !links_applied);
        let (durable_changed, entries) = stage_updates(&bases, updates);
        let reconciliation = self
            .stage_shadow_reconciliation(transaction, &projections, &identities)
            .await?;
        let staged: RecordUpdates = entries.iter().cloned().collect();
        let mut queued = self
            .storage
            .load_mutation_queue()
            .await
            .map_err(EngineError::Storage)?;
        queued.retain(|entry| entry.id != transaction);
        let replacement = self
            .rebuild_queued_layers_with_staged(queued, None, &staged)
            .await?;
        candidates.extend(layer_keys(&replacement));
        let mut settled_bases = self.load_bases(&candidates).await?;
        settled_bases.extend(staged);
        let after = effective_records(&settled_bases, &replacement, &candidates);
        if !self
            .storage
            .complete_mutation_with_shadow(
                transaction,
                claim,
                entries.clone(),
                projections,
                reconciliation,
            )
            .await
            .map_err(EngineError::Storage)?
        {
            return Err(EngineError::StaleMutationClaim(transaction));
        }
        let revision = self.advance_revision()?;
        self.update_loaded_search_catalogs(&entries);
        for (key, record) in entries {
            self.hot.put(key, record);
        }

        // No fallible reads after settlement: publish exactly the staged view.
        self.optimistic = replacement;
        live_projection_keys.extend(live_query::projection_keys(&self.optimistic));
        let live_records = candidates.clone();
        let visible_changed: BTreeSet<EntityKey<'static>> = candidates
            .into_iter()
            .filter(|key| before.get(key) != after.get(key))
            .collect();
        let affected_ops = self.deps.ops_for_keys(visible_changed.iter());
        self.live_queries.record(
            revision,
            live_query::Changes {
                records: live_records,
                projections: live_projection_keys,
            },
        );
        Ok(WriteResult {
            field_changes: Some(crate::field_changes::between(&before, &after)),
            identity_errors,
            revision,
            revision_advanced: true,
            search_changed_buckets: None,
            changed: durable_changed,
            affected_ops,
            reset: false,
            revalidations,
            mutation_uuid,
        })
    }

    /// Permanently fails the claimed head with an explicit supersession outcome.
    pub async fn rollback_optimistic_write_with_outcome(
        &mut self,
        transaction: OptimisticTransactionId,
        claim: MutationClaimToken,
    ) -> Result<RollbackOptimisticWriteResult, EngineError<S::Error>> {
        self.hydrate_optimistic().await?;
        let layer = self
            .optimistic
            .iter()
            .find(|layer| layer.id == transaction)
            .ok_or(EngineError::UnknownTransaction(transaction))?;
        let replacement_transaction_id = if layer.superseded {
            Some(
                self.optimistic
                    .iter()
                    .find(|candidate| candidate.uuid == layer.uuid && !candidate.superseded)
                    .map(|candidate| candidate.id)
                    .ok_or(EngineError::UnknownTransaction(transaction))?,
            )
        } else {
            None
        };
        let write_result = self.rollback_optimistic_write(transaction, claim).await?;
        match replacement_transaction_id {
            Some(replacement_transaction_id) => Ok(
                RollbackOptimisticWriteResult::DiscardedSuperseded(SupersededMutationResult {
                    write_result,
                    replacement_transaction_id,
                }),
            ),
            None => Ok(RollbackOptimisticWriteResult::RolledBack(write_result)),
        }
    }

    /// Permanently fails the claimed head, atomically removing its queue row
    /// and optimistic layer.
    pub async fn rollback_optimistic_write(
        &mut self,
        transaction: OptimisticTransactionId,
        claim: MutationClaimToken,
    ) -> Result<WriteResult, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        let mut live_projection_keys = live_query::projection_keys(&self.optimistic);
        self.hydrate_optimistic().await?;
        let layer = self
            .optimistic
            .iter()
            .find(|layer| layer.id == transaction)
            .ok_or(EngineError::UnknownTransaction(transaction))?;
        let mutation_uuid = Some(layer.uuid.to_string());
        // Failure can mean the server changed elsewhere (for example a draft
        // was sent). Hydration already rebased these durable query variables.
        let revalidations = layer.revalidations.clone();
        let mut candidates = layer_keys(&self.optimistic);
        let bases = self.load_bases(&candidates).await?;
        let before = effective_records(&bases, &self.optimistic, &candidates);
        let reconciliation = self
            .stage_shadow_reconciliation(transaction, &[], &IdentityMap::new())
            .await?;
        let mut queued = self
            .storage
            .load_mutation_queue()
            .await
            .map_err(EngineError::Storage)?;
        queued.retain(|entry| entry.id != transaction);
        let replacement = self.rebuild_queued_layers(queued).await?;
        candidates.extend(layer_keys(&replacement));
        let current_bases = self.load_bases(&candidates).await?;
        let after = effective_records(&current_bases, &replacement, &candidates);
        if !self
            .storage
            .discard_mutation_with_shadow(transaction, claim, reconciliation)
            .await
            .map_err(EngineError::Storage)?
        {
            return Err(EngineError::StaleMutationClaim(transaction));
        }
        let revision = self.advance_revision()?;
        self.optimistic = replacement;
        live_projection_keys.extend(live_query::projection_keys(&self.optimistic));
        let live_records = candidates.clone();
        let visible_changed: BTreeSet<EntityKey<'static>> = candidates
            .into_iter()
            .filter(|key| before.get(key) != after.get(key))
            .collect();
        let affected_ops = self.deps.ops_for_keys(visible_changed.iter());
        self.live_queries.record(
            revision,
            live_query::Changes {
                records: live_records,
                projections: live_projection_keys,
            },
        );
        Ok(WriteResult {
            field_changes: Some(crate::field_changes::between(&before, &after)),
            identity_errors: Vec::new(),
            revision,
            revision_advanced: true,
            search_changed_buckets: None,
            changed: BTreeSet::new(),
            affected_ops,
            reset: false,
            revalidations,
            mutation_uuid,
        })
    }

    /// Loads every normalized record reached while resolving query-rooted
    /// link updates, including records currently outside the hot tier.
    async fn load_link_patch_bases(
        &mut self,
        mut candidates: BTreeSet<EntityKey<'static>>,
        layers: &[OptimisticLayer],
        pending_updates: &RecordUpdates,
        patches: &[OptimisticLinkPatch],
        staged: &RecordUpdates,
    ) -> Result<
        (
            BTreeSet<EntityKey<'static>>,
            HashMap<EntityKey<'static>, Record>,
        ),
        EngineError<S::Error>,
    > {
        candidates.extend(patches.iter().map(OptimisticLinkPatch::root_key));
        loop {
            let mut bases = self.load_bases(&candidates).await?;
            for key in &candidates {
                if let Some(record) = staged.get(key) {
                    bases.insert(key.clone(), record.clone());
                }
            }
            let composed = effective_records(&bases, layers, &candidates);
            let mut effective = present_records(composed);
            merge_updates_into_effective(&mut effective, pending_updates);
            // Inserted references validate against a loaded record, so a
            // patch referencing a durable entity absent from the updates
            // must pull that entity's base in too. A genuinely record-less
            // key stays absent after loading and the filter below stops the
            // loop from retrying it.
            let missing: BTreeSet<_> = patches
                .iter()
                .flat_map(|patch| missing_patch_records(&self.schema, &effective, patch))
                .chain(patches.iter().filter_map(|patch| {
                    let inserted = patch.operation.inserted_entity_key()?;
                    (!effective.contains_key(inserted)).then(|| inserted.clone())
                }))
                .filter(|key| !candidates.contains(key))
                .collect();
            if missing.is_empty() {
                return Ok((candidates, bases));
            }
            candidates.extend(missing);
        }
    }

    /// Loads current durable records (hot tier, then storage) for `keys`
    /// without touching LRU recency or persisting anything.
    async fn load_bases(
        &mut self,
        keys: &BTreeSet<EntityKey<'static>>,
    ) -> Result<HashMap<EntityKey<'static>, Record>, EngineError<S::Error>> {
        let mut out = HashMap::new();
        let mut missing = Vec::new();
        for key in keys {
            match self.hot.peek(key) {
                Some(record) => {
                    out.insert(key.clone(), record.clone());
                }
                None => missing.push(key.clone()),
            }
        }
        if !missing.is_empty() {
            let fetched = self
                .storage
                .get_batch(&missing)
                .await
                .map_err(EngineError::Storage)?;
            for (key, record) in missing.into_iter().zip(fetched) {
                if let Some(r) = record {
                    out.insert(key, r);
                }
            }
        }
        Ok(out)
    }

    /// Recovers cached argument variants of one generated query field.
    ///
    /// Only records needed to resolve the selected field's normalized owner
    /// are loaded. The recovered variants are not denormalized.
    pub async fn inspect_query_variants(
        &mut self,
        inspection: &QueryInspection,
    ) -> Result<Vec<CachedQueryVariant>, EngineError<S::Error>> {
        self.hydrate_optimistic().await?;
        let operation = Self::document(&mut self.docs, &inspection.query)?
            .operation(inspection.operation_name.as_deref())?
            .clone();
        let prepared = prepare(&self.schema, &operation, &inspection.path)?;

        let mut candidates = BTreeSet::from([EntityKey::root()]);
        let variants = loop {
            let bases = self.load_bases(&candidates).await?;
            let effective =
                present_records(effective_records(&bases, &self.optimistic, &candidates));
            match resolve_owner(&self.schema, &effective, &operation, &inspection.path)? {
                OwnerResolution::Owner(owner) => break recover_variants(&owner, &prepared)?,
                OwnerResolution::Absent => return Ok(Vec::new()),
                OwnerResolution::NeedRecord(key) if !candidates.contains(&key) => {
                    candidates.insert(key.into_owned());
                }
                OwnerResolution::NeedRecord(_) => return Ok(Vec::new()),
            }
        };
        Ok(variants
            .into_iter()
            .map(|variables| CachedQueryVariant { variables })
            .collect())
    }

    /// Enumerates and materializes cached argument variants of one generated
    /// query field.
    ///
    /// Normalized owners, canonical field keys, cold records, and optimistic
    /// layers remain internal. Every recovered variable set is read through
    /// the ordinary denormalizer so inspection has cache-only read semantics.
    pub async fn inspect_query(
        &mut self,
        inspection: &QueryInspection,
    ) -> Result<Vec<CachedQueryInstance>, EngineError<S::Error>> {
        let variants = self.inspect_query_variants(inspection).await?;
        let mut instances = Vec::with_capacity(variants.len());
        for CachedQueryVariant { variables } in variants {
            if !matches_variable_filters(&variables, &inspection.variable_filters) {
                continue;
            }
            let value = match self
                .read_query(
                    None,
                    &inspection.query,
                    inspection.operation_name.as_deref(),
                    &variables,
                )
                .await?
            {
                ReadResult::Hit { data } => Some(selected_result_value(&data, &inspection.path)?),
                ReadResult::Miss => None,
            };
            instances.push(CachedQueryInstance { variables, value });
        }
        Ok(instances)
    }

    /// Reacts to a reset performed by *another* engine instance sharing the
    /// same storage (cross-tab broadcast): drops all local in-memory state
    /// and returns every local active operation for re-execution.
    pub fn external_reset(&mut self) -> Result<Revisioned<BTreeSet<OpId>>, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        self.hot.clear();
        self.live_queries = live_query::LiveQueries::default();
        self.window_observations = window_observation::WindowObservations::default();
        self.query_watches = watch_query::QueryWatches::default();
        self.docs.clear();
        self.optimistic.clear();
        self.search_catalogs.clear();
        // Another engine may have rebound the shared storage and changed the
        // durable queue, so both identity and optimism must re-hydrate.
        self.optimistic_hydrated = false;
        self.identity = IdentityState::NotHydrated;
        let affected = self.deps.all_ops();
        self.advance_revision()?;
        Ok(self.revisioned(affected))
    }

    /// Unregisters an active operation (urql teardown).
    pub fn teardown_operation(&mut self, op_id: OpId) {
        self.deps.remove_op(op_id);
        self.query_watches.remove(op_id);
    }

    /// Handles records changed *outside* this engine instance (another tab's
    /// engine in the dedicated-worker fallback topology, or push-driven
    /// invalidation): evicts them from the hot tier so the next read hits
    /// storage, and returns the local active operations that depend on them.
    pub fn invalidate_keys<'k>(
        &mut self,
        keys: impl IntoIterator<Item = &'k EntityKey<'static>>,
    ) -> Result<Revisioned<BTreeSet<OpId>>, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        let affected = self.invalidate_keys_inner(keys);
        self.advance_revision()?;
        Ok(self.revisioned(affected))
    }

    fn invalidate_keys_inner<'k>(
        &mut self,
        keys: impl IntoIterator<Item = &'k EntityKey<'static>>,
    ) -> BTreeSet<OpId> {
        let mut affected = BTreeSet::new();
        for key in keys {
            self.hot.pop(key);
            affected.extend(self.deps.ops_for_keys([key]));
        }
        // The durable projection was updated by the writing context. Reload
        // only the compact catalog on the next text search.
        self.search_catalogs.clear();
        affected
    }

    /// Deletes locally stale records from both durable and hot tiers and
    /// returns active operations that traversed those records.
    ///
    /// Use this for an explicit server-provided cache-deletion effect.
    /// Cross-engine notifications for records already written to shared
    /// storage should use
    /// [`Self::invalidate_keys`] instead.
    pub async fn delete_keys(
        &mut self,
        keys: &[EntityKey<'static>],
    ) -> Result<Revisioned<BTreeSet<OpId>>, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        let affected = self.deps.ops_for_keys(keys.iter());
        self.storage
            .delete_batch(keys)
            .await
            .map_err(EngineError::Storage)?;
        for key in keys {
            self.hot.pop(key);
            self.search_catalogs.remove(key);
        }
        self.advance_revision()?;
        Ok(self.revisioned(affected))
    }

    /// Drops all cached state (for example, on logout), including any pending
    /// optimistic layers.
    pub async fn clear(&mut self) -> Result<CacheRevision, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        self.storage.clear().await.map_err(EngineError::Storage)?;
        self.hot.clear();
        self.live_queries = live_query::LiveQueries::default();
        self.window_observations = window_observation::WindowObservations::default();
        self.query_watches = watch_query::QueryWatches::default();
        self.optimistic.clear();
        self.optimistic_hydrated = true;
        self.search_catalogs.clear();
        self.deps = DepIndex::new();
        // The wipe below removes the binding record too.
        self.identity = IdentityState::Missing;
        self.advance_revision()
    }

    pub fn active_ops(&self) -> usize {
        self.deps.active_ops()
    }

    /// Returns payload-free durable mutation queue diagnostics.
    pub async fn queue_diagnostics(&self) -> Result<QueueDiagnostics, EngineError<S::Error>> {
        self.storage
            .queue_diagnostics()
            .await
            .map_err(EngineError::Storage)
    }

    /// Access to the underlying storage for non-consuming diagnostics.
    pub fn storage(&self) -> &S {
        &self.storage
    }

    /// Consumes the engine and returns its owned storage.
    ///
    /// Hosts must use this transition for storage lifecycles that require
    /// exclusive ownership, such as proving that a browser database connection
    /// is closed before preserving or physically resetting its OPFS files.
    pub fn into_storage(self) -> S {
        self.storage
    }

    /// Memoized document parse. Takes the cache (not `&mut self`) so callers
    /// can keep the returned borrow while using other engine fields.
    fn document<'d>(
        docs: &'d mut LruCache<String, Document>,
        query: &str,
    ) -> Result<&'d Document, DocumentError> {
        docs.try_get_or_insert_ref(query, || Document::parse(query))
    }
}

impl<S: PredicateIndexStorage> Engine<S> {
    /// Atomically persist normalized records and generic projection lifecycle changes.
    pub async fn put_records_with_projections(
        &mut self,
        origin_op: Option<OpId>,
        entries: Vec<(EntityKey<'static>, Record)>,
        projections: Vec<ProjectionMutation>,
    ) -> Result<WriteResult, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        let mut updates = RecordUpdates::new();
        for (key, record) in entries {
            updates.entry(key).or_default().merge(record);
        }
        let PersistedChanges {
            field_changes,
            changed,
            revision,
            revision_advanced,
            ..
        } = self.persist_updates(updates, projections).await?;
        let mut affected_ops = self.deps.ops_for_keys(changed.iter());
        if let Some(origin_op) = origin_op {
            affected_ops.remove(&origin_op);
        }
        Ok(WriteResult {
            field_changes: self.optimistic.is_empty().then_some(field_changes),
            identity_errors: Vec::new(),
            revision,
            revision_advanced,
            search_changed_buckets: None,
            changed,
            affected_ops,
            reset: false,
            revalidations: Vec::new(),
            mutation_uuid: None,
        })
    }

    /// Mark projections incomplete atomically without changing normalized base records.
    pub async fn mark_projections_incomplete(
        &mut self,
        projections: Vec<ProjectionMutation>,
    ) -> Result<CacheRevision, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        self.storage
            .put_batch_with_projections(Vec::new(), projections)
            .await
            .map_err(EngineError::Storage)?;
        self.advance_revision()
    }

    /// Marks generic projections incomplete and invalidates normalized hot-tier records
    /// as one externally observed logical view mutation.
    pub async fn invalidate_keys_with_projections(
        &mut self,
        keys: &[EntityKey<'static>],
        projections: Vec<ProjectionMutation>,
    ) -> Result<Revisioned<BTreeSet<OpId>>, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        self.storage
            .put_batch_with_projections(Vec::new(), projections)
            .await
            .map_err(EngineError::Storage)?;
        let affected = self.invalidate_keys_inner(keys.iter());
        self.advance_revision()?;
        Ok(self.revisioned(affected))
    }

    /// Delete normalized records and generic projections in one storage transaction.
    pub async fn delete_keys_with_projections(
        &mut self,
        keys: &[EntityKey<'static>],
        projection_keys: &[PredicateRecordKey],
    ) -> Result<Revisioned<BTreeSet<OpId>>, EngineError<S::Error>> {
        self.delete_keys_with_projection_changes(
            keys,
            projection_keys
                .iter()
                .cloned()
                .map(ProjectionMutation::Delete)
                .collect(),
        )
        .await
    }

    /// Atomically delete records and update projections that depend on them.
    pub async fn delete_keys_with_projection_changes(
        &mut self,
        keys: &[EntityKey<'static>],
        projections: Vec<ProjectionMutation>,
    ) -> Result<Revisioned<BTreeSet<OpId>>, EngineError<S::Error>> {
        self.ensure_revision_can_advance()?;
        let affected = self.deps.ops_for_keys(keys.iter());
        self.storage
            .delete_batch_with_projection_changes(keys, projections)
            .await
            .map_err(EngineError::Storage)?;
        for key in keys {
            self.hot.pop(key);
            self.search_catalogs.remove(key);
        }
        self.advance_revision()?;
        Ok(self.revisioned(affected))
    }

    /// Reconcile server-page membership and local candidates at one engine revision.
    pub async fn reconcile_predicate_index(
        &mut self,
        query: &ValidatedIndexQuery,
        baseline: &[PredicateBaselineEntry],
    ) -> Result<Revisioned<PredicateReconciliation>, EngineError<S::Error>> {
        if baseline.len() > MAX_RECONCILIATION_BASELINE {
            return Err(RecordSelectionError::TooManyKeys {
                count: baseline.len(),
                max: MAX_RECONCILIATION_BASELINE,
            }
            .into());
        }
        let value = self
            .storage
            .reconcile_predicate_index(query, baseline)
            .await
            .map_err(EngineError::Storage)?;
        Ok(self.revisioned(value))
    }

    /// Execute a complete generic exact-index query over authoritative and optimistic projections.
    pub async fn query_predicate_index(
        &mut self,
        query: &ValidatedIndexQuery,
    ) -> Result<Revisioned<PredicateQueryResult>, EngineError<S::Error>> {
        let value = self.query_predicate_index_value(query).await?;
        Ok(self.revisioned(value))
    }

    async fn query_predicate_index_value(
        &mut self,
        query: &ValidatedIndexQuery,
    ) -> Result<PredicateQueryResult, EngineError<S::Error>> {
        self.storage
            .query_predicate_index(query)
            .await
            .map_err(EngineError::Storage)
    }
}

/// Read view over the durable tiers plus the optimistic composition. Uses
/// `peek` (no recency mutation) — recency is refreshed once per read from
/// the dep set.
struct EngineSource<'a> {
    hot: &'a LruCache<EntityKey<'static>, Record>,
    /// Durable records batch-fetched from storage during this read.
    fetched: &'a HashMap<EntityKey<'static>, Record>,
    /// Optimistically touched keys: durable base + layers, pre-merged.
    /// Takes precedence over both durable tiers.
    composed: &'a HashMap<EntityKey<'static>, Record>,
}

impl RecordSource for EngineSource<'_> {
    fn get(&self, key: &EntityKey<'static>) -> Option<&Record> {
        self.composed
            .get(key)
            .or_else(|| self.fetched.get(key))
            .or_else(|| self.hot.peek(key))
    }
}

/// All active optimistic layers' updates merged in creation order (later
/// layers override earlier ones field-by-field).
fn merged_optimistic(layers: &[OptimisticLayer]) -> BTreeMap<EntityKey<'static>, Record> {
    let mut out: BTreeMap<EntityKey<'static>, Record> = BTreeMap::new();
    for layer in layers {
        for (key, record) in &layer.updates {
            out.entry(key.clone()).or_default().merge(record.clone());
        }
    }
    out
}

/// Merges partial response updates into already-loaded durable bases without
/// mutating the hot tier or storage. The caller can then atomically settle a
/// queued mutation before publishing the staged records in memory.
fn stage_updates(
    bases: &HashMap<EntityKey<'static>, Record>,
    updates: RecordUpdates,
) -> (
    BTreeSet<EntityKey<'static>>,
    Vec<(EntityKey<'static>, Record)>,
) {
    let mut changed = BTreeSet::new();
    let mut entries = Vec::with_capacity(updates.len());
    for (key, update) in updates {
        let merged = match bases.get(&key) {
            Some(existing) => {
                let mut merged = existing.clone();
                if merged.merge(update) {
                    changed.insert(key.clone());
                }
                merged
            }
            None => {
                changed.insert(key.clone());
                update
            }
        };
        entries.push((key, merged));
    }
    (changed, entries)
}

/// Effective visible records for `keys`: durable base + every active layer
/// merged in order. `None` when the key exists nowhere.
fn effective_records(
    bases: &HashMap<EntityKey<'static>, Record>,
    layers: &[OptimisticLayer],
    keys: &BTreeSet<EntityKey<'static>>,
) -> HashMap<EntityKey<'static>, Option<Record>> {
    keys.iter()
        .map(|key| {
            let mut record: Option<Record> = bases.get(key).cloned();
            for layer in layers {
                if let Some(update) = layer.updates.get(key) {
                    record
                        .get_or_insert_with(Record::default)
                        .merge(update.clone());
                }
            }
            (key.clone(), record)
        })
        .collect()
}

fn present_records(
    records: HashMap<EntityKey<'static>, Option<Record>>,
) -> HashMap<EntityKey<'static>, Record> {
    records
        .into_iter()
        .filter_map(|(key, record)| record.map(|record| (key, record)))
        .collect()
}

fn merge_updates_into_effective(
    effective: &mut HashMap<EntityKey<'static>, Record>,
    updates: &RecordUpdates,
) {
    for (key, update) in updates {
        effective
            .entry(key.clone())
            .or_default()
            .merge(update.clone());
    }
}

fn layer_keys(layers: &[OptimisticLayer]) -> BTreeSet<EntityKey<'static>> {
    layers
        .iter()
        .flat_map(|layer| layer.updates.keys().cloned())
        .collect()
}

fn cursor_allows(cursor: Option<&SearchCursor>, document: &SearchDocument) -> bool {
    cursor.is_none_or(|cursor| {
        document.timestamp_ms < cursor.timestamp_ms
            || (document.timestamp_ms == cursor.timestamp_ms
                && document.record_key > cursor.record_key)
    })
}

fn deduplicate_revalidations(
    revalidations: impl IntoIterator<Item = QueryRevalidation>,
) -> Vec<QueryRevalidation> {
    let mut unique = BTreeSet::new();
    for mut revalidation in revalidations {
        if let Ok(variables) = serde_json::from_str::<Json>(&revalidation.variables_json) {
            revalidation.variables_json = canonical_json(&variables);
        }
        unique.insert(revalidation);
    }
    unique.into_iter().collect()
}