Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

15.7 KB
Raw Blame
//! Maintained fragment views over the existing predicate index.
//!
//! The engine retains row projections and their dependencies. Writes invalidate
//! only dependent rows; projection changes reevaluate indexed membership. A read
//! publishes one coherent delta, including rollback, at the engine revision.

use super::*;
use serde::{Deserialize, Serialize};
use std::{collections::VecDeque, sync::Arc};

const VIEW_CAPACITY: usize = 32;
const JOURNAL_CAPACITY: usize = 128;

pub(super) type RowDependencies = BTreeMap<EntityKey<'static>, BTreeSet<EntityKey<'static>>>;

/// A bounded server baseline plus a locally executable filter and projection.
#[derive(Clone, PartialEq, Eq)]
pub struct LiveQuerySpec {
    /// Validated filtering, ordering, and candidate limit.
    pub query: ValidatedIndexQuery,
    /// Positive membership evidence for this exact server query.
    pub baseline: Vec<PredicateBaselineEntry>,
    /// Selected fields, including nested normalized records.
    pub selection: Arc<RecordSelection>,
}

/// Identity of one retained live-view specification, not a persistent cache key.
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(transparent)]
pub struct LiveQueryRegistration(Uuid);

/// Changes since the subscriber's last accepted view revision.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct LiveQueryUpdate {
    /// Identifies this retained specification; invalid after replacement or eviction.
    pub registration_id: LiveQueryRegistration,
    /// Immutable proof identity for this exact evaluated window.
    pub observation_id: String,
    /// Exact engine revision observed by this atomic read.
    pub revision: String,
    /// The subscriber must discard previous rows (first read, eviction or gap).
    pub reset: bool,
    /// New display order; omitted when membership and order did not change.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub keys: Option<Vec<PredicateRecordKey>>,
    /// Changed or newly materialized rows. Unchanged projections are omitted.
    pub upserts: Vec<SelectedRecord>,
    /// Field changes within already materialized rows, using response paths.
    pub patches: Vec<LiveRowPatch>,
    /// Previously materialized rows that no longer have a complete projection.
    pub removed: Vec<EntityKey<'static>>,
    /// Baseline members whose current predicate cannot be evaluated safely.
    pub retained_keys: Vec<PredicateRecordKey>,
    /// Whether optimistic projection state contributes to membership.
    pub optimistic: bool,
}

/// A response object property or list index (aliases are already resolved).
#[derive(Debug, Clone, Serialize)]
#[serde(untagged)]
pub enum ResponsePathSegment {
    /// Object property.
    Field(String),
    /// Array position.
    Index(usize),
}

/// Replacement of one value inside a selected record.
#[derive(Debug, Serialize)]
pub struct LiveFieldPatch {
    /// Path relative to the selected record; empty replaces the whole record.
    pub path: Vec<ResponsePathSegment>,
    /// Current effective value after all optimistic layers.
    pub value: Json,
}

/// Field and identity metadata changes for one already-visible record.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct LiveRowPatch {
    /// Canonical normalized key.
    pub record_key: EntityKey<'static>,
    /// Replacements applied atomically in response order.
    pub fields: Vec<LiveFieldPatch>,
    /// Current queue identity metadata.
    pub identity: identity::IdentityStatus,
}

fn diff_fields(
    before: &Json,
    after: &Json,
    path: &mut Vec<ResponsePathSegment>,
    out: &mut Vec<LiveFieldPatch>,
) {
    if before == after {
        return;
    }
    match (before, after) {
        (Json::Object(before), Json::Object(after))
            if before.keys().all(|key| after.contains_key(key)) =>
        {
            for (key, value) in after {
                path.push(ResponsePathSegment::Field(key.clone()));
                if let Some(previous) = before.get(key) {
                    diff_fields(previous, value, path, out);
                } else {
                    out.push(LiveFieldPatch {
                        path: path.clone(),
                        value: value.clone(),
                    });
                }
                path.pop();
            }
        }
        (Json::Array(before), Json::Array(after))
            if before.len() == after.len()
                && before
                    .iter()
                    .zip(after)
                    .all(|(a, b)| a.get("id") == b.get("id")) =>
        {
            for (index, (a, b)) in before.iter().zip(after).enumerate() {
                path.push(ResponsePathSegment::Index(index));
                diff_fields(a, b, path, out);
                path.pop();
            }
        }
        _ => out.push(LiveFieldPatch {
            path: path.clone(),
            value: after.clone(),
        }),
    }
}

#[derive(Default)]
pub(super) struct Changes {
    pub records: BTreeSet<EntityKey<'static>>,
    pub projections: BTreeSet<PredicateRecordKey>,
}

struct View {
    registration_id: LiveQueryRegistration,
    spec: LiveQuerySpec,
    revision: CacheRevision,
    membership: PredicateReconciliation,
    rows: BTreeMap<EntityKey<'static>, SelectedRecord>,
    dependencies: RowDependencies,
    dependents: BTreeMap<EntityKey<'static>, BTreeSet<EntityKey<'static>>>,
}

impl View {
    fn replace_dependencies(
        &mut self,
        root: &EntityKey<'static>,
        next: Option<BTreeSet<EntityKey<'static>>>,
    ) {
        if let Some(previous) = self.dependencies.remove(root) {
            for key in previous {
                if let Some(roots) = self.dependents.get_mut(&key) {
                    roots.remove(root);
                    if roots.is_empty() {
                        self.dependents.remove(&key);
                    }
                }
            }
        }
        if let Some(next) = next {
            for key in &next {
                self.dependents
                    .entry(key.clone())
                    .or_default()
                    .insert(root.clone());
            }
            self.dependencies.insert(root.clone(), next);
        }
    }
}

pub(super) struct LiveQueries {
    views: LruCache<String, View>,
    // None is a conservative barrier (reset, external invalidation, or a write
    // that failed after advancing its revision). Never infer an empty change.
    journal: VecDeque<(CacheRevision, Option<Changes>)>,
}

impl Default for LiveQueries {
    fn default() -> Self {
        Self {
            views: LruCache::new(NonZeroUsize::new(VIEW_CAPACITY).unwrap()),
            journal: VecDeque::new(),
        }
    }
}

impl LiveQueries {
    pub(super) fn advance(&mut self, revision: CacheRevision) {
        self.journal.push_back((revision, None));
        if self.journal.len() > JOURNAL_CAPACITY {
            self.journal.pop_front();
        }
    }

    pub(super) fn record(&mut self, revision: CacheRevision, changes: Changes) {
        if let Some((last, entry)) = self.journal.back_mut()
            && *last == revision
        {
            *entry = Some(changes);
        }
    }

    pub(super) fn extend(
        &mut self,
        revision: CacheRevision,
        records: impl IntoIterator<Item = EntityKey<'static>>,
        projections: impl IntoIterator<Item = PredicateRecordKey>,
    ) {
        if let Some((last, Some(entry))) = self.journal.back_mut()
            && *last == revision
        {
            entry.records.extend(records);
            entry.projections.extend(projections);
        }
    }

    pub(super) fn changes_since(
        &self,
        since: CacheRevision,
        now: CacheRevision,
    ) -> Option<Changes> {
        let mut cursor = since;
        let mut changes = Changes::default();
        for (revision, entry) in self
            .journal
            .iter()
            .filter(|(revision, _)| *revision > since)
        {
            if cursor.checked_successor()? != *revision {
                return None;
            }
            let entry = entry.as_ref()?;
            changes.records.extend(entry.records.iter().cloned());
            changes
                .projections
                .extend(entry.projections.iter().cloned());
            cursor = *revision;
        }
        (cursor == now).then_some(changes)
    }
}

impl<S: PredicateIndexStorage> Engine<S> {
    /// Maintains an ordered fragment view under a caller-owned subscription id.
    ///
    /// Reusing an id with a different spec produces a full replacement. A stale
    /// subscriber revision also gets a replacement, never an inapplicable delta.
    /// Views are bounded and evictable; no correctness depends on their retention.
    pub async fn read_live_query(
        &mut self,
        id: &str,
        spec: LiveQuerySpec,
        since: Option<CacheRevision>,
    ) -> Result<LiveQueryUpdate, EngineError<S::Error>> {
        if id.len() > 128 || id.is_empty() {
            return Err(RecordSelectionError::InvalidKey.into());
        }
        self.hydrate_optimistic().await?;
        self.read_registered_live_query(id, spec, since).await
    }

    /// Reads a retained specification without retransmitting it. A missing or
    /// replaced registration returns None so the caller can register it again.
    /// Lookup follows durable optimistic-state hydration and shares the read's
    /// engine lease. A replaced or cleared specification cannot reuse this token.
    pub async fn resume_live_query(
        &mut self,
        id: &str,
        registration_id: LiveQueryRegistration,
        since: Option<CacheRevision>,
    ) -> Result<Option<LiveQueryUpdate>, EngineError<S::Error>> {
        self.hydrate_optimistic().await?;
        let Some(view) = self
            .live_queries
            .views
            .peek(id)
            .filter(|view| view.registration_id == registration_id)
        else {
            return Ok(None);
        };
        let spec = view.spec.clone();
        self.read_registered_live_query(id, spec, since)
            .await
            .map(Some)
    }

    async fn read_registered_live_query(
        &mut self,
        id: &str,
        spec: LiveQuerySpec,
        since: Option<CacheRevision>,
    ) -> Result<LiveQueryUpdate, EngineError<S::Error>> {
        let old = self
            .live_queries
            .views
            .pop(id)
            .filter(|view| view.spec == spec);
        let changes = old.as_ref().and_then(|view| {
            self.live_queries
                .changes_since(view.revision, self.revision)
        });
        let reset = old.as_ref().is_none_or(|view| Some(view.revision) != since);
        let membership = if let (Some(old), Some(changes)) = (&old, &changes)
            && changes.projections.is_empty()
        {
            old.membership.clone()
        } else {
            self.reconcile_predicate_index(&spec.query, &spec.baseline)
                .await?
                .value
        };
        let order_changed = reset
            || old
                .as_ref()
                .is_none_or(|view| view.membership.keys != membership.keys);
        let mut view = old.unwrap_or_else(|| View {
            registration_id: LiveQueryRegistration(Uuid::now_v7()),
            spec: spec.clone(),
            revision: self.revision,
            membership: membership.clone(),
            rows: BTreeMap::new(),
            dependencies: BTreeMap::new(),
            dependents: BTreeMap::new(),
        });
        let keys: BTreeSet<_> = membership
            .keys
            .iter()
            .map(|key| EntityKey(key.as_str().to_owned().into()))
            .collect();
        let mut dirty = BTreeSet::new();
        if let Some(changes) = &changes {
            for changed in &changes.records {
                dirty.extend(view.dependents.get(changed).into_iter().flatten().cloned());
            }
            // New members have never been projected, including members that
            // were formerly outside a bounded candidate page.
            dirty.extend(
                keys.iter()
                    .filter(|key| !view.dependencies.contains_key(*key))
                    .cloned(),
            );
        } else {
            dirty.extend(keys.iter().cloned());
        }
        dirty.retain(|key| keys.contains(key));
        let removed_members: Vec<_> = view
            .dependencies
            .keys()
            .filter(|key| !keys.contains(*key))
            .cloned()
            .collect();
        let mut removed = Vec::new();
        for key in removed_members {
            view.replace_dependencies(&key, None);
            if view.rows.remove(&key).is_some() {
                removed.push(key);
            }
        }
        let mut upserts = Vec::new();
        let mut patches = Vec::new();
        let dirty: Vec<_> = dirty.into_iter().collect();
        for chunk in dirty.chunks(MAX_RECORD_SELECTION_KEYS) {
            let (selected, mut tracked) = self.read_records_tracked(&spec.selection, chunk).await?;
            let mut selected: BTreeMap<_, _> = selected
                .into_iter()
                .map(|row| (row.record_key.clone(), row))
                .collect();
            for key in chunk {
                view.replace_dependencies(key, tracked.remove(key));
                match selected.remove(key) {
                    Some(row) => {
                        if view.rows.get(key) != Some(&row) {
                            if let Some(previous) = view.rows.get(key) {
                                let mut fields = Vec::new();
                                diff_fields(
                                    &previous.record,
                                    &row.record,
                                    &mut Vec::new(),
                                    &mut fields,
                                );
                                patches.push(LiveRowPatch {
                                    record_key: key.clone(),
                                    fields,
                                    identity: row.identity.clone(),
                                });
                            } else {
                                upserts.push(row.clone());
                            }
                            view.rows.insert(key.clone(), row);
                        }
                    }
                    None => {
                        if view.rows.remove(key).is_some() {
                            removed.push(key.clone());
                        }
                    }
                }
            }
        }
        if reset {
            upserts = view.rows.values().cloned().collect();
            removed.clear();
            patches.clear();
        }
        let observation_id = self
            .observe_live_query_window(&spec, &membership, &view.dependencies)
            .await?;
        let result = LiveQueryUpdate {
            registration_id: view.registration_id,
            observation_id,
            revision: self.revision.to_string(),
            reset,
            keys: order_changed.then(|| membership.keys.clone()),
            upserts,
            patches,
            removed,
            retained_keys: membership.retained_keys.clone(),
            optimistic: membership.optimistic,
        };
        view.membership = membership;
        view.revision = self.revision;
        self.live_queries.views.put(id.to_owned(), view);
        Ok(result)
    }

    /// Releases the in-memory view; persisted records and mutations are unaffected.
    pub fn release_live_query(&mut self, id: &str) {
        self.live_queries.views.pop(id);
    }
}

#[cfg(test)]
mod test;

pub(super) fn projection_keys(layers: &[OptimisticLayer]) -> BTreeSet<PredicateRecordKey> {
    layers
        .iter()
        .flat_map(|layer| &layer.projection_mutations)
        .map(|mutation| mutation.record_key().clone())
        .collect()
}