Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
//! The per-email-link calendar change log: watermarks, delta pages, and the
//! policy that turns raw log rows into a bounded, gap-free page.
//!
//! Every calendar write transaction appends its changes to the log of the
//! email link the changed rows sync through, under a per-link sequence. A
//! client remembers the last sequence it applied for each visible link and
//! asks for everything after it; visibility is resolved at read time from the
//! links the viewer owns or is delegated.
use std::{
collections::{BTreeMap, HashMap, HashSet},
future::Future,
};
use chrono::{DateTime, Utc};
use rootcause::Report;
use uuid::Uuid;
use super::models::{
ActorInboxes, CalendarEvent, CalendarOccurrence, OccurrenceException, OwnedInbox,
VisibleCalendar,
};
#[cfg(test)]
mod test;
/// Most distinct events one delta page hydrates.
pub const MAX_DELTA_EVENTS: usize = 500;
/// Most occurrences one delta page carries. A page always carries at least
/// one event so a single very long series still makes progress.
pub const MAX_DELTA_OCCURRENCES: usize = 5000;
/// Most log rows read for one link per page.
pub const MAX_DELTA_ROWS_PER_LINK: usize = 2000;
/// How long log rows are kept. A client whose watermark is older must reset.
pub const CHANGE_LOG_RETENTION: chrono::Duration = chrono::Duration::days(30);
/// Most log rows one retention batch removes.
pub const CHANGE_LOG_PRUNE_BATCH: usize = 10_000;
/// One logged change to an event or a calendar.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum CalendarChangeOp {
/// The event was created or rewritten.
UpsertEvent(Uuid),
/// The event no longer exists.
DeleteEvent(Uuid),
/// The calendar was created, restored, or changed a visible field.
UpsertCalendar(Uuid),
/// The calendar was removed.
DeleteCalendar(Uuid),
}
/// One row of a link's change log.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CalendarChange {
/// Position in the link's log, starting at 1.
pub seq: i64,
/// What changed.
pub op: CalendarChangeOp,
}
/// How far a client has applied one link's log.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CalendarLinkWatermark {
/// Email link identifier.
pub link_id: Uuid,
/// Last applied sequence; 0 before anything was logged.
pub seq: i64,
}
/// A client's position in every visible link's log.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct CalendarWatermark(BTreeMap<Uuid, i64>);
impl CalendarWatermark {
/// Build a watermark from per-link positions. A link listed twice keeps
/// its lowest position, so nothing between the two is skipped.
pub fn from_links(links: impl IntoIterator<Item = CalendarLinkWatermark>) -> Self {
let mut positions = BTreeMap::new();
for link in links {
positions
.entry(link.link_id)
.and_modify(|seq: &mut i64| *seq = (*seq).min(link.seq))
.or_insert(link.seq);
}
Self(positions)
}
/// Whether no link is tracked.
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// The position for `link_id`, when tracked.
pub fn seq(&self, link_id: Uuid) -> Option<i64> {
self.0.get(&link_id).copied()
}
/// Per-link positions ordered by link.
pub fn links(&self) -> impl Iterator<Item = CalendarLinkWatermark> + '_ {
self.0
.iter()
.map(|(&link_id, &seq)| CalendarLinkWatermark { link_id, seq })
}
}
/// One visible link's log, read in a single snapshot.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CalendarLinkLog {
/// Email link identifier.
pub link_id: Uuid,
/// Last sequence written for the link; 0 when nothing was ever logged.
pub counter: i64,
/// Lowest position a client may resume from. Rows at or below it were
/// pruned, so a client behind it has lost changes.
pub floor: i64,
/// Rows after the requested position in sequence order, at most the
/// requested limit. Empty when the link was not requested.
pub changes: Vec<CalendarChange>,
}
/// One instance of a changed event.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EventOccurrence {
/// The materialized instance.
pub occurrence: CalendarOccurrence,
/// The instance's exception content, empty when it has none.
pub exception: OccurrenceException,
}
/// A changed event with its complete current set of visible occurrences.
/// Clients replace everything they hold for the event with this.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CalendarEventChange {
/// The series event.
pub event: CalendarEvent,
/// Email link the event syncs through.
pub link_id: Uuid,
/// Every non-cancelled materialized occurrence, ordered by start.
pub occurrences: Vec<EventOccurrence>,
}
/// The changed events a page could afford to load.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct EventChangeLoad {
/// Considered events that are still visible with a visible occurrence.
pub events: Vec<CalendarEventChange>,
/// How many requested ids, from the front, were considered. A considered
/// id missing from `events` is gone; later ids were not read.
pub considered: usize,
}
/// One bounded page of changes after a watermark.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct CalendarChangesPage {
/// Events created or changed, with their full occurrence sets.
pub events: Vec<CalendarEventChange>,
/// Events that no longer exist or no longer have a visible occurrence.
pub deleted_event_ids: Vec<Uuid>,
/// Calendars created or changed.
pub calendars: Vec<VisibleCalendar>,
/// Calendars that were removed.
pub deleted_calendar_ids: Vec<Uuid>,
/// The position after this page, one entry per currently visible link.
/// A link missing here is no longer visible.
pub new_watermark: CalendarWatermark,
/// Whether more changes follow `new_watermark`.
pub has_more: bool,
/// Whether the client must drop everything it holds and refetch: its
/// watermark is older than retention, or a link became visible that it
/// never tracked.
pub reset_required: bool,
}
/// Persistence reads behind the change log.
pub trait CalendarChangeRepository: Send + Sync + 'static {
/// Every link the viewer can see with its current counter.
fn current_watermark(
&self,
viewer: &str,
) -> impl Future<Output = Result<CalendarWatermark, Report>> + Send;
/// Every link the viewer can see with its counter, floor, and up to
/// `limit` rows after the position `since` holds for it, read in one
/// snapshot. Links `since` does not track carry no rows.
fn link_logs(
&self,
viewer: &str,
since: &CalendarWatermark,
limit: usize,
) -> impl Future<Output = Result<Vec<CalendarLinkLog>, Report>> + Send;
/// The current state of the longest prefix of `event_ids` whose
/// materialized occurrences total at most `max_occurrences`, and always
/// at least the first id. Within that prefix, events the viewer cannot
/// see, that were deleted or cancelled, or that lost every visible
/// occurrence are left out.
fn load_event_changes(
&self,
viewer: &str,
event_ids: &[Uuid],
max_occurrences: usize,
) -> impl Future<Output = Result<EventChangeLoad, Report>> + Send;
/// The calendars among `calendar_ids` that are still visible.
fn list_calendars_by_ids(
&self,
viewer: &str,
calendar_ids: &[Uuid],
) -> impl Future<Output = Result<Vec<VisibleCalendar>, Report>> + Send;
/// Every connected inbox the viewer owns.
fn owned_inboxes(
&self,
viewer: &str,
) -> impl Future<Output = Result<Vec<OwnedInbox>, Report>> + Send;
}
/// Persistence port for change-log retention.
pub trait CalendarChangeLogPruner: Send + Sync + 'static {
/// Remove up to `batch` log rows created before `cutoff`, oldest first,
/// and the counters of links that no longer exist. Returns the number of
/// log rows removed.
fn prune_change_log(
&self,
cutoff: DateTime<Utc>,
batch: usize,
) -> impl Future<Output = Result<usize, Report>> + Send;
}
/// Inbound service port for reading the change log.
pub trait CalendarChangeQueryService: Send + Sync + 'static {
/// The viewer's current position in every visible link's log.
fn current_watermark(
&self,
viewer: &str,
) -> impl Future<Output = Result<CalendarWatermark, Report>> + Send;
/// One bounded page of changes after `since`. An empty `since` returns
/// only the current watermark.
fn changes_since(
&self,
viewer: &str,
since: CalendarWatermark,
) -> impl Future<Output = Result<CalendarChangesPage, Report>> + Send;
/// The current state of one event with its complete set of visible
/// occurrences, or `None` when the viewer cannot see it or it has no
/// visible occurrence left.
fn event_change(
&self,
viewer: &str,
event_id: Uuid,
) -> impl Future<Output = Result<Option<CalendarEventChange>, Report>> + Send;
}
/// Change-log reads with the delta policy applied.
pub struct CalendarChangeService<R> {
repository: R,
}
impl<R: CalendarChangeRepository> CalendarChangeService<R> {
/// Construct the service.
pub fn new(repository: R) -> Self {
Self { repository }
}
/// The viewer's current position in every visible link's log.
#[tracing::instrument(skip(self, viewer), err)]
pub async fn current_watermark(&self, viewer: &str) -> Result<CalendarWatermark, Report> {
self.repository.current_watermark(viewer).await
}
/// One bounded page of changes after `since`.
#[tracing::instrument(skip(self, viewer, since), err)]
pub async fn changes_since(
&self,
viewer: &str,
since: CalendarWatermark,
) -> Result<CalendarChangesPage, Report> {
if since.is_empty() {
return Ok(CalendarChangesPage {
new_watermark: self.repository.current_watermark(viewer).await?,
..CalendarChangesPage::default()
});
}
let logs = self
.repository
.link_logs(viewer, &since, MAX_DELTA_ROWS_PER_LINK + 1)
.await?;
if requires_reset(&logs, &since) {
return Ok(CalendarChangesPage {
new_watermark: counters(&logs),
reset_required: true,
..CalendarChangesPage::default()
});
}
let walk = walk_logs(&logs, &since, MAX_DELTA_EVENTS);
let upserted: Vec<Uuid> = walk.upserted_events().collect();
let load = if upserted.is_empty() {
EventChangeLoad::default()
} else {
self.repository
.load_event_changes(viewer, &upserted, MAX_DELTA_OCCURRENCES)
.await?
};
// Events past the loaded prefix wait for the next page: walking again
// with a smaller budget stops each link right before the first of them.
let walk = match upserted.get(load.considered) {
Some(&unread) => walk_logs(&logs, &since, walk.event_position(unread)),
None => walk,
};
let mut hydrated: HashMap<Uuid, CalendarEventChange> = load
.events
.into_iter()
.map(|change| (change.event.id, change))
.collect();
let upserted_calendars: Vec<Uuid> = walk.upserted_calendars().collect();
let calendars = if upserted_calendars.is_empty() {
Vec::new()
} else {
self.repository
.list_calendars_by_ids(viewer, &upserted_calendars)
.await?
};
let listed_calendars: HashSet<Uuid> =
calendars.iter().map(|calendar| calendar.id).collect();
let mut page = CalendarChangesPage {
calendars,
new_watermark: walk.watermark,
has_more: walk.truncated,
..CalendarChangesPage::default()
};
for op in walk.ops {
match op {
CalendarChangeOp::UpsertEvent(id) => match hydrated.remove(&id) {
Some(change) => page.events.push(change),
None => page.deleted_event_ids.push(id),
},
CalendarChangeOp::DeleteEvent(id) => page.deleted_event_ids.push(id),
CalendarChangeOp::UpsertCalendar(id) if !listed_calendars.contains(&id) => {
page.deleted_calendar_ids.push(id)
}
CalendarChangeOp::UpsertCalendar(_) => {}
CalendarChangeOp::DeleteCalendar(id) => page.deleted_calendar_ids.push(id),
}
}
if let Some(viewer) = ActorInboxes::from_owned(self.repository.owned_inboxes(viewer).await?)
{
for change in &mut page.events {
mark_viewer(&viewer, change);
}
}
Ok(page)
}
/// The current state of one event with its complete set of visible
/// occurrences.
#[tracing::instrument(skip(self, viewer), err)]
pub async fn event_change(
&self,
viewer: &str,
event_id: Uuid,
) -> Result<Option<CalendarEventChange>, Report> {
let Some(mut change) = self
.repository
.load_event_changes(viewer, &[event_id], MAX_DELTA_OCCURRENCES)
.await?
.events
.pop()
else {
return Ok(None);
};
if let Some(viewer) = ActorInboxes::from_owned(self.repository.owned_inboxes(viewer).await?)
{
mark_viewer(&viewer, &mut change);
}
Ok(Some(change))
}
}
fn mark_viewer(viewer: &ActorInboxes, change: &mut CalendarEventChange) {
viewer.mark_event(&mut change.event, change.link_id);
for instance in &mut change.occurrences {
if let Some(attendees) = &mut instance.exception.attendees {
viewer.mark_attendees(attendees, change.link_id);
}
}
}
impl<R: CalendarChangeRepository> CalendarChangeQueryService for CalendarChangeService<R> {
fn current_watermark(
&self,
viewer: &str,
) -> impl Future<Output = Result<CalendarWatermark, Report>> + Send {
CalendarChangeService::current_watermark(self, viewer)
}
fn changes_since(
&self,
viewer: &str,
since: CalendarWatermark,
) -> impl Future<Output = Result<CalendarChangesPage, Report>> + Send {
CalendarChangeService::changes_since(self, viewer, since)
}
fn event_change(
&self,
viewer: &str,
event_id: Uuid,
) -> impl Future<Output = Result<Option<CalendarEventChange>, Report>> + Send {
CalendarChangeService::event_change(self, viewer, event_id)
}
}
/// A client must reset when it fell behind retention on any link, or when a
/// link became visible that it never tracked: that link's data predates its
/// log, so replaying the log cannot reproduce it.
fn requires_reset(logs: &[CalendarLinkLog], since: &CalendarWatermark) -> bool {
logs.iter().any(|log| match since.seq(log.link_id) {
None => true,
Some(seq) => seq < log.floor,
})
}
fn counters(logs: &[CalendarLinkLog]) -> CalendarWatermark {
CalendarWatermark::from_links(logs.iter().map(|log| CalendarLinkWatermark {
link_id: log.link_id,
seq: log.counter,
}))
}
/// The coalesced changes a page covers and where it stops.
#[derive(Debug, PartialEq, Eq)]
struct LogWalk {
/// The latest change per target, in first-seen order.
ops: Vec<CalendarChangeOp>,
watermark: CalendarWatermark,
truncated: bool,
}
impl LogWalk {
fn upserted_events(&self) -> impl Iterator<Item = Uuid> + '_ {
self.ops.iter().filter_map(|op| match op {
CalendarChangeOp::UpsertEvent(id) => Some(*id),
_ => None,
})
}
/// How many distinct events the walk covers before `event_id`.
fn event_position(&self, event_id: Uuid) -> usize {
self.ops
.iter()
.filter(|op| matches!(target(**op), Target::Event(_)))
.take_while(|op| target(**op) != Target::Event(event_id))
.count()
}
fn upserted_calendars(&self) -> impl Iterator<Item = Uuid> + '_ {
self.ops.iter().filter_map(|op| match op {
CalendarChangeOp::UpsertCalendar(id) => Some(*id),
_ => None,
})
}
}
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
enum Target {
Event(Uuid),
Calendar(Uuid),
}
fn target(op: CalendarChangeOp) -> Target {
match op {
CalendarChangeOp::UpsertEvent(id) | CalendarChangeOp::DeleteEvent(id) => Target::Event(id),
CalendarChangeOp::UpsertCalendar(id) | CalendarChangeOp::DeleteCalendar(id) => {
Target::Calendar(id)
}
}
}
/// Walk each link's rows in sequence order, links in id order, until
/// `max_events` distinct events are covered. A link stops on a sequence
/// boundary: its watermark advances only past rows the page covers, so a
/// truncated page resumes exactly where it stopped.
fn walk_logs(logs: &[CalendarLinkLog], since: &CalendarWatermark, max_events: usize) -> LogWalk {
let mut sorted: Vec<&CalendarLinkLog> = logs.iter().collect();
sorted.sort_by_key(|log| log.link_id);
let mut order: Vec<Target> = Vec::new();
let mut latest: HashMap<Target, CalendarChangeOp> = HashMap::new();
let mut events = 0;
let mut truncated = false;
let mut watermark = Vec::with_capacity(sorted.len());
for log in sorted {
let start = since.seq(log.link_id).unwrap_or(log.counter);
let mut position = start;
let mut stopped = log.changes.len() > MAX_DELTA_ROWS_PER_LINK;
for change in log.changes.iter().take(MAX_DELTA_ROWS_PER_LINK) {
let key = target(change.op);
let new_event = matches!(key, Target::Event(_)) && !latest.contains_key(&key);
if new_event && events == max_events {
stopped = true;
break;
}
if new_event {
events += 1;
}
if latest.insert(key, change.op).is_none() {
order.push(key);
}
position = change.seq;
}
truncated |= stopped;
watermark.push(CalendarLinkWatermark {
link_id: log.link_id,
seq: if stopped { position } else { log.counter },
});
}
LogWalk {
ops: order.into_iter().map(|key| latest[&key]).collect(),
watermark: CalendarWatermark::from_links(watermark),
truncated,
}
}
/// Removes change-log rows older than [`CHANGE_LOG_RETENTION`].
pub struct CalendarChangeLogRetention<R> {
repository: R,
}
impl<R: CalendarChangeLogPruner> CalendarChangeLogRetention<R> {
/// Construct the retention job from its persistence port.
pub fn new(repository: R) -> Self {
Self { repository }
}
/// Prune everything older than retention as of `now`, in batches.
/// Returns the number of rows removed.
#[tracing::instrument(skip(self), err)]
pub async fn run_once(&self, now: DateTime<Utc>) -> Result<usize, Report> {
let cutoff = now - CHANGE_LOG_RETENTION;
let mut removed = 0;
loop {
let batch = self
.repository
.prune_change_log(cutoff, CHANGE_LOG_PRUNE_BATCH)
.await?;
removed += batch;
if batch < CHANGE_LOG_PRUNE_BATCH {
return Ok(removed);
}
}
}
}