Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
//! Durable mailbox enumeration independent from provider-specific notifications.
use email_api_client::domain::models::{
EmailApiError, MailFolder, MailboxChangePage, MailboxMessage, ProviderId, StreamPosition,
StreamToken,
};
use progress::{ImportCompleted, ImportTally};
use std::future::Future;
use std::time::Instant;
use telemetry::{claim_failed, elapsed_ms, record_deferred, record_done, record_failed, span_int};
use tracing::{Instrument, Span};
use uuid::Uuid;
pub mod catalog;
pub mod commands;
pub mod contacts;
pub mod credentials;
pub mod drafts;
pub mod gmail_history;
pub mod health;
pub mod initialization;
pub mod lifecycle;
pub mod progress;
pub mod projection;
pub mod settings;
pub mod telemetry;
#[cfg(test)]
mod test;
pub mod watches;
#[derive(Debug, thiserror::Error)]
pub enum MailboxError {
#[error("{0}")]
InvalidInput(&'static str),
#[error(transparent)]
Provider(#[from] EmailApiError),
#[error("mailbox persistence is unavailable")]
Persistence,
/// A database call failed; the cause carries no row data.
#[error("mailbox persistence is unavailable: {0}")]
Database(DatabaseFailure),
/// A projection's downstream effect (event publishing, a queue, another
/// service) failed or is not ready; the effect logs its own cause.
#[error("a mailbox projection effect is unavailable")]
Effect,
#[error("mailbox generation or worker lease changed")]
Stale,
/// The provider no longer reports the message where it was; a later
/// attempt either finds it or settles that it is gone.
#[error("mailbox message location has not settled")]
LocationUnsettled,
}
/// What failed underneath a database call. The database's own message is
/// left out because it can quote row data.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DatabaseFailure {
/// The database rejected the statement with this SQLSTATE.
Sqlstate(String),
/// No pooled connection became free in time.
PoolTimedOut,
/// The connection failed or closed.
Connection,
/// A returned value could not be decoded.
Decode,
/// Any other driver failure.
Other,
}
impl std::fmt::Display for DatabaseFailure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Sqlstate(code) => write!(f, "sqlstate {code}"),
Self::PoolTimedOut => f.write_str("connection pool timed out"),
Self::Connection => f.write_str("connection failed"),
Self::Decode => f.write_str("decode failed"),
Self::Other => f.write_str("driver error"),
}
}
}
/// Both generations are resolved from storage, never selected by the client.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct MailboxKey {
pub link_id: Uuid,
pub sync_generation: i64,
pub grant_generation: i64,
}
#[derive(Debug)]
pub struct StreamLease {
pub id: Uuid,
pub mailbox: MailboxKey,
pub folder: ProviderId,
pub position: Option<StreamToken>,
pub initial_complete: bool,
pub lease_id: Uuid,
pub fence: i64,
}
#[derive(Debug)]
pub struct MessageLease {
pub mailbox: MailboxKey,
pub provider_id: ProviderId,
pub revision: i64,
pub lease_id: Uuid,
pub is_import: bool,
pub attempts: i32,
}
#[derive(Debug)]
pub struct CatalogLease {
pub id: Uuid,
pub mailbox: MailboxKey,
pub lease_id: Uuid,
pub fence: i64,
}
/// What a committed page changed.
#[derive(Debug, Default)]
pub struct CommittedPage {
/// Reconciliation rows the page queued or requeued.
pub queued: u64,
/// Set when this page finished the mailbox's import run.
pub import_completed: Option<ImportCompleted>,
}
pub enum SyncLease<'a> {
Catalog(&'a CatalogLease),
Stream(&'a StreamLease),
Message(&'a MessageLease),
}
pub trait MailboxSyncRepository: Send + Sync + 'static {
/// Extend only a still-current lease while a provider operation is in flight.
fn renew(&self, lease: SyncLease<'_>) -> impl Future<Output = Result<(), MailboxError>> + Send;
fn claim_catalog(
&self,
lease_id: Uuid,
) -> impl Future<Output = Result<Option<CatalogLease>, MailboxError>> + Send;
/// Replaces a complete catalog only under its lease fence. Missing folders
/// trigger message reconciliation, never deletion based on a partial scan.
fn commit_catalog(
&self,
lease: &CatalogLease,
folders: &[MailFolder],
) -> impl Future<Output = Result<Option<ImportCompleted>, MailboxError>> + Send;
fn release_catalog(
&self,
lease: &CatalogLease,
retry_after_seconds: u32,
) -> impl Future<Output = Result<(), MailboxError>> + Send;
fn claim_stream(
&self,
lease_id: Uuid,
) -> impl Future<Output = Result<Option<StreamLease>, MailboxError>> + Send;
/// Atomically records every changed/removed ID and advances the opaque
/// cursor only while this generation and lease fence are still current.
fn commit_page(
&self,
lease: &StreamLease,
page: &MailboxChangePage,
) -> impl Future<Output = Result<CommittedPage, MailboxError>> + Send;
/// Reset only this expired stream; this operation never removes messages.
fn reset_stream(
&self,
lease: &StreamLease,
) -> impl Future<Output = Result<(), MailboxError>> + Send;
fn release_stream(
&self,
lease: &StreamLease,
retry_after_seconds: u32,
) -> impl Future<Output = Result<(), MailboxError>> + Send;
fn claim_message(
&self,
lease_id: Uuid,
) -> impl Future<Output = Result<Option<MessageLease>, MailboxError>> + Send;
fn folders(
&self,
mailbox: MailboxKey,
) -> impl Future<Output = Result<Vec<MailFolder>, MailboxError>> + Send;
/// Finishes only the leased revision. A newer wakeup must remain pending.
/// Reports the import run this message finished, once per run.
fn complete_message(
&self,
lease: &MessageLease,
) -> impl Future<Output = Result<Option<ImportCompleted>, MailboxError>> + Send;
fn release_message(
&self,
lease: &MessageLease,
retry_after_seconds: u32,
) -> impl Future<Output = Result<(), MailboxError>> + Send;
}
pub trait MailboxGateway: Send + Sync + 'static {
fn folders(
&self,
mailbox: MailboxKey,
) -> impl Future<Output = Result<Vec<MailFolder>, EmailApiError>> + Send;
fn changes(
&self,
mailbox: MailboxKey,
folder: &ProviderId,
position: Option<&StreamToken>,
) -> impl Future<Output = Result<MailboxChangePage, EmailApiError>> + Send;
fn message(
&self,
mailbox: MailboxKey,
message: &ProviderId,
folders: &[MailFolder],
) -> impl Future<Output = Result<Option<MailboxMessage>, EmailApiError>> + Send;
/// Provider requests admitted for the mailbox's sync generation, when known.
fn admitted_requests(&self, mailbox: MailboxKey) -> impl Future<Output = Option<u64>> + Send;
}
/// The shared ingestion path writes content, normalized state and a projection
/// outbox atomically. Import work must not generate incoming-mail notifications.
pub trait MailboxIngest: Send + Sync + 'static {
fn ingest(
&self,
lease: &MessageLease,
snapshot: MailboxMessage,
) -> impl Future<Output = Result<(), MailboxError>> + Send;
/// A mailbox-wide GET returned 404. Preserve recoverable content, reconcile
/// membership, and recheck before any destructive retention decision.
fn absent(&self, lease: &MessageLease)
-> impl Future<Output = Result<(), MailboxError>> + Send;
}
pub struct MailboxSyncService<R, P, I> {
repo: R,
provider: P,
ingest: I,
imports: ImportTally,
}
impl<R, P, I> MailboxSyncService<R, P, I> {
pub fn new(repo: R, provider: P, ingest: I) -> Self {
Self {
repo,
provider,
ingest,
imports: ImportTally::default(),
}
}
/// The in-memory import counts the health loop flushes.
pub fn import_tally(&self) -> ImportTally {
self.imports.clone()
}
}
impl<R: MailboxSyncRepository, P: MailboxGateway, I: MailboxIngest> MailboxSyncService<R, P, I> {
pub async fn discover_once(&self) -> Result<bool, MailboxError> {
let Some(lease) = self
.repo
.claim_catalog(macro_uuid::generate_uuid_v7())
.await
.map_err(|error| claim_failed("discover", error))?
else {
return Ok(false);
};
let span = tracing::info_span!(
"outlook.sync.discover",
mailbox.link_id = %lease.mailbox.link_id,
mailbox.sync_generation = lease.mailbox.sync_generation,
outlook.stream_kind = "folder_catalog",
outlook.folders = tracing::field::Empty,
outlook.outcome = tracing::field::Empty,
outlook.defer_reason = tracing::field::Empty,
outlook.defer_seconds = tracing::field::Empty,
otel.status_description = tracing::field::Empty,
);
self.discover(&lease).instrument(span).await
}
async fn discover(&self, lease: &CatalogLease) -> Result<bool, MailboxError> {
let span = Span::current();
let outcome = maintain_lease(
async { Ok(self.provider.folders(lease.mailbox).await?) },
|| self.repo.renew(SyncLease::Catalog(lease)),
)
.await;
let result = match outcome {
Ok(folders) => {
span.record("outlook.folders", span_int(folders.len()));
self.repo.commit_catalog(lease, &folders).await
}
Err(MailboxError::Provider(error)) => {
let delay = retry_seconds(&error);
let error = MailboxError::from(error);
self.repo
.release_catalog(lease, delay)
.await
.inspect_err(|release| {
record_failed(&span, lease.mailbox.link_id, release);
})?;
record_deferred(&span, lease.mailbox.link_id, &error, delay);
return Err(error);
}
Err(error) => Err(error),
};
match result {
Ok(import_completed) => {
record_done(&span, "imported");
if let Some(completed) = import_completed {
self.report_import_completed(&completed).await;
}
Ok(true)
}
Err(error) => {
record_failed(&span, lease.mailbox.link_id, &error);
Err(error)
}
}
}
/// Process one bounded provider page. Notifications only make streams due;
/// periodic polling recovers missed, duplicate and out-of-order notifications.
pub async fn enumerate_once(&self) -> Result<bool, MailboxError> {
let Some(lease) = self
.repo
.claim_stream(macro_uuid::generate_uuid_v7())
.await
.map_err(|error| claim_failed("enumerate", error))?
else {
return Ok(false);
};
let span = tracing::info_span!(
"outlook.sync.enumerate",
mailbox.link_id = %lease.mailbox.link_id,
mailbox.sync_generation = lease.mailbox.sync_generation,
outlook.stream_kind = "mail_folder",
outlook.page_items = tracing::field::Empty,
outlook.rows_queued = tracing::field::Empty,
outlook.delta_advanced = false,
outlook.folder_initial_complete = lease.initial_complete,
outlook.cursor_reset = false,
outlook.outcome = tracing::field::Empty,
outlook.defer_reason = tracing::field::Empty,
outlook.defer_seconds = tracing::field::Empty,
otel.status_description = tracing::field::Empty,
);
self.enumerate(&lease).instrument(span).await
}
async fn enumerate(&self, lease: &StreamLease) -> Result<bool, MailboxError> {
let span = Span::current();
let outcome = maintain_lease(
async {
Ok(self
.provider
.changes(lease.mailbox, &lease.folder, lease.position.as_ref())
.await?)
},
|| self.repo.renew(SyncLease::Stream(lease)),
)
.await;
let result = match outcome {
Ok(page) => {
span.record(
"outlook.page_items",
span_int(page.changed.len() + page.removed.len()),
);
let checkpoint = matches!(page.position, StreamPosition::Checkpoint(_));
self.repo.commit_page(lease, &page).await.map(|committed| {
span.record("outlook.rows_queued", span_int(committed.queued));
span.record("outlook.delta_advanced", true);
span.record(
"outlook.folder_initial_complete",
lease.initial_complete || checkpoint,
);
committed.import_completed
})
}
Err(MailboxError::Provider(EmailApiError::OutdatedCursor)) => {
span.record("outlook.cursor_reset", true);
self.repo.reset_stream(lease).await.map(|()| None)
}
Err(MailboxError::Provider(error)) => {
let delay = retry_seconds(&error);
let error = MailboxError::from(error);
self.repo
.release_stream(lease, delay)
.await
.inspect_err(|release| {
record_failed(&span, lease.mailbox.link_id, release);
})?;
record_deferred(&span, lease.mailbox.link_id, &error, delay);
return Err(error);
}
Err(error) => Err(error),
};
match result {
Ok(import_completed) => {
record_done(&span, "imported");
if let Some(completed) = import_completed {
self.report_import_completed(&completed).await;
}
Ok(true)
}
Err(error) => {
record_failed(&span, lease.mailbox.link_id, &error);
Err(error)
}
}
}
pub async fn reconcile_once(&self) -> Result<bool, MailboxError> {
let Some(lease) = self
.repo
.claim_message(macro_uuid::generate_uuid_v7())
.await
.map_err(|error| claim_failed("reconcile", error))?
else {
return Ok(false);
};
let span = tracing::info_span!(
"outlook.sync.reconcile",
mailbox.link_id = %lease.mailbox.link_id,
mailbox.sync_generation = lease.mailbox.sync_generation,
outlook.is_import = lease.is_import,
outlook.attempts = lease.attempts,
outlook.graph_calls = tracing::field::Empty,
outlook.attachments = tracing::field::Empty,
outlook.db_write_ms = tracing::field::Empty,
outlook.outcome = tracing::field::Empty,
outlook.defer_reason = tracing::field::Empty,
outlook.defer_seconds = tracing::field::Empty,
otel.status_description = tracing::field::Empty,
);
self.reconcile(&lease).instrument(span).await
}
async fn reconcile(&self, lease: &MessageLease) -> Result<bool, MailboxError> {
let span = Span::current();
let operation = async {
let folders = self.repo.folders(lease.mailbox).await?;
let (snapshot, graph_calls) = email_api_client::domain::request_count::counted(
self.provider
.message(lease.mailbox, &lease.provider_id, &folders),
)
.await;
span.record("outlook.graph_calls", i64::from(graph_calls));
let started = Instant::now();
let outcome = match snapshot? {
Some(snapshot) => {
span.record(
"outlook.attachments",
span_int(snapshot.content.message.attachments.len()),
);
self.ingest.ingest(lease, snapshot).await?;
"imported"
}
None if lease.attempts < 3 => return Err(MailboxError::LocationUnsettled),
None => {
self.ingest.absent(lease).await?;
"absent"
}
};
let import_completed = self.repo.complete_message(lease).await?;
span.record("outlook.db_write_ms", elapsed_ms(started));
Ok((outcome, import_completed))
};
let result = maintain_lease(operation, || self.repo.renew(SyncLease::Message(lease))).await;
let error = match result {
Ok((outcome, import_completed)) => {
record_done(&span, outcome);
if outcome == "imported" {
self.imports
.record(lease.mailbox.link_id, chrono::Utc::now());
}
if let Some(completed) = import_completed {
self.report_import_completed(&completed).await;
}
return Ok(true);
}
Err(error) => error,
};
let delay = match &error {
MailboxError::Provider(error) => retry_seconds(error),
_ => 30,
};
self.repo
.release_message(lease, delay)
.await
.inspect_err(|release| {
record_failed(&span, lease.mailbox.link_id, release);
})?;
if matches!(error, MailboxError::Stale) {
record_failed(&span, lease.mailbox.link_id, &error);
} else {
record_deferred(&span, lease.mailbox.link_id, &error, delay);
}
Err(error)
}
async fn report_import_completed(&self, completed: &ImportCompleted) {
let graph_calls = self.provider.admitted_requests(completed.mailbox).await;
progress::log_import_completed(completed, graph_calls);
}
}
/// Keep long catalogue scans and uploads fenced without imposing an arbitrary
/// mailbox-size limit. Losing ownership cancels the operation, never its successor.
pub(crate) async fn maintain_lease<F, H, HF, T>(
operation: F,
mut renew: H,
) -> Result<T, MailboxError>
where
F: Future<Output = Result<T, MailboxError>>,
H: FnMut() -> HF,
HF: Future<Output = Result<(), MailboxError>>,
{
tokio::pin!(operation);
loop {
tokio::select! {
result = &mut operation => return result,
_ = tokio::time::sleep(std::time::Duration::from_secs(30)) => renew().await?,
}
}
}
fn retry_seconds(error: &EmailApiError) -> u32 {
match error {
EmailApiError::RateLimited {
retry_after: Some(delay),
..
} => delay.as_secs().clamp(1, 86_400) as u32,
EmailApiError::AuthRequired | EmailApiError::Forbidden => 300,
EmailApiError::Permanent { .. } => 600,
_ => 30,
}
}