Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
use crate::pubsub::context::PubSubContext;
use crate::pubsub::util::{build_contact_connections, cg_refresh_email};
use contacts::domain::ports::ContactsIngress;
use email_db_client::backfill::job::update::BackfillCompletion;
use models_email::api::refresh::{BackfillStatus, RefreshEmailEvent};
use models_email::db::address::EmailRecipientType;
use models_email::service::attachment::{
AttachmentUploadArgs, AttachmentUploadDestination, AttachmentUploadMetadata,
};
use models_email::service::backfill::{
BackfillAttachmentPayload, BackfillMessagePayload, BackfillOperation, BackfillPubsubMessage,
JobScopedPayload, UpdateMetadataPayload,
};
use models_email::service::link::Link;
use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
use uuid::Uuid;
/// Emit a `refresh_email` event once per this many completed threads, plus on
/// the first completed thread and at job completion, rather than per-thread.
/// The first-thread emit ensures backfills smaller than this interval still
/// surface a progress event before completing.
const REFRESH_EMAIL_THREAD_INTERVAL: i32 = 50;
#[cfg(test)]
mod test;
/// called when a thread has completed processing. checks if it is the last thread to be processed
/// for the job, and if so, performs the necessary actions for job completion.
#[tracing::instrument(skip(ctx))]
pub async fn incr_completed_threads(
ctx: &PubSubContext,
link: &Link,
job_id: Uuid,
) -> Result<(), ProcessingError> {
let progress = ctx
.redis_client
.incr_completed_threads(job_id)
.await
.map_err(|e| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::RedisQueryFailed,
source: e.context("Failed to increment completed thread count"),
})
})?;
if progress.job_complete {
tracing::info!(
job_id = job_id.to_string(),
"All threads for job have been processed"
);
handle_job_completed(ctx, job_id, None).await?;
} else if progress.completed_threads == 1
|| progress.completed_threads % REFRESH_EMAIL_THREAD_INTERVAL == 0
{
cg_refresh_email(
&ctx.connection_gateway_client,
link.macro_id.as_ref(),
RefreshEmailEvent::BackfillProgress {
link_id: link.id,
status: BackfillStatus::Progress,
completed_threads: progress.completed_threads,
total_threads: progress.total_threads,
},
)
.await;
}
Ok(())
}
/// If this message was the last one in the thread to be processed, proceed to next stage of backfill for thread
#[tracing::instrument(skip(ctx))]
pub async fn incr_completed_messages(
ctx: &PubSubContext,
link: &Link,
job_id: Uuid,
p: &BackfillMessagePayload,
) -> Result<(), ProcessingError> {
let thread_backfill_complete = ctx
.redis_client
.incr_completed_messages(job_id, &p.thread_provider_id)
.await
.map_err(|e| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::RedisQueryFailed,
source: e.context(format!(
"Handle completed message for {} failed",
p.message_provider_id
)),
})
})?;
if thread_backfill_complete {
handle_thread_completed(ctx, link, job_id, p).await?;
}
Ok(())
}
/// performs actions when all threads and messages have been backfilled for the user.
#[tracing::instrument(skip(ctx))]
pub(crate) async fn handle_job_completed(
ctx: &PubSubContext,
job_id: Uuid,
init_lease_token: Option<Uuid>,
) -> Result<(), ProcessingError> {
tracing::info!("Backfill complete for job {}", job_id);
let outcome = email_db_client::backfill::job::update::complete_backfill_job(
&ctx.db,
job_id,
init_lease_token,
)
.await
.map_err(|e| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: e.context("Failed to complete email backfill job"),
})
})?;
completion_result(job_id, outcome)
}
fn completion_result(job_id: Uuid, outcome: BackfillCompletion) -> Result<(), ProcessingError> {
match outcome {
BackfillCompletion::Completed | BackfillCompletion::AlreadyTerminal => Ok(()),
BackfillCompletion::LeaseLost => Err(ProcessingError::Retryable(DetailedError {
reason: FailureReason::EmailBackfillInitBusy,
source: anyhow::anyhow!("email backfill completion lease was lost for job {job_id}"),
})),
BackfillCompletion::NotFound => Err(ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::BackfillJobNotFound,
source: anyhow::anyhow!("email backfill job {job_id} was not found"),
})),
}
}
/// Runs durable completion effects published from the completion outbox.
#[tracing::instrument(skip(ctx))]
pub(crate) async fn finalize_backfill(
ctx: &PubSubContext,
link_id: Uuid,
job_id: Uuid,
) -> Result<(), ProcessingError> {
let Some(lease_token) =
email_db_client::backfill::job::update::claim_completion_effects(&ctx.db, job_id)
.await
.map_err(|error| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: error.context("Failed to claim backfill completion effects"),
})
})?
else {
let pending =
email_db_client::backfill::job::update::completion_effects_pending(&ctx.db, job_id)
.await
.map_err(|error| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: error.context("Failed to load backfill completion effects"),
})
})?;
if !pending {
return Ok(());
}
return Err(ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: anyhow::anyhow!("backfill completion effects are already leased"),
}));
};
let result = finalize_claimed_backfill(ctx, link_id, job_id, lease_token).await;
if result.is_err() {
email_db_client::backfill::job::update::release_completion_effects(
&ctx.db,
job_id,
lease_token,
)
.await
.inspect_err(|error| {
tracing::error!(
error = ?error,
job_id = %job_id,
"failed to release backfill completion-effects lease"
);
})
.ok();
}
result
}
async fn finalize_claimed_backfill(
ctx: &PubSubContext,
link_id: Uuid,
job_id: Uuid,
lease_token: Uuid,
) -> Result<(), ProcessingError> {
let Some(job) = email_db_client::backfill::job::get::get_backfill_job(&ctx.db, job_id)
.await
.map_err(|error| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: error.context("Failed to load completed backfill job"),
})
})?
else {
return Ok(());
};
match job.status {
models_email::service::backfill::BackfillJobStatus::Complete => {}
// A job that failed or was cancelled after its completion message
// published will never become Complete; retrying such a delivery
// only burns receives, so retire its effects instead.
models_email::service::backfill::BackfillJobStatus::Failed
| models_email::service::backfill::BackfillJobStatus::Cancelled => {
tracing::warn!(%job_id, status = ?job.status, "retiring completion effects for a terminal, non-complete job");
return retire_completion_effects(ctx, job_id, lease_token).await;
}
models_email::service::backfill::BackfillJobStatus::Init
| models_email::service::backfill::BackfillJobStatus::InProgress => {
return Err(ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: anyhow::anyhow!(
"backfill completion effects published before job completion"
),
}));
}
}
let Some(canonical_link_id) = job.link_id else {
return retire_completion_effects(ctx, job_id, lease_token).await;
};
if canonical_link_id != link_id {
tracing::warn!(
message_link_id = %link_id,
canonical_link_id = %canonical_link_id,
"using the completed backfill job's canonical link"
);
}
let Some(link) = email_db_client::links::get::fetch_link_by_id(&ctx.db, canonical_link_id)
.await
.map_err(|error| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: error.context("Failed to load link for backfill completion"),
})
})?
else {
return retire_completion_effects(ctx, job_id, lease_token).await;
};
let effects = async {
handle_attachment_upload(ctx, &link, job_id)
.await
.map_err(completion_effect_retry)?;
handle_contacts_sync(ctx, &link)
.await
.map_err(completion_effect_retry)?;
// Best effort: the mailbox now has sent mail to rank, so open direct
// messages with the people the user emails most.
crate::util::contact_dms::ensure_top_contact_dms(&ctx.db, &link).await;
cg_refresh_email(
&ctx.connection_gateway_client,
link.macro_id.as_ref(),
RefreshEmailEvent::BackfillProgress {
link_id: canonical_link_id,
status: BackfillStatus::Complete,
completed_threads: job.total_threads,
total_threads: job.total_threads,
},
)
.await;
Ok(())
};
tokio::pin!(effects);
let lease_heartbeat = maintain_completion_effects_lease(ctx.db.clone(), job_id, lease_token);
tokio::pin!(lease_heartbeat);
tokio::select! {
biased;
lease_result = &mut lease_heartbeat => lease_result?,
effects_result = &mut effects => effects_result?,
}
retire_completion_effects(ctx, job_id, lease_token).await
}
async fn maintain_completion_effects_lease(
db: sqlx::PgPool,
job_id: Uuid,
lease_token: Uuid,
) -> Result<(), ProcessingError> {
loop {
tokio::time::sleep(std::time::Duration::from_secs(30)).await;
let renewed = email_db_client::backfill::job::update::renew_completion_effects(
&db,
job_id,
lease_token,
)
.await
.map_err(|error| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: error.context("Failed to renew backfill completion-effects lease"),
})
})?;
if !renewed {
return Err(ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: anyhow::anyhow!("backfill completion-effects lease was lost"),
}));
}
}
}
async fn retire_completion_effects(
ctx: &PubSubContext,
job_id: Uuid,
lease_token: Uuid,
) -> Result<(), ProcessingError> {
email_db_client::backfill::job::update::mark_completion_effects_complete(
&ctx.db,
job_id,
lease_token,
)
.await
.map_err(|error| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: error.context("Failed to finish backfill completion effects"),
})
})
}
fn completion_effect_retry(error: ProcessingError) -> ProcessingError {
match error {
ProcessingError::NonRetryable(detail) => ProcessingError::Retryable(detail),
error => error,
}
}
#[tracing::instrument(skip(ctx))]
async fn handle_attachment_upload(
ctx: &PubSubContext,
link: &Link,
job_id: Uuid,
) -> Result<(), ProcessingError> {
if cfg!(not(feature = "attachment_upload")) {
return Ok(());
}
let attachments =
email_db_client::attachments::provider::upload::fetch_job_attachments_for_backfill(
&ctx.db, link.id,
)
.await
.map_err(|e| {
ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: e.context("Failed to fetch job attachment backfill metadata".to_string()),
})
})?;
if !attachments.is_empty() {
tracing::debug!(
"Found {} condition 5 attachments to backfill for job {}",
attachments.len(),
job_id
);
send_attachment_backfill_messages(ctx, link.id, job_id, attachments).await?;
}
Ok(())
}
#[tracing::instrument(skip(ctx))]
async fn handle_contacts_sync(ctx: &PubSubContext, link: &Link) -> Result<(), ProcessingError> {
if cfg!(not(feature = "contacts_sync")) {
return Ok(());
}
let email_addresses =
email_db_client::contacts::get::fetch_contacts_emails_by_link_id(&ctx.db, link.id)
.await
.map_err(|e| {
ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: e.context("Failed to fetch contact email addresses".to_string()),
})
})?;
let length = email_addresses.len();
tracing::info!(
"Populating {} contacts for macro email {}",
length,
link.macro_id
);
let connections = build_contact_connections(
&link.macro_id,
link.id,
email_addresses.iter().map(String::as_str),
);
ctx.contacts_ingress
.enqueue_contact_connections(connections)
.await
.map_err(|e| {
ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::SqsEnqueueFailed,
source: anyhow::anyhow!("{e:?}").context(format!(
"Failed to enqueue {length} contact connections for link {}",
link.id
)),
})
})?;
tracing::info!(
"Successfully populated {} contacts for macro email {}",
length,
link.macro_id
);
Ok(())
}
/// when a thread is done being backfilled, update its metadata and backfill its attachments.
#[tracing::instrument(skip(ctx))]
async fn handle_thread_completed(
ctx: &PubSubContext,
link: &Link,
job_id: Uuid,
p: &BackfillMessagePayload,
) -> Result<(), ProcessingError> {
enqueue_thread_metadata_update(ctx, link, job_id, &p.thread_provider_id, p.thread_db_id)
.await?;
handle_thread_attachment_upload(ctx, link, job_id, p.thread_db_id).await
}
/// Hands a fully written thread to its UpdateThreadMetadata step, which
/// computes the thread's metadata and counts it as completed.
#[tracing::instrument(skip(ctx))]
pub async fn enqueue_thread_metadata_update(
ctx: &PubSubContext,
link: &Link,
job_id: Uuid,
thread_provider_id: &str,
thread_db_id: Uuid,
) -> Result<(), ProcessingError> {
let ps_message = BackfillPubsubMessage {
backfill_operation: BackfillOperation::UpdateThreadMetadata(JobScopedPayload {
link_id: link.id,
job_id,
payload: UpdateMetadataPayload {
thread_provider_id: thread_provider_id.to_string(),
thread_db_id,
},
}),
};
ctx.sqs_client
.enqueue_email_backfill_message(ps_message)
.await
.map_err(|e| {
ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::SqsEnqueueFailed,
source: e.context("Failed to enqueue metadata message".to_string()),
})
})
}
/// Creates BackfillAttachmentPayload messages and enqueues them via SQS for the given attachments
#[tracing::instrument(skip(ctx, attachments))]
async fn send_attachment_backfill_messages(
ctx: &PubSubContext,
link_id: Uuid,
job_id: Uuid,
attachments: Vec<AttachmentUploadMetadata>,
) -> Result<(), ProcessingError> {
if attachments.is_empty() {
return Ok(());
}
let message_ids = attachments
.iter()
.map(|a| a.message_db_id)
.collect::<Vec<_>>();
let message_recipients =
email_db_client::contacts::get::fetch_db_recipients_in_bulk(&ctx.db, &message_ids)
.await
.map_err(|e| {
ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: e.context(
"Failed to fetch db recipients for thread attachment backfill".to_string(),
),
})
})?;
for attachment in attachments {
// get the email addresses of the recipients of the message
let recipient_emails: Vec<String> = message_recipients
.get(&attachment.message_db_id)
.map(|v| v.as_slice())
.unwrap_or(&[])
.iter()
.filter(|(_, recipient_type)| *recipient_type == EmailRecipientType::To)
.filter_map(|(contact, _)| contact.email_address.clone())
.collect();
let upload_destination = if matches!(
attachment.mime_type.split('/').next(),
Some("image" | "video")
) {
AttachmentUploadDestination::Sfs
} else {
AttachmentUploadDestination::Dss
};
let attachment_upload_args = AttachmentUploadArgs {
recipient_emails,
attachment_metadata: attachment,
backfill: true,
upload_destination,
};
let new_payload = BackfillAttachmentPayload {
metadata: attachment_upload_args,
};
let ps_message = BackfillPubsubMessage {
backfill_operation: BackfillOperation::BackfillAttachment(JobScopedPayload {
link_id,
job_id,
payload: new_payload,
}),
};
ctx.sqs_client
.enqueue_email_backfill_message(ps_message)
.await
.map_err(|e| {
ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::SqsEnqueueFailed,
source: e.context("Failed to enqueue attachment backfill message".to_string()),
})
})?;
}
Ok(())
}
/// Enqueues the uploads of a backfilled thread's attachments.
#[tracing::instrument(skip(ctx))]
pub async fn handle_thread_attachment_upload(
ctx: &PubSubContext,
link: &Link,
job_id: Uuid,
thread_db_id: Uuid,
) -> Result<(), ProcessingError> {
if cfg!(not(feature = "attachment_upload")) {
return Ok(());
}
let (attachments, attachments2) = tokio::try_join!(
email_db_client::attachments::provider::upload::thread_document_atts_for_backfill(
&ctx.db,
thread_db_id,
),
email_db_client::attachments::provider::upload::thread_media_atts_for_backfill(
&ctx.db,
thread_db_id,
)
)
.map_err(|e| {
ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: e.context("Failed to fetch thread attachment backfill metadata".to_string()),
})
})?;
let mut all_attachments = attachments;
all_attachments.extend(attachments2);
if !all_attachments.is_empty() {
tracing::debug!(
"Found {} attachments to backfill for thread {}",
all_attachments.len(),
thread_db_id
);
send_attachment_backfill_messages(ctx, link.id, job_id, all_attachments).await?;
}
Ok(())
}