Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
use crate::pubsub::backfill::{
backfill_attachment, backfill_message, backfill_thread, deferral, depopulate_crm_contact,
depopulate_crm_for_user, error_handlers, increment_counters, init, list_threads,
populate_crm_contact, populate_crm_for_user, seed_sent_contact, update_metadata,
};
use crate::pubsub::context::PubSubContext;
use anyhow::Context;
use models_email::email::service::backfill::{
BackfillJob, BackfillJobStatus, BackfillOperation, BackfillPubsubMessage, JobScopedPayload,
};
use models_email::email::service::link;
use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
use sqs_worker::cleanup_message;
use std::time::Duration;
use uuid::Uuid;
// Process a single message from the backfill queue
pub async fn process_message(
ctx: PubSubContext,
message: &aws_sdk_sqs::types::Message,
) -> anyhow::Result<()> {
// Malformed JSON is NOT retryable.
let data = match extract_backfill_message(message) {
Ok(data) => data,
Err(e) => {
tracing::error!(error = %e, "Failed to extract message, this is non-retryable.");
if let Err(cleanup_err) = cleanup_message(&ctx.sqs_worker, message).await {
tracing::error!(error = %cleanup_err, "Failed to clean up message after extraction error");
}
return Err(e);
}
};
let processing_result =
keep_invisible(message, &ctx.sqs_worker, inner_process_message(&ctx, &data)).await;
match processing_result {
// Processing success. Clean up the message
Ok(()) => {
cleanup_message(&ctx.sqs_worker, message).await?;
Ok(())
}
// A permanent failure occurred. We clean up the message to prevent it from being retried
Err(ProcessingError::NonRetryable(e)) => {
error_handlers::handle_non_retryable_error(&ctx, message, &data, &e).await
}
// A temporary failure occurred. We log it and don't clean up the message, so it gets retried
Err(ProcessingError::Retryable(e)) => match deferral::rate_limit(&e) {
Some(retry_after) => {
error_handlers::defer_rate_limited_operation(&ctx, message, &data, &e, retry_after)
.await
}
None => error_handlers::handle_retryable_error(&data, &e).await,
},
}
}
/// How often a running operation pushes its message's visibility out again.
const VISIBILITY_HEARTBEAT: Duration = Duration::from_secs(20);
/// The backfill queue's visibility timeout, restored by every heartbeat.
const VISIBILITY_TIMEOUT_SECONDS: i32 = 60;
/// Runs `work` while keeping its message hidden from other workers.
///
/// A whole thread, or any operation slowed by a struggling database, can
/// outlast the queue's visibility timeout. Delivered again while the first
/// copy still runs, the second copy would find the thread written and count
/// it, and the first would count it again through its metadata step,
/// completing the job early. A failed heartbeat only logs: the message then
/// reappears as it would have without one.
async fn keep_invisible<F: Future>(
message: &aws_sdk_sqs::types::Message,
sqs_worker: &sqs_worker::SQSWorker,
work: F,
) -> F::Output {
let Some(receipt_handle) = message.receipt_handle() else {
return work.await;
};
let mut work = std::pin::pin!(work);
let mut heartbeats = tokio::time::interval_at(
tokio::time::Instant::now() + VISIBILITY_HEARTBEAT,
VISIBILITY_HEARTBEAT,
);
loop {
tokio::select! {
output = &mut work => return output,
_ = heartbeats.tick() => {
if let Err(error) = sqs_worker
.change_message_visibility(receipt_handle, VISIBILITY_TIMEOUT_SECONDS)
.await
{
tracing::warn!(error = ?error, "failed to keep a running backfill operation hidden");
}
}
}
}
}
#[tracing::instrument(skip(ctx))]
async fn inner_process_message(
ctx: &PubSubContext,
data: &BackfillPubsubMessage,
) -> Result<(), ProcessingError> {
match &data.backfill_operation {
BackfillOperation::Init(scope) => {
let Some(JobContextNoToken { link, backfill_job }) =
fetch_job_context_no_token(ctx, scope, false).await?
else {
return Ok(());
};
init::init_backfill(ctx, scope, &link, &backfill_job).await
}
BackfillOperation::ListThreads(scope) => {
let Some(JobContextNoToken { link, backfill_job }) =
fetch_job_context_no_token(ctx, scope, false).await?
else {
return Ok(());
};
list_threads::list_threads(ctx, scope, &link, &backfill_job).await
}
BackfillOperation::BackfillThread(scope) => {
let Some(JobContextNoToken { link, .. }) =
fetch_job_context_no_token(ctx, scope, false).await?
else {
return Ok(());
};
backfill_thread::backfill_thread(ctx, scope, &link).await
}
BackfillOperation::BackfillMessage(scope) => {
let Some(JobContextNoToken { link, .. }) =
fetch_job_context_no_token(ctx, scope, false).await?
else {
// BackfillMessage owns per-thread progress in addition to
// the job-level progress that fetch_job_context already
// cleared on cancellation. Drop it so a re-run of the
// cancelled job doesn't see stale thread counters.
let _ = ctx
.redis_client
.delete_backfill_thread_progress(
scope.job_id,
&scope.payload.thread_provider_id,
)
.await;
return Ok(());
};
backfill_message::backfill_message(ctx, scope, &link).await
}
BackfillOperation::UpdateThreadMetadata(scope) => {
// UpdateThreadMetadata is a DB-only step; skip the Gmail token
// fetch so a revoked token can't fail this handler with
// AccessTokenFetchFailed.
let Some(JobContextNoToken { link, .. }) =
fetch_job_context_no_token(ctx, scope, false).await?
else {
return Ok(());
};
update_metadata::update_thread_metadata(ctx, scope, &link).await
}
BackfillOperation::BackfillAttachment(scope) => {
let Some(JobContextNoToken { link, .. }) =
fetch_job_context_no_token(ctx, scope, true).await?
else {
return Ok(());
};
backfill_attachment::backfill_attachment(ctx, &link, &scope.payload).await
}
BackfillOperation::FinalizeBackfill(scope) => {
increment_counters::finalize_backfill(ctx, scope.link_id, scope.job_id).await
}
BackfillOperation::SeedSentContact(scope) => {
let Some(JobContextNoToken { link, .. }) =
fetch_job_context_no_token(ctx, scope, false).await?
else {
return Ok(());
};
seed_sent_contact::seed_sent_contact(ctx, scope, &link).await
}
BackfillOperation::PopulateCrmContact(scope) => {
let Some(link) = fetch_link_if_present(ctx, scope.link_id).await? else {
return Ok(());
};
populate_crm_contact::populate_crm_contact(ctx, &link, &scope.payload).await
}
BackfillOperation::DepopulateCrmContact(scope) => {
let Some(link) = fetch_link_if_present(ctx, scope.link_id).await? else {
return Ok(());
};
depopulate_crm_contact::depopulate_crm_contact(ctx, &link, &scope.payload).await
}
BackfillOperation::PopulateCrmForUser(payload) => {
populate_crm_for_user::populate_crm_for_user(ctx, payload).await
}
BackfillOperation::DepopulateCrmForUser(payload) => {
depopulate_crm_for_user::depopulate_crm_for_user(ctx, payload).await
}
}
}
/// Same as [`JobContext`] but without a Gmail access token. Used by
/// handlers that do not require a raw Gmail access token. Provider operations
/// can still fetch tokens through `email_api`.
struct JobContextNoToken {
link: link::Link,
backfill_job: BackfillJob,
}
/// Shared prefetch for DB-only job-scoped handlers: loads the backfill
/// job, short-circuits on cancellation (also cleaning up the job-level
/// redis progress key), and loads the link. Returns `Ok(None)` when the
/// parent backfill job is failed or cancelled, and normally when it is
/// complete. Completion follow-up operations may opt into running after the
/// terminal transition. Variant-specific cancellation cleanup (e.g.
/// per-thread progress keys) belongs at the call site, not here.
async fn fetch_job_context_no_token<P>(
ctx: &PubSubContext,
scope: &JobScopedPayload<P>,
allow_complete: bool,
) -> Result<Option<JobContextNoToken>, ProcessingError> {
let backfill_job = email_db_client::backfill::job::get::get_backfill_job(&ctx.db, scope.job_id)
.await
.map_err(|e| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: e.context("Failed to fetch backfill job"),
})
})?
.ok_or_else(|| {
ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::BackfillJobNotFound,
source: anyhow::anyhow!("Backfill job not found"),
})
})?;
if matches!(
backfill_job.status,
BackfillJobStatus::Cancelled | BackfillJobStatus::Failed
) || (backfill_job.status == BackfillJobStatus::Complete && !allow_complete)
{
let _ = ctx
.redis_client
.delete_backfill_job_progress(scope.job_id)
.await;
return Ok(None);
}
let link = fetch_link(ctx, scope.link_id).await?;
Ok(Some(JobContextNoToken { link, backfill_job }))
}
/// Looks up a link by id, mapping the absence into a NonRetryable error
/// (the message names a link that doesn't exist; retrying won't help).
async fn fetch_link(ctx: &PubSubContext, link_id: Uuid) -> Result<link::Link, ProcessingError> {
match email_db_client::links::get::fetch_link_by_id(&ctx.db, link_id).await {
Ok(Some(link)) => Ok(link),
Ok(None) => {
let err_msg = format!("Link not found for link_id: {link_id}");
tracing::error!("{}", err_msg);
Err(ProcessingError::NonRetryable(DetailedError {
reason: FailureReason::LinkNotFound,
source: anyhow::anyhow!(err_msg),
}))
}
Err(e) => Err(ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: e,
})),
}
}
/// Looks up the link a link-scoped CRM message was fanned out for.
///
/// Returns `None` once the link is deleted: link teardown already removed the
/// link's CRM rows, so messages still queued for it have nothing left to do.
async fn fetch_link_if_present(
ctx: &PubSubContext,
link_id: Uuid,
) -> Result<Option<link::Link>, ProcessingError> {
let link = email_db_client::links::get::fetch_link_by_id(&ctx.db, link_id)
.await
.map_err(|e| {
ProcessingError::Retryable(DetailedError {
reason: FailureReason::DatabaseQueryFailed,
source: e,
})
})?;
if link.is_none() {
tracing::debug!(%link_id, "Link was deleted; dropping link-scoped CRM message");
}
Ok(link)
}
/// Extracts backfill message from the SQS message body
#[tracing::instrument(skip(message))]
fn extract_backfill_message(
message: &aws_sdk_sqs::types::Message,
) -> anyhow::Result<BackfillPubsubMessage> {
let message_body = message.body().context("message body not found")?;
// Deserialize the JSON string into a BackfillPubsubMessage
let backfill_message: BackfillPubsubMessage = serde_json::from_str(message_body)
.context("Failed to deserialize message body to BackfillPubsubMessage")?;
Ok(backfill_message)
}