Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

12.2 KB
Raw Blame
use crate::pubsub::backfill::{
    backfill_attachment, backfill_message, backfill_thread, deferral, depopulate_crm_contact,
    depopulate_crm_for_user, error_handlers, increment_counters, init, list_threads,
    populate_crm_contact, populate_crm_for_user, seed_sent_contact, update_metadata,
};
use crate::pubsub::context::PubSubContext;
use anyhow::Context;
use models_email::email::service::backfill::{
    BackfillJob, BackfillJobStatus, BackfillOperation, BackfillPubsubMessage, JobScopedPayload,
};
use models_email::email::service::link;
use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
use sqs_worker::cleanup_message;
use std::time::Duration;
use uuid::Uuid;

// Process a single message from the backfill queue
pub async fn process_message(
    ctx: PubSubContext,
    message: &aws_sdk_sqs::types::Message,
) -> anyhow::Result<()> {
    // Malformed JSON is NOT retryable.
    let data = match extract_backfill_message(message) {
        Ok(data) => data,
        Err(e) => {
            tracing::error!(error = %e, "Failed to extract message, this is non-retryable.");
            if let Err(cleanup_err) = cleanup_message(&ctx.sqs_worker, message).await {
                tracing::error!(error = %cleanup_err, "Failed to clean up message after extraction error");
            }
            return Err(e);
        }
    };

    let processing_result =
        keep_invisible(message, &ctx.sqs_worker, inner_process_message(&ctx, &data)).await;

    match processing_result {
        // Processing success. Clean up the message
        Ok(()) => {
            cleanup_message(&ctx.sqs_worker, message).await?;
            Ok(())
        }

        // A permanent failure occurred. We clean up the message to prevent it from being retried
        Err(ProcessingError::NonRetryable(e)) => {
            error_handlers::handle_non_retryable_error(&ctx, message, &data, &e).await
        }

        // A temporary failure occurred. We log it and don't clean up the message, so it gets retried
        Err(ProcessingError::Retryable(e)) => match deferral::rate_limit(&e) {
            Some(retry_after) => {
                error_handlers::defer_rate_limited_operation(&ctx, message, &data, &e, retry_after)
                    .await
            }
            None => error_handlers::handle_retryable_error(&data, &e).await,
        },
    }
}

/// How often a running operation pushes its message's visibility out again.
const VISIBILITY_HEARTBEAT: Duration = Duration::from_secs(20);
/// The backfill queue's visibility timeout, restored by every heartbeat.
const VISIBILITY_TIMEOUT_SECONDS: i32 = 60;

/// Runs `work` while keeping its message hidden from other workers.
///
/// A whole thread, or any operation slowed by a struggling database, can
/// outlast the queue's visibility timeout. Delivered again while the first
/// copy still runs, the second copy would find the thread written and count
/// it, and the first would count it again through its metadata step,
/// completing the job early. A failed heartbeat only logs: the message then
/// reappears as it would have without one.
async fn keep_invisible<F: Future>(
    message: &aws_sdk_sqs::types::Message,
    sqs_worker: &sqs_worker::SQSWorker,
    work: F,
) -> F::Output {
    let Some(receipt_handle) = message.receipt_handle() else {
        return work.await;
    };
    let mut work = std::pin::pin!(work);
    let mut heartbeats = tokio::time::interval_at(
        tokio::time::Instant::now() + VISIBILITY_HEARTBEAT,
        VISIBILITY_HEARTBEAT,
    );
    loop {
        tokio::select! {
            output = &mut work => return output,
            _ = heartbeats.tick() => {
                if let Err(error) = sqs_worker
                    .change_message_visibility(receipt_handle, VISIBILITY_TIMEOUT_SECONDS)
                    .await
                {
                    tracing::warn!(error = ?error, "failed to keep a running backfill operation hidden");
                }
            }
        }
    }
}

#[tracing::instrument(skip(ctx))]
async fn inner_process_message(
    ctx: &PubSubContext,
    data: &BackfillPubsubMessage,
) -> Result<(), ProcessingError> {
    match &data.backfill_operation {
        BackfillOperation::Init(scope) => {
            let Some(JobContextNoToken { link, backfill_job }) =
                fetch_job_context_no_token(ctx, scope, false).await?
            else {
                return Ok(());
            };
            init::init_backfill(ctx, scope, &link, &backfill_job).await
        }
        BackfillOperation::ListThreads(scope) => {
            let Some(JobContextNoToken { link, backfill_job }) =
                fetch_job_context_no_token(ctx, scope, false).await?
            else {
                return Ok(());
            };
            list_threads::list_threads(ctx, scope, &link, &backfill_job).await
        }
        BackfillOperation::BackfillThread(scope) => {
            let Some(JobContextNoToken { link, .. }) =
                fetch_job_context_no_token(ctx, scope, false).await?
            else {
                return Ok(());
            };
            backfill_thread::backfill_thread(ctx, scope, &link).await
        }
        BackfillOperation::BackfillMessage(scope) => {
            let Some(JobContextNoToken { link, .. }) =
                fetch_job_context_no_token(ctx, scope, false).await?
            else {
                // BackfillMessage owns per-thread progress in addition to
                // the job-level progress that fetch_job_context already
                // cleared on cancellation. Drop it so a re-run of the
                // cancelled job doesn't see stale thread counters.
                let _ = ctx
                    .redis_client
                    .delete_backfill_thread_progress(
                        scope.job_id,
                        &scope.payload.thread_provider_id,
                    )
                    .await;
                return Ok(());
            };
            backfill_message::backfill_message(ctx, scope, &link).await
        }
        BackfillOperation::UpdateThreadMetadata(scope) => {
            // UpdateThreadMetadata is a DB-only step; skip the Gmail token
            // fetch so a revoked token can't fail this handler with
            // AccessTokenFetchFailed.
            let Some(JobContextNoToken { link, .. }) =
                fetch_job_context_no_token(ctx, scope, false).await?
            else {
                return Ok(());
            };
            update_metadata::update_thread_metadata(ctx, scope, &link).await
        }
        BackfillOperation::BackfillAttachment(scope) => {
            let Some(JobContextNoToken { link, .. }) =
                fetch_job_context_no_token(ctx, scope, true).await?
            else {
                return Ok(());
            };
            backfill_attachment::backfill_attachment(ctx, &link, &scope.payload).await
        }
        BackfillOperation::FinalizeBackfill(scope) => {
            increment_counters::finalize_backfill(ctx, scope.link_id, scope.job_id).await
        }
        BackfillOperation::SeedSentContact(scope) => {
            let Some(JobContextNoToken { link, .. }) =
                fetch_job_context_no_token(ctx, scope, false).await?
            else {
                return Ok(());
            };
            seed_sent_contact::seed_sent_contact(ctx, scope, &link).await
        }
        BackfillOperation::PopulateCrmContact(scope) => {
            let Some(link) = fetch_link_if_present(ctx, scope.link_id).await? else {
                return Ok(());
            };
            populate_crm_contact::populate_crm_contact(ctx, &link, &scope.payload).await
        }
        BackfillOperation::DepopulateCrmContact(scope) => {
            let Some(link) = fetch_link_if_present(ctx, scope.link_id).await? else {
                return Ok(());
            };
            depopulate_crm_contact::depopulate_crm_contact(ctx, &link, &scope.payload).await
        }
        BackfillOperation::PopulateCrmForUser(payload) => {
            populate_crm_for_user::populate_crm_for_user(ctx, payload).await
        }
        BackfillOperation::DepopulateCrmForUser(payload) => {
            depopulate_crm_for_user::depopulate_crm_for_user(ctx, payload).await
        }
    }
}

/// Same as [`JobContext`] but without a Gmail access token. Used by
/// handlers that do not require a raw Gmail access token. Provider operations
/// can still fetch tokens through `email_api`.
struct JobContextNoToken {
    link: link::Link,
    backfill_job: BackfillJob,
}

/// Shared prefetch for DB-only job-scoped handlers: loads the backfill
/// job, short-circuits on cancellation (also cleaning up the job-level
/// redis progress key), and loads the link. Returns `Ok(None)` when the
/// parent backfill job is failed or cancelled, and normally when it is
/// complete. Completion follow-up operations may opt into running after the
/// terminal transition. Variant-specific cancellation cleanup (e.g.
/// per-thread progress keys) belongs at the call site, not here.
async fn fetch_job_context_no_token<P>(
    ctx: &PubSubContext,
    scope: &JobScopedPayload<P>,
    allow_complete: bool,
) -> Result<Option<JobContextNoToken>, ProcessingError> {
    let backfill_job = email_db_client::backfill::job::get::get_backfill_job(&ctx.db, scope.job_id)
        .await
        .map_err(|e| {
            ProcessingError::Retryable(DetailedError {
                reason: FailureReason::DatabaseQueryFailed,
                source: e.context("Failed to fetch backfill job"),
            })
        })?
        .ok_or_else(|| {
            ProcessingError::NonRetryable(DetailedError {
                reason: FailureReason::BackfillJobNotFound,
                source: anyhow::anyhow!("Backfill job not found"),
            })
        })?;

    if matches!(
        backfill_job.status,
        BackfillJobStatus::Cancelled | BackfillJobStatus::Failed
    ) || (backfill_job.status == BackfillJobStatus::Complete && !allow_complete)
    {
        let _ = ctx
            .redis_client
            .delete_backfill_job_progress(scope.job_id)
            .await;
        return Ok(None);
    }

    let link = fetch_link(ctx, scope.link_id).await?;

    Ok(Some(JobContextNoToken { link, backfill_job }))
}

/// Looks up a link by id, mapping the absence into a NonRetryable error
/// (the message names a link that doesn't exist; retrying won't help).
async fn fetch_link(ctx: &PubSubContext, link_id: Uuid) -> Result<link::Link, ProcessingError> {
    match email_db_client::links::get::fetch_link_by_id(&ctx.db, link_id).await {
        Ok(Some(link)) => Ok(link),
        Ok(None) => {
            let err_msg = format!("Link not found for link_id: {link_id}");
            tracing::error!("{}", err_msg);
            Err(ProcessingError::NonRetryable(DetailedError {
                reason: FailureReason::LinkNotFound,
                source: anyhow::anyhow!(err_msg),
            }))
        }
        Err(e) => Err(ProcessingError::Retryable(DetailedError {
            reason: FailureReason::DatabaseQueryFailed,
            source: e,
        })),
    }
}

/// Looks up the link a link-scoped CRM message was fanned out for.
///
/// Returns `None` once the link is deleted: link teardown already removed the
/// link's CRM rows, so messages still queued for it have nothing left to do.
async fn fetch_link_if_present(
    ctx: &PubSubContext,
    link_id: Uuid,
) -> Result<Option<link::Link>, ProcessingError> {
    let link = email_db_client::links::get::fetch_link_by_id(&ctx.db, link_id)
        .await
        .map_err(|e| {
            ProcessingError::Retryable(DetailedError {
                reason: FailureReason::DatabaseQueryFailed,
                source: e,
            })
        })?;
    if link.is_none() {
        tracing::debug!(%link_id, "Link was deleted; dropping link-scoped CRM message");
    }
    Ok(link)
}

/// Extracts backfill message from the SQS message body
#[tracing::instrument(skip(message))]
fn extract_backfill_message(
    message: &aws_sdk_sqs::types::Message,
) -> anyhow::Result<BackfillPubsubMessage> {
    let message_body = message.body().context("message body not found")?;

    // Deserialize the JSON string into a BackfillPubsubMessage
    let backfill_message: BackfillPubsubMessage = serde_json::from_str(message_body)
        .context("Failed to deserialize message body to BackfillPubsubMessage")?;

    Ok(backfill_message)
}