Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

12.3 KB
Raw Blame
use super::email_api_error::map_email_api_error;
use super::increment_counters;
use crate::pubsub::context::PubSubContext;
use models_email::email::service::backfill::{
    BackfillJob, BackfillOperation, BackfillPubsubMessage, BackfillThreadPayload, JobScopedPayload,
};
use models_email::email::service::link;
use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
use models_email::email::service::thread::ListThreadsPayload;
use models_email::gmail::labels::SystemLabelID;
use std::cmp::min;

// the max size allowed by the gmail api
const BACKFILL_THREAD_BATCH_SIZE: u32 = 500;

// How many of the user's most important (CATEGORY_PERSONAL) threads the
// priority pass seeds first, before handing off to the normal sweep.
const PRIORITY_PASS_THREAD_LIMIT: u32 = 200;

// How many of the user's most recent sent messages we seed the contacts
// service from during the priority pass (the gmail api caps this at 500).
#[cfg(feature = "contacts_sync")]
const SENT_CONTACT_SEED_LIMIT: u32 = 200;

/// This step is invoked by Init.
/// Each ListThreads operation gets a batch of 500 thread_ids from the gmail api
/// and sends a BackfillThread message for each thread_id in the batch. If there
/// are still threads left to fetch, it will trigger another ListThreads message
/// to be created, looping until all threads requiring population have been listed.
///
/// The very first ListThreads (enqueued by Init) is a `priority_pass`: it lists
/// only the user's most important threads first, then hands off to this normal
/// most-recent-to-least sweep.
pub async fn list_threads(
    ctx: &PubSubContext,
    scope: &JobScopedPayload<ListThreadsPayload>,
    link: &link::Link,
    job: &BackfillJob,
) -> Result<(), ProcessingError> {
    if scope.payload.priority_pass {
        return list_priority_threads(ctx, scope, link, job).await;
    }

    let p = &scope.payload;
    let total_threads = job.total_threads;
    let threads_retrieved_count = job.threads_retrieved_count;

    let num_threads_to_list = min(
        BACKFILL_THREAD_BATCH_SIZE as i32,
        total_threads - threads_retrieved_count,
    );

    // get batch of thread ids
    let thread_list = ctx
        .email_api
        .list_threads(
            link.id,
            num_threads_to_list as u32,
            p.next_page_token.as_deref(),
            &[],
        )
        .await
        .map_err(|error| map_email_api_error(error, "Failed to list provider threads"))?;

    // pass along token if it exists for fetching next batch of thread_ids
    let next_page_token = thread_list.next_page_token.clone();
    let threads = thread_list.threads;

    // add the threads we just discovered to the job counter
    email_db_client::backfill::job::update::update_job_threads_retrieved_count(
        &ctx.db,
        scope.job_id,
        threads.len() as i32,
    )
    .await
    .map_err(|e| {
        ProcessingError::Retryable(DetailedError {
            reason: FailureReason::DatabaseQueryFailed,
            source: e.context("Failed to update backfill job num_threads"),
        })
    })?;

    // send a pubsub message for each discovered thread
    for thread in &threads {
        let thread_sqs_msg = BackfillPubsubMessage {
            backfill_operation: BackfillOperation::BackfillThread(JobScopedPayload {
                link_id: scope.link_id,
                job_id: scope.job_id,
                payload: BackfillThreadPayload {
                    thread_provider_id: thread.provider_id.clone(),
                    refresh_existing: p.refresh_existing,
                },
            }),
        };

        ctx.sqs_client
            .enqueue_email_backfill_message(thread_sqs_msg)
            .await
            .map_err(|e| {
                ProcessingError::NonRetryable(DetailedError {
                    reason: FailureReason::SqsEnqueueFailed,
                    source: e.context(format!("Failed to enqueue thread {}", thread.provider_id)),
                })
            })?;
    }

    // if we have more threads to fetch, send another pubsub message
    if next_page_token.is_some() {
        let list_thread_msg = BackfillPubsubMessage {
            backfill_operation: BackfillOperation::ListThreads(JobScopedPayload {
                link_id: scope.link_id,
                job_id: scope.job_id,
                payload: ListThreadsPayload {
                    next_page_token,
                    priority_pass: false,
                    refresh_existing: p.refresh_existing,
                },
            }),
        };

        ctx.sqs_client
            .enqueue_email_backfill_message(list_thread_msg)
            .await
            .map_err(|e| {
                ProcessingError::NonRetryable(DetailedError {
                    reason: FailureReason::SqsEnqueueFailed,
                    source: e.context("Failed to enqueue list threads message".to_string()),
                })
            })?;
    } else if threads.is_empty() && threads_retrieved_count == 0 {
        // A truly empty mailbox is completed in init_backfill (total_threads
        // is 0 there). Reaching here means the profile reported threads but
        // the listing returned none, so no per-thread message was ever
        // enqueued and the per-thread completion path will never finalize the
        // job. Complete it here.
        increment_counters::handle_job_completed(ctx, scope.job_id, None).await?;
        let _ = ctx
            .redis_client
            .delete_backfill_job_progress(scope.job_id)
            .await
            .inspect_err(|e| tracing::error!(error = ?e, "Failed to delete backfill job progress"));
    }

    Ok(())
}

/// The priority first pass (see [`ListThreadsPayload::priority_pass`]). Does a
/// single CATEGORY_PERSONAL-filtered listing of up to `PRIORITY_PASS_THREAD_LIMIT`
/// threads, enqueues a BackfillThread for each, bumps the job's redis total to
/// account for the normal sweep re-covering these same threads, then hands off
/// to the normal sweep. Unlike the normal pass it does not paginate and does
/// not touch `threads_retrieved_count` (that counter drives the normal sweep's
/// batch math and must stay equal to the real mailbox count).
async fn list_priority_threads(
    ctx: &PubSubContext,
    scope: &JobScopedPayload<ListThreadsPayload>,
    link: &link::Link,
    job: &BackfillJob,
) -> Result<(), ProcessingError> {
    // Never request more than the job's overall thread budget (which already
    // accounts for any requested limit set at job creation).
    let num_threads_to_list = min(PRIORITY_PASS_THREAD_LIMIT as i32, job.total_threads);

    if num_threads_to_list > 0 {
        // CATEGORY_PERSONAL is Gmail's "important to a human" bucket; seeding
        // these first means a brand-new user sees signal immediately.
        let thread_list = ctx
            .email_api
            .list_threads(
                link.id,
                num_threads_to_list as u32,
                None,
                &[SystemLabelID::CategoryPersonal.as_str()],
            )
            .await
            .map_err(|error| {
                map_email_api_error(error, "Failed to list priority provider threads")
            })?;

        let threads = thread_list.threads;

        // Bump the job total to cover these extra BackfillThread messages: the
        // normal sweep re-covers the same threads and re-increments the
        // completed counter via the skip path, so without this the
        // `completed >= total` check would finalize the job early. Bump before
        // enqueuing so no priority thread can complete against a stale total;
        // after this only non-retryable enqueues remain, so a retry can't
        // double-bump.
        if !threads.is_empty() {
            ctx.redis_client
                .add_to_total_threads(scope.job_id, threads.len() as i32)
                .await
                .map_err(|e| {
                    ProcessingError::Retryable(DetailedError {
                        reason: FailureReason::RedisQueryFailed,
                        source: e.context("Failed to bump total_threads for priority pass"),
                    })
                })?;
        }

        for thread in &threads {
            let thread_sqs_msg = BackfillPubsubMessage {
                backfill_operation: BackfillOperation::BackfillThread(JobScopedPayload {
                    link_id: scope.link_id,
                    job_id: scope.job_id,
                    payload: BackfillThreadPayload {
                        thread_provider_id: thread.provider_id.clone(),
                        refresh_existing: false,
                    },
                }),
            };

            ctx.sqs_client
                .enqueue_email_backfill_message(thread_sqs_msg)
                .await
                .map_err(|e| {
                    ProcessingError::NonRetryable(DetailedError {
                        reason: FailureReason::SqsEnqueueFailed,
                        source: e.context(format!(
                            "Failed to enqueue priority thread {}",
                            thread.provider_id
                        )),
                    })
                })?;
        }
    }

    // Seed the contacts service from the user's recent sent mail so a new user
    // has contacts before full backfill completes. Best-effort: this runs after
    // the priority-pass total_threads bump, so it must never return a retryable
    // error (that would re-run the pass and double-bump the counter).
    #[cfg(feature = "contacts_sync")]
    enqueue_sent_contact_seeds(ctx, scope, link).await;

    // Hand off to the normal most-recent-to-least sweep from the beginning. This
    // runs regardless of how many priority threads were found.
    let list_thread_msg = BackfillPubsubMessage {
        backfill_operation: BackfillOperation::ListThreads(JobScopedPayload {
            link_id: scope.link_id,
            job_id: scope.job_id,
            payload: ListThreadsPayload {
                next_page_token: None,
                priority_pass: false,
                refresh_existing: scope.payload.refresh_existing,
            },
        }),
    };

    ctx.sqs_client
        .enqueue_email_backfill_message(list_thread_msg)
        .await
        .map_err(|e| {
            ProcessingError::NonRetryable(DetailedError {
                reason: FailureReason::SqsEnqueueFailed,
                source: e.context(
                    "Failed to enqueue normal ListThreads after priority pass".to_string(),
                ),
            })
        })?;

    Ok(())
}

/// Lists the user's most recent sent messages and fans out one
/// [`BackfillOperation::SeedSentContact`] per message so the contacts service
/// gets seeded early in the backfill (see [`list_priority_threads`]).
///
/// Best-effort by design: it runs after the priority pass has already bumped
/// `total_threads`, so it must not propagate a retryable error (that would
/// re-run the whole pass and double-bump). On any failure it logs and returns —
/// `handle_contacts_sync` re-seeds the full contact set at job completion.
#[cfg(feature = "contacts_sync")]
async fn enqueue_sent_contact_seeds(
    ctx: &PubSubContext,
    scope: &JobScopedPayload<ListThreadsPayload>,
    link: &link::Link,
) {
    use models_email::email::service::backfill::SeedSentContactPayload;

    let message_ids = match ctx
        .email_api
        .list_messages(
            link.id,
            SENT_CONTACT_SEED_LIMIT,
            &[SystemLabelID::Sent.as_str()],
        )
        .await
    {
        Ok(ids) => ids,
        Err(e) => {
            tracing::error!(error = ?e, link_id = %link.id, "Skipping sent-contact seed: failed to list provider sent messages");
            return;
        }
    };

    for message_provider_id in message_ids {
        let msg = BackfillPubsubMessage {
            backfill_operation: BackfillOperation::SeedSentContact(JobScopedPayload {
                link_id: scope.link_id,
                job_id: scope.job_id,
                payload: SeedSentContactPayload {
                    message_provider_id: message_provider_id.clone(),
                },
            }),
        };

        if let Err(e) = ctx.sqs_client.enqueue_email_backfill_message(msg).await {
            tracing::error!(error = ?e, link_id = %link.id, message_id = %message_provider_id, "Failed to enqueue sent-contact seed; skipping remaining");
            return;
        }
    }
}