Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): stop additional-inbox connects failing on backfill rate limit (#4840)
gbirmana63f9830b9f6
diff --git a/.sqlx/query-2237ef2e86f1c48cef5ebd2fe896c84624135cd808de26975538992ca7d3b69f.json b/.sqlx/query-2237ef2e86f1c48cef5ebd2fe896c84624135cd808de26975538992ca7d3b69f.jsondeleted file mode 100644index ee636ccb21..0000000000--- a/.sqlx/query-2237ef2e86f1c48cef5ebd2fe896c84624135cd808de26975538992ca7d3b69f.json+++ /dev/null@@ -1,83 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n SELECT\n id,\n link_id,\n fusionauth_user_id,\n threads_requested_limit,\n total_threads,\n threads_retrieved_count,\n status as \"status: db::backfill::BackfillJobStatus\",\n created_at,\n updated_at\n FROM email_backfill_jobs\n WHERE fusionauth_user_id = $1\n AND created_at > NOW() - INTERVAL '24 hours'\n AND status NOT IN ('Cancelled', 'Failed')\n ORDER BY created_at DESC\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "id",- "type_info": "Uuid"- },- {- "ordinal": 1,- "name": "link_id",- "type_info": "Uuid"- },- {- "ordinal": 2,- "name": "fusionauth_user_id",- "type_info": "Text"- },- {- "ordinal": 3,- "name": "threads_requested_limit",- "type_info": "Int4"- },- {- "ordinal": 4,- "name": "total_threads",- "type_info": "Int4"- },- {- "ordinal": 5,- "name": "threads_retrieved_count",- "type_info": "Int4"- },- {- "ordinal": 6,- "name": "status: db::backfill::BackfillJobStatus",- "type_info": {- "Custom": {- "name": "email_backfill_job_status",- "kind": {- "Enum": [- "Init",- "InProgress",- "Complete",- "Cancelled",- "Failed"- ]- }- }- }- },- {- "ordinal": 7,- "name": "created_at",- "type_info": "Timestamptz"- },- {- "ordinal": 8,- "name": "updated_at",- "type_info": "Timestamptz"- }- ],- "parameters": {- "Left": [- "Text"- ]- },- "nullable": [- false,- true,- false,- true,- false,- false,- false,- false,- false- ]- },- "hash": "2237ef2e86f1c48cef5ebd2fe896c84624135cd808de26975538992ca7d3b69f"-}diff --git a/apps/web/src/features/auth/EmailAuth.tsx b/apps/web/src/features/auth/EmailAuth.tsxindex 04b9b3160f..85c74151c5 100644--- a/apps/web/src/features/auth/EmailAuth.tsx+++ b/apps/web/src/features/auth/EmailAuth.tsx@@ -169,8 +169,15 @@ function EmailLinkCallback(props: Pick<EmailAuthParams, 'successPath'>) {});return;}+ if (err.tag === 'NoGmailGrant') {+ toast.failure(+ 'Gmail access was not granted. Please allow all requested permissions and try again.'+ );+ navigateToEmailSettings();+ return;+ }toast.failure('Failed to add inbox');- navigateToSuccess();+ navigateToEmailSettings();});};diff --git a/apps/web/src/lib/core/email-link/index.ts b/apps/web/src/lib/core/email-link/index.tsindex 07b64bc2d2..0ca45880e4 100644--- a/apps/web/src/lib/core/email-link/index.ts+++ b/apps/web/src/lib/core/email-link/index.ts@@ -195,6 +195,21 @@ function isPaymentRequired(errors: ReadonlyArray<{ code: string }>): boolean {return errors.some((error) => error.code === 'PAYMENT_REQUIRED');}+/**+ * The backend answers `POST /link/gmail` with 429 when the user has too many+ * incomplete link attempts in flight (each abandoned OAuth leaves a pending row+ * that expires after 24h). The auth client maps that to `TOO_MANY_PENDING_LINKS`+ * so the flow can explain the wait instead of a dead-end generic failure.+ */+function isTooManyPendingLinks(+ errors: ReadonlyArray<{ code: string }>+): boolean {+ return errors.some((error) => error.code === 'TOO_MANY_PENDING_LINKS');+}++const TOO_MANY_PENDING_LINKS_MESSAGE =+ 'Too many inbox connections in progress.';+/*** Starts the add-inbox flow: fetches the Gmail link authorization URL and* navigates the browser to the OAuth consent page. The callback returns to@@ -244,6 +259,10 @@ export function useAddInboxFlow() {showPaywall(PaywallKey.MULTI_INBOX);return;}+ if (isTooManyPendingLinks(result.error)) {+ toast.failure(TOO_MANY_PENDING_LINKS_MESSAGE);+ return;+ }toast.failure('Failed to start Gmail link flow');return;}@@ -285,6 +304,8 @@ export function useAddInboxFlow() {window.location.href = result.value.authorization_url;} else if (isPaymentRequired(result.error)) {showPaywall(PaywallKey.MULTI_INBOX);+ } else if (isTooManyPendingLinks(result.error)) {+ toast.failure(TOO_MANY_PENDING_LINKS_MESSAGE);} else {toast.failure('Failed to start Gmail link flow');}diff --git a/apps/web/src/lib/service-clients/service-auth/client.ts b/apps/web/src/lib/service-clients/service-auth/client.tsindex 341e9508d3..6b974b0891 100644--- a/apps/web/src/lib/service-clients/service-auth/client.ts+++ b/apps/web/src/lib/service-clients/service-auth/client.ts@@ -549,18 +549,30 @@ export const authServiceClient = {? `${authHost}/link/gmail?original_url=${encodeURIComponent(originalUrl)}`: `${authHost}/link/gmail`;return (- await fetchWithAuth<InitGmailLinkResponse, 'PAYMENT_REQUIRED'>(url, {+ await fetchWithAuth<+ InitGmailLinkResponse,+ 'PAYMENT_REQUIRED' | 'TOO_MANY_PENDING_LINKS'+ >(url, {method: 'POST',// The backend returns 402 when the user isn't entitled to additional- // inboxes; surface it as a distinct code so the add-inbox flow can open- // the paywall instead of showing a generic failure.- errorResponseHandler: async (response) =>- response.status === 402- ? { code: 'PAYMENT_REQUIRED', message: 'Payment required' }- : {- code: 'HTTP_ERROR',- message: `HTTP error! status: ${response.status}`,- },+ // inboxes, and 429 when they have too many incomplete link attempts in+ // flight. Surface each as a distinct code so the add-inbox flow can open+ // the paywall or explain the wait instead of a generic failure.+ errorResponseHandler: async (response) => {+ if (response.status === 402) {+ return { code: 'PAYMENT_REQUIRED', message: 'Payment required' };+ }+ if (response.status === 429) {+ return {+ code: 'TOO_MANY_PENDING_LINKS',+ message: 'Too many pending inbox connections',+ };+ }+ return {+ code: 'HTTP_ERROR',+ message: `HTTP error! status: ${response.status}`,+ };+ },})).map((result) => result);},diff --git a/crates/email_db_client/src/backfill/job/get.rs b/crates/email_db_client/src/backfill/job/get.rsindex 26e58ec612..90c1ec421b 100644--- a/crates/email_db_client/src/backfill/job/get.rs+++ b/crates/email_db_client/src/backfill/job/get.rs@@ -126,44 +126,6 @@ pub async fn get_latest_backfill_job_by_link_id(Ok(record.map(Into::into))}-/// Retrieves all backfill jobs created in the last 24 hours for a given macro ID-#[tracing::instrument(skip(pool), err)]-pub async fn get_recent_jobs_by_fusionauth_user_id(- pool: &PgPool,- fusionauth_user_id: &str,-) -> anyhow::Result<Vec<service::backfill::BackfillJob>> {- // Jobs created in the last 24h for this user, excluding terminal (Cancelled/Failed)- // statuses, used for the connect rate limit.- let records = sqlx::query_as!(- db::backfill::BackfillJob,- r#"- SELECT- id,- link_id,- fusionauth_user_id,- threads_requested_limit,- total_threads,- threads_retrieved_count,- status as "status: db::backfill::BackfillJobStatus",- created_at,- updated_at- FROM email_backfill_jobs- WHERE fusionauth_user_id = $1- AND created_at > NOW() - INTERVAL '24 hours'- AND status NOT IN ('Cancelled', 'Failed')- ORDER BY created_at DESC- "#,- fusionauth_user_id- )- .fetch_all(pool)- .await?;-- // Convert all database records to service models- let jobs = records.into_iter().map(Into::into).collect();-- Ok(jobs)-}-/// Retrieves a user's most recent backfill jobs, newest first. Spans all of the/// user's links and all statuses; `link_id` is intentionally not used so jobs/// survive a link being deleted and recreated. Capped at 100 so frequentdiff --git a/crates/email_db_client/src/backfill/job/get/test.rs b/crates/email_db_client/src/backfill/job/get/test.rsindex 17cdca6a19..f2af3d0889 100644--- a/crates/email_db_client/src/backfill/job/get/test.rs+++ b/crates/email_db_client/src/backfill/job/get/test.rs@@ -1,7 +1,6 @@-use super::{get_all_jobs_by_fusionauth_user_id, get_recent_jobs_by_fusionauth_user_id};+use super::get_all_jobs_by_fusionauth_user_id;use macro_db_migrator::MACRO_DB_MIGRATIONS;use models_email::email::db::backfill::BackfillJobStatus as DbStatus;-use models_email::email::service::backfill::BackfillJobStatus;use sqlx::types::Uuid;use sqlx::{Pool, Postgres};@@ -19,37 +18,6 @@ async fn insert_job(pool: &Pool<Postgres>, fusion: &str, status: DbStatus, hours.unwrap();}-#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]-async fn recent_jobs_excludes_terminal_and_old_and_other_users(- pool: Pool<Postgres>,-) -> anyhow::Result<()> {- const FUSION: &str = "fusion-rate-limit-user";- const OTHER: &str = "fusion-other-user";-- // Recent, non-terminal → counted.- insert_job(&pool, FUSION, DbStatus::Init, 1).await;- insert_job(&pool, FUSION, DbStatus::InProgress, 2).await;- insert_job(&pool, FUSION, DbStatus::Complete, 3).await;- // Recent but terminal → excluded so connect/disconnect churn does not count.- insert_job(&pool, FUSION, DbStatus::Cancelled, 1).await;- insert_job(&pool, FUSION, DbStatus::Failed, 2).await;- // Older than the 24h window → excluded.- insert_job(&pool, FUSION, DbStatus::Init, 25).await;- // Different user → excluded.- insert_job(&pool, OTHER, DbStatus::Init, 1).await;-- let jobs = get_recent_jobs_by_fusionauth_user_id(&pool, FUSION).await?;-- assert_eq!(jobs.len(), 3);- assert!(jobs.iter().all(|j| j.fusionauth_user_id == FUSION));- assert!(jobs.iter().all(|j| !matches!(- j.status,- BackfillJobStatus::Cancelled | BackfillJobStatus::Failed- )));-- Ok(())-}-#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn all_jobs_returns_users_jobs_any_status_newest_first(pool: Pool<Postgres>,diff --git a/services/email_service/src/api/email/init.rs b/services/email_service/src/api/email/init.rsindex 7983325de9..645bcb1e01 100644--- a/services/email_service/src/api/email/init.rs+++ b/services/email_service/src/api/email/init.rs@@ -40,9 +40,6 @@ pub enum InitError {#[error("No Gmail grant available to provision from")]NoGmailGrant,- #[error("Job limit exceeded")]- TooManyJobs,-#[error("Failed to enqueue backfill message")]EnqueueError,@@ -89,7 +86,6 @@ impl InitError {| InitError::NoGmailGrant| InitError::BadRequest(_)| InitError::Parse(_) => StatusCode::BAD_REQUEST,- InitError::TooManyJobs => StatusCode::TOO_MANY_REQUESTS,InitError::EnqueueError | InitError::DatabaseError(_) | InitError::GmailError(_) => {StatusCode::INTERNAL_SERVER_ERROR}@@ -201,20 +197,48 @@ pub async fn handler(// Init runs on every authentication, so its expected no-op outcomes (400s)// must not error-log. The span skips the auto err event and the result is// classified here, inside the span, where user fields still attach.+ let link_id = query.link_id;+ let db = ctx.db.clone();let result = init_user(ctx, query, authorization).await;if let Err(e) = &result {let status = e.status_code();if status.is_server_error() {tracing::error!(error = ?e, "init failed");- } else if status == StatusCode::TOO_MANY_REQUESTS {- tracing::warn!(error = %e, "init rate limited");} else {tracing::debug!(error = %e, "init declined");}+ cleanup_in_progress_link_on_failure(&db, link_id, e).await;}result}+/// Whether a failed `/email/init` should delete its `in_progress_user_link` row. True for+/// every terminal failure except `SharedInboxConflict`, which is held open on purpose so the+/// `force_share` retry can reuse the same `link_id`.+fn should_clean_up_in_progress_link(error: &InitError) -> bool {+ !matches!(error, InitError::SharedInboxConflict { .. })+}++/// A failed `/email/init` otherwise leaves the `in_progress_user_link` row behind — only the+/// success paths delete it — where it counts toward the `/link/gmail` start cap and can lock+/// the user out of further attempts.+async fn cleanup_in_progress_link_on_failure(+ db: &sqlx::Pool<sqlx::Postgres>,+ link_id: Option<Uuid>,+ error: &InitError,+) {+ if let Some(link_id) = link_id+ && should_clean_up_in_progress_link(error)+ {+ macro_db_client::in_progress_user_link::delete_in_progress_user_link(db, &link_id)+ .await+ .inspect_err(|del_err| {+ tracing::warn!(error = ?del_err, ?link_id, "Failed to clean up in_progress_user_link after failed init");+ })+ .ok();+ }+}+async fn init_user(ctx: ApiContext,Query(InitParams {@@ -339,8 +363,6 @@ async fn init_user(.context("child macro user disappeared before self-link bootstrap")?.to_string();- enforce_backfill_rate_limit(&ctx.db, &child_fusion_id, &linked_email).await?;-let gmail_token =fetch_gmail_token_for_email(&ctx, &child_fusion_id, &linked_email).await?;let watch_response = register_watch_recovering(&ctx.gmail_client, &gmail_token).await?;@@ -517,9 +539,6 @@ async fn init_user(.into_response());}- enforce_backfill_rate_limit(&ctx.db, &user_context.fusion_user_id, &linked_email)- .await?;-let gmail_token =fetch_gmail_token_for_email(&ctx, &user_context.fusion_user_id, &linked_email).await?;@@ -570,8 +589,6 @@ async fn init_user(let email = extract_email_with_response(&user_context.user_id).map_err(|_| InitError::BadRequest("Failed to extract email".to_string()))?;- enforce_backfill_rate_limit(&ctx.db, &user_context.fusion_user_id, &email).await?;-let gmail_token =fetch_gmail_token_for_email(&ctx, &user_context.fusion_user_id, &email).await?;@@ -720,28 +737,6 @@ async fn init_user(.into_response())}-/// Rejects a connect when this Gmail identity already has 10+ recent backfill jobs in-/// the last 24h (`@macro.com` is exempt). Enforced before the link and Gmail watch are-/// created so a rejected connect never has to tear down a half-provisioned inbox.-async fn enforce_backfill_rate_limit(- db: &sqlx::PgPool,- fusion_user_id: &str,- email_address: &str,-) -> Result<(), InitError> {- let recent_jobs = email_db_client::backfill::job::get::get_recent_jobs_by_fusionauth_user_id(- db,- fusion_user_id,- )- .await- .context("Failed to fetch recent backfill jobs")?;-- if recent_jobs.len() >= 10 && !email_address.ends_with("@macro.com") {- return Err(InitError::TooManyJobs);- }-- Ok(())-}-/// Registers a Gmail watch, recovering from the one-watch-per-mailbox limit. A watch/// left over from a prior connect (e.g. a disconnect that could not reach Gmail to stop/// it) makes a fresh registration fail with 400 "Only one user push notification clientdiff --git a/services/email_service/src/api/email/init/test.rs b/services/email_service/src/api/email/init/test.rsindex 735162f4ee..ebcf2f1284 100644--- a/services/email_service/src/api/email/init/test.rs+++ b/services/email_service/src/api/email/init/test.rs@@ -68,3 +68,31 @@ fn other_watch_failures_stay_gmail_errors() {));}}++#[test]+fn only_shared_inbox_conflict_keeps_the_in_progress_link() {+ // SharedInboxConflict is held open for the force_share retry, so its row must be kept;+ // every other terminal failure must clean up the row so it stops counting toward the+ // /link/gmail start cap. The delete itself is covered by macro_db_client's own tests.+ assert!(+ !should_clean_up_in_progress_link(&InitError::SharedInboxConflict {+ email_address: "[email protected]".to_string(),+ existing_owner_email: "[email protected]".to_string(),+ existing_link_id: Uuid::new_v4(),+ }),+ "SharedInboxConflict must keep the row for the force_share retry"+ );++ for err in [+ InitError::AlreadyInitialized,+ InitError::NoGmailGrant,+ InitError::EnqueueError,+ InitError::BadRequest("bad".to_string()),+ InitError::GmailError(GmailError::Unauthorized),+ ] {+ assert!(+ should_clean_up_in_progress_link(&err),+ "terminal failure {err:?} should clean up the in_progress row"+ );+ }+}