Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): stop additional-inbox connects failing on backfill rate limit (#4840)

gbirmana63f9830b9f6
diff --git a/.sqlx/query-2237ef2e86f1c48cef5ebd2fe896c84624135cd808de26975538992ca7d3b69f.json b/.sqlx/query-2237ef2e86f1c48cef5ebd2fe896c84624135cd808de26975538992ca7d3b69f.json
deleted file mode 100644
index ee636ccb21..0000000000
--- a/.sqlx/query-2237ef2e86f1c48cef5ebd2fe896c84624135cd808de26975538992ca7d3b69f.json
+++ /dev/null
@@ -1,83 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT\n id,\n link_id,\n fusionauth_user_id,\n threads_requested_limit,\n total_threads,\n threads_retrieved_count,\n status as \"status: db::backfill::BackfillJobStatus\",\n created_at,\n updated_at\n FROM email_backfill_jobs\n WHERE fusionauth_user_id = $1\n AND created_at > NOW() - INTERVAL '24 hours'\n AND status NOT IN ('Cancelled', 'Failed')\n ORDER BY created_at DESC\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 1,
- "name": "link_id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 2,
- "name": "fusionauth_user_id",
- "type_info": "Text"
- },
- {
- "ordinal": 3,
- "name": "threads_requested_limit",
- "type_info": "Int4"
- },
- {
- "ordinal": 4,
- "name": "total_threads",
- "type_info": "Int4"
- },
- {
- "ordinal": 5,
- "name": "threads_retrieved_count",
- "type_info": "Int4"
- },
- {
- "ordinal": 6,
- "name": "status: db::backfill::BackfillJobStatus",
- "type_info": {
- "Custom": {
- "name": "email_backfill_job_status",
- "kind": {
- "Enum": [
- "Init",
- "InProgress",
- "Complete",
- "Cancelled",
- "Failed"
- ]
- }
- }
- }
- },
- {
- "ordinal": 7,
- "name": "created_at",
- "type_info": "Timestamptz"
- },
- {
- "ordinal": 8,
- "name": "updated_at",
- "type_info": "Timestamptz"
- }
- ],
- "parameters": {
- "Left": [
- "Text"
- ]
- },
- "nullable": [
- false,
- true,
- false,
- true,
- false,
- false,
- false,
- false,
- false
- ]
- },
- "hash": "2237ef2e86f1c48cef5ebd2fe896c84624135cd808de26975538992ca7d3b69f"
-}
diff --git a/apps/web/src/features/auth/EmailAuth.tsx b/apps/web/src/features/auth/EmailAuth.tsx
index 04b9b3160f..85c74151c5 100644
--- a/apps/web/src/features/auth/EmailAuth.tsx
+++ b/apps/web/src/features/auth/EmailAuth.tsx
@@ -169,8 +169,15 @@ function EmailLinkCallback(props: Pick<EmailAuthParams, 'successPath'>) {
});
return;
}
+ if (err.tag === 'NoGmailGrant') {
+ toast.failure(
+ 'Gmail access was not granted. Please allow all requested permissions and try again.'
+ );
+ navigateToEmailSettings();
+ return;
+ }
toast.failure('Failed to add inbox');
- navigateToSuccess();
+ navigateToEmailSettings();
}
);
};
diff --git a/apps/web/src/lib/core/email-link/index.ts b/apps/web/src/lib/core/email-link/index.ts
index 07b64bc2d2..0ca45880e4 100644
--- a/apps/web/src/lib/core/email-link/index.ts
+++ b/apps/web/src/lib/core/email-link/index.ts
@@ -195,6 +195,21 @@ function isPaymentRequired(errors: ReadonlyArray<{ code: string }>): boolean {
return errors.some((error) => error.code === 'PAYMENT_REQUIRED');
}
+/**
+ * The backend answers `POST /link/gmail` with 429 when the user has too many
+ * incomplete link attempts in flight (each abandoned OAuth leaves a pending row
+ * that expires after 24h). The auth client maps that to `TOO_MANY_PENDING_LINKS`
+ * so the flow can explain the wait instead of a dead-end generic failure.
+ */
+function isTooManyPendingLinks(
+ errors: ReadonlyArray<{ code: string }>
+): boolean {
+ return errors.some((error) => error.code === 'TOO_MANY_PENDING_LINKS');
+}
+
+const TOO_MANY_PENDING_LINKS_MESSAGE =
+ 'Too many inbox connections in progress.';
+
/**
* Starts the add-inbox flow: fetches the Gmail link authorization URL and
* navigates the browser to the OAuth consent page. The callback returns to
@@ -244,6 +259,10 @@ export function useAddInboxFlow() {
showPaywall(PaywallKey.MULTI_INBOX);
return;
}
+ if (isTooManyPendingLinks(result.error)) {
+ toast.failure(TOO_MANY_PENDING_LINKS_MESSAGE);
+ return;
+ }
toast.failure('Failed to start Gmail link flow');
return;
}
@@ -285,6 +304,8 @@ export function useAddInboxFlow() {
window.location.href = result.value.authorization_url;
} else if (isPaymentRequired(result.error)) {
showPaywall(PaywallKey.MULTI_INBOX);
+ } else if (isTooManyPendingLinks(result.error)) {
+ toast.failure(TOO_MANY_PENDING_LINKS_MESSAGE);
} else {
toast.failure('Failed to start Gmail link flow');
}
diff --git a/apps/web/src/lib/service-clients/service-auth/client.ts b/apps/web/src/lib/service-clients/service-auth/client.ts
index 341e9508d3..6b974b0891 100644
--- a/apps/web/src/lib/service-clients/service-auth/client.ts
+++ b/apps/web/src/lib/service-clients/service-auth/client.ts
@@ -549,18 +549,30 @@ export const authServiceClient = {
? `${authHost}/link/gmail?original_url=${encodeURIComponent(originalUrl)}`
: `${authHost}/link/gmail`;
return (
- await fetchWithAuth<InitGmailLinkResponse, 'PAYMENT_REQUIRED'>(url, {
+ await fetchWithAuth<
+ InitGmailLinkResponse,
+ 'PAYMENT_REQUIRED' | 'TOO_MANY_PENDING_LINKS'
+ >(url, {
method: 'POST',
// The backend returns 402 when the user isn't entitled to additional
- // inboxes; surface it as a distinct code so the add-inbox flow can open
- // the paywall instead of showing a generic failure.
- errorResponseHandler: async (response) =>
- response.status === 402
- ? { code: 'PAYMENT_REQUIRED', message: 'Payment required' }
- : {
- code: 'HTTP_ERROR',
- message: `HTTP error! status: ${response.status}`,
- },
+ // inboxes, and 429 when they have too many incomplete link attempts in
+ // flight. Surface each as a distinct code so the add-inbox flow can open
+ // the paywall or explain the wait instead of a generic failure.
+ errorResponseHandler: async (response) => {
+ if (response.status === 402) {
+ return { code: 'PAYMENT_REQUIRED', message: 'Payment required' };
+ }
+ if (response.status === 429) {
+ return {
+ code: 'TOO_MANY_PENDING_LINKS',
+ message: 'Too many pending inbox connections',
+ };
+ }
+ return {
+ code: 'HTTP_ERROR',
+ message: `HTTP error! status: ${response.status}`,
+ };
+ },
})
).map((result) => result);
},
diff --git a/crates/email_db_client/src/backfill/job/get.rs b/crates/email_db_client/src/backfill/job/get.rs
index 26e58ec612..90c1ec421b 100644
--- a/crates/email_db_client/src/backfill/job/get.rs
+++ b/crates/email_db_client/src/backfill/job/get.rs
@@ -126,44 +126,6 @@ pub async fn get_latest_backfill_job_by_link_id(
Ok(record.map(Into::into))
}
-/// Retrieves all backfill jobs created in the last 24 hours for a given macro ID
-#[tracing::instrument(skip(pool), err)]
-pub async fn get_recent_jobs_by_fusionauth_user_id(
- pool: &PgPool,
- fusionauth_user_id: &str,
-) -> anyhow::Result<Vec<service::backfill::BackfillJob>> {
- // Jobs created in the last 24h for this user, excluding terminal (Cancelled/Failed)
- // statuses, used for the connect rate limit.
- let records = sqlx::query_as!(
- db::backfill::BackfillJob,
- r#"
- SELECT
- id,
- link_id,
- fusionauth_user_id,
- threads_requested_limit,
- total_threads,
- threads_retrieved_count,
- status as "status: db::backfill::BackfillJobStatus",
- created_at,
- updated_at
- FROM email_backfill_jobs
- WHERE fusionauth_user_id = $1
- AND created_at > NOW() - INTERVAL '24 hours'
- AND status NOT IN ('Cancelled', 'Failed')
- ORDER BY created_at DESC
- "#,
- fusionauth_user_id
- )
- .fetch_all(pool)
- .await?;
-
- // Convert all database records to service models
- let jobs = records.into_iter().map(Into::into).collect();
-
- Ok(jobs)
-}
-
/// Retrieves a user's most recent backfill jobs, newest first. Spans all of the
/// user's links and all statuses; `link_id` is intentionally not used so jobs
/// survive a link being deleted and recreated. Capped at 100 so frequent
diff --git a/crates/email_db_client/src/backfill/job/get/test.rs b/crates/email_db_client/src/backfill/job/get/test.rs
index 17cdca6a19..f2af3d0889 100644
--- a/crates/email_db_client/src/backfill/job/get/test.rs
+++ b/crates/email_db_client/src/backfill/job/get/test.rs
@@ -1,7 +1,6 @@
-use super::{get_all_jobs_by_fusionauth_user_id, get_recent_jobs_by_fusionauth_user_id};
+use super::get_all_jobs_by_fusionauth_user_id;
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use models_email::email::db::backfill::BackfillJobStatus as DbStatus;
-use models_email::email::service::backfill::BackfillJobStatus;
use sqlx::types::Uuid;
use sqlx::{Pool, Postgres};
@@ -19,37 +18,6 @@ async fn insert_job(pool: &Pool<Postgres>, fusion: &str, status: DbStatus, hours
.unwrap();
}
-#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
-async fn recent_jobs_excludes_terminal_and_old_and_other_users(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- const FUSION: &str = "fusion-rate-limit-user";
- const OTHER: &str = "fusion-other-user";
-
- // Recent, non-terminal → counted.
- insert_job(&pool, FUSION, DbStatus::Init, 1).await;
- insert_job(&pool, FUSION, DbStatus::InProgress, 2).await;
- insert_job(&pool, FUSION, DbStatus::Complete, 3).await;
- // Recent but terminal → excluded so connect/disconnect churn does not count.
- insert_job(&pool, FUSION, DbStatus::Cancelled, 1).await;
- insert_job(&pool, FUSION, DbStatus::Failed, 2).await;
- // Older than the 24h window → excluded.
- insert_job(&pool, FUSION, DbStatus::Init, 25).await;
- // Different user → excluded.
- insert_job(&pool, OTHER, DbStatus::Init, 1).await;
-
- let jobs = get_recent_jobs_by_fusionauth_user_id(&pool, FUSION).await?;
-
- assert_eq!(jobs.len(), 3);
- assert!(jobs.iter().all(|j| j.fusionauth_user_id == FUSION));
- assert!(jobs.iter().all(|j| !matches!(
- j.status,
- BackfillJobStatus::Cancelled | BackfillJobStatus::Failed
- )));
-
- Ok(())
-}
-
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn all_jobs_returns_users_jobs_any_status_newest_first(
pool: Pool<Postgres>,
diff --git a/services/email_service/src/api/email/init.rs b/services/email_service/src/api/email/init.rs
index 7983325de9..645bcb1e01 100644
--- a/services/email_service/src/api/email/init.rs
+++ b/services/email_service/src/api/email/init.rs
@@ -40,9 +40,6 @@ pub enum InitError {
#[error("No Gmail grant available to provision from")]
NoGmailGrant,
- #[error("Job limit exceeded")]
- TooManyJobs,
-
#[error("Failed to enqueue backfill message")]
EnqueueError,
@@ -89,7 +86,6 @@ impl InitError {
| InitError::NoGmailGrant
| InitError::BadRequest(_)
| InitError::Parse(_) => StatusCode::BAD_REQUEST,
- InitError::TooManyJobs => StatusCode::TOO_MANY_REQUESTS,
InitError::EnqueueError | InitError::DatabaseError(_) | InitError::GmailError(_) => {
StatusCode::INTERNAL_SERVER_ERROR
}
@@ -201,20 +197,48 @@ pub async fn handler(
// Init runs on every authentication, so its expected no-op outcomes (400s)
// must not error-log. The span skips the auto err event and the result is
// classified here, inside the span, where user fields still attach.
+ let link_id = query.link_id;
+ let db = ctx.db.clone();
let result = init_user(ctx, query, authorization).await;
if let Err(e) = &result {
let status = e.status_code();
if status.is_server_error() {
tracing::error!(error = ?e, "init failed");
- } else if status == StatusCode::TOO_MANY_REQUESTS {
- tracing::warn!(error = %e, "init rate limited");
} else {
tracing::debug!(error = %e, "init declined");
}
+ cleanup_in_progress_link_on_failure(&db, link_id, e).await;
}
result
}
+/// Whether a failed `/email/init` should delete its `in_progress_user_link` row. True for
+/// every terminal failure except `SharedInboxConflict`, which is held open on purpose so the
+/// `force_share` retry can reuse the same `link_id`.
+fn should_clean_up_in_progress_link(error: &InitError) -> bool {
+ !matches!(error, InitError::SharedInboxConflict { .. })
+}
+
+/// A failed `/email/init` otherwise leaves the `in_progress_user_link` row behind — only the
+/// success paths delete it — where it counts toward the `/link/gmail` start cap and can lock
+/// the user out of further attempts.
+async fn cleanup_in_progress_link_on_failure(
+ db: &sqlx::Pool<sqlx::Postgres>,
+ link_id: Option<Uuid>,
+ error: &InitError,
+) {
+ if let Some(link_id) = link_id
+ && should_clean_up_in_progress_link(error)
+ {
+ macro_db_client::in_progress_user_link::delete_in_progress_user_link(db, &link_id)
+ .await
+ .inspect_err(|del_err| {
+ tracing::warn!(error = ?del_err, ?link_id, "Failed to clean up in_progress_user_link after failed init");
+ })
+ .ok();
+ }
+}
+
async fn init_user(
ctx: ApiContext,
Query(InitParams {
@@ -339,8 +363,6 @@ async fn init_user(
.context("child macro user disappeared before self-link bootstrap")?
.to_string();
- enforce_backfill_rate_limit(&ctx.db, &child_fusion_id, &linked_email).await?;
-
let gmail_token =
fetch_gmail_token_for_email(&ctx, &child_fusion_id, &linked_email).await?;
let watch_response = register_watch_recovering(&ctx.gmail_client, &gmail_token).await?;
@@ -517,9 +539,6 @@ async fn init_user(
.into_response());
}
- enforce_backfill_rate_limit(&ctx.db, &user_context.fusion_user_id, &linked_email)
- .await?;
-
let gmail_token =
fetch_gmail_token_for_email(&ctx, &user_context.fusion_user_id, &linked_email)
.await?;
@@ -570,8 +589,6 @@ async fn init_user(
let email = extract_email_with_response(&user_context.user_id)
.map_err(|_| InitError::BadRequest("Failed to extract email".to_string()))?;
- enforce_backfill_rate_limit(&ctx.db, &user_context.fusion_user_id, &email).await?;
-
let gmail_token =
fetch_gmail_token_for_email(&ctx, &user_context.fusion_user_id, &email).await?;
@@ -720,28 +737,6 @@ async fn init_user(
.into_response())
}
-/// Rejects a connect when this Gmail identity already has 10+ recent backfill jobs in
-/// the last 24h (`@macro.com` is exempt). Enforced before the link and Gmail watch are
-/// created so a rejected connect never has to tear down a half-provisioned inbox.
-async fn enforce_backfill_rate_limit(
- db: &sqlx::PgPool,
- fusion_user_id: &str,
- email_address: &str,
-) -> Result<(), InitError> {
- let recent_jobs = email_db_client::backfill::job::get::get_recent_jobs_by_fusionauth_user_id(
- db,
- fusion_user_id,
- )
- .await
- .context("Failed to fetch recent backfill jobs")?;
-
- if recent_jobs.len() >= 10 && !email_address.ends_with("@macro.com") {
- return Err(InitError::TooManyJobs);
- }
-
- Ok(())
-}
-
/// Registers a Gmail watch, recovering from the one-watch-per-mailbox limit. A watch
/// left over from a prior connect (e.g. a disconnect that could not reach Gmail to stop
/// it) makes a fresh registration fail with 400 "Only one user push notification client
diff --git a/services/email_service/src/api/email/init/test.rs b/services/email_service/src/api/email/init/test.rs
index 735162f4ee..ebcf2f1284 100644
--- a/services/email_service/src/api/email/init/test.rs
+++ b/services/email_service/src/api/email/init/test.rs
@@ -68,3 +68,31 @@ fn other_watch_failures_stay_gmail_errors() {
));
}
}
+
+#[test]
+fn only_shared_inbox_conflict_keeps_the_in_progress_link() {
+ // SharedInboxConflict is held open for the force_share retry, so its row must be kept;
+ // every other terminal failure must clean up the row so it stops counting toward the
+ // /link/gmail start cap. The delete itself is covered by macro_db_client's own tests.
+ assert!(
+ !should_clean_up_in_progress_link(&InitError::SharedInboxConflict {
+ email_address: "[email protected]".to_string(),
+ existing_owner_email: "[email protected]".to_string(),
+ existing_link_id: Uuid::new_v4(),
+ }),
+ "SharedInboxConflict must keep the row for the force_share retry"
+ );
+
+ for err in [
+ InitError::AlreadyInitialized,
+ InitError::NoGmailGrant,
+ InitError::EnqueueError,
+ InitError::BadRequest("bad".to_string()),
+ InitError::GmailError(GmailError::Unauthorized),
+ ] {
+ assert!(
+ should_clean_up_in_progress_link(&err),
+ "terminal failure {err:?} should clean up the in_progress row"
+ );
+ }
+}