Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): log expected Gmail rate-limit refusals below error (#7891)
teodc6455a0018a
diff --git a/crates/email_api_client/src/domain/service.rs b/crates/email_api_client/src/domain/service.rsindex a8d6908afc..4004ff78cf 100644--- a/crates/email_api_client/src/domain/service.rs+++ b/crates/email_api_client/src/domain/service.rs@@ -2,7 +2,9 @@use uuid::Uuid;-use super::models::{AccessToken, ApiOperationKind, EmailApiError, TokenError, TokenFreshness};+use super::models::{+ AccessToken, ApiOperationKind, EmailApiError, RateLimitOrigin, TokenError, TokenFreshness,+};use super::ports::{ProviderRateLimiter, ProviderTokenSource};mod attachments;@@ -60,6 +62,8 @@ where.map_err(map_token_error)}+ /// Admits an operation and acquires its access token, logging a failure in+ /// the calling operation's span.async fn prepare(&self,link_id: Uuid,@@ -69,12 +73,41 @@ where// paying the full token dance (SELECT + Redis + possible auth-service// refresh + health write) per refused attempt. The limiter does not// consume provider quota on denial.+ self.check_rate_limit(link_id, operation).await?;++ self.get_access_token(link_id, TokenFreshness::Cached)+ .await+ .inspect_err(log_operation_error)+ }++ /// Applies local quota policy, logging a refusal in the calling+ /// operation's span.+ async fn check_rate_limit(+ &self,+ link_id: Uuid,+ operation: ApiOperationKind,+ ) -> Result<(), EmailApiError> {self.rate_limiter.check_rate_limit(link_id, operation).await- .map_err(EmailApiError::from)?;+ .map_err(EmailApiError::from)+ .inspect_err(log_operation_error)+ }+}- self.get_access_token(link_id, TokenFreshness::Cached).await+/// Logs a failed operation in the current span.+///+/// Unlike `#[instrument(err)]`, whose level is fixed, this distinguishes a+/// local rate-limit refusal: expected backpressure where no provider request+/// was made and the caller decides whether to retry or drop. It is logged at+/// DEBUG; every other failure, including provider throttling, stays at ERROR.+fn log_operation_error(error: &EmailApiError) {+ match error {+ EmailApiError::RateLimited {+ origin: RateLimitOrigin::Local,+ ..+ } => tracing::debug!(error = %error),+ _ => tracing::error!(error = %error),}}diff --git a/crates/email_api_client/src/domain/service/attachments.rs b/crates/email_api_client/src/domain/service/attachments.rsindex 22a8fe7d1f..106e640267 100644--- a/crates/email_api_client/src/domain/service/attachments.rs+++ b/crates/email_api_client/src/domain/service/attachments.rs@@ -2,7 +2,7 @@ use uuid::Uuid;use super::super::models::EmailApiError;use super::super::ports::{MailboxAttachmentClient, ProviderRateLimiter, ProviderTokenSource};-use super::{ApiOperationKind, EmailApiClientServiceImpl};+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};impl<R, T, L> EmailApiClientServiceImpl<R, T, L>where@@ -11,7 +11,7 @@ whereL: ProviderRateLimiter,{/// Downloads an attachment from a provider message.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn get_attachment(&self,link_id: Uuid,@@ -25,6 +25,7 @@ whereself.repository.get_attachment(&access_token, provider_message_id, provider_attachment_id).await+ .inspect_err(log_operation_error)}}diff --git a/crates/email_api_client/src/domain/service/blocklist.rs b/crates/email_api_client/src/domain/service/blocklist.rsindex dbbdc690bd..4c2046c167 100644--- a/crates/email_api_client/src/domain/service/blocklist.rs+++ b/crates/email_api_client/src/domain/service/blocklist.rs@@ -2,7 +2,7 @@ use uuid::Uuid;use super::super::models::EmailApiError;use super::super::ports::{MailboxBlocklistClient, ProviderRateLimiter, ProviderTokenSource};-use super::{ApiOperationKind, EmailApiClientServiceImpl};+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};impl<R, T, L> EmailApiClientServiceImpl<R, T, L>where@@ -11,7 +11,7 @@ whereL: ProviderRateLimiter,{/// Ensures messages from an email address are sent to trash.- #[tracing::instrument(skip(self, email_address), err)]+ #[tracing::instrument(skip(self, email_address))]pub async fn block_sender(&self,link_id: Uuid,@@ -22,10 +22,11 @@ whereself.repository.block_sender(&access_token, email_address).await+ .inspect_err(log_operation_error)}/// Removes the blocked-sender rule for an email address, when present.- #[tracing::instrument(skip(self, email_address), err)]+ #[tracing::instrument(skip(self, email_address))]pub async fn unblock_sender(&self,link_id: Uuid,@@ -38,16 +39,20 @@ whereself.repository.unblock_sender(&access_token, email_address).await+ .inspect_err(log_operation_error)}/// Lists email addresses covered by blocked-sender rules.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn list_blocked_senders(&self, link_id: Uuid) -> Result<Vec<String>, EmailApiError> {let access_token = self.prepare(link_id, ApiOperationKind::ListBlockedSenders).await?;- self.repository.list_blocked_senders(&access_token).await+ self.repository+ .list_blocked_senders(&access_token)+ .await+ .inspect_err(log_operation_error)}}diff --git a/crates/email_api_client/src/domain/service/contacts.rs b/crates/email_api_client/src/domain/service/contacts.rsindex 458911d297..38e356281f 100644--- a/crates/email_api_client/src/domain/service/contacts.rs+++ b/crates/email_api_client/src/domain/service/contacts.rs@@ -3,7 +3,7 @@ use uuid::Uuid;use super::super::models::EmailApiError;use super::super::ports::{MailboxContactsClient, ProviderRateLimiter, ProviderTokenSource};-use super::{ApiOperationKind, EmailApiClientServiceImpl};+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};impl<R, T, L> EmailApiClientServiceImpl<R, T, L>where@@ -12,7 +12,7 @@ whereL: ProviderRateLimiter,{/// Fetches the linked mailbox owner's contact record.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn get_self_contact(&self, link_id: Uuid) -> Result<Contact, EmailApiError> {let access_token = self.prepare(link_id, ApiOperationKind::ListContacts)@@ -21,6 +21,7 @@ whereself.repository.get_self_contact(&access_token, link_id).await+ .inspect_err(log_operation_error)}/// Lists primary contacts, optionally continuing an incremental synchronization.diff --git a/crates/email_api_client/src/domain/service/labels.rs b/crates/email_api_client/src/domain/service/labels.rsindex b6689284b0..21d5d83f41 100644--- a/crates/email_api_client/src/domain/service/labels.rs+++ b/crates/email_api_client/src/domain/service/labels.rs@@ -3,7 +3,7 @@ use uuid::Uuid;use super::super::models::EmailApiError;use super::super::ports::{MailboxLabelClient, ProviderRateLimiter, ProviderTokenSource};-use super::{ApiOperationKind, EmailApiClientServiceImpl};+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};impl<R, T, L> EmailApiClientServiceImpl<R, T, L>where@@ -12,15 +12,18 @@ whereL: ProviderRateLimiter,{/// Lists all labels for a linked mailbox.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn list_labels(&self, link_id: Uuid) -> Result<Vec<Label>, EmailApiError> {let access_token = self.prepare(link_id, ApiOperationKind::ListLabels).await?;- self.repository.list_labels(&access_token, link_id).await+ self.repository+ .list_labels(&access_token, link_id)+ .await+ .inspect_err(log_operation_error)}/// Creates a user label.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn create_label(&self,link_id: Uuid,@@ -31,10 +34,11 @@ whereself.repository.create_label(&access_token, link_id, label_name).await+ .inspect_err(log_operation_error)}/// Deletes a provider label.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn delete_label(&self,link_id: Uuid,@@ -45,6 +49,7 @@ whereself.repository.delete_label(&access_token, provider_label_id).await+ .inspect_err(log_operation_error)}}diff --git a/crates/email_api_client/src/domain/service/messages.rs b/crates/email_api_client/src/domain/service/messages.rsindex 8141541525..bbe932efc3 100644--- a/crates/email_api_client/src/domain/service/messages.rs+++ b/crates/email_api_client/src/domain/service/messages.rs@@ -5,7 +5,7 @@ use super::super::models::{CalendarPart, EmailApiError, MessageWithCalendarPartsuse super::super::ports::{MailboxCalendarClient, MailboxMessageClient, ProviderRateLimiter, ProviderTokenSource,};-use super::{ApiOperationKind, EmailApiClientServiceImpl};+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};impl<R, T, L> EmailApiClientServiceImpl<R, T, L>where@@ -15,7 +15,7 @@ where{/// Fetches and normalizes one provider message, including any calendar/// invitation parts, in a single provider read.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn get_message(&self,link_id: Uuid,@@ -26,10 +26,11 @@ whereself.repository.get_message(&access_token, link_id, provider_message_id).await+ .inspect_err(log_operation_error)}/// Fetches the provider label identifiers attached to one message.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn get_message_label_ids(&self,link_id: Uuid,@@ -40,10 +41,11 @@ whereself.repository.get_message_label_ids(&access_token, provider_message_id).await+ .inspect_err(log_operation_error)}/// Lists message identifiers carrying all requested provider labels.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn list_messages(&self,link_id: Uuid,@@ -57,10 +59,11 @@ whereself.repository.list_messages(&access_token, limit, label_ids).await+ .inspect_err(log_operation_error)}/// Lists message identifiers belonging to one provider thread.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn get_message_ids_for_thread(&self,link_id: Uuid,@@ -71,10 +74,11 @@ whereself.repository.get_message_ids_for_thread(&access_token, provider_thread_id).await+ .inspect_err(log_operation_error)}/// Fetches and normalizes every message in one provider thread.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn get_thread(&self,link_id: Uuid,@@ -85,10 +89,11 @@ whereself.repository.get_thread(&access_token, link_id, provider_thread_id).await+ .inspect_err(log_operation_error)}/// Lists one page of provider threads.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn list_threads(&self,link_id: Uuid,@@ -101,10 +106,11 @@ whereself.repository.list_threads(&access_token, limit, next_page_token, label_ids).await+ .inspect_err(log_operation_error)}/// Applies provider label additions and removals to one message.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn modify_message_labels(&self,link_id: Uuid,@@ -124,6 +130,7 @@ wherelabels_to_remove,).await+ .inspect_err(log_operation_error)}}@@ -139,7 +146,7 @@ where/// This charges a full message read; ingest paths that already fetched the/// message should consume [`MessageWithCalendarParts::calendar_parts`]/// instead. This lookup exists for durable re-extraction jobs.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn get_calendar_parts(&self,link_id: Uuid,@@ -149,6 +156,7 @@ whereself.repository.get_calendar_parts(&access_token, provider_message_id).await+ .inspect_err(log_operation_error)}}diff --git a/crates/email_api_client/src/domain/service/send.rs b/crates/email_api_client/src/domain/service/send.rsindex b6c870c5dc..4791ebca17 100644--- a/crates/email_api_client/src/domain/service/send.rs+++ b/crates/email_api_client/src/domain/service/send.rs@@ -2,7 +2,7 @@ use uuid::Uuid;use super::super::models::{EmailApiError, SendRequest, SentIds};use super::super::ports::{MailboxSendClient, ProviderRateLimiter, ProviderTokenSource};-use super::{ApiOperationKind, EmailApiClientServiceImpl};+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};impl<R, T, L> EmailApiClientServiceImpl<R, T, L>where@@ -11,7 +11,7 @@ whereL: ProviderRateLimiter,{/// Sends a message through the linked mailbox.- #[tracing::instrument(skip(self, request), err)]+ #[tracing::instrument(skip(self, request))]pub async fn send_message(&self,link_id: Uuid,@@ -23,6 +23,7 @@ whereself.repository.send_message(&access_token, request, provider_thread_id).await+ .inspect_err(log_operation_error)}}diff --git a/crates/email_api_client/src/domain/service/subscription.rs b/crates/email_api_client/src/domain/service/subscription.rsindex e9338c1630..ec95ee3c78 100644--- a/crates/email_api_client/src/domain/service/subscription.rs+++ b/crates/email_api_client/src/domain/service/subscription.rs@@ -3,7 +3,7 @@ use uuid::Uuid;use super::super::models::{EmailApiError, ProviderSubscription, TokenFreshness};use super::super::ports::{MailboxSubscriptionClient, ProviderRateLimiter, ProviderTokenSource};-use super::{ApiOperationKind, EmailApiClientServiceImpl};+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};impl<R, T, L> EmailApiClientServiceImpl<R, T, L>where@@ -12,42 +12,50 @@ whereL: ProviderRateLimiter,{/// Registers or renews the linked mailbox's notification subscription.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn register_subscription(&self,link_id: Uuid,) -> Result<ProviderSubscription, EmailApiError> {let access_token = self.prepare(link_id, ApiOperationKind::Subscribe).await?;- self.repository.subscribe(&access_token).await+ self.repository+ .subscribe(&access_token)+ .await+ .inspect_err(log_operation_error)}/// Registers a subscription using a freshly fetched token.////// This is intended for mailbox initialization, where a cached access token/// could outlive a revoked provider grant.- #[tracing::instrument(skip(self, link), fields(link_id = %link.id), err)]+ #[tracing::instrument(skip(self, link), fields(link_id = %link.id))]pub async fn register_subscription_without_cache(&self,link: &Link,) -> Result<ProviderSubscription, EmailApiError> {- self.rate_limiter- .check_rate_limit(link.id, ApiOperationKind::Subscribe)- .await- .map_err(EmailApiError::from)?;+ self.check_rate_limit(link.id, ApiOperationKind::Subscribe)+ .await?;let access_token = self.token_source.get_access_token_for_link(link, TokenFreshness::Fresh).await- .map_err(super::map_token_error)?;+ .map_err(super::map_token_error)+ .inspect_err(log_operation_error)?;- self.repository.subscribe(&access_token).await+ self.repository+ .subscribe(&access_token)+ .await+ .inspect_err(log_operation_error)}/// Stops the linked mailbox's notification subscription.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn stop_subscription(&self, link_id: Uuid) -> Result<(), EmailApiError> {let access_token = self.prepare(link_id, ApiOperationKind::Unsubscribe).await?;- self.repository.unsubscribe(&access_token).await+ self.repository+ .unsubscribe(&access_token)+ .await+ .inspect_err(log_operation_error)}/// Stops the mailbox's subscription without token-health side effects.@@ -55,19 +63,21 @@ where/// This is intended for link teardown: a grant that died since the last/// probe must not set `needs_reauth` or fan out a reauthorization/// notification for an inbox that is being intentionally removed.- #[tracing::instrument(skip(self, link), fields(link_id = %link.id), err)]+ #[tracing::instrument(skip(self, link), fields(link_id = %link.id))]pub async fn stop_subscription_for_link(&self, link: &Link) -> Result<(), EmailApiError> {- self.rate_limiter- .check_rate_limit(link.id, ApiOperationKind::Unsubscribe)- .await- .map_err(EmailApiError::from)?;+ self.check_rate_limit(link.id, ApiOperationKind::Unsubscribe)+ .await?;let access_token = self.token_source.get_access_token_health_neutral(link, TokenFreshness::Cached).await- .map_err(super::map_token_error)?;+ .map_err(super::map_token_error)+ .inspect_err(log_operation_error)?;- self.repository.unsubscribe(&access_token).await+ self.repository+ .unsubscribe(&access_token)+ .await+ .inspect_err(log_operation_error)}}diff --git a/crates/email_api_client/src/domain/service/sync.rs b/crates/email_api_client/src/domain/service/sync.rsindex 92750574e6..48c3f86417 100644--- a/crates/email_api_client/src/domain/service/sync.rs+++ b/crates/email_api_client/src/domain/service/sync.rs@@ -2,7 +2,7 @@ use uuid::Uuid;use super::super::models::{ChangeBatch, EmailApiError, SyncCursor};use super::super::ports::{MailboxSyncClient, ProviderRateLimiter, ProviderTokenSource};-use super::{ApiOperationKind, EmailApiClientServiceImpl};+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};impl<R, T, L> EmailApiClientServiceImpl<R, T, L>where@@ -11,15 +11,18 @@ whereL: ProviderRateLimiter,{/// Returns the number of threads reported by a linked mailbox.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn get_thread_count(&self, link_id: Uuid) -> Result<u64, EmailApiError> {let access_token = self.prepare(link_id, ApiOperationKind::GetProfile).await?;- self.repository.get_thread_count(&access_token).await+ self.repository+ .get_thread_count(&access_token)+ .await+ .inspect_err(log_operation_error)}/// Lists mailbox changes following `cursor`.- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(skip(self))]pub async fn list_changes(&self,link_id: Uuid,@@ -27,7 +30,10 @@ where) -> Result<ChangeBatch, EmailApiError> {let access_token = self.prepare(link_id, ApiOperationKind::ListChanges).await?;- self.repository.list_changes(&access_token, cursor).await+ self.repository+ .list_changes(&access_token, cursor)+ .await+ .inspect_err(log_operation_error)}}diff --git a/services/email_service/src/pubsub/backfill/error_handlers.rs b/services/email_service/src/pubsub/backfill/error_handlers.rsindex 0fe0d01fe6..e6d40fb24d 100644--- a/services/email_service/src/pubsub/backfill/error_handlers.rs+++ b/services/email_service/src/pubsub/backfill/error_handlers.rs@@ -7,7 +7,7 @@ use models_email::api::refresh::{BackfillStatus, RefreshEmailEvent};use models_email::email::service::backfill::{BackfillMessagePayload, BackfillOperation, BackfillPubsubMessage, JobScopedPayload,};-use models_email::email::service::pubsub::DetailedError;+use models_email::email::service::pubsub::{DetailedError, FailureReason};use sqs_worker::cleanup_message;use uuid::Uuid;@@ -119,6 +119,13 @@ pub async fn handle_retryable_error(let error_chain = format!("{:#}", _e.source);tracing::Span::current().record("error", &error_chain);+ // Rate limits are expected backpressure, redelivered after the visibility+ // timeout; the email API client logs provider throttling at ERROR.+ if _e.reason == FailureReason::GmailApiRateLimited {+ tracing::debug!("Rate-limited backfill operation will be redelivered");+ return Ok(());+ }+match &data.backfill_operation {BackfillOperation::Init(_) => {tracing::warn!("Retryable error in Init")diff --git a/services/email_service/src/pubsub/gmail_ops/error_handlers.rs b/services/email_service/src/pubsub/gmail_ops/error_handlers.rsindex d45fbc00fe..a89c56cc98 100644--- a/services/email_service/src/pubsub/gmail_ops/error_handlers.rs+++ b/services/email_service/src/pubsub/gmail_ops/error_handlers.rs@@ -1,6 +1,6 @@use crate::pubsub::gmail_ops::worker::GmailOpsContext;use models_email::gmail::gmail_ops::GmailOpsPubsubMessage;-use models_email::service::pubsub::{DetailedError, ProcessingError};+use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingError};use sqs_worker::cleanup_message;/// Handles non-retryable errors by cleaning up the SQS message.@@ -11,7 +11,13 @@ pub async fn handle_non_retryable_error(data: &GmailOpsPubsubMessage,e: &DetailedError,) -> anyhow::Result<()> {- tracing::error!(error = ?e, payload = ?data.operation, "Non-retryable error processing gmail ops message. The message will be deleted.");+ // A rate-limited operation is only non-retryable here once it has been+ // re-enqueued to the retry queue, so deleting it loses nothing.+ if e.reason == FailureReason::GmailApiRateLimited {+ tracing::debug!(error = ?e, payload = ?data.operation, "Rate-limited gmail ops message deferred. The message will be deleted.");+ } else {+ tracing::error!(error = ?e, payload = ?data.operation, "Non-retryable error processing gmail ops message. The message will be deleted.");+ }cleanup_message(&ctx.sqs_worker, message).await?;Ok(())diff --git a/services/email_service/src/pubsub/inbox_sync/error_handlers.rs b/services/email_service/src/pubsub/inbox_sync/error_handlers.rsindex 5d5072bc6b..39bc31ce3d 100644--- a/services/email_service/src/pubsub/inbox_sync/error_handlers.rs+++ b/services/email_service/src/pubsub/inbox_sync/error_handlers.rs@@ -1,6 +1,6 @@use crate::pubsub::context::PubSubContext;use models_email::gmail::inbox_sync::InboxSyncPubsubMessage;-use models_email::service::pubsub::{DetailedError, ProcessingError};+use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingError};use sqs_worker::cleanup_message;/// Handles non-retryable errors by updating the appropriate status in the database and cleaning up the SQS message@@ -11,7 +11,14 @@ pub async fn handle_non_retryable_error(data: &InboxSyncPubsubMessage,e: &DetailedError,) -> anyhow::Result<()> {- tracing::error!(error = %e, payload = format!("{:?}", data.operation), "Non-retryable error processing inbox sync message. The message will be deleted.");+ // A rate-limited operation is only non-retryable here once it has been+ // re-enqueued to the retry queue, or when it is a notification that a+ // later one supersedes, so deleting it loses nothing.+ if e.reason == FailureReason::GmailApiRateLimited {+ tracing::debug!(error = %e, payload = format!("{:?}", data.operation), "Rate-limited inbox sync message deferred. The message will be deleted.");+ } else {+ tracing::error!(error = %e, payload = format!("{:?}", data.operation), "Non-retryable error processing inbox sync message. The message will be deleted.");+ }cleanup_message(&ctx.sqs_worker, message).await?;Ok(())