Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): log expected Gmail rate-limit refusals below error (#7891)

teodc6455a0018a
diff --git a/crates/email_api_client/src/domain/service.rs b/crates/email_api_client/src/domain/service.rs
index a8d6908afc..4004ff78cf 100644
--- a/crates/email_api_client/src/domain/service.rs
+++ b/crates/email_api_client/src/domain/service.rs
@@ -2,7 +2,9 @@
use uuid::Uuid;
-use super::models::{AccessToken, ApiOperationKind, EmailApiError, TokenError, TokenFreshness};
+use super::models::{
+ AccessToken, ApiOperationKind, EmailApiError, RateLimitOrigin, TokenError, TokenFreshness,
+};
use super::ports::{ProviderRateLimiter, ProviderTokenSource};
mod attachments;
@@ -60,6 +62,8 @@ where
.map_err(map_token_error)
}
+ /// Admits an operation and acquires its access token, logging a failure in
+ /// the calling operation's span.
async fn prepare(
&self,
link_id: Uuid,
@@ -69,12 +73,41 @@ where
// paying the full token dance (SELECT + Redis + possible auth-service
// refresh + health write) per refused attempt. The limiter does not
// consume provider quota on denial.
+ self.check_rate_limit(link_id, operation).await?;
+
+ self.get_access_token(link_id, TokenFreshness::Cached)
+ .await
+ .inspect_err(log_operation_error)
+ }
+
+ /// Applies local quota policy, logging a refusal in the calling
+ /// operation's span.
+ async fn check_rate_limit(
+ &self,
+ link_id: Uuid,
+ operation: ApiOperationKind,
+ ) -> Result<(), EmailApiError> {
self.rate_limiter
.check_rate_limit(link_id, operation)
.await
- .map_err(EmailApiError::from)?;
+ .map_err(EmailApiError::from)
+ .inspect_err(log_operation_error)
+ }
+}
- self.get_access_token(link_id, TokenFreshness::Cached).await
+/// Logs a failed operation in the current span.
+///
+/// Unlike `#[instrument(err)]`, whose level is fixed, this distinguishes a
+/// local rate-limit refusal: expected backpressure where no provider request
+/// was made and the caller decides whether to retry or drop. It is logged at
+/// DEBUG; every other failure, including provider throttling, stays at ERROR.
+fn log_operation_error(error: &EmailApiError) {
+ match error {
+ EmailApiError::RateLimited {
+ origin: RateLimitOrigin::Local,
+ ..
+ } => tracing::debug!(error = %error),
+ _ => tracing::error!(error = %error),
}
}
diff --git a/crates/email_api_client/src/domain/service/attachments.rs b/crates/email_api_client/src/domain/service/attachments.rs
index 22a8fe7d1f..106e640267 100644
--- a/crates/email_api_client/src/domain/service/attachments.rs
+++ b/crates/email_api_client/src/domain/service/attachments.rs
@@ -2,7 +2,7 @@ use uuid::Uuid;
use super::super::models::EmailApiError;
use super::super::ports::{MailboxAttachmentClient, ProviderRateLimiter, ProviderTokenSource};
-use super::{ApiOperationKind, EmailApiClientServiceImpl};
+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
@@ -11,7 +11,7 @@ where
L: ProviderRateLimiter,
{
/// Downloads an attachment from a provider message.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn get_attachment(
&self,
link_id: Uuid,
@@ -25,6 +25,7 @@ where
self.repository
.get_attachment(&access_token, provider_message_id, provider_attachment_id)
.await
+ .inspect_err(log_operation_error)
}
}
diff --git a/crates/email_api_client/src/domain/service/blocklist.rs b/crates/email_api_client/src/domain/service/blocklist.rs
index dbbdc690bd..4c2046c167 100644
--- a/crates/email_api_client/src/domain/service/blocklist.rs
+++ b/crates/email_api_client/src/domain/service/blocklist.rs
@@ -2,7 +2,7 @@ use uuid::Uuid;
use super::super::models::EmailApiError;
use super::super::ports::{MailboxBlocklistClient, ProviderRateLimiter, ProviderTokenSource};
-use super::{ApiOperationKind, EmailApiClientServiceImpl};
+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
@@ -11,7 +11,7 @@ where
L: ProviderRateLimiter,
{
/// Ensures messages from an email address are sent to trash.
- #[tracing::instrument(skip(self, email_address), err)]
+ #[tracing::instrument(skip(self, email_address))]
pub async fn block_sender(
&self,
link_id: Uuid,
@@ -22,10 +22,11 @@ where
self.repository
.block_sender(&access_token, email_address)
.await
+ .inspect_err(log_operation_error)
}
/// Removes the blocked-sender rule for an email address, when present.
- #[tracing::instrument(skip(self, email_address), err)]
+ #[tracing::instrument(skip(self, email_address))]
pub async fn unblock_sender(
&self,
link_id: Uuid,
@@ -38,16 +39,20 @@ where
self.repository
.unblock_sender(&access_token, email_address)
.await
+ .inspect_err(log_operation_error)
}
/// Lists email addresses covered by blocked-sender rules.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn list_blocked_senders(&self, link_id: Uuid) -> Result<Vec<String>, EmailApiError> {
let access_token = self
.prepare(link_id, ApiOperationKind::ListBlockedSenders)
.await?;
- self.repository.list_blocked_senders(&access_token).await
+ self.repository
+ .list_blocked_senders(&access_token)
+ .await
+ .inspect_err(log_operation_error)
}
}
diff --git a/crates/email_api_client/src/domain/service/contacts.rs b/crates/email_api_client/src/domain/service/contacts.rs
index 458911d297..38e356281f 100644
--- a/crates/email_api_client/src/domain/service/contacts.rs
+++ b/crates/email_api_client/src/domain/service/contacts.rs
@@ -3,7 +3,7 @@ use uuid::Uuid;
use super::super::models::EmailApiError;
use super::super::ports::{MailboxContactsClient, ProviderRateLimiter, ProviderTokenSource};
-use super::{ApiOperationKind, EmailApiClientServiceImpl};
+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
@@ -12,7 +12,7 @@ where
L: ProviderRateLimiter,
{
/// Fetches the linked mailbox owner's contact record.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn get_self_contact(&self, link_id: Uuid) -> Result<Contact, EmailApiError> {
let access_token = self
.prepare(link_id, ApiOperationKind::ListContacts)
@@ -21,6 +21,7 @@ where
self.repository
.get_self_contact(&access_token, link_id)
.await
+ .inspect_err(log_operation_error)
}
/// Lists primary contacts, optionally continuing an incremental synchronization.
diff --git a/crates/email_api_client/src/domain/service/labels.rs b/crates/email_api_client/src/domain/service/labels.rs
index b6689284b0..21d5d83f41 100644
--- a/crates/email_api_client/src/domain/service/labels.rs
+++ b/crates/email_api_client/src/domain/service/labels.rs
@@ -3,7 +3,7 @@ use uuid::Uuid;
use super::super::models::EmailApiError;
use super::super::ports::{MailboxLabelClient, ProviderRateLimiter, ProviderTokenSource};
-use super::{ApiOperationKind, EmailApiClientServiceImpl};
+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
@@ -12,15 +12,18 @@ where
L: ProviderRateLimiter,
{
/// Lists all labels for a linked mailbox.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn list_labels(&self, link_id: Uuid) -> Result<Vec<Label>, EmailApiError> {
let access_token = self.prepare(link_id, ApiOperationKind::ListLabels).await?;
- self.repository.list_labels(&access_token, link_id).await
+ self.repository
+ .list_labels(&access_token, link_id)
+ .await
+ .inspect_err(log_operation_error)
}
/// Creates a user label.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn create_label(
&self,
link_id: Uuid,
@@ -31,10 +34,11 @@ where
self.repository
.create_label(&access_token, link_id, label_name)
.await
+ .inspect_err(log_operation_error)
}
/// Deletes a provider label.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn delete_label(
&self,
link_id: Uuid,
@@ -45,6 +49,7 @@ where
self.repository
.delete_label(&access_token, provider_label_id)
.await
+ .inspect_err(log_operation_error)
}
}
diff --git a/crates/email_api_client/src/domain/service/messages.rs b/crates/email_api_client/src/domain/service/messages.rs
index 8141541525..bbe932efc3 100644
--- a/crates/email_api_client/src/domain/service/messages.rs
+++ b/crates/email_api_client/src/domain/service/messages.rs
@@ -5,7 +5,7 @@ use super::super::models::{CalendarPart, EmailApiError, MessageWithCalendarParts
use super::super::ports::{
MailboxCalendarClient, MailboxMessageClient, ProviderRateLimiter, ProviderTokenSource,
};
-use super::{ApiOperationKind, EmailApiClientServiceImpl};
+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
@@ -15,7 +15,7 @@ where
{
/// Fetches and normalizes one provider message, including any calendar
/// invitation parts, in a single provider read.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn get_message(
&self,
link_id: Uuid,
@@ -26,10 +26,11 @@ where
self.repository
.get_message(&access_token, link_id, provider_message_id)
.await
+ .inspect_err(log_operation_error)
}
/// Fetches the provider label identifiers attached to one message.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn get_message_label_ids(
&self,
link_id: Uuid,
@@ -40,10 +41,11 @@ where
self.repository
.get_message_label_ids(&access_token, provider_message_id)
.await
+ .inspect_err(log_operation_error)
}
/// Lists message identifiers carrying all requested provider labels.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn list_messages(
&self,
link_id: Uuid,
@@ -57,10 +59,11 @@ where
self.repository
.list_messages(&access_token, limit, label_ids)
.await
+ .inspect_err(log_operation_error)
}
/// Lists message identifiers belonging to one provider thread.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn get_message_ids_for_thread(
&self,
link_id: Uuid,
@@ -71,10 +74,11 @@ where
self.repository
.get_message_ids_for_thread(&access_token, provider_thread_id)
.await
+ .inspect_err(log_operation_error)
}
/// Fetches and normalizes every message in one provider thread.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn get_thread(
&self,
link_id: Uuid,
@@ -85,10 +89,11 @@ where
self.repository
.get_thread(&access_token, link_id, provider_thread_id)
.await
+ .inspect_err(log_operation_error)
}
/// Lists one page of provider threads.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn list_threads(
&self,
link_id: Uuid,
@@ -101,10 +106,11 @@ where
self.repository
.list_threads(&access_token, limit, next_page_token, label_ids)
.await
+ .inspect_err(log_operation_error)
}
/// Applies provider label additions and removals to one message.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn modify_message_labels(
&self,
link_id: Uuid,
@@ -124,6 +130,7 @@ where
labels_to_remove,
)
.await
+ .inspect_err(log_operation_error)
}
}
@@ -139,7 +146,7 @@ where
/// This charges a full message read; ingest paths that already fetched the
/// message should consume [`MessageWithCalendarParts::calendar_parts`]
/// instead. This lookup exists for durable re-extraction jobs.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn get_calendar_parts(
&self,
link_id: Uuid,
@@ -149,6 +156,7 @@ where
self.repository
.get_calendar_parts(&access_token, provider_message_id)
.await
+ .inspect_err(log_operation_error)
}
}
diff --git a/crates/email_api_client/src/domain/service/send.rs b/crates/email_api_client/src/domain/service/send.rs
index b6c870c5dc..4791ebca17 100644
--- a/crates/email_api_client/src/domain/service/send.rs
+++ b/crates/email_api_client/src/domain/service/send.rs
@@ -2,7 +2,7 @@ use uuid::Uuid;
use super::super::models::{EmailApiError, SendRequest, SentIds};
use super::super::ports::{MailboxSendClient, ProviderRateLimiter, ProviderTokenSource};
-use super::{ApiOperationKind, EmailApiClientServiceImpl};
+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
@@ -11,7 +11,7 @@ where
L: ProviderRateLimiter,
{
/// Sends a message through the linked mailbox.
- #[tracing::instrument(skip(self, request), err)]
+ #[tracing::instrument(skip(self, request))]
pub async fn send_message(
&self,
link_id: Uuid,
@@ -23,6 +23,7 @@ where
self.repository
.send_message(&access_token, request, provider_thread_id)
.await
+ .inspect_err(log_operation_error)
}
}
diff --git a/crates/email_api_client/src/domain/service/subscription.rs b/crates/email_api_client/src/domain/service/subscription.rs
index e9338c1630..ec95ee3c78 100644
--- a/crates/email_api_client/src/domain/service/subscription.rs
+++ b/crates/email_api_client/src/domain/service/subscription.rs
@@ -3,7 +3,7 @@ use uuid::Uuid;
use super::super::models::{EmailApiError, ProviderSubscription, TokenFreshness};
use super::super::ports::{MailboxSubscriptionClient, ProviderRateLimiter, ProviderTokenSource};
-use super::{ApiOperationKind, EmailApiClientServiceImpl};
+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
@@ -12,42 +12,50 @@ where
L: ProviderRateLimiter,
{
/// Registers or renews the linked mailbox's notification subscription.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn register_subscription(
&self,
link_id: Uuid,
) -> Result<ProviderSubscription, EmailApiError> {
let access_token = self.prepare(link_id, ApiOperationKind::Subscribe).await?;
- self.repository.subscribe(&access_token).await
+ self.repository
+ .subscribe(&access_token)
+ .await
+ .inspect_err(log_operation_error)
}
/// Registers a subscription using a freshly fetched token.
///
/// This is intended for mailbox initialization, where a cached access token
/// could outlive a revoked provider grant.
- #[tracing::instrument(skip(self, link), fields(link_id = %link.id), err)]
+ #[tracing::instrument(skip(self, link), fields(link_id = %link.id))]
pub async fn register_subscription_without_cache(
&self,
link: &Link,
) -> Result<ProviderSubscription, EmailApiError> {
- self.rate_limiter
- .check_rate_limit(link.id, ApiOperationKind::Subscribe)
- .await
- .map_err(EmailApiError::from)?;
+ self.check_rate_limit(link.id, ApiOperationKind::Subscribe)
+ .await?;
let access_token = self
.token_source
.get_access_token_for_link(link, TokenFreshness::Fresh)
.await
- .map_err(super::map_token_error)?;
+ .map_err(super::map_token_error)
+ .inspect_err(log_operation_error)?;
- self.repository.subscribe(&access_token).await
+ self.repository
+ .subscribe(&access_token)
+ .await
+ .inspect_err(log_operation_error)
}
/// Stops the linked mailbox's notification subscription.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn stop_subscription(&self, link_id: Uuid) -> Result<(), EmailApiError> {
let access_token = self.prepare(link_id, ApiOperationKind::Unsubscribe).await?;
- self.repository.unsubscribe(&access_token).await
+ self.repository
+ .unsubscribe(&access_token)
+ .await
+ .inspect_err(log_operation_error)
}
/// Stops the mailbox's subscription without token-health side effects.
@@ -55,19 +63,21 @@ where
/// This is intended for link teardown: a grant that died since the last
/// probe must not set `needs_reauth` or fan out a reauthorization
/// notification for an inbox that is being intentionally removed.
- #[tracing::instrument(skip(self, link), fields(link_id = %link.id), err)]
+ #[tracing::instrument(skip(self, link), fields(link_id = %link.id))]
pub async fn stop_subscription_for_link(&self, link: &Link) -> Result<(), EmailApiError> {
- self.rate_limiter
- .check_rate_limit(link.id, ApiOperationKind::Unsubscribe)
- .await
- .map_err(EmailApiError::from)?;
+ self.check_rate_limit(link.id, ApiOperationKind::Unsubscribe)
+ .await?;
let access_token = self
.token_source
.get_access_token_health_neutral(link, TokenFreshness::Cached)
.await
- .map_err(super::map_token_error)?;
+ .map_err(super::map_token_error)
+ .inspect_err(log_operation_error)?;
- self.repository.unsubscribe(&access_token).await
+ self.repository
+ .unsubscribe(&access_token)
+ .await
+ .inspect_err(log_operation_error)
}
}
diff --git a/crates/email_api_client/src/domain/service/sync.rs b/crates/email_api_client/src/domain/service/sync.rs
index 92750574e6..48c3f86417 100644
--- a/crates/email_api_client/src/domain/service/sync.rs
+++ b/crates/email_api_client/src/domain/service/sync.rs
@@ -2,7 +2,7 @@ use uuid::Uuid;
use super::super::models::{ChangeBatch, EmailApiError, SyncCursor};
use super::super::ports::{MailboxSyncClient, ProviderRateLimiter, ProviderTokenSource};
-use super::{ApiOperationKind, EmailApiClientServiceImpl};
+use super::{ApiOperationKind, EmailApiClientServiceImpl, log_operation_error};
impl<R, T, L> EmailApiClientServiceImpl<R, T, L>
where
@@ -11,15 +11,18 @@ where
L: ProviderRateLimiter,
{
/// Returns the number of threads reported by a linked mailbox.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn get_thread_count(&self, link_id: Uuid) -> Result<u64, EmailApiError> {
let access_token = self.prepare(link_id, ApiOperationKind::GetProfile).await?;
- self.repository.get_thread_count(&access_token).await
+ self.repository
+ .get_thread_count(&access_token)
+ .await
+ .inspect_err(log_operation_error)
}
/// Lists mailbox changes following `cursor`.
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(skip(self))]
pub async fn list_changes(
&self,
link_id: Uuid,
@@ -27,7 +30,10 @@ where
) -> Result<ChangeBatch, EmailApiError> {
let access_token = self.prepare(link_id, ApiOperationKind::ListChanges).await?;
- self.repository.list_changes(&access_token, cursor).await
+ self.repository
+ .list_changes(&access_token, cursor)
+ .await
+ .inspect_err(log_operation_error)
}
}
diff --git a/services/email_service/src/pubsub/backfill/error_handlers.rs b/services/email_service/src/pubsub/backfill/error_handlers.rs
index 0fe0d01fe6..e6d40fb24d 100644
--- a/services/email_service/src/pubsub/backfill/error_handlers.rs
+++ b/services/email_service/src/pubsub/backfill/error_handlers.rs
@@ -7,7 +7,7 @@ use models_email::api::refresh::{BackfillStatus, RefreshEmailEvent};
use models_email::email::service::backfill::{
BackfillMessagePayload, BackfillOperation, BackfillPubsubMessage, JobScopedPayload,
};
-use models_email::email::service::pubsub::DetailedError;
+use models_email::email::service::pubsub::{DetailedError, FailureReason};
use sqs_worker::cleanup_message;
use uuid::Uuid;
@@ -119,6 +119,13 @@ pub async fn handle_retryable_error(
let error_chain = format!("{:#}", _e.source);
tracing::Span::current().record("error", &error_chain);
+ // Rate limits are expected backpressure, redelivered after the visibility
+ // timeout; the email API client logs provider throttling at ERROR.
+ if _e.reason == FailureReason::GmailApiRateLimited {
+ tracing::debug!("Rate-limited backfill operation will be redelivered");
+ return Ok(());
+ }
+
match &data.backfill_operation {
BackfillOperation::Init(_) => {
tracing::warn!("Retryable error in Init")
diff --git a/services/email_service/src/pubsub/gmail_ops/error_handlers.rs b/services/email_service/src/pubsub/gmail_ops/error_handlers.rs
index d45fbc00fe..a89c56cc98 100644
--- a/services/email_service/src/pubsub/gmail_ops/error_handlers.rs
+++ b/services/email_service/src/pubsub/gmail_ops/error_handlers.rs
@@ -1,6 +1,6 @@
use crate::pubsub::gmail_ops::worker::GmailOpsContext;
use models_email::gmail::gmail_ops::GmailOpsPubsubMessage;
-use models_email::service::pubsub::{DetailedError, ProcessingError};
+use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
use sqs_worker::cleanup_message;
/// Handles non-retryable errors by cleaning up the SQS message.
@@ -11,7 +11,13 @@ pub async fn handle_non_retryable_error(
data: &GmailOpsPubsubMessage,
e: &DetailedError,
) -> anyhow::Result<()> {
- tracing::error!(error = ?e, payload = ?data.operation, "Non-retryable error processing gmail ops message. The message will be deleted.");
+ // A rate-limited operation is only non-retryable here once it has been
+ // re-enqueued to the retry queue, so deleting it loses nothing.
+ if e.reason == FailureReason::GmailApiRateLimited {
+ tracing::debug!(error = ?e, payload = ?data.operation, "Rate-limited gmail ops message deferred. The message will be deleted.");
+ } else {
+ tracing::error!(error = ?e, payload = ?data.operation, "Non-retryable error processing gmail ops message. The message will be deleted.");
+ }
cleanup_message(&ctx.sqs_worker, message).await?;
Ok(())
diff --git a/services/email_service/src/pubsub/inbox_sync/error_handlers.rs b/services/email_service/src/pubsub/inbox_sync/error_handlers.rs
index 5d5072bc6b..39bc31ce3d 100644
--- a/services/email_service/src/pubsub/inbox_sync/error_handlers.rs
+++ b/services/email_service/src/pubsub/inbox_sync/error_handlers.rs
@@ -1,6 +1,6 @@
use crate::pubsub::context::PubSubContext;
use models_email::gmail::inbox_sync::InboxSyncPubsubMessage;
-use models_email::service::pubsub::{DetailedError, ProcessingError};
+use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
use sqs_worker::cleanup_message;
/// Handles non-retryable errors by updating the appropriate status in the database and cleaning up the SQS message
@@ -11,7 +11,14 @@ pub async fn handle_non_retryable_error(
data: &InboxSyncPubsubMessage,
e: &DetailedError,
) -> anyhow::Result<()> {
- tracing::error!(error = %e, payload = format!("{:?}", data.operation), "Non-retryable error processing inbox sync message. The message will be deleted.");
+ // A rate-limited operation is only non-retryable here once it has been
+ // re-enqueued to the retry queue, or when it is a notification that a
+ // later one supersedes, so deleting it loses nothing.
+ if e.reason == FailureReason::GmailApiRateLimited {
+ tracing::debug!(error = %e, payload = format!("{:?}", data.operation), "Rate-limited inbox sync message deferred. The message will be deleted.");
+ } else {
+ tracing::error!(error = %e, payload = format!("{:?}", data.operation), "Non-retryable error processing inbox sync message. The message will be deleted.");
+ }
cleanup_message(&ctx.sqs_worker, message).await?;
Ok(())