Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Use tokio-retry for async retry loops (#5144)
Sean Aye508db73d67e8
diff --git a/Cargo.lock b/Cargo.lockindex 4aec9f4fb7..9bc18fd46b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4488,6 +4488,7 @@ dependencies = ["task_dedup","thiserror 2.0.18","tokio",+ "tokio-retry","tower 0.5.3","tracing","tracing-subscriber",@@ -4967,6 +4968,7 @@ dependencies = ["system_properties","thiserror 2.0.18","tokio",+ "tokio-retry","tower 0.5.3","tower-http","tracing",@@ -15674,6 +15676,7 @@ dependencies = ["sqs_client","thiserror 2.0.18","tokio",+ "tokio-retry","tower 0.5.3","tracing","url",diff --git a/crates/documents/Cargo.toml b/crates/documents/Cargo.tomlindex 224bed4553..540760660d 100644--- a/crates/documents/Cargo.toml+++ b/crates/documents/Cargo.toml@@ -62,7 +62,7 @@ outbound = ["dep:sqlx","ports",]-ports = ["dep:tokio"]+ports = ["dep:tokio", "dep:tokio-retry"]service = ["dep:entity_access_management", "dep:foreign_entity", "ports"][dependencies]@@ -110,6 +110,7 @@ s3_key = { path = "../s3_key" }# Async runtime (for ports)tokio = { workspace = true, optional = true }+tokio-retry = { workspace = true, optional = true }# Axum dependencies (for inbound)axum = { workspace = true, optional = true }diff --git a/crates/documents/src/domain/markdown_backfill.rs b/crates/documents/src/domain/markdown_backfill.rsindex bcacbc8834..f6cfa040a7 100644--- a/crates/documents/src/domain/markdown_backfill.rs+++ b/crates/documents/src/domain/markdown_backfill.rs@@ -4,7 +4,8 @@ use std::{future::Future, time::Duration};use anyhow::Context;use futures::stream::{self, StreamExt};-use tokio::time::{sleep, timeout};+use tokio::time::timeout;+use tokio_retry::Retry;use crate::domain::models::DocumentError;use crate::domain::ports::markdown::MarkdownInitializationPort;@@ -445,37 +446,37 @@ whereS: SyncServiceProbe,{let attempts = retries + 1;-- for attempt_index in 0..attempts {- let attempt = attempt_index + 1;-- match timeout(request_timeout, sync_service.exists(document_id)).await {- Ok(Ok(exists)) => return Ok(exists),- Ok(Err(error)) if attempt < attempts => {- tracing::debug!(%document_id, attempt, attempts, error = ?error, "sync-service exists request failed; retrying");- }- Ok(Err(error)) => {- return Err(error).with_context(|| {- format!("sync-service exists request failed after {attempts} attempts")- });- }- Err(error) if attempt < attempts => {- tracing::debug!(%document_id, attempt, attempts, timeout_secs = request_timeout.as_secs(), error = ?error, "sync-service exists request timed out; retrying");- }- Err(error) => {- return Err(error).with_context(|| {- format!(- "sync-service exists request timed out after {attempts} attempts of {}s",- request_timeout.as_secs()- )- });+ let retry_strategy = (1..attempts).map(|attempt| Duration::from_millis(250 * attempt as u64));+ let mut attempt = 0usize;++ Retry::start(retry_strategy, || {+ attempt += 1;+ async move {+ match timeout(request_timeout, sync_service.exists(document_id)).await {+ Ok(Ok(exists)) => Ok(exists),+ Ok(Err(error)) => {+ if attempt < attempts {+ tracing::debug!(%document_id, attempt, attempts, error = ?error, "sync-service exists request failed; retrying");+ }+ Err(error).with_context(|| {+ format!("sync-service exists request failed after {attempts} attempts")+ })+ }+ Err(error) => {+ if attempt < attempts {+ tracing::debug!(%document_id, attempt, attempts, timeout_secs = request_timeout.as_secs(), error = ?error, "sync-service exists request timed out; retrying");+ }+ Err(error).with_context(|| {+ format!(+ "sync-service exists request timed out after {attempts} attempts of {}s",+ request_timeout.as_secs()+ )+ })+ }}}-- sleep(Duration::from_millis(250 * attempt as u64)).await;- }-- unreachable!("retry loop should return before exhausting attempts")+ })+ .await}fn sync_snapshot_already_exists(error: &DocumentError) -> bool {diff --git a/crates/documents/src/outbound/markdown_init.rs b/crates/documents/src/outbound/markdown_init.rsindex 42d40517b1..de61de2d57 100644--- a/crates/documents/src/outbound/markdown_init.rs+++ b/crates/documents/src/outbound/markdown_init.rs@@ -6,9 +6,11 @@ const MARKDOWN_GOLDEN_SNAPSHOT: &[u8] =use std::future::Future;use std::sync::Arc;+use std::time::Duration;use lexical_client::LexicalClient;use sync_service_client::SyncServiceClient;+use tokio_retry::{Retry, strategy::FixedInterval};use crate::domain::models::DocumentError;use crate::domain::ports::markdown::{@@ -77,22 +79,35 @@ wherelet sync_service_client = self.sync_service_client.clone();let document_id = document_id.to_owned();tokio::spawn(async move {- const MAX_ATTEMPTS: u32 = 3;- const RETRY_DELAY: u64 = 1;- for attempt in 1..=MAX_ATTEMPTS {- match sync_service_client- .initialize_from_snapshot(&document_id, loro_snapshot.as_slice())- .await- {- Ok(()) => return,- Err(e) if attempt < MAX_ATTEMPTS => {- tracing::warn!(error=?e, attempt, "failed to initialize sync service from snapshot, retrying in 1s");- tokio::time::sleep(std::time::Duration::from_secs(RETRY_DELAY)).await;+ const MAX_ATTEMPTS: usize = 3;+ const RETRY_DELAY: Duration = Duration::from_secs(1);++ let loro_snapshot: Arc<[u8]> = loro_snapshot.into();+ let mut attempt = 0usize;+ let result = Retry::start(+ FixedInterval::new(RETRY_DELAY).take(MAX_ATTEMPTS - 1),+ || {+ attempt += 1;+ let sync_service_client = Arc::clone(&sync_service_client);+ let document_id = document_id.clone();+ let loro_snapshot = Arc::clone(&loro_snapshot);+ async move {+ let result = sync_service_client+ .initialize_from_snapshot(&document_id, &loro_snapshot)+ .await;+ if let Err(error) = &result+ && attempt < MAX_ATTEMPTS+ {+ tracing::warn!(error=?error, attempt, "failed to initialize sync service from snapshot, retrying in 1s");+ }+ result}- Err(e) => {- tracing::error!(error=?e, "failed to initialize sync service from snapshot after {MAX_ATTEMPTS} attempts");- }- }+ },+ )+ .await;++ if let Err(error) = result {+ tracing::error!(error=?error, "failed to initialize sync service from snapshot after {MAX_ATTEMPTS} attempts");}});diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rsindex 9a3962b5ff..c604c2d2ac 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs@@ -18,6 +18,7 @@ use model_entity::{Entity, EntityType};use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};use rootcause::prelude::{Report, ResultExt as _};+use tokio_retry::{Retry, strategy::ExponentialBackoff};use crate::domain::{ports::{SoupItemReader, SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},@@ -42,6 +43,12 @@ const MAX_SERVICE_ATTEMPTS: u32 = 5;/// Delay before the first retry; each subsequent delay doubles.const SERVICE_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);+fn service_retry_strategy() -> impl Iterator<Item = Duration> {+ ExponentialBackoff::from_millis(2)+ .factor(500)+ .take((MAX_SERVICE_ATTEMPTS - 1) as usize)+}+fn entity_from_document_event(event: &DocumentMacroEvent) -> Option<Entity<'static>> {match &event.event().event {DocumentTopicEvent::Updated(metadata) => {@@ -109,37 +116,38 @@ async fn notify_with_retry<S: SoupRealtimeService>(partition: i32,offset: i64,) -> Result<(), Report> {- let mut delay = SERVICE_RETRY_BASE_DELAY;- let mut attempt = 1u32;-- loop {- tracing::trace!(attempt, "notifying realtime Soup recipients");- match service.notify_users(entity.clone()).await {- Ok(()) => {- tracing::trace!(attempt, "realtime Soup recipients notified");- return Ok(());- }- Err(error) if attempt < MAX_SERVICE_ATTEMPTS => {- tracing::warn!(- error = ?error,- attempt,- delay_secs = delay.as_secs_f32(),- "realtime Soup fan-out failed, retrying"- );- tokio::time::sleep(delay).await;- delay *= 2;- attempt += 1;- }- Err(error) => {- return Err(error- .context(format!(- "realtime Soup fan-out failed after {MAX_SERVICE_ATTEMPTS} attempts \- (partition {partition} offset {offset})"- ))- .into_dynamic());+ let mut attempt = 0u32;+ Retry::start(service_retry_strategy(), || {+ attempt += 1;+ let entity = entity.clone();+ async move {+ tracing::trace!(attempt, "notifying realtime Soup recipients");+ let result = service.notify_users(entity).await;+ match &result {+ Ok(()) => tracing::trace!(attempt, "realtime Soup recipients notified"),+ Err(error) if attempt < MAX_SERVICE_ATTEMPTS => {+ let delay = SERVICE_RETRY_BASE_DELAY * 2u32.pow(attempt - 1);+ tracing::warn!(+ error = ?error,+ attempt,+ delay_secs = delay.as_secs_f32(),+ "realtime Soup fan-out failed, retrying"+ );+ }+ Err(_) => {}}+ result}- }+ })+ .await+ .map_err(|error| {+ error+ .context(format!(+ "realtime Soup fan-out failed after {MAX_SERVICE_ATTEMPTS} attempts \+ (partition {partition} offset {offset})"+ ))+ .into_dynamic()+ })}fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {diff --git a/crates/webhook/Cargo.toml b/crates/webhook/Cargo.tomlindex 104b3b8ad8..9e701bd160 100644--- a/crates/webhook/Cargo.toml+++ b/crates/webhook/Cargo.toml@@ -9,6 +9,7 @@ consumer = ["dep:kafka_util","dep:rdkafka","dep:tokio",+ "dep:tokio-retry","ingestion","macro_event_broker/outbound",]@@ -82,6 +83,7 @@ sqlx = { workspace = true, optional = true }sqs_client = { path = "../sqs_client", default-features = false, optional = true }thiserror = { workspace = true }tokio = { workspace = true, features = ["macros", "time"], optional = true }+tokio-retry = { workspace = true, optional = true }tracing = { workspace = true }url = { workspace = true, optional = true }utoipa = { workspace = true, optional = true }diff --git a/crates/webhook/src/inbound/kafka_consumer.rs b/crates/webhook/src/inbound/kafka_consumer.rsindex d2bdb6d455..9ad9219557 100644--- a/crates/webhook/src/inbound/kafka_consumer.rs+++ b/crates/webhook/src/inbound/kafka_consumer.rs@@ -21,7 +21,10 @@#[cfg(test)]mod test;-use crate::domain::{events::WebhookMacroEvent, ingestion::WebhookEventIngestionService};+use crate::domain::{+ events::WebhookMacroEvent,+ ingestion::{WebhookEventIngestionError, WebhookEventIngestionService},+};use anyhow::Context as _;use channels::domain::broker_events::ChannelMacroEvent;use documents::domain::events::DocumentMacroEvent;@@ -33,6 +36,7 @@ use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message};use std::future::Future;use std::time::Duration;+use tokio_retry::{RetryIf, strategy::ExponentialBackoff};/// Consumer group for webhook event ingestion. Offsets are committed under/// this group, so restarts resume where the previous run left off.@@ -62,6 +66,12 @@ const MAX_INGEST_ATTEMPTS: u32 = 5;/// consumer from its group.const INGEST_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);+fn ingest_retry_strategy() -> impl Iterator<Item = Duration> {+ ExponentialBackoff::from_millis(2)+ .factor(500)+ .take((MAX_INGEST_ATTEMPTS - 1) as usize)+}+/// Commit `message`'s offset, logging the outcome.fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>) {match consumer.inner().commit_message(message, CommitMode::Async) {@@ -91,58 +101,67 @@ async fn ingest_with_retry<S: WebhookEventIngestionService>(partition: i32,offset: i64,) -> anyhow::Result<()> {- let mut delay = INGEST_RETRY_BASE_DELAY;- let mut attempt = 1u32;- loop {- tracing::trace!(partition, offset, attempt, "ingesting broker event");- let result = match event {- DeclaredMacroEvent::DocumentMacroEvent(event) => {- service.ingest_document_event(event.event().clone()).await- }- DeclaredMacroEvent::ChannelMacroEvent(event) => {- service.ingest_channel_event(event.event().clone()).await- }- DeclaredMacroEvent::WebhookMacroEvent(event) => {- service.ingest_webhook_event(event.event().clone()).await- }- };- match result {- Ok(()) => {- tracing::trace!(partition, offset, attempt, "broker event ingested");- return Ok(());- }- Err(e) if !e.is_transient() => {- tracing::error!(- error = ?e,- partition,- offset,- "dropping broker event after non-retryable ingestion failure"- );- return Ok(());- }- Err(e) if attempt < MAX_INGEST_ATTEMPTS => {- tracing::warn!(- error = ?e,- partition,- offset,- attempt,- delay_secs = delay.as_secs_f32(),- "transient ingestion failure, retrying"- );- tokio::time::sleep(delay).await;- delay *= 2;- attempt += 1;- }- Err(e) => {- return Err(e).with_context(|| {- format!(- "transient ingestion failure persisted after \- {MAX_INGEST_ATTEMPTS} attempts \- (partition {partition} offset {offset})"- )- });+ let mut attempt = 0u32;+ let result = RetryIf::start(+ ingest_retry_strategy(),+ || {+ attempt += 1;+ async move {+ tracing::trace!(partition, offset, attempt, "ingesting broker event");+ let result = match event {+ DeclaredMacroEvent::DocumentMacroEvent(event) => {+ service.ingest_document_event(event.event().clone()).await+ }+ DeclaredMacroEvent::ChannelMacroEvent(event) => {+ service.ingest_channel_event(event.event().clone()).await+ }+ DeclaredMacroEvent::WebhookMacroEvent(event) => {+ service.ingest_webhook_event(event.event().clone()).await+ }+ };++ match &result {+ Ok(()) => {+ tracing::trace!(partition, offset, attempt, "broker event ingested")+ }+ Err(error) if error.is_transient() && attempt < MAX_INGEST_ATTEMPTS => {+ let delay = INGEST_RETRY_BASE_DELAY * 2u32.pow(attempt - 1);+ tracing::warn!(+ error = ?error,+ partition,+ offset,+ attempt,+ delay_secs = delay.as_secs_f32(),+ "transient ingestion failure, retrying"+ );+ }+ Err(_) => {}+ }+ result}+ },+ |error: &WebhookEventIngestionError| error.is_transient(),+ )+ .await;++ match result {+ Ok(()) => Ok(()),+ Err(error) if !error.is_transient() => {+ tracing::error!(+ error = ?error,+ partition,+ offset,+ "dropping broker event after non-retryable ingestion failure"+ );+ Ok(())}+ Err(error) => Err(error).with_context(|| {+ format!(+ "transient ingestion failure persisted after \+ {MAX_INGEST_ATTEMPTS} attempts \+ (partition {partition} offset {offset})"+ )+ }),}}diff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.tomlindex 295b7b276c..38f6817c11 100644--- a/services/email_service/Cargo.toml+++ b/services/email_service/Cargo.toml@@ -166,6 +166,7 @@ strum_macros = { workspace = true }system_properties = { path = "../../crates/system_properties" }thiserror = { workspace = true }tokio = { workspace = true, features = ["signal"] }+tokio-retry.workspace = truetower = { workspace = true }tower-http = { workspace = true, features = ["compression-gzip",diff --git a/services/email_service/src/bin/backfill_sfs_attachments/upload.rs b/services/email_service/src/bin/backfill_sfs_attachments/upload.rsindex f907380d87..2d6e330701 100644--- a/services/email_service/src/bin/backfill_sfs_attachments/upload.rs+++ b/services/email_service/src/bin/backfill_sfs_attachments/upload.rs@@ -2,6 +2,7 @@ use anyhow::Context;use models_email::service::attachment::{AttachmentSfs, AttachmentUploadMetadata};use sqlx::PgPool;use static_file_service_client::put_file::PutFileResponse;+use tokio_retry::{Retry, strategy::ExponentialBackoff};use tracing::instrument;use uuid::Uuid;@@ -57,40 +58,37 @@ impl AttachmentProcessor {bytes: bytes::Bytes,mime_type: String,) -> anyhow::Result<PutFileResponse> {- let mut last_err: Option<anyhow::Error> = None;-- for attempt in 1usize..=5 {- let res = self- .sfs_client- .put_file_with_bytes("a", bytes.clone(), mime_type.clone())- .await;-- match res {- Ok(ok) => return Ok(ok),- Err(e) => {- println!(- "SFS upload failed (attempt {}/5). Will{} retry. Error: {:?}",- attempt,- if attempt < 5 { "" } else { " not" },- e- );-- let err = anyhow::anyhow!("{e:?}").context(format!(- "Failed to upload attachment to SFS (attempt {attempt}/5)"- ));- last_err = Some(err);-- if attempt < 5 {- // exponential backoff: 200ms, 400ms, 800ms, 1600ms- let backoff_ms =- 200u64.saturating_mul(2u64.saturating_pow((attempt - 1) as u32));- tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await;+ const MAX_ATTEMPTS: usize = 5;++ let retry_strategy = ExponentialBackoff::from_millis(2)+ .factor(100)+ .take(MAX_ATTEMPTS - 1);+ let mut attempt = 0usize;++ Retry::start(retry_strategy, || {+ attempt += 1;+ let bytes = bytes.clone();+ let mime_type = mime_type.clone();+ async move {+ match self+ .sfs_client+ .put_file_with_bytes("a", bytes, mime_type)+ .await+ {+ Ok(response) => Ok(response),+ Err(error) => {+ println!(+ "SFS upload failed (attempt {attempt}/{MAX_ATTEMPTS}). Will{} retry. Error: {error:?}",+ if attempt < MAX_ATTEMPTS { "" } else { " not" },+ );+ Err(anyhow::anyhow!("{error:?}").context(format!(+ "Failed to upload attachment to SFS (attempt {attempt}/{MAX_ATTEMPTS})"+ )))}}}- }-- Err(last_err.unwrap_or_else(|| anyhow::anyhow!("Failed to upload attachment to SFS")))+ })+ .await}async fn persist_sfs_metadata(diff --git a/services/email_service/src/pubsub/link_manager/process.rs b/services/email_service/src/pubsub/link_manager/process.rsindex 3bcb3338d2..f1e9b2e5f5 100644--- a/services/email_service/src/pubsub/link_manager/process.rs+++ b/services/email_service/src/pubsub/link_manager/process.rs@@ -21,6 +21,8 @@ use notification::domain::service::NotificationIngress;use sqs_client::search::SearchQueueMessage;use sqs_client::search::email::EmailLinkMessage;use sqs_worker::cleanup_message;+use std::time::Duration;+use tokio_retry::RetryIf;#[tracing::instrument(skip(ctx, message), err)]pub async fn process_message(@@ -139,30 +141,45 @@ async fn get_link_or_skip(/// so don't retry it — the watch then lingers until Gmail expires it or the next connect/// stops it.async fn fetch_teardown_token(ctx: &LinkManagerContext, link: &Link) -> Option<String> {- const MAX_ATTEMPTS: u32 = 3;-- for attempt in 1..=MAX_ATTEMPTS {- match fetch_gmail_access_token_from_link(link, &ctx.redis_client, &ctx.auth_service_client)- .await- {- Ok(token) => return Some(token),- Err(e) if is_forbidden_error(&e) => {- tracing::warn!(error=?e, link_id=%link.id, "Gmail access revoked; cannot stop watch (it will expire on its own)");- return None;- }- Err(e) if attempt < MAX_ATTEMPTS => {- tracing::warn!(error=?e, attempt, link_id=%link.id, "Transient failure fetching token to stop Gmail watch; retrying");- tokio::time::sleep(std::time::Duration::from_millis(200 * u64::from(attempt)))- .await;- }- Err(e) => {- tracing::warn!(error=?e, link_id=%link.id, "Could not fetch token to stop Gmail watch after retries; watch will linger until it expires or the next connect stops it");- return None;+ const MAX_ATTEMPTS: usize = 3;++ let retry_strategy = [Duration::from_millis(200), Duration::from_millis(400)];+ let mut attempt = 0usize;+ let result = RetryIf::start(+ retry_strategy,+ || {+ attempt += 1;+ async move {+ let result = fetch_gmail_access_token_from_link(+ link,+ &ctx.redis_client,+ &ctx.auth_service_client,+ )+ .await;+ if let Err(error) = &result+ && !is_forbidden_error(error)+ && attempt < MAX_ATTEMPTS+ {+ tracing::warn!(error=?error, attempt, link_id=%link.id, "Transient failure fetching token to stop Gmail watch; retrying");+ }+ result}+ },+ |error: &anyhow::Error| !is_forbidden_error(error),+ )+ .await;++ match result {+ Ok(token) => Some(token),+ Err(error) if is_forbidden_error(&error) => {+ tracing::warn!(error=?error, link_id=%link.id, "Gmail access revoked; cannot stop watch (it will expire on its own)");+ None+ }+ Err(error) => {+ tracing::warn!(error=?error, link_id=%link.id, "Could not fetch token to stop Gmail watch after retries; watch will linger until it expires or the next connect stops it");+ None}}-- None}/// Handles the Refresh operation: renews Gmail watch subscription and syncs contacts.