Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Use tokio-retry for async retry loops (#5144)

Sean Aye508db73d67e8
diff --git a/Cargo.lock b/Cargo.lock
index 4aec9f4fb7..9bc18fd46b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4488,6 +4488,7 @@ dependencies = [
"task_dedup",
"thiserror 2.0.18",
"tokio",
+ "tokio-retry",
"tower 0.5.3",
"tracing",
"tracing-subscriber",
@@ -4967,6 +4968,7 @@ dependencies = [
"system_properties",
"thiserror 2.0.18",
"tokio",
+ "tokio-retry",
"tower 0.5.3",
"tower-http",
"tracing",
@@ -15674,6 +15676,7 @@ dependencies = [
"sqs_client",
"thiserror 2.0.18",
"tokio",
+ "tokio-retry",
"tower 0.5.3",
"tracing",
"url",
diff --git a/crates/documents/Cargo.toml b/crates/documents/Cargo.toml
index 224bed4553..540760660d 100644
--- a/crates/documents/Cargo.toml
+++ b/crates/documents/Cargo.toml
@@ -62,7 +62,7 @@ outbound = [
"dep:sqlx",
"ports",
]
-ports = ["dep:tokio"]
+ports = ["dep:tokio", "dep:tokio-retry"]
service = ["dep:entity_access_management", "dep:foreign_entity", "ports"]
[dependencies]
@@ -110,6 +110,7 @@ s3_key = { path = "../s3_key" }
# Async runtime (for ports)
tokio = { workspace = true, optional = true }
+tokio-retry = { workspace = true, optional = true }
# Axum dependencies (for inbound)
axum = { workspace = true, optional = true }
diff --git a/crates/documents/src/domain/markdown_backfill.rs b/crates/documents/src/domain/markdown_backfill.rs
index bcacbc8834..f6cfa040a7 100644
--- a/crates/documents/src/domain/markdown_backfill.rs
+++ b/crates/documents/src/domain/markdown_backfill.rs
@@ -4,7 +4,8 @@ use std::{future::Future, time::Duration};
use anyhow::Context;
use futures::stream::{self, StreamExt};
-use tokio::time::{sleep, timeout};
+use tokio::time::timeout;
+use tokio_retry::Retry;
use crate::domain::models::DocumentError;
use crate::domain::ports::markdown::MarkdownInitializationPort;
@@ -445,37 +446,37 @@ where
S: SyncServiceProbe,
{
let attempts = retries + 1;
-
- for attempt_index in 0..attempts {
- let attempt = attempt_index + 1;
-
- match timeout(request_timeout, sync_service.exists(document_id)).await {
- Ok(Ok(exists)) => return Ok(exists),
- Ok(Err(error)) if attempt < attempts => {
- tracing::debug!(%document_id, attempt, attempts, error = ?error, "sync-service exists request failed; retrying");
- }
- Ok(Err(error)) => {
- return Err(error).with_context(|| {
- format!("sync-service exists request failed after {attempts} attempts")
- });
- }
- Err(error) if attempt < attempts => {
- tracing::debug!(%document_id, attempt, attempts, timeout_secs = request_timeout.as_secs(), error = ?error, "sync-service exists request timed out; retrying");
- }
- Err(error) => {
- return Err(error).with_context(|| {
- format!(
- "sync-service exists request timed out after {attempts} attempts of {}s",
- request_timeout.as_secs()
- )
- });
+ let retry_strategy = (1..attempts).map(|attempt| Duration::from_millis(250 * attempt as u64));
+ let mut attempt = 0usize;
+
+ Retry::start(retry_strategy, || {
+ attempt += 1;
+ async move {
+ match timeout(request_timeout, sync_service.exists(document_id)).await {
+ Ok(Ok(exists)) => Ok(exists),
+ Ok(Err(error)) => {
+ if attempt < attempts {
+ tracing::debug!(%document_id, attempt, attempts, error = ?error, "sync-service exists request failed; retrying");
+ }
+ Err(error).with_context(|| {
+ format!("sync-service exists request failed after {attempts} attempts")
+ })
+ }
+ Err(error) => {
+ if attempt < attempts {
+ tracing::debug!(%document_id, attempt, attempts, timeout_secs = request_timeout.as_secs(), error = ?error, "sync-service exists request timed out; retrying");
+ }
+ Err(error).with_context(|| {
+ format!(
+ "sync-service exists request timed out after {attempts} attempts of {}s",
+ request_timeout.as_secs()
+ )
+ })
+ }
}
}
-
- sleep(Duration::from_millis(250 * attempt as u64)).await;
- }
-
- unreachable!("retry loop should return before exhausting attempts")
+ })
+ .await
}
fn sync_snapshot_already_exists(error: &DocumentError) -> bool {
diff --git a/crates/documents/src/outbound/markdown_init.rs b/crates/documents/src/outbound/markdown_init.rs
index 42d40517b1..de61de2d57 100644
--- a/crates/documents/src/outbound/markdown_init.rs
+++ b/crates/documents/src/outbound/markdown_init.rs
@@ -6,9 +6,11 @@ const MARKDOWN_GOLDEN_SNAPSHOT: &[u8] =
use std::future::Future;
use std::sync::Arc;
+use std::time::Duration;
use lexical_client::LexicalClient;
use sync_service_client::SyncServiceClient;
+use tokio_retry::{Retry, strategy::FixedInterval};
use crate::domain::models::DocumentError;
use crate::domain::ports::markdown::{
@@ -77,22 +79,35 @@ where
let sync_service_client = self.sync_service_client.clone();
let document_id = document_id.to_owned();
tokio::spawn(async move {
- const MAX_ATTEMPTS: u32 = 3;
- const RETRY_DELAY: u64 = 1;
- for attempt in 1..=MAX_ATTEMPTS {
- match sync_service_client
- .initialize_from_snapshot(&document_id, loro_snapshot.as_slice())
- .await
- {
- Ok(()) => return,
- Err(e) if attempt < MAX_ATTEMPTS => {
- tracing::warn!(error=?e, attempt, "failed to initialize sync service from snapshot, retrying in 1s");
- tokio::time::sleep(std::time::Duration::from_secs(RETRY_DELAY)).await;
+ const MAX_ATTEMPTS: usize = 3;
+ const RETRY_DELAY: Duration = Duration::from_secs(1);
+
+ let loro_snapshot: Arc<[u8]> = loro_snapshot.into();
+ let mut attempt = 0usize;
+ let result = Retry::start(
+ FixedInterval::new(RETRY_DELAY).take(MAX_ATTEMPTS - 1),
+ || {
+ attempt += 1;
+ let sync_service_client = Arc::clone(&sync_service_client);
+ let document_id = document_id.clone();
+ let loro_snapshot = Arc::clone(&loro_snapshot);
+ async move {
+ let result = sync_service_client
+ .initialize_from_snapshot(&document_id, &loro_snapshot)
+ .await;
+ if let Err(error) = &result
+ && attempt < MAX_ATTEMPTS
+ {
+ tracing::warn!(error=?error, attempt, "failed to initialize sync service from snapshot, retrying in 1s");
+ }
+ result
}
- Err(e) => {
- tracing::error!(error=?e, "failed to initialize sync service from snapshot after {MAX_ATTEMPTS} attempts");
- }
- }
+ },
+ )
+ .await;
+
+ if let Err(error) = result {
+ tracing::error!(error=?error, "failed to initialize sync service from snapshot after {MAX_ATTEMPTS} attempts");
}
});
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rs
index 9a3962b5ff..c604c2d2ac 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs
@@ -18,6 +18,7 @@ use model_entity::{Entity, EntityType};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use rootcause::prelude::{Report, ResultExt as _};
+use tokio_retry::{Retry, strategy::ExponentialBackoff};
use crate::domain::{
ports::{SoupItemReader, SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},
@@ -42,6 +43,12 @@ const MAX_SERVICE_ATTEMPTS: u32 = 5;
/// Delay before the first retry; each subsequent delay doubles.
const SERVICE_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
+fn service_retry_strategy() -> impl Iterator<Item = Duration> {
+ ExponentialBackoff::from_millis(2)
+ .factor(500)
+ .take((MAX_SERVICE_ATTEMPTS - 1) as usize)
+}
+
fn entity_from_document_event(event: &DocumentMacroEvent) -> Option<Entity<'static>> {
match &event.event().event {
DocumentTopicEvent::Updated(metadata) => {
@@ -109,37 +116,38 @@ async fn notify_with_retry<S: SoupRealtimeService>(
partition: i32,
offset: i64,
) -> Result<(), Report> {
- let mut delay = SERVICE_RETRY_BASE_DELAY;
- let mut attempt = 1u32;
-
- loop {
- tracing::trace!(attempt, "notifying realtime Soup recipients");
- match service.notify_users(entity.clone()).await {
- Ok(()) => {
- tracing::trace!(attempt, "realtime Soup recipients notified");
- return Ok(());
- }
- Err(error) if attempt < MAX_SERVICE_ATTEMPTS => {
- tracing::warn!(
- error = ?error,
- attempt,
- delay_secs = delay.as_secs_f32(),
- "realtime Soup fan-out failed, retrying"
- );
- tokio::time::sleep(delay).await;
- delay *= 2;
- attempt += 1;
- }
- Err(error) => {
- return Err(error
- .context(format!(
- "realtime Soup fan-out failed after {MAX_SERVICE_ATTEMPTS} attempts \
- (partition {partition} offset {offset})"
- ))
- .into_dynamic());
+ let mut attempt = 0u32;
+ Retry::start(service_retry_strategy(), || {
+ attempt += 1;
+ let entity = entity.clone();
+ async move {
+ tracing::trace!(attempt, "notifying realtime Soup recipients");
+ let result = service.notify_users(entity).await;
+ match &result {
+ Ok(()) => tracing::trace!(attempt, "realtime Soup recipients notified"),
+ Err(error) if attempt < MAX_SERVICE_ATTEMPTS => {
+ let delay = SERVICE_RETRY_BASE_DELAY * 2u32.pow(attempt - 1);
+ tracing::warn!(
+ error = ?error,
+ attempt,
+ delay_secs = delay.as_secs_f32(),
+ "realtime Soup fan-out failed, retrying"
+ );
+ }
+ Err(_) => {}
}
+ result
}
- }
+ })
+ .await
+ .map_err(|error| {
+ error
+ .context(format!(
+ "realtime Soup fan-out failed after {MAX_SERVICE_ATTEMPTS} attempts \
+ (partition {partition} offset {offset})"
+ ))
+ .into_dynamic()
+ })
}
fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {
diff --git a/crates/webhook/Cargo.toml b/crates/webhook/Cargo.toml
index 104b3b8ad8..9e701bd160 100644
--- a/crates/webhook/Cargo.toml
+++ b/crates/webhook/Cargo.toml
@@ -9,6 +9,7 @@ consumer = [
"dep:kafka_util",
"dep:rdkafka",
"dep:tokio",
+ "dep:tokio-retry",
"ingestion",
"macro_event_broker/outbound",
]
@@ -82,6 +83,7 @@ sqlx = { workspace = true, optional = true }
sqs_client = { path = "../sqs_client", default-features = false, optional = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["macros", "time"], optional = true }
+tokio-retry = { workspace = true, optional = true }
tracing = { workspace = true }
url = { workspace = true, optional = true }
utoipa = { workspace = true, optional = true }
diff --git a/crates/webhook/src/inbound/kafka_consumer.rs b/crates/webhook/src/inbound/kafka_consumer.rs
index d2bdb6d455..9ad9219557 100644
--- a/crates/webhook/src/inbound/kafka_consumer.rs
+++ b/crates/webhook/src/inbound/kafka_consumer.rs
@@ -21,7 +21,10 @@
#[cfg(test)]
mod test;
-use crate::domain::{events::WebhookMacroEvent, ingestion::WebhookEventIngestionService};
+use crate::domain::{
+ events::WebhookMacroEvent,
+ ingestion::{WebhookEventIngestionError, WebhookEventIngestionService},
+};
use anyhow::Context as _;
use channels::domain::broker_events::ChannelMacroEvent;
use documents::domain::events::DocumentMacroEvent;
@@ -33,6 +36,7 @@ use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message};
use std::future::Future;
use std::time::Duration;
+use tokio_retry::{RetryIf, strategy::ExponentialBackoff};
/// Consumer group for webhook event ingestion. Offsets are committed under
/// this group, so restarts resume where the previous run left off.
@@ -62,6 +66,12 @@ const MAX_INGEST_ATTEMPTS: u32 = 5;
/// consumer from its group.
const INGEST_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
+fn ingest_retry_strategy() -> impl Iterator<Item = Duration> {
+ ExponentialBackoff::from_millis(2)
+ .factor(500)
+ .take((MAX_INGEST_ATTEMPTS - 1) as usize)
+}
+
/// Commit `message`'s offset, logging the outcome.
fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>) {
match consumer.inner().commit_message(message, CommitMode::Async) {
@@ -91,58 +101,67 @@ async fn ingest_with_retry<S: WebhookEventIngestionService>(
partition: i32,
offset: i64,
) -> anyhow::Result<()> {
- let mut delay = INGEST_RETRY_BASE_DELAY;
- let mut attempt = 1u32;
- loop {
- tracing::trace!(partition, offset, attempt, "ingesting broker event");
- let result = match event {
- DeclaredMacroEvent::DocumentMacroEvent(event) => {
- service.ingest_document_event(event.event().clone()).await
- }
- DeclaredMacroEvent::ChannelMacroEvent(event) => {
- service.ingest_channel_event(event.event().clone()).await
- }
- DeclaredMacroEvent::WebhookMacroEvent(event) => {
- service.ingest_webhook_event(event.event().clone()).await
- }
- };
- match result {
- Ok(()) => {
- tracing::trace!(partition, offset, attempt, "broker event ingested");
- return Ok(());
- }
- Err(e) if !e.is_transient() => {
- tracing::error!(
- error = ?e,
- partition,
- offset,
- "dropping broker event after non-retryable ingestion failure"
- );
- return Ok(());
- }
- Err(e) if attempt < MAX_INGEST_ATTEMPTS => {
- tracing::warn!(
- error = ?e,
- partition,
- offset,
- attempt,
- delay_secs = delay.as_secs_f32(),
- "transient ingestion failure, retrying"
- );
- tokio::time::sleep(delay).await;
- delay *= 2;
- attempt += 1;
- }
- Err(e) => {
- return Err(e).with_context(|| {
- format!(
- "transient ingestion failure persisted after \
- {MAX_INGEST_ATTEMPTS} attempts \
- (partition {partition} offset {offset})"
- )
- });
+ let mut attempt = 0u32;
+ let result = RetryIf::start(
+ ingest_retry_strategy(),
+ || {
+ attempt += 1;
+ async move {
+ tracing::trace!(partition, offset, attempt, "ingesting broker event");
+ let result = match event {
+ DeclaredMacroEvent::DocumentMacroEvent(event) => {
+ service.ingest_document_event(event.event().clone()).await
+ }
+ DeclaredMacroEvent::ChannelMacroEvent(event) => {
+ service.ingest_channel_event(event.event().clone()).await
+ }
+ DeclaredMacroEvent::WebhookMacroEvent(event) => {
+ service.ingest_webhook_event(event.event().clone()).await
+ }
+ };
+
+ match &result {
+ Ok(()) => {
+ tracing::trace!(partition, offset, attempt, "broker event ingested")
+ }
+ Err(error) if error.is_transient() && attempt < MAX_INGEST_ATTEMPTS => {
+ let delay = INGEST_RETRY_BASE_DELAY * 2u32.pow(attempt - 1);
+ tracing::warn!(
+ error = ?error,
+ partition,
+ offset,
+ attempt,
+ delay_secs = delay.as_secs_f32(),
+ "transient ingestion failure, retrying"
+ );
+ }
+ Err(_) => {}
+ }
+ result
}
+ },
+ |error: &WebhookEventIngestionError| error.is_transient(),
+ )
+ .await;
+
+ match result {
+ Ok(()) => Ok(()),
+ Err(error) if !error.is_transient() => {
+ tracing::error!(
+ error = ?error,
+ partition,
+ offset,
+ "dropping broker event after non-retryable ingestion failure"
+ );
+ Ok(())
}
+ Err(error) => Err(error).with_context(|| {
+ format!(
+ "transient ingestion failure persisted after \
+ {MAX_INGEST_ATTEMPTS} attempts \
+ (partition {partition} offset {offset})"
+ )
+ }),
}
}
diff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.toml
index 295b7b276c..38f6817c11 100644
--- a/services/email_service/Cargo.toml
+++ b/services/email_service/Cargo.toml
@@ -166,6 +166,7 @@ strum_macros = { workspace = true }
system_properties = { path = "../../crates/system_properties" }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["signal"] }
+tokio-retry.workspace = true
tower = { workspace = true }
tower-http = { workspace = true, features = [
"compression-gzip",
diff --git a/services/email_service/src/bin/backfill_sfs_attachments/upload.rs b/services/email_service/src/bin/backfill_sfs_attachments/upload.rs
index f907380d87..2d6e330701 100644
--- a/services/email_service/src/bin/backfill_sfs_attachments/upload.rs
+++ b/services/email_service/src/bin/backfill_sfs_attachments/upload.rs
@@ -2,6 +2,7 @@ use anyhow::Context;
use models_email::service::attachment::{AttachmentSfs, AttachmentUploadMetadata};
use sqlx::PgPool;
use static_file_service_client::put_file::PutFileResponse;
+use tokio_retry::{Retry, strategy::ExponentialBackoff};
use tracing::instrument;
use uuid::Uuid;
@@ -57,40 +58,37 @@ impl AttachmentProcessor {
bytes: bytes::Bytes,
mime_type: String,
) -> anyhow::Result<PutFileResponse> {
- let mut last_err: Option<anyhow::Error> = None;
-
- for attempt in 1usize..=5 {
- let res = self
- .sfs_client
- .put_file_with_bytes("a", bytes.clone(), mime_type.clone())
- .await;
-
- match res {
- Ok(ok) => return Ok(ok),
- Err(e) => {
- println!(
- "SFS upload failed (attempt {}/5). Will{} retry. Error: {:?}",
- attempt,
- if attempt < 5 { "" } else { " not" },
- e
- );
-
- let err = anyhow::anyhow!("{e:?}").context(format!(
- "Failed to upload attachment to SFS (attempt {attempt}/5)"
- ));
- last_err = Some(err);
-
- if attempt < 5 {
- // exponential backoff: 200ms, 400ms, 800ms, 1600ms
- let backoff_ms =
- 200u64.saturating_mul(2u64.saturating_pow((attempt - 1) as u32));
- tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await;
+ const MAX_ATTEMPTS: usize = 5;
+
+ let retry_strategy = ExponentialBackoff::from_millis(2)
+ .factor(100)
+ .take(MAX_ATTEMPTS - 1);
+ let mut attempt = 0usize;
+
+ Retry::start(retry_strategy, || {
+ attempt += 1;
+ let bytes = bytes.clone();
+ let mime_type = mime_type.clone();
+ async move {
+ match self
+ .sfs_client
+ .put_file_with_bytes("a", bytes, mime_type)
+ .await
+ {
+ Ok(response) => Ok(response),
+ Err(error) => {
+ println!(
+ "SFS upload failed (attempt {attempt}/{MAX_ATTEMPTS}). Will{} retry. Error: {error:?}",
+ if attempt < MAX_ATTEMPTS { "" } else { " not" },
+ );
+ Err(anyhow::anyhow!("{error:?}").context(format!(
+ "Failed to upload attachment to SFS (attempt {attempt}/{MAX_ATTEMPTS})"
+ )))
}
}
}
- }
-
- Err(last_err.unwrap_or_else(|| anyhow::anyhow!("Failed to upload attachment to SFS")))
+ })
+ .await
}
async fn persist_sfs_metadata(
diff --git a/services/email_service/src/pubsub/link_manager/process.rs b/services/email_service/src/pubsub/link_manager/process.rs
index 3bcb3338d2..f1e9b2e5f5 100644
--- a/services/email_service/src/pubsub/link_manager/process.rs
+++ b/services/email_service/src/pubsub/link_manager/process.rs
@@ -21,6 +21,8 @@ use notification::domain::service::NotificationIngress;
use sqs_client::search::SearchQueueMessage;
use sqs_client::search::email::EmailLinkMessage;
use sqs_worker::cleanup_message;
+use std::time::Duration;
+use tokio_retry::RetryIf;
#[tracing::instrument(skip(ctx, message), err)]
pub async fn process_message(
@@ -139,30 +141,45 @@ async fn get_link_or_skip(
/// so don't retry it — the watch then lingers until Gmail expires it or the next connect
/// stops it.
async fn fetch_teardown_token(ctx: &LinkManagerContext, link: &Link) -> Option<String> {
- const MAX_ATTEMPTS: u32 = 3;
-
- for attempt in 1..=MAX_ATTEMPTS {
- match fetch_gmail_access_token_from_link(link, &ctx.redis_client, &ctx.auth_service_client)
- .await
- {
- Ok(token) => return Some(token),
- Err(e) if is_forbidden_error(&e) => {
- tracing::warn!(error=?e, link_id=%link.id, "Gmail access revoked; cannot stop watch (it will expire on its own)");
- return None;
- }
- Err(e) if attempt < MAX_ATTEMPTS => {
- tracing::warn!(error=?e, attempt, link_id=%link.id, "Transient failure fetching token to stop Gmail watch; retrying");
- tokio::time::sleep(std::time::Duration::from_millis(200 * u64::from(attempt)))
- .await;
- }
- Err(e) => {
- tracing::warn!(error=?e, link_id=%link.id, "Could not fetch token to stop Gmail watch after retries; watch will linger until it expires or the next connect stops it");
- return None;
+ const MAX_ATTEMPTS: usize = 3;
+
+ let retry_strategy = [Duration::from_millis(200), Duration::from_millis(400)];
+ let mut attempt = 0usize;
+ let result = RetryIf::start(
+ retry_strategy,
+ || {
+ attempt += 1;
+ async move {
+ let result = fetch_gmail_access_token_from_link(
+ link,
+ &ctx.redis_client,
+ &ctx.auth_service_client,
+ )
+ .await;
+ if let Err(error) = &result
+ && !is_forbidden_error(error)
+ && attempt < MAX_ATTEMPTS
+ {
+ tracing::warn!(error=?error, attempt, link_id=%link.id, "Transient failure fetching token to stop Gmail watch; retrying");
+ }
+ result
}
+ },
+ |error: &anyhow::Error| !is_forbidden_error(error),
+ )
+ .await;
+
+ match result {
+ Ok(token) => Some(token),
+ Err(error) if is_forbidden_error(&error) => {
+ tracing::warn!(error=?error, link_id=%link.id, "Gmail access revoked; cannot stop watch (it will expire on its own)");
+ None
+ }
+ Err(error) => {
+ tracing::warn!(error=?error, link_id=%link.id, "Could not fetch token to stop Gmail watch after retries; watch will linger until it expires or the next connect stops it");
+ None
}
}
-
- None
}
/// Handles the Refresh operation: renews Gmail watch subscription and syncs contacts.