Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/chore/realtime pr feedback (#5134)
Sean Ayec0536bcd2b7e
diff --git a/Cargo.lock b/Cargo.lockindex 13ab36dc3f..25af734a1e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -13495,6 +13495,7 @@ dependencies = ["serde_json","soup","tokio",+ "tokio-retry","tracing","uuid","workspace-hack",@@ -14447,6 +14448,16 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "tokio-retry"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470"+dependencies = [+ "pin-project-lite",+ "tokio",+]+[[package]]name = "tokio-rustls"version = "0.24.1"diff --git a/Cargo.toml b/Cargo.tomlindex f94b60d4bf..7fd9c994fe 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -223,6 +223,7 @@ strum = { version = "0.27.1", features = ["derive"] }thiserror = "2.0.0"time = "0.3.37"tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] }+tokio-retry = { version = "0.3.2", default-features = false }tokio-stream = { version = "0.1.17" }tokio-tungstenite = { version = "0.28", default-features = false, features = ["connect",diff --git a/crates/soup_realtime/Cargo.toml b/crates/soup_realtime/Cargo.tomlindex 328f230c50..692f4f1686 100644--- a/crates/soup_realtime/Cargo.toml+++ b/crates/soup_realtime/Cargo.toml@@ -24,6 +24,7 @@ ports = ["dep:futures","dep:rootcause","dep:tokio",+ "dep:tokio-retry","dep:tracing",]@@ -43,6 +44,7 @@ rootcause = { workspace = true, optional = true }serde = { workspace = true }soup = { path = "../soup", default-features = false, features = ["ports"], optional = true }tokio = { workspace = true, features = ["sync", "time"], optional = true }+tokio-retry = { workspace = true, optional = true }tracing = { workspace = true, optional = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rsindex cdf613cf0f..dcc446ec28 100644--- a/crates/soup_realtime/src/domain/service.rs+++ b/crates/soup_realtime/src/domain/service.rs@@ -3,7 +3,7 @@#[cfg(test)]mod test;-use std::{collections::HashSet, num::NonZeroUsize, sync::Arc};+use std::{collections::HashSet, num::NonZeroUsize, sync::Arc, time::Duration};use broadcast::{BroadcastManager, GlobalSpawner};use futures::{StreamExt as _, stream};@@ -11,6 +11,7 @@ use macro_user_id::user_id::MacroUserIdStr;use model_entity::Entity;use models_soup::item::SoupItem;use rootcause::prelude::{Report, ResultExt as _};+use tokio_retry::{Retry, strategy::ExponentialBackoff};use super::{models::SoupRealtimeMessage,@@ -24,6 +25,15 @@ use super::{const BROADCAST_BUFFER_CAPACITY: NonZeroUsize = NonZeroUsize::new(64).unwrap();/// Number of messages buffered for each individual subscriber.const SUBSCRIBER_BUFFER_CAPACITY: NonZeroUsize = NonZeroUsize::new(16).unwrap();+/// Total receive attempts before the consumer returns for supervision.+const MAX_RECEIVE_ATTEMPTS: usize = 5;++/// Retries after one, two, four, and eight seconds.+fn receive_retry_strategy() -> impl Iterator<Item = Duration> {+ ExponentialBackoff::from_millis(2)+ .factor(500)+ .take(MAX_RECEIVE_ATTEMPTS - 1)+}/// Service for distributing recipient-targeted realtime Soup messages.pub struct SoupRealtimeConsumerService<C>@@ -66,11 +76,14 @@ where#[tracing::instrument(skip(self), err)]pub async fn run(&self) -> Result<(), Report> {loop {- let SoupRealtimeMessage { user_id, item } = self- .consumer- .recv()- .await- .context("failed to receive realtime Soup message")?;+ let SoupRealtimeMessage { user_id, item } = Retry::start(+ receive_retry_strategy(),+ || self.consumer.recv(),+ )+ .await+ .context(format!(+ "failed to receive realtime Soup message after {MAX_RECEIVE_ATTEMPTS} attempts"+ ))?;match self.broadcasts.publish(&user_id, Arc::new(item)) {Ok(subscriber_count) => {diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rsindex 6829ef355c..e71cc0e645 100644--- a/crates/soup_realtime/src/domain/service/test.rs+++ b/crates/soup_realtime/src/domain/service/test.rs@@ -71,10 +71,12 @@ impl SoupItemReader for FakeReader {struct FakeRealtimeConsumer {messages: Mutex<VecDeque<Result<SoupRealtimeMessage, Report>>>,+ calls: Arc<AtomicUsize>,}impl SoupRealtimeConsumer for FakeRealtimeConsumer {async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {+ self.calls.fetch_add(1, Ordering::SeqCst);self.messages.lock().expect("consumer messages lock")@@ -189,12 +191,14 @@ fn item_response()}-#[tokio::test]+#[tokio::test(start_paused = true)]async fn consumer_service_distributes_items_only_to_their_users() {let one = user("one");let two = user("two");+ let receive_calls = Arc::new(AtomicUsize::new(0));let consumer = FakeRealtimeConsumer {messages: Mutex::new(VecDeque::from([+ Err(rootcause::report!("transient receive failure")),Ok(SoupRealtimeMessage::new(one.clone(),document_item(DOCUMENT_ID, "For one", None),@@ -204,6 +208,7 @@ async fn consumer_service_distributes_items_only_to_their_users() {document_item(OTHER_DOCUMENT_ID, "For two", None),)),])),+ calls: Arc::clone(&receive_calls),};let service = Arc::new(SoupRealtimeConsumerService::new(consumer));let mut one_first = service.subscribe(one.clone());@@ -218,6 +223,11 @@ async fn consumer_service_distributes_items_only_to_their_users() {.expect("consumer task joins").expect_err("fake consumer eventually stops");assert!(run_error.to_string().contains("failed to receive"));+ assert_eq!(+ receive_calls.load(Ordering::SeqCst),+ 3 + MAX_RECEIVE_ATTEMPTS,+ "a successful message resets the receive retry strategy"+ );let one_first_item = tokio::time::timeout(Duration::from_secs(1), one_first.recv()).awaitdiff --git a/crates/soup_realtime/src/outbound/soup_consumer.rs b/crates/soup_realtime/src/outbound/soup_consumer.rsindex 6199e08029..db6848c3f8 100644--- a/crates/soup_realtime/src/outbound/soup_consumer.rs+++ b/crates/soup_realtime/src/outbound/soup_consumer.rs@@ -78,6 +78,7 @@ impl SoupTopicConsumer {}impl SoupRealtimeConsumer for SoupTopicConsumer {+ #[tracing::instrument(err, skip(self))]async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {SoupTopicConsumer::recv(self).await}diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.tomlindex a43ac39d05..6bb533f996 100644--- a/services/document_storage_service/Cargo.toml+++ b/services/document_storage_service/Cargo.toml@@ -20,19 +20,8 @@ path = "src/doppler_config.rs"[features]connection_gateway = []-default = [- "connection_gateway",- "delete_document_worker",- "graphql",- "location_check",-]+default = ["connection_gateway", "delete_document_worker", "location_check"]delete_document_worker = []-graphql = [- "dep:async-graphql",- "dep:async-graphql-axum",- "dep:complete_graph",- "dep:soup_realtime",-]local = ["local_auth"]local_auth = ["macro_authorization/local_auth"]location_check = []@@ -42,8 +31,8 @@ ai_tools = { path = "../../crates/ai_tools" }ai_usage = { path = "../../crates/ai_usage" }analytics_client = { path = "../../crates/analytics_client" }anyhow = { workspace = true }-async-graphql = { workspace = true, optional = true }-async-graphql-axum = { version = "7.0.17", optional = true }+async-graphql = { workspace = true }+async-graphql-axum = "7.0.17"aws-sdk-dynamodb = { workspace = true }aws-sdk-s3 = { workspace = true }aws-sdk-secretsmanager = { workspace = true }@@ -73,7 +62,7 @@ chrono = { workspace = true }clap = { workspace = true }cloudfront_sign = { workspace = true }comms_db_client = { path = "../../crates/comms_db_client" }-complete_graph = { path = "../../crates/complete_graph", optional = true }+complete_graph = { path = "../../crates/complete_graph" }connection = { path = "../../crates/connection", default-features = false, features = ["outbound",] }@@ -115,7 +104,9 @@ lazy_static = { workspace = true }lexical_client = { path = "../../crates/lexical_client" }lexical_mention_extractor = { path = "../../crates/lexical_mention_extractor" }macro_auth = { path = "../../crates/macro_auth" }-macro_authorization = { path = "../../crates/macro_authorization", features = ["postgres"] }+macro_authorization = { path = "../../crates/macro_authorization", features = [+ "postgres",+] }macro_aws_config = { path = "../../crates/macro_aws_config", features = ["s3"] }macro_axum_utils = { path = "../../crates/macro_axum_utils" }macro_config = { path = "../../crates/macro_config" }@@ -181,7 +172,7 @@ soup = { path = "../../crates/soup", features = [soup_realtime = { path = "../../crates/soup_realtime", default-features = false, features = ["consumer","outbound",-], optional = true }+] }sqlx = { workspace = true }sqs_client = { path = "../../crates/sqs_client", default-features = false, features = ["document",diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex 1c6c1df6bf..960c06e9d5 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -95,7 +95,6 @@ use soup::{domain::service::SoupImpl, inbound::axum_router::SoupRouterState,outbound::pg_soup_repo::PgSoupRepo,};-#[cfg(feature = "graphql")]use soup_realtime::{domain::service::SoupRealtimeConsumerService, outbound::soup_consumer::SoupTopicConsumer,};@@ -150,13 +149,11 @@ type DssSoupState =SoupRouterState<DssSoupService, DssEmailService, EntityAccessService, AuthorizationService>;/// Realtime Soup consumer service used by GraphQL subscriptions.-#[cfg(feature = "graphql")]pub(crate) type DssSoupRealtimeService = SoupRealtimeConsumerService<SoupTopicConsumer>;/// GraphQL Soup schema wired to the DSS services; the `ApiContext` state/// parameter lets GraphQL resolvers run the same axum extractors as the REST/// routes, lazily, against the stored request parts.-#[cfg(feature = "graphql")]pub(crate) type DssGraphqlSoupSchema = complete_graph::SharedSoupSchema<DssSoupService,DssSoupRealtimeService,@@ -419,9 +416,7 @@ pub(crate) struct ApiContext {pub dynamodb_client: Arc<DynamodbClient>,pub dynamo_db: aws_sdk_dynamodb::Client,pub soup_router_state: DssSoupState,- #[cfg(feature = "graphql")]pub graphql_soup_schema: DssGraphqlSoupSchema,- #[cfg(feature = "graphql")]pub graphql_notification_reader: Arc<ai_tools::ToolNotificationService>,pub favorites_state: DssFavoritesState,pub favorites_service: Arc<FavoritesServiceType>,diff --git a/services/document_storage_service/src/api/mod.rs b/services/document_storage_service/src/api/mod.rsindex 72afafaaba..c04d541d47 100644--- a/services/document_storage_service/src/api/mod.rs+++ b/services/document_storage_service/src/api/mod.rs@@ -29,7 +29,6 @@ mod middleware;mod activity;mod annotations;mod documents;-#[cfg(feature = "graphql")]mod graphql_soup;mod health;mod history;@@ -82,10 +81,8 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {}fn items_router(state: ApiContext) -> Router<ApiContext> {- let router = soup::inbound::axum_router::soup_router(state.soup_router_state.clone());- #[cfg(feature = "graphql")]- let router = router.merge(graphql_soup::router());- router+ soup::inbound::axum_router::soup_router(state.soup_router_state.clone())+ .merge(graphql_soup::router())}fn api_router(state: ApiContext) -> Router {diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex 1e315e13ad..cb18d639e9 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -90,7 +90,6 @@ use macro_service_urls::{};use macro_sha_count_client::Redis;use notification::domain::service::SqsNotificationIngress;-#[cfg(feature = "graphql")]use notification::domain::service::{NotificationReaderService, PlatformArnConfig};use notification::outbound::queue::SqsQueue;use opensearch_client::OpensearchClient;@@ -111,7 +110,6 @@ use soup::{domain::service::SoupImpl, inbound::axum_router::SoupRouterState,outbound::pg_soup_repo::PgSoupRepo,};-#[cfg(feature = "graphql")]use soup_realtime::{domain::service::{SoupRealtimeConsumerService, SoupRealtimeServiceImpl},outbound::{@@ -336,7 +334,6 @@ async fn main() -> anyhow::Result<()> {let notification_ingress_service = Arc::new(SqsNotificationIngress {queue: ingress_queue.clone(),});- #[cfg(feature = "graphql")]let graphql_notification_reader: Arc<ai_tools::ToolNotificationService> =Arc::new(NotificationReaderService {repository: notification::outbound::repository::DbNotificationRepository::new(@@ -862,13 +859,11 @@ async fn main() -> anyhow::Result<()> {foreign_entity_service_for_soup,));- #[cfg(feature = "graphql")]let soup_realtime_service = Arc::new(SoupRealtimeConsumerService::new(SoupTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")})?,));- #[cfg(feature = "graphql")]tokio::spawn({let soup_realtime_service = Arc::clone(&soup_realtime_service);async move {@@ -884,7 +879,6 @@ async fn main() -> anyhow::Result<()> {}});- #[cfg(feature = "graphql")]tokio::spawn({let brokers = config.kafka_brokers.as_ref().to_string();let entity_access_service = entity_access_service.as_ref().clone();@@ -934,12 +928,10 @@ async fn main() -> anyhow::Result<()> {authorization_state.clone(),),favorites_service,- #[cfg(feature = "graphql")]graphql_soup_schema: complete_graph::build_schema_from_arcs(soup_service,soup_realtime_service,),- #[cfg(feature = "graphql")]graphql_notification_reader,github_sync_service: Arc::new(github_sync_service_impl),foreign_entity_state,