Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/chore/realtime pr feedback (#5134)

Sean Ayec0536bcd2b7e
diff --git a/Cargo.lock b/Cargo.lock
index 13ab36dc3f..25af734a1e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -13495,6 +13495,7 @@ dependencies = [
"serde_json",
"soup",
"tokio",
+ "tokio-retry",
"tracing",
"uuid",
"workspace-hack",
@@ -14447,6 +14448,16 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "tokio-retry"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "tokio-rustls"
version = "0.24.1"
diff --git a/Cargo.toml b/Cargo.toml
index f94b60d4bf..7fd9c994fe 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -223,6 +223,7 @@ strum = { version = "0.27.1", features = ["derive"] }
thiserror = "2.0.0"
time = "0.3.37"
tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread"] }
+tokio-retry = { version = "0.3.2", default-features = false }
tokio-stream = { version = "0.1.17" }
tokio-tungstenite = { version = "0.28", default-features = false, features = [
"connect",
diff --git a/crates/soup_realtime/Cargo.toml b/crates/soup_realtime/Cargo.toml
index 328f230c50..692f4f1686 100644
--- a/crates/soup_realtime/Cargo.toml
+++ b/crates/soup_realtime/Cargo.toml
@@ -24,6 +24,7 @@ ports = [
"dep:futures",
"dep:rootcause",
"dep:tokio",
+ "dep:tokio-retry",
"dep:tracing",
]
@@ -43,6 +44,7 @@ rootcause = { workspace = true, optional = true }
serde = { workspace = true }
soup = { path = "../soup", default-features = false, features = ["ports"], optional = true }
tokio = { workspace = true, features = ["sync", "time"], optional = true }
+tokio-retry = { workspace = true, optional = true }
tracing = { workspace = true, optional = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rs
index cdf613cf0f..dcc446ec28 100644
--- a/crates/soup_realtime/src/domain/service.rs
+++ b/crates/soup_realtime/src/domain/service.rs
@@ -3,7 +3,7 @@
#[cfg(test)]
mod test;
-use std::{collections::HashSet, num::NonZeroUsize, sync::Arc};
+use std::{collections::HashSet, num::NonZeroUsize, sync::Arc, time::Duration};
use broadcast::{BroadcastManager, GlobalSpawner};
use futures::{StreamExt as _, stream};
@@ -11,6 +11,7 @@ use macro_user_id::user_id::MacroUserIdStr;
use model_entity::Entity;
use models_soup::item::SoupItem;
use rootcause::prelude::{Report, ResultExt as _};
+use tokio_retry::{Retry, strategy::ExponentialBackoff};
use super::{
models::SoupRealtimeMessage,
@@ -24,6 +25,15 @@ use super::{
const BROADCAST_BUFFER_CAPACITY: NonZeroUsize = NonZeroUsize::new(64).unwrap();
/// Number of messages buffered for each individual subscriber.
const SUBSCRIBER_BUFFER_CAPACITY: NonZeroUsize = NonZeroUsize::new(16).unwrap();
+/// Total receive attempts before the consumer returns for supervision.
+const MAX_RECEIVE_ATTEMPTS: usize = 5;
+
+/// Retries after one, two, four, and eight seconds.
+fn receive_retry_strategy() -> impl Iterator<Item = Duration> {
+ ExponentialBackoff::from_millis(2)
+ .factor(500)
+ .take(MAX_RECEIVE_ATTEMPTS - 1)
+}
/// Service for distributing recipient-targeted realtime Soup messages.
pub struct SoupRealtimeConsumerService<C>
@@ -66,11 +76,14 @@ where
#[tracing::instrument(skip(self), err)]
pub async fn run(&self) -> Result<(), Report> {
loop {
- let SoupRealtimeMessage { user_id, item } = self
- .consumer
- .recv()
- .await
- .context("failed to receive realtime Soup message")?;
+ let SoupRealtimeMessage { user_id, item } = Retry::start(
+ receive_retry_strategy(),
+ || self.consumer.recv(),
+ )
+ .await
+ .context(format!(
+ "failed to receive realtime Soup message after {MAX_RECEIVE_ATTEMPTS} attempts"
+ ))?;
match self.broadcasts.publish(&user_id, Arc::new(item)) {
Ok(subscriber_count) => {
diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rs
index 6829ef355c..e71cc0e645 100644
--- a/crates/soup_realtime/src/domain/service/test.rs
+++ b/crates/soup_realtime/src/domain/service/test.rs
@@ -71,10 +71,12 @@ impl SoupItemReader for FakeReader {
struct FakeRealtimeConsumer {
messages: Mutex<VecDeque<Result<SoupRealtimeMessage, Report>>>,
+ calls: Arc<AtomicUsize>,
}
impl SoupRealtimeConsumer for FakeRealtimeConsumer {
async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {
+ self.calls.fetch_add(1, Ordering::SeqCst);
self.messages
.lock()
.expect("consumer messages lock")
@@ -189,12 +191,14 @@ fn item_response(
)
}
-#[tokio::test]
+#[tokio::test(start_paused = true)]
async fn consumer_service_distributes_items_only_to_their_users() {
let one = user("one");
let two = user("two");
+ let receive_calls = Arc::new(AtomicUsize::new(0));
let consumer = FakeRealtimeConsumer {
messages: Mutex::new(VecDeque::from([
+ Err(rootcause::report!("transient receive failure")),
Ok(SoupRealtimeMessage::new(
one.clone(),
document_item(DOCUMENT_ID, "For one", None),
@@ -204,6 +208,7 @@ async fn consumer_service_distributes_items_only_to_their_users() {
document_item(OTHER_DOCUMENT_ID, "For two", None),
)),
])),
+ calls: Arc::clone(&receive_calls),
};
let service = Arc::new(SoupRealtimeConsumerService::new(consumer));
let mut one_first = service.subscribe(one.clone());
@@ -218,6 +223,11 @@ async fn consumer_service_distributes_items_only_to_their_users() {
.expect("consumer task joins")
.expect_err("fake consumer eventually stops");
assert!(run_error.to_string().contains("failed to receive"));
+ assert_eq!(
+ receive_calls.load(Ordering::SeqCst),
+ 3 + MAX_RECEIVE_ATTEMPTS,
+ "a successful message resets the receive retry strategy"
+ );
let one_first_item = tokio::time::timeout(Duration::from_secs(1), one_first.recv())
.await
diff --git a/crates/soup_realtime/src/outbound/soup_consumer.rs b/crates/soup_realtime/src/outbound/soup_consumer.rs
index 6199e08029..db6848c3f8 100644
--- a/crates/soup_realtime/src/outbound/soup_consumer.rs
+++ b/crates/soup_realtime/src/outbound/soup_consumer.rs
@@ -78,6 +78,7 @@ impl SoupTopicConsumer {
}
impl SoupRealtimeConsumer for SoupTopicConsumer {
+ #[tracing::instrument(err, skip(self))]
async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {
SoupTopicConsumer::recv(self).await
}
diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.toml
index a43ac39d05..6bb533f996 100644
--- a/services/document_storage_service/Cargo.toml
+++ b/services/document_storage_service/Cargo.toml
@@ -20,19 +20,8 @@ path = "src/doppler_config.rs"
[features]
connection_gateway = []
-default = [
- "connection_gateway",
- "delete_document_worker",
- "graphql",
- "location_check",
-]
+default = ["connection_gateway", "delete_document_worker", "location_check"]
delete_document_worker = []
-graphql = [
- "dep:async-graphql",
- "dep:async-graphql-axum",
- "dep:complete_graph",
- "dep:soup_realtime",
-]
local = ["local_auth"]
local_auth = ["macro_authorization/local_auth"]
location_check = []
@@ -42,8 +31,8 @@ ai_tools = { path = "../../crates/ai_tools" }
ai_usage = { path = "../../crates/ai_usage" }
analytics_client = { path = "../../crates/analytics_client" }
anyhow = { workspace = true }
-async-graphql = { workspace = true, optional = true }
-async-graphql-axum = { version = "7.0.17", optional = true }
+async-graphql = { workspace = true }
+async-graphql-axum = "7.0.17"
aws-sdk-dynamodb = { workspace = true }
aws-sdk-s3 = { workspace = true }
aws-sdk-secretsmanager = { workspace = true }
@@ -73,7 +62,7 @@ chrono = { workspace = true }
clap = { workspace = true }
cloudfront_sign = { workspace = true }
comms_db_client = { path = "../../crates/comms_db_client" }
-complete_graph = { path = "../../crates/complete_graph", optional = true }
+complete_graph = { path = "../../crates/complete_graph" }
connection = { path = "../../crates/connection", default-features = false, features = [
"outbound",
] }
@@ -115,7 +104,9 @@ lazy_static = { workspace = true }
lexical_client = { path = "../../crates/lexical_client" }
lexical_mention_extractor = { path = "../../crates/lexical_mention_extractor" }
macro_auth = { path = "../../crates/macro_auth" }
-macro_authorization = { path = "../../crates/macro_authorization", features = ["postgres"] }
+macro_authorization = { path = "../../crates/macro_authorization", features = [
+ "postgres",
+] }
macro_aws_config = { path = "../../crates/macro_aws_config", features = ["s3"] }
macro_axum_utils = { path = "../../crates/macro_axum_utils" }
macro_config = { path = "../../crates/macro_config" }
@@ -181,7 +172,7 @@ soup = { path = "../../crates/soup", features = [
soup_realtime = { path = "../../crates/soup_realtime", default-features = false, features = [
"consumer",
"outbound",
-], optional = true }
+] }
sqlx = { workspace = true }
sqs_client = { path = "../../crates/sqs_client", default-features = false, features = [
"document",
diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rs
index 1c6c1df6bf..960c06e9d5 100644
--- a/services/document_storage_service/src/api/context.rs
+++ b/services/document_storage_service/src/api/context.rs
@@ -95,7 +95,6 @@ use soup::{
domain::service::SoupImpl, inbound::axum_router::SoupRouterState,
outbound::pg_soup_repo::PgSoupRepo,
};
-#[cfg(feature = "graphql")]
use soup_realtime::{
domain::service::SoupRealtimeConsumerService, outbound::soup_consumer::SoupTopicConsumer,
};
@@ -150,13 +149,11 @@ type DssSoupState =
SoupRouterState<DssSoupService, DssEmailService, EntityAccessService, AuthorizationService>;
/// Realtime Soup consumer service used by GraphQL subscriptions.
-#[cfg(feature = "graphql")]
pub(crate) type DssSoupRealtimeService = SoupRealtimeConsumerService<SoupTopicConsumer>;
/// GraphQL Soup schema wired to the DSS services; the `ApiContext` state
/// parameter lets GraphQL resolvers run the same axum extractors as the REST
/// routes, lazily, against the stored request parts.
-#[cfg(feature = "graphql")]
pub(crate) type DssGraphqlSoupSchema = complete_graph::SharedSoupSchema<
DssSoupService,
DssSoupRealtimeService,
@@ -419,9 +416,7 @@ pub(crate) struct ApiContext {
pub dynamodb_client: Arc<DynamodbClient>,
pub dynamo_db: aws_sdk_dynamodb::Client,
pub soup_router_state: DssSoupState,
- #[cfg(feature = "graphql")]
pub graphql_soup_schema: DssGraphqlSoupSchema,
- #[cfg(feature = "graphql")]
pub graphql_notification_reader: Arc<ai_tools::ToolNotificationService>,
pub favorites_state: DssFavoritesState,
pub favorites_service: Arc<FavoritesServiceType>,
diff --git a/services/document_storage_service/src/api/mod.rs b/services/document_storage_service/src/api/mod.rs
index 72afafaaba..c04d541d47 100644
--- a/services/document_storage_service/src/api/mod.rs
+++ b/services/document_storage_service/src/api/mod.rs
@@ -29,7 +29,6 @@ mod middleware;
mod activity;
mod annotations;
mod documents;
-#[cfg(feature = "graphql")]
mod graphql_soup;
mod health;
mod history;
@@ -82,10 +81,8 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {
}
fn items_router(state: ApiContext) -> Router<ApiContext> {
- let router = soup::inbound::axum_router::soup_router(state.soup_router_state.clone());
- #[cfg(feature = "graphql")]
- let router = router.merge(graphql_soup::router());
- router
+ soup::inbound::axum_router::soup_router(state.soup_router_state.clone())
+ .merge(graphql_soup::router())
}
fn api_router(state: ApiContext) -> Router {
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index 1e315e13ad..cb18d639e9 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -90,7 +90,6 @@ use macro_service_urls::{
};
use macro_sha_count_client::Redis;
use notification::domain::service::SqsNotificationIngress;
-#[cfg(feature = "graphql")]
use notification::domain::service::{NotificationReaderService, PlatformArnConfig};
use notification::outbound::queue::SqsQueue;
use opensearch_client::OpensearchClient;
@@ -111,7 +110,6 @@ use soup::{
domain::service::SoupImpl, inbound::axum_router::SoupRouterState,
outbound::pg_soup_repo::PgSoupRepo,
};
-#[cfg(feature = "graphql")]
use soup_realtime::{
domain::service::{SoupRealtimeConsumerService, SoupRealtimeServiceImpl},
outbound::{
@@ -336,7 +334,6 @@ async fn main() -> anyhow::Result<()> {
let notification_ingress_service = Arc::new(SqsNotificationIngress {
queue: ingress_queue.clone(),
});
- #[cfg(feature = "graphql")]
let graphql_notification_reader: Arc<ai_tools::ToolNotificationService> =
Arc::new(NotificationReaderService {
repository: notification::outbound::repository::DbNotificationRepository::new(
@@ -862,13 +859,11 @@ async fn main() -> anyhow::Result<()> {
foreign_entity_service_for_soup,
));
- #[cfg(feature = "graphql")]
let soup_realtime_service = Arc::new(SoupRealtimeConsumerService::new(
SoupTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {
anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")
})?,
));
- #[cfg(feature = "graphql")]
tokio::spawn({
let soup_realtime_service = Arc::clone(&soup_realtime_service);
async move {
@@ -884,7 +879,6 @@ async fn main() -> anyhow::Result<()> {
}
});
- #[cfg(feature = "graphql")]
tokio::spawn({
let brokers = config.kafka_brokers.as_ref().to_string();
let entity_access_service = entity_access_service.as_ref().clone();
@@ -934,12 +928,10 @@ async fn main() -> anyhow::Result<()> {
authorization_state.clone(),
),
favorites_service,
- #[cfg(feature = "graphql")]
graphql_soup_schema: complete_graph::build_schema_from_arcs(
soup_service,
soup_realtime_service,
),
- #[cfg(feature = "graphql")]
graphql_notification_reader,
github_sync_service: Arc::new(github_sync_service_impl),
foreign_entity_state,