Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/feat/expose soup subscription (#5118)
Sean Ayec6754bbb4c79
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 05eff0dcde..0ddc1ffc31 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -927,6 +927,7 @@"crates/attachment","crates/authentication_service_client","crates/bot_id",+ "crates/broadcast","crates/call","crates/channel_sender","crates/channels",@@ -938,9 +939,11 @@"crates/cowlike","crates/crm","crates/document_sub_type",+ "crates/documents","crates/email","crates/email_db_client","crates/email_utils",+ "crates/embedding","crates/entity_access","crates/entity_access_db_utils","crates/entity_access_management",@@ -956,14 +959,17 @@"crates/item_filters","crates/kafka_util","crates/last_online_tracker",+ "crates/lexical_client","crates/macro_auth","crates/macro_authorization","crates/macro_aws_config",+ "crates/macro_entrypoint","crates/macro_env","crates/macro_env_var","crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sync_service_jwt","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -978,23 +984,33 @@"crates/models_email","crates/models_entity_access_management","crates/models_grouping",+ "crates/models_opensearch","crates/models_pagination","crates/models_permissions","crates/models_properties",+ "crates/models_search",+ "crates/models_search_cursor","crates/models_soup","crates/non_empty","crates/notification",+ "crates/opensearch_client",+ "crates/opensearch_query_builder","crates/prompt","crates/properties","crates/rate_limit","crates/readonly_pool","crates/remote_env_var","crates/s3_key",+ "crates/secretsmanager_client","crates/share_permission_db_utils","crates/soup",+ "crates/soup_realtime","crates/sqs_client","crates/sqs_worker",+ "crates/sync_service",+ "crates/sync_service_client","crates/system_properties",+ "crates/task_dedup","crates/unfurl","crates/workspace-hack"],@@ -1490,6 +1506,7 @@"crates/authentication_service_client","crates/bot_id","crates/bots",+ "crates/broadcast","crates/cal","crates/call","crates/channel_bots",@@ -1597,6 +1614,7 @@"crates/secretsmanager_client","crates/share_permission_db_utils","crates/soup",+ "crates/soup_realtime","crates/sqs_client","crates/sqs_worker","crates/sync_service",@@ -2767,6 +2785,7 @@"crates/attachment","crates/authentication_service_client","crates/bot_id",+ "crates/broadcast","crates/call","crates/channel_sender","crates/channels",@@ -2777,9 +2796,11 @@"crates/cowlike","crates/crm","crates/document_sub_type",+ "crates/documents","crates/email","crates/email_db_client","crates/email_utils",+ "crates/embedding","crates/entity_access","crates/entity_access_db_utils","crates/entity_access_management",@@ -2792,14 +2813,17 @@"crates/item_filters","crates/kafka_util","crates/last_online_tracker",+ "crates/lexical_client","crates/macro_auth","crates/macro_authorization","crates/macro_aws_config",+ "crates/macro_entrypoint","crates/macro_env","crates/macro_env_var","crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sync_service_jwt","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -2814,23 +2838,33 @@"crates/models_email","crates/models_entity_access_management","crates/models_grouping",+ "crates/models_opensearch","crates/models_pagination","crates/models_permissions","crates/models_properties",+ "crates/models_search",+ "crates/models_search_cursor","crates/models_soup","crates/non_empty","crates/notification",+ "crates/opensearch_client",+ "crates/opensearch_query_builder","crates/prompt","crates/properties","crates/rate_limit","crates/readonly_pool","crates/remote_env_var","crates/s3_key",+ "crates/secretsmanager_client","crates/share_permission_db_utils","crates/soup",+ "crates/soup_realtime","crates/sqs_client","crates/sqs_worker",+ "crates/sync_service",+ "crates/sync_service_client","crates/system_properties",+ "crates/task_dedup","crates/unfurl","crates/workspace-hack"],@@ -5081,6 +5115,7 @@"crates/attachment","crates/authentication_service_client","crates/bot_id",+ "crates/broadcast","crates/call","crates/channel_sender","crates/channels",diff --git a/Cargo.lock b/Cargo.lockindex 38e5eb3cc7..3cbc9982ac 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2897,6 +2897,7 @@ dependencies = ["rootcause","serde","soup",+ "soup_realtime","tokio","uuid","workspace-hack",@@ -4317,6 +4318,7 @@ dependencies = ["serde","serde_json","soup",+ "soup_realtime","sqlx","sqs_client","sqs_worker",@@ -6063,6 +6065,7 @@ name = "graphql_soup"version = "0.1.0"dependencies = ["async-graphql",+ "async-stream","axum","axum-extra","chrono",@@ -6070,6 +6073,7 @@ dependencies = ["email","entity_access","filter_ast",+ "futures","graphql_common","item_filters","macro_authorization",@@ -6082,6 +6086,7 @@ dependencies = ["serde","serde_json","soup",+ "soup_realtime","uuid","workspace-hack",]@@ -13467,6 +13472,7 @@ dependencies = [name = "soup_realtime"version = "0.1.0"dependencies = [+ "broadcast","chrono","documents","entity_access",diff --git a/crates/complete_graph/Cargo.toml b/crates/complete_graph/Cargo.tomlindex ffe1ee266b..9df69def8c 100644--- a/crates/complete_graph/Cargo.toml+++ b/crates/complete_graph/Cargo.toml@@ -25,14 +25,17 @@ macro_authorization = { path = "../macro_authorization", features = ["axum","outbound",] }+macro_user_id = { path = "../macro_user_id" }model-entity = { path = "../model-entity" }soup = { path = "../soup", default-features = false, features = ["ports"] }+soup_realtime = { path = "../soup_realtime", default-features = false, features = [+ "ports",+] }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]apollo-compiler = { workspace = true }-macro_user_id = { path = "../macro_user_id" }model_user = { path = "../model_user" }models_pagination = { path = "../models_pagination" }models_properties = { path = "../models_properties" }diff --git a/crates/complete_graph/src/lib.rs b/crates/complete_graph/src/lib.rsindex 777ceae15b..698ef07a01 100644--- a/crates/complete_graph/src/lib.rs+++ b/crates/complete_graph/src/lib.rs@@ -12,7 +12,7 @@ mod schema;mod sdl_test;pub use edges::{SoupEdges, SoupEmailThreadEdges};-pub use graphql_common::GraphqlSoupRequestParts;+pub use graphql_common::GraphqlRequestParts;pub use graphql_email::{EmailContentKey, EmailContentLoad, EmailContentLoader, EmailServiceEmailContentReader,NoOpSoupEmailContentEdgeReader, SoupEmailContentEdgeReader, email_content_loader,@@ -27,5 +27,6 @@ pub use graphql_properties::{};pub use schema::{SchemaOnlySoupSchema, SchemaOnlyState, SharedSoupSchema, SoupQueryRoot, SoupSchema,- build_schema, build_schema_from_arc, build_schema_with_service,+ SoupSubscriptionRoot, build_schema, build_schema_from_arc, build_schema_from_arcs,+ build_schema_with_service, build_schema_with_services,};diff --git a/crates/complete_graph/src/schema.rs b/crates/complete_graph/src/schema.rsindex fd7d430498..71649b895c 100644--- a/crates/complete_graph/src/schema.rs+++ b/crates/complete_graph/src/schema.rs@@ -3,7 +3,7 @@ mod test;use std::{marker::PhantomData, sync::Arc};-use async_graphql::{Context, EmptySubscription, Object, Schema};+use async_graphql::{Context, Object, Schema, Subscription};use axum::extract::FromRef;use email::{domain::ports::{EmailService, NoOpEmailService},@@ -18,36 +18,43 @@ use graphql_properties::{PropertiesMutationRoot,};use graphql_soup::{- GroupedSoup, GroupedSoupInput, SoupInput, SoupPage, resolve_grouped_soup, resolve_soup,+ GraphqlSoupItem, GroupedSoup, GroupedSoupInput, SoupInput, SoupPage, resolve_grouped_soup,+ resolve_soup, resolve_soup_updates,};use macro_authorization::{InternalAuthConfig, MacroAuthorizationService, MacroAuthorizationServiceImpl,MacroAuthorizationState, NoopMacroAuthJwtValidator,};+use macro_user_id::user_id::MacroUserIdStr;use soup::domain::ports::{NoOpSoupService, SoupService};+use soup_realtime::domain::ports::{+ NoOpSoupRealtimeSubscriptionService, SoupRealtimeSubscriptionService,+};use crate::SoupEdges;/// GraphQL Soup schema type.///-/// `S` is the soup service, `E` the email service, `EAS` the entity access-/// service, `Auth` the authorization service, `St` the embedding axum router-/// state, `W` the property mutation writer, `NR` the notification edge reader,-/// `PR` the property edge reader, and `ER` the email-content edge reader.-pub type SoupSchema<S, E, EAS, Auth, St, W, NR, PR, ER> = Schema<+/// `S` is the soup query service, `R` the realtime subscription service, `E`+/// the email service, `EAS` the entity access service, `Auth` the authorization+/// service, `St` the embedding axum router state, `W` the property mutation+/// writer, `NR` the notification edge reader, `PR` the property edge reader,+/// and `ER` the email-content edge reader.+pub type SoupSchema<S, R, E, EAS, Auth, St, W, NR, PR, ER> = Schema<SoupQueryRoot<S, E, EAS, Auth, St, NR, PR, ER>,PropertiesMutationRoot<W>,- EmptySubscription,+ SoupSubscriptionRoot<R, Auth, St, NR, PR, ER>,>;-/// GraphQL Soup schema type backed by a shared soup service.-pub type SharedSoupSchema<S, E, EAS, Auth, St, W, NR, PR, ER> =- SoupSchema<Arc<S>, E, EAS, Auth, St, W, NR, PR, ER>;+/// GraphQL Soup schema type backed by shared query and realtime services.+pub type SharedSoupSchema<S, R, E, EAS, Auth, St, W, NR, PR, ER> =+ SoupSchema<Arc<S>, Arc<R>, E, EAS, Auth, St, W, NR, PR, ER>;/// GraphQL Soup schema type backed by the no-op services, used only for/// SDL export or introspection.pub type SchemaOnlySoupSchema = SoupSchema<NoOpSoupService,+ NoOpSoupRealtimeSubscriptionService,NoOpEmailService,NoOpEntityAccessService,SchemaOnlyAuthorizationService,@@ -113,6 +120,25 @@ impl<S, E, EAS, Auth, St, NR, PR, ER> SoupQueryRoot<S, E, EAS, Auth, St, NR, PR,}}+/// Root GraphQL subscription object for realtime Soup updates.+pub struct SoupSubscriptionRoot<R, Auth, St, NR, PR, ER> {+ /// Realtime Soup service used by user-scoped subscriptions.+ service: R,+ /// Associates the root with authorization, state, and edge reader types.+ #[allow(clippy::type_complexity)]+ _marker: PhantomData<fn() -> (Auth, St, NR, PR, ER)>,+}++impl<R, Auth, St, NR, PR, ER> SoupSubscriptionRoot<R, Auth, St, NR, PR, ER> {+ /// Creates a root GraphQL subscription object.+ pub fn new(service: R) -> Self {+ Self {+ service,+ _marker: PhantomData,+ }+ }+}+/// The authenticated user (viewer). All user-scoped data hangs off this/// object so clients (and their normalized caches) observe data ownership/// structurally rather than implicitly through the session.@@ -121,17 +147,19 @@ pub struct GraphqlUser<S, E, EAS, Auth, St, NR, PR, ER> {service: S,/// Associates the user object with its adapter and reader types._marker: ServicesMarker<E, EAS, Auth, St, NR, PR, ER>,+ /// The [MacroUserIdStr] of the resolving user+ user_id: MacroUserIdStr<'static>,}/// Build a GraphQL schema for Soup suitable for SDL export or introspection.pub fn build_schema() -> SchemaOnlySoupSchema {- build_schema_with_service(NoOpSoupService)+ build_schema_with_services(NoOpSoupService, NoOpSoupRealtimeSubscriptionService)}-/// Build a GraphQL schema for Soup backed by the provided service.+/// Build a GraphQL schema backed by a query service and no-op realtime service.pub fn build_schema_with_service<S, E, EAS, Auth, St, W, NR, PR, ER>(service: S,-) -> SoupSchema<S, E, EAS, Auth, St, W, NR, PR, ER>+) -> SoupSchema<S, NoOpSoupRealtimeSubscriptionService, E, EAS, Auth, St, W, NR, PR, ER>whereS: SoupService + Clone,E: EmailService,@@ -145,19 +173,42 @@ whereNR: SoupNotificationEdgeReader,PR: EntityPropertyReader,ER: SoupEmailContentEdgeReader,+{+ build_schema_with_services(service, NoOpSoupRealtimeSubscriptionService)+}++/// Build a GraphQL schema backed by query and realtime Soup services.+pub fn build_schema_with_services<S, R, E, EAS, Auth, St, W, NR, PR, ER>(+ service: S,+ realtime_service: R,+) -> SoupSchema<S, R, E, EAS, Auth, St, W, NR, PR, ER>+where+ S: SoupService + Clone,+ R: SoupRealtimeSubscriptionService + Clone,+ E: EmailService,+ EAS: EntityAccessService,+ Auth: MacroAuthorizationService,+ St: Clone + Send + Sync + 'static,+ EmailRouterState<E>: FromRef<St>,+ Arc<EAS>: FromRef<St>,+ MacroAuthorizationState<Auth>: FromRef<St>,+ W: EntityPropertyWriter,+ NR: SoupNotificationEdgeReader,+ PR: EntityPropertyReader,+ ER: SoupEmailContentEdgeReader,{Schema::build(SoupQueryRoot::new(service),PropertiesMutationRoot::<W>::new(),- EmptySubscription,+ SoupSubscriptionRoot::new(realtime_service),).finish()}-/// Build a GraphQL schema for Soup backed by an `Arc`-shared service.+/// Build a GraphQL schema backed by an `Arc`-shared query service.pub fn build_schema_from_arc<S, E, EAS, Auth, St, W, NR, PR, ER>(service: Arc<S>,-) -> SharedSoupSchema<S, E, EAS, Auth, St, W, NR, PR, ER>+) -> SoupSchema<Arc<S>, NoOpSoupRealtimeSubscriptionService, E, EAS, Auth, St, W, NR, PR, ER>whereS: SoupService,E: EmailService,@@ -175,6 +226,29 @@ wherebuild_schema_with_service(service)}+/// Build a GraphQL schema backed by `Arc`-shared query and realtime services.+pub fn build_schema_from_arcs<S, R, E, EAS, Auth, St, W, NR, PR, ER>(+ service: Arc<S>,+ realtime_service: Arc<R>,+) -> SharedSoupSchema<S, R, E, EAS, Auth, St, W, NR, PR, ER>+where+ S: SoupService,+ R: SoupRealtimeSubscriptionService,+ E: EmailService,+ EAS: EntityAccessService,+ Auth: MacroAuthorizationService,+ St: Clone + Send + Sync + 'static,+ EmailRouterState<E>: FromRef<St>,+ Arc<EAS>: FromRef<St>,+ MacroAuthorizationState<Auth>: FromRef<St>,+ W: EntityPropertyWriter,+ NR: SoupNotificationEdgeReader,+ PR: EntityPropertyReader,+ ER: SoupEmailContentEdgeReader,+{+ build_schema_with_services(service, realtime_service)+}+/// Root entry point for the complete GraphQL API.#[Object]impl<S, E, EAS, Auth, St, NR, PR, ER> SoupQueryRoot<S, E, EAS, Auth, St, NR, PR, ER>@@ -192,11 +266,40 @@ whereER: SoupEmailContentEdgeReader,{/// The authenticated user.- async fn user(&self) -> GraphqlUser<S, E, EAS, Auth, St, NR, PR, ER> {- GraphqlUser {+ async fn user(+ &self,+ ctx: &Context<'_>,+ ) -> async_graphql::Result<GraphqlUser<S, E, EAS, Auth, St, NR, PR, ER>> {+ let user_id = require_authorized_user::<Auth, St>(ctx).await?;++ Ok(GraphqlUser {service: self.service.clone(),_marker: PhantomData,- }+ user_id,+ })+ }+}++/// Root entry point for realtime Soup subscriptions.+#[Subscription]+impl<R, Auth, St, NR, PR, ER> SoupSubscriptionRoot<R, Auth, St, NR, PR, ER>+where+ R: SoupRealtimeSubscriptionService,+ Auth: MacroAuthorizationService,+ St: Clone + Send + Sync + 'static,+ MacroAuthorizationState<Auth>: FromRef<St>,+ NR: SoupNotificationEdgeReader,+ PR: EntityPropertyReader,+ ER: SoupEmailContentEdgeReader,+{+ /// Subscribe to realtime Soup updates for the authenticated user.+ async fn soup_updates(+ &self,+ ctx: &Context<'_>,+ ) -> async_graphql::Result<+ impl async_graphql::futures_util::Stream<Item = GraphqlSoupItem<SoupEdges<NR, PR, ER>>>,+ > {+ resolve_soup_updates::<R, Auth, St, SoupEdges<NR, PR, ER>>(&self.service, ctx).await}}@@ -217,9 +320,8 @@ whereER: SoupEmailContentEdgeReader,{/// Stable id of the authenticated user.- async fn id(&self, ctx: &Context<'_>) -> async_graphql::Result<async_graphql::ID> {- let macro_user_id = require_authorized_user::<Auth, St>(ctx).await?;- Ok(async_graphql::ID(macro_user_id.to_string()))+ async fn id(&self) -> async_graphql::ID {+ async_graphql::ID(self.user_id.to_string())}/// Fetch Soup items nested into grouping bins.diff --git a/crates/complete_graph/src/schema/test.rs b/crates/complete_graph/src/schema/test.rsindex f13c3bfb21..da88994568 100644--- a/crates/complete_graph/src/schema/test.rs+++ b/crates/complete_graph/src/schema/test.rs@@ -1,3 +1,4 @@+use std::sync::Mutex;use std::sync::atomic::{AtomicUsize, Ordering};use axum::http::{Request as HttpRequest, header};@@ -10,7 +11,7 @@ use entity_access::domain::models::{AccessError, AccessLevel, BotId, CallChannelInfo, EditAccessLevel, EntityAccessReceipt,EntityPermission, EntityType, RequiredPermission, TeamRole, UserTeamInfo, ViewAccessLevel,};-use graphql_common::GraphqlSoupRequestParts;+use graphql_common::GraphqlRequestParts;use macro_authorization::{INTERNAL_API_KEY_HEADER, INTERNAL_MACRO_USER_ID_HEADER, InternalIdentityClaims,MacroAuthorizationError, MacroAuthorizationService, MacroAuthorizationState,@@ -31,6 +32,27 @@ const VALID_USER_ID: &str = "macro|[email protected]";const INTERNAL_USER_ID: &str = "macro|[email protected]";const VALID_INTERNAL_KEY: &str = "valid-internal-key";+#[derive(Clone)]+struct TestRealtimeSubscriptionService {+ #[allow(clippy::type_complexity)]+ receiver: Arc<Mutex<Option<tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>>>>>,+ subscribed_user: Arc<Mutex<Option<MacroUserIdStr<'static>>>>,+}++impl SoupRealtimeSubscriptionService for TestRealtimeSubscriptionService {+ fn subscribe(+ &self,+ user_id: MacroUserIdStr<'static>,+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {+ *self.subscribed_user.lock().expect("subscribed user lock") = Some(user_id);+ self.receiver+ .lock()+ .expect("subscription receiver lock")+ .take()+ .expect("test subscription is opened once")+ }+}+#[derive(Clone, Default)]struct CountingSoupService {raw_calls: Arc<AtomicUsize>,@@ -472,6 +494,7 @@ impl FromRef<TestState> for Arc<CountingEntityAccessService> {struct TestHarness {schema: SoupSchema<CountingSoupService,+ NoOpSoupRealtimeSubscriptionService,CountingEmailService,CountingEntityAccessService,FakeAuthorizationService,@@ -555,12 +578,66 @@ impl TestHarness {parts: axum::http::request::Parts,) -> async_graphql::Response {let request = async_graphql::Request::new(query)- .data(GraphqlSoupRequestParts::new(parts))+ .data(GraphqlRequestParts::new(parts)).data(self.state.clone());self.schema.execute(request).await}}+#[tokio::test]+async fn soup_updates_subscribes_as_the_authenticated_user() {+ use async_graphql::futures_util::{StreamExt as _, pin_mut};++ let user_id = MacroUserIdStr::parse_from_str(VALID_USER_ID).unwrap();+ let subscribed_user = Arc::new(Mutex::new(None));+ let (sender, receiver) = tokio::sync::mpsc::channel(1);+ let realtime = TestRealtimeSubscriptionService {+ receiver: Arc::new(Mutex::new(Some(receiver))),+ subscribed_user: Arc::clone(&subscribed_user),+ };+ let schema: SoupSchema<+ NoOpSoupService,+ TestRealtimeSubscriptionService,+ NoOpEmailService,+ NoOpEntityAccessService,+ SchemaOnlyAuthorizationService,+ SchemaOnlyState,+ NoOpEntityPropertyWriter,+ NoOpSoupNotificationEdgeReader,+ NoOpEntityPropertyReader,+ NoOpSoupEmailContentEdgeReader,+ > = build_schema_with_services(NoOpSoupService, realtime);+ let request = async_graphql::Request::new(+ "subscription { soupUpdates { id entityType entity { __typename ... on GraphqlSoupDocument { id name } } } }",+ )+ .data(user_id.clone());+ let responses = schema.execute_stream(request);+ pin_mut!(responses);++ let document_id = Uuid::from_u128(42);+ sender+ .send(Arc::new(grouped_document(document_id).map_extra(|_| ())))+ .await+ .expect("subscription remains open");+ let response = responses.next().await.expect("one subscription response");++ assert!(response.errors.is_empty(), "{:?}", response.errors);+ let data = response.data.into_json().expect("response data is JSON");+ assert_eq!(data["soupUpdates"]["id"], document_id.to_string());+ assert_eq!(data["soupUpdates"]["entityType"], "DOCUMENT");+ assert_eq!(+ data["soupUpdates"]["entity"]["name"],+ format!("Document {document_id}")+ );+ assert_eq!(+ subscribed_user+ .lock()+ .expect("subscribed user lock")+ .as_ref(),+ Some(&user_id)+ );+}+#[tokio::test]async fn user_id_resolves_without_touching_services() {let harness = harness();diff --git a/crates/complete_graph/src/sdl_test.rs b/crates/complete_graph/src/sdl_test.rsindex a232e557e8..b37a7e8ed0 100644--- a/crates/complete_graph/src/sdl_test.rs+++ b/crates/complete_graph/src/sdl_test.rs@@ -34,6 +34,8 @@ fn soup_response_schema_exposes_frontend_fields() {"type GraphqlSoupChat {","deletedAt: String","type GraphqlSoupProject {",+ "type SoupSubscriptionRoot {",+ "soupUpdates: GraphqlSoupItem!",] {assert_sdl_line(&sdl, expected);}diff --git a/crates/graphql_common/src/authorization.rs b/crates/graphql_common/src/authorization.rsindex 4812f9d478..ec8a487184 100644--- a/crates/graphql_common/src/authorization.rs+++ b/crates/graphql_common/src/authorization.rs@@ -21,6 +21,10 @@ whereMacroAuthorizationState<Auth>: FromRef<St>,St: Clone + Send + Sync + 'static,{+ if let Some(user_id) = ctx.data_opt::<MacroUserIdStr<'static>>() {+ return Ok(user_id.clone());+ }+let Cached(authorization) =extract_part::<Cached<OptionalMacroAuthorizationExtractor<Auth>>, St>(ctx).await?;diff --git a/crates/graphql_common/src/authorization/test.rs b/crates/graphql_common/src/authorization/test.rsindex c27815ad16..acb7080c6c 100644--- a/crates/graphql_common/src/authorization/test.rs+++ b/crates/graphql_common/src/authorization/test.rs@@ -16,7 +16,7 @@ use model_user::UserContext;use rootcause::Report;use super::*;-use crate::GraphqlSoupRequestParts;+use crate::GraphqlRequestParts;const VALID_USER_ID: &str = "macro|[email protected]";@@ -105,7 +105,7 @@ async fn execute(};let schema = Schema::build(TestQuery, EmptyMutation, EmptySubscription).finish();let request = async_graphql::Request::new(query)- .data(GraphqlSoupRequestParts::new(parts))+ .data(GraphqlRequestParts::new(parts)).data(state);schema.execute(request).awaitdiff --git a/crates/graphql_common/src/extract.rs b/crates/graphql_common/src/extract.rsindex 8de307a1aa..6ebfd2e166 100644--- a/crates/graphql_common/src/extract.rs+++ b/crates/graphql_common/src/extract.rs@@ -1,7 +1,7 @@use async_graphql::Context;use axum::extract::FromRequestParts;-use crate::request_context::GraphqlSoupRequestParts;+use crate::request_context::GraphqlRequestParts;/// Run an axum extractor against the request parts stored in the GraphQL/// context, using the embedding router state `St` also stored there.@@ -11,7 +11,7 @@ whereT::Rejection: std::fmt::Display,St: Clone + Send + Sync + 'static,{- let parts = ctx.data::<GraphqlSoupRequestParts>()?;+ let parts = ctx.data::<GraphqlRequestParts>()?;let state = ctx.data::<St>()?;parts.extract_with_state::<T, St>(state)diff --git a/crates/graphql_common/src/lib.rs b/crates/graphql_common/src/lib.rsindex eec3c05e39..3b6b013e29 100644--- a/crates/graphql_common/src/lib.rs+++ b/crates/graphql_common/src/lib.rs@@ -28,4 +28,4 @@ pub use property_filter::{GraphqlPropertiesBinaryExpr, GraphqlPropertiesExpr, GraphqlPropertiesLiteral,GraphqlPropertyEntityType, GraphqlPropertyMatchValue,};-pub use request_context::GraphqlSoupRequestParts;+pub use request_context::GraphqlRequestParts;diff --git a/crates/graphql_common/src/request_context.rs b/crates/graphql_common/src/request_context.rsindex 1ebfe0093e..a9aae30de7 100644--- a/crates/graphql_common/src/request_context.rs+++ b/crates/graphql_common/src/request_context.rs@@ -1,6 +1,6 @@use axum::{extract::FromRequestParts, http::request::Parts};-/// Request-scoped HTTP request parts for a Soup GraphQL query.+/// Request-scoped HTTP request parts for a Soup GraphQL operation.////// Resolvers run axum `FromRequestParts` extractors against these parts on/// demand instead of the embedding service loading everything upfront. This@@ -10,12 +10,12 @@ use axum::{extract::FromRequestParts, http::request::Parts};/// Extraction results that should be computed once per request should be/// wrapped in `axum_extra::extract::Cached`, which memoizes inside the parts'/// extensions exactly like the REST routes do.-pub struct GraphqlSoupRequestParts {+pub struct GraphqlRequestParts {/// Mutable HTTP request parts shared by lazy resolver extractors.parts: tokio::sync::Mutex<Parts>,}-impl GraphqlSoupRequestParts {+impl GraphqlRequestParts {/// Wrap the parts of the incoming HTTP request.pub fn new(parts: Parts) -> Self {Self {diff --git a/crates/graphql_soup/Cargo.toml b/crates/graphql_soup/Cargo.tomlindex b8e6e82229..056d76310b 100644--- a/crates/graphql_soup/Cargo.toml+++ b/crates/graphql_soup/Cargo.toml@@ -6,6 +6,8 @@ version = "0.1.0"[dependencies]async-graphql = { workspace = true, features = ["dataloader"] }+async-stream = { workspace = true }+futures = { workspace = true }axum = { workspace = true }axum-extra = { workspace = true }chrono = { workspace = true }@@ -31,5 +33,6 @@ models_soup = { path = "../models_soup" }serde = { workspace = true }serde_json = { workspace = true }soup = { path = "../soup", default-features = false, features = ["ports"] }+soup_realtime = { path = "../soup_realtime", default-features = false, features = ["ports"] }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/graphql_soup/src/lib.rs b/crates/graphql_soup/src/lib.rsindex 2dc1437b54..8c8560e498 100644--- a/crates/graphql_soup/src/lib.rs+++ b/crates/graphql_soup/src/lib.rs@@ -13,7 +13,7 @@ mod objects;/// Top-level Soup query resolver.mod resolvers;-pub use graphql_common::{GraphqlSoupEntityType, GraphqlSoupRequestParts};+pub use graphql_common::{GraphqlRequestParts, GraphqlSoupEntityType};pub use inputs::{GraphqlSimpleSortMethod, GroupedSoupInput, SoupInput};pub use objects::{GraphqlSoupBin, GraphqlSoupCall, GraphqlSoupChannel, GraphqlSoupChannelThread, GraphqlSoupChat,@@ -21,4 +21,4 @@ pub use objects::{GraphqlSoupEntity, GraphqlSoupForeignEntity, GraphqlSoupItem, GraphqlSoupProject, GroupedSoup,SoupEntityEdges, SoupPage,};-pub use resolvers::{resolve_grouped_soup, resolve_soup};+pub use resolvers::{resolve_grouped_soup, resolve_soup, resolve_soup_updates};diff --git a/crates/graphql_soup/src/resolvers.rs b/crates/graphql_soup/src/resolvers.rsindex 6efaff51f0..ac8e903820 100644--- a/crates/graphql_soup/src/resolvers.rs+++ b/crates/graphql_soup/src/resolvers.rs@@ -11,16 +11,40 @@ use entity_access::{domain::{models::MemberTeamRole, ports::EntityAccessService},inbound::axum_extractors::OptionalMacroUserTeamExtractorV2,};+use futures::Stream;use graphql_common::{extract_part, require_authorized_user};use macro_authorization::{MacroAuthorizationService, MacroAuthorizationState};use models_pagination::TypeEraseCursor;use soup::domain::{models::grouping::NestedSoupGroups, ports::SoupService};+use soup_realtime::domain::ports::SoupRealtimeSubscriptionService;use crate::{inputs::{GroupedSoupInput, SoupInput},- objects::{GroupedSoup, SoupEntityEdges, SoupPage},+ objects::{GraphqlSoupItem, GroupedSoup, SoupEntityEdges, SoupPage},};+/// Subscribe to realtime Soup updates for the authenticated user.+pub async fn resolve_soup_updates<R, Auth, St, Edges>(+ service: &R,+ ctx: &Context<'_>,+) -> async_graphql::Result<impl Stream<Item = GraphqlSoupItem<Edges>> + Send + 'static>+where+ R: SoupRealtimeSubscriptionService,+ Auth: MacroAuthorizationService,+ MacroAuthorizationState<Auth>: FromRef<St>,+ St: Clone + Send + Sync + 'static,+ Edges: SoupEntityEdges,+{+ let macro_user_id = require_authorized_user::<Auth, St>(ctx).await?;+ let mut receiver = service.subscribe(macro_user_id);++ Ok(async_stream::stream! {+ while let Some(item) = receiver.recv().await {+ yield GraphqlSoupItem::from(item.as_ref().clone());+ }+ })+}+/// Resolve Soup items nested into grouping bins for the authenticated user.pub async fn resolve_grouped_soup<S, Auth, St, Edges>(service: &S,diff --git a/crates/models_soup/src/call_record.rs b/crates/models_soup/src/call_record.rsindex e5ab475cb8..ac464fc96a 100644--- a/crates/models_soup/src/call_record.rs+++ b/crates/models_soup/src/call_record.rs@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};use uuid::Uuid;/// A participant in a call record, as displayed in Soup.-#[derive(Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[serde(rename_all = "camelCase")]pub struct SoupCallRecordParticipant {@@ -19,7 +19,7 @@ pub struct SoupCallRecordParticipant {/// A call record as displayed in Soup. Excludes room_name, egress_id,/// and transcript — fields that are irrelevant for the soup feed.-#[derive(Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[serde(rename_all = "camelCase")]pub struct SoupCallRecord<T = ()> {diff --git a/crates/models_soup/src/comms.rs b/crates/models_soup/src/comms.rsindex c4a8de34a9..eeacaf66e8 100644--- a/crates/models_soup/src/comms.rs+++ b/crates/models_soup/src/comms.rs@@ -105,7 +105,7 @@ pub struct LatestMessage {}/// A channel as displayed in Soup.-#[derive(Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]pub struct SoupChannel {/// Channel metadata and participants.@@ -124,7 +124,7 @@ pub struct SoupChannel {////// This mirrors the public channel-message API shape so soup consumers can/// render a thread root the same way they render a channel timeline message.-#[derive(Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]pub struct SoupChannelThread {/// Message id.diff --git a/crates/models_soup/src/crm_company.rs b/crates/models_soup/src/crm_company.rsindex 3bd0bcd001..bee5d664ae 100644--- a/crates/models_soup/src/crm_company.rs+++ b/crates/models_soup/src/crm_company.rs@@ -5,7 +5,7 @@ use uuid::Uuid;/// A CRM domain as displayed in Soup. Mirrors the crm crate's/// [`CrmDomain`] with a stable wire shape that the FE can rely on.-#[derive(Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[serde(rename_all = "camelCase")]pub struct SoupCrmDomain {@@ -33,7 +33,7 @@ impl From<CrmDomain> for SoupCrmDomain {/// A CRM company as displayed in Soup. Carries the core company/// fields plus display metadata resolved from `crm_domain_directory`/// against the primary (earliest-created) domain.-#[derive(Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[serde(rename_all = "camelCase")]pub struct SoupCrmCompany<T = ()> {diff --git a/crates/models_soup/src/email_thread.rs b/crates/models_soup/src/email_thread.rsindex c9d3aa4b8d..08cc2ed985 100644--- a/crates/models_soup/src/email_thread.rs+++ b/crates/models_soup/src/email_thread.rs@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};use uuid::Uuid;/// A contact participating in an email thread as displayed in Soup.-#[derive(Debug, Doppleganger, Serialize, Deserialize)]+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]#[dg(backward = email::domain::models::Contact)]@@ -24,7 +24,7 @@ pub struct SoupContact {}/// An email label as displayed in Soup.-#[derive(Debug, Doppleganger, Serialize, Deserialize)]+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]#[dg(backward = email::domain::models::Label)]@@ -49,7 +49,7 @@ pub struct SoupLabel {}/// Gmail-style message list visibility for a label.-#[derive(Debug, Doppleganger, Serialize, Deserialize)]+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]#[dg(backward = email::domain::models::MessageListVisibility)]@@ -62,7 +62,7 @@ pub enum SoupMessageListVisibility {}/// Gmail-style label list visibility.-#[derive(Debug, Doppleganger, Serialize, Deserialize)]+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]#[dg(backward = email::domain::models::LabelListVisibility)]@@ -77,7 +77,7 @@ pub enum SoupLabelListVisibility {}/// Whether an email label is provider-created or user-created.-#[derive(Debug, Doppleganger, Serialize, Deserialize)]+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]#[dg(backward = email::domain::models::LabelType)]@@ -90,7 +90,7 @@ pub enum SoupLabelType {}/// An email attachment as displayed in Soup.-#[derive(Debug, Doppleganger, Serialize, Deserialize)]+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]#[dg(backward = email::domain::models::Attachment)]@@ -115,7 +115,7 @@ pub struct SoupAttachment {}/// Email thread preview data as displayed in Soup.-#[derive(Debug, Doppleganger, Serialize, Deserialize)]+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]#[dg(backward = email::domain::models::EmailThreadPreview)]@@ -159,7 +159,7 @@ pub struct SoupEmailThreadPreview {}/// Email thread preview enriched with related metadata for Soup.-#[derive(Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[serde(rename_all = "camelCase")]pub struct SoupEnrichedEmailThreadPreview<T = ()> {diff --git a/crates/models_soup/src/foreign_entity.rs b/crates/models_soup/src/foreign_entity.rsindex ffd5b253d7..77655c7bc6 100644--- a/crates/models_soup/src/foreign_entity.rs+++ b/crates/models_soup/src/foreign_entity.rs@@ -4,7 +4,7 @@ use serde_json::Value;use uuid::Uuid;/// A foreign entity record as displayed in Soup.-#[derive(Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]#[serde(rename_all = "camelCase")]pub struct SoupForeignEntity {diff --git a/crates/models_soup/src/item.rs b/crates/models_soup/src/item.rsindex fa9e60af2d..5cd58b747b 100644--- a/crates/models_soup/src/item.rs+++ b/crates/models_soup/src/item.rs@@ -19,7 +19,7 @@ use uuid::Uuid;mod test;/// A single item in the Soup feed.-#[derive(Serialize, Deserialize, Debug)]+#[derive(Clone, Serialize, Deserialize, Debug)]#[serde(rename_all = "camelCase", tag = "tag", content = "data")]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]pub enum SoupItem<T = ()> {diff --git a/crates/soup_realtime/Cargo.toml b/crates/soup_realtime/Cargo.tomlindex a314e849e6..328f230c50 100644--- a/crates/soup_realtime/Cargo.toml+++ b/crates/soup_realtime/Cargo.toml@@ -19,9 +19,16 @@ outbound = ["dep:soup","ports",]-ports = ["dep:futures", "dep:rootcause", "dep:tracing"]+ports = [+ "dep:broadcast",+ "dep:futures",+ "dep:rootcause",+ "dep:tokio",+ "dep:tracing",+][dependencies]+broadcast = { path = "../broadcast", optional = true }documents = { path = "../documents", default-features = false, optional = true }entity_access = { path = "../entity_access", default-features = false, features = ["ports"], optional = true }futures = { workspace = true, optional = true }@@ -35,7 +42,7 @@ rdkafka = { workspace = true, optional = true }rootcause = { workspace = true, optional = true }serde = { workspace = true }soup = { path = "../soup", default-features = false, features = ["ports"], optional = true }-tokio = { workspace = true, features = ["time"], optional = true }+tokio = { workspace = true, features = ["sync", "time"], optional = true }tracing = { workspace = true, optional = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/soup_realtime/README.md b/crates/soup_realtime/README.mdindex 2aa71b4ac3..d5c3e589b6 100644--- a/crates/soup_realtime/README.md+++ b/crates/soup_realtime/README.md@@ -5,3 +5,5 @@ Hexagonal components for expanding `document.updated` Kafka events into one fullThis crate intentionally does not start a consumer or wire itself into a service binary. A later composition root can combine the domain service with the entity-access, Soup repository, Kafka publisher, and Kafka consumer adapters exposed here.Delivery is at least once. If publication partially succeeds and processing is retried, downstream consumers may receive duplicate messages.++`SoupRealtimeConsumerService` can run a realtime consumer adapter and distribute received items through instance-local, user-keyed subscriptions. Subscription values are shared as `Arc<SoupItem<()>>` so multiple subscribers do not require cloning the underlying item.diff --git a/crates/soup_realtime/src/domain/ports.rs b/crates/soup_realtime/src/domain/ports.rsindex a67f874ff2..5a43e5d60a 100644--- a/crates/soup_realtime/src/domain/ports.rs+++ b/crates/soup_realtime/src/domain/ports.rs@@ -1,6 +1,6 @@-//! Ports used by realtime Soup fan-out.+//! Ports used by realtime Soup services.-use std::future::Future;+use std::{future::Future, sync::Arc};use macro_user_id::user_id::MacroUserIdStr;use model_entity::Entity;@@ -18,6 +18,47 @@ pub trait SoupRealtimeService: Send + Sync + 'static {) -> impl Future<Output = Result<(), Report>> + Send;}+/// Receives complete recipient-targeted Soup messages.+pub trait SoupRealtimeConsumer: Send + Sync + 'static {+ /// Waits for and returns the next realtime Soup message.+ fn recv(&self) -> impl Future<Output = Result<SoupRealtimeMessage, Report>> + Send;+}++/// Provides user-scoped subscriptions to received realtime Soup items.+pub trait SoupRealtimeSubscriptionService: Send + Sync + 'static {+ /// Subscribes to realtime Soup items addressed to `user_id`.+ fn subscribe(+ &self,+ user_id: MacroUserIdStr<'static>,+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>>;+}++impl<S> SoupRealtimeSubscriptionService for Arc<S>+where+ S: SoupRealtimeSubscriptionService,+{+ fn subscribe(+ &self,+ user_id: MacroUserIdStr<'static>,+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {+ self.as_ref().subscribe(user_id)+ }+}++/// No-op subscription service used when only the GraphQL schema shape is needed.+#[derive(Clone, Copy, Debug, Default)]+pub struct NoOpSoupRealtimeSubscriptionService;++impl SoupRealtimeSubscriptionService for NoOpSoupRealtimeSubscriptionService {+ fn subscribe(+ &self,+ _user_id: MacroUserIdStr<'static>,+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {+ let (_sender, receiver) = tokio::sync::mpsc::channel(1);+ receiver+ }+}+/// Resolves the users who currently have access to an entity.pub trait UserAccessExpander: Send + Sync + 'static {/// Returns all current user accessors for `entity`.diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rsindex 947b495bc1..cdf613cf0f 100644--- a/crates/soup_realtime/src/domain/service.rs+++ b/crates/soup_realtime/src/domain/service.rs@@ -1,20 +1,99 @@-//! Realtime Soup hydration and fan-out orchestration.+//! Realtime Soup orchestration.#[cfg(test)]mod test;-use std::collections::HashSet;+use std::{collections::HashSet, num::NonZeroUsize, sync::Arc};+use broadcast::{BroadcastManager, GlobalSpawner};use futures::{StreamExt as _, stream};+use macro_user_id::user_id::MacroUserIdStr;use model_entity::Entity;use models_soup::item::SoupItem;use rootcause::prelude::{Report, ResultExt as _};use super::{models::SoupRealtimeMessage,- ports::{SoupItemReader, SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},+ ports::{+ SoupItemReader, SoupRealtimeConsumer, SoupRealtimePublisher, SoupRealtimeService,+ SoupRealtimeSubscriptionService, UserAccessExpander,+ },};+/// Number of messages retained by each user-keyed broadcast channel.+const BROADCAST_BUFFER_CAPACITY: NonZeroUsize = NonZeroUsize::new(64).unwrap();+/// Number of messages buffered for each individual subscriber.+const SUBSCRIBER_BUFFER_CAPACITY: NonZeroUsize = NonZeroUsize::new(16).unwrap();++/// Service for distributing recipient-targeted realtime Soup messages.+pub struct SoupRealtimeConsumerService<C>+where+ C: SoupRealtimeConsumer,+{+ consumer: C,+ broadcasts: BroadcastManager<GlobalSpawner, MacroUserIdStr<'static>, Arc<SoupItem<()>>>,+}++impl<C> SoupRealtimeConsumerService<C>+where+ C: SoupRealtimeConsumer,+{+ /// Creates a realtime Soup consumer service backed by `consumer`.+ pub fn new(consumer: C) -> Self {+ Self {+ consumer,+ broadcasts: BroadcastManager::new(GlobalSpawner, BROADCAST_BUFFER_CAPACITY),+ }+ }++ /// Subscribes to realtime Soup items addressed to `user_id`.+ ///+ /// The returned receiver is closed if its buffer fills, ensuring a slow+ /// subscriber cannot delay the shared consumer or other subscribers.+ #[must_use]+ pub fn subscribe(+ &self,+ user_id: MacroUserIdStr<'static>,+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {+ self.broadcasts+ .subscribe(user_id, SUBSCRIBER_BUFFER_CAPACITY)+ }++ /// Receives messages and distributes them to subscribers until reception fails.+ ///+ /// Callers should run this future in a supervised task. A message for a user+ /// without active subscribers is intentionally dropped.+ #[tracing::instrument(skip(self), err)]+ pub async fn run(&self) -> Result<(), Report> {+ loop {+ let SoupRealtimeMessage { user_id, item } = self+ .consumer+ .recv()+ .await+ .context("failed to receive realtime Soup message")?;++ match self.broadcasts.publish(&user_id, Arc::new(item)) {+ Ok(subscriber_count) => {+ tracing::trace!(subscriber_count, "distributed realtime Soup message")+ }+ Err(_) => tracing::trace!("dropping realtime Soup message without subscribers"),+ }+ }+ }+}++impl<C> SoupRealtimeSubscriptionService for SoupRealtimeConsumerService<C>+where+ C: SoupRealtimeConsumer,+{+ fn subscribe(+ &self,+ user_id: MacroUserIdStr<'static>,+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {+ SoupRealtimeConsumerService::subscribe(self, user_id)+ }+}+/// Maximum number of Kafka publications polled concurrently.const PUBLISH_CONCURRENCY: usize = 16;diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rsindex f9bf0e77c0..6829ef355c 100644--- a/crates/soup_realtime/src/domain/service/test.rs+++ b/crates/soup_realtime/src/domain/service/test.rs@@ -1,6 +1,7 @@-use std::collections::{HashMap, HashSet};+use std::collections::{HashMap, HashSet, VecDeque};use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::{Arc, Mutex};+use std::time::Duration;use chrono::{DateTime, Utc};use macro_user_id::user_id::MacroUserIdStr;@@ -9,7 +10,9 @@ use models_soup::{document::SoupDocument, item::SoupItem};use uuid::Uuid;use super::*;-use crate::domain::ports::{SoupItemReader, SoupRealtimePublisher, UserAccessExpander};+use crate::domain::ports::{+ SoupItemReader, SoupRealtimeConsumer, SoupRealtimePublisher, UserAccessExpander,+};const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";const OTHER_DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000002";@@ -66,6 +69,20 @@ impl SoupItemReader for FakeReader {}}+struct FakeRealtimeConsumer {+ messages: Mutex<VecDeque<Result<SoupRealtimeMessage, Report>>>,+}++impl SoupRealtimeConsumer for FakeRealtimeConsumer {+ async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {+ self.messages+ .lock()+ .expect("consumer messages lock")+ .pop_front()+ .unwrap_or_else(|| Err(rootcause::report!("consumer stopped")))+ }+}+struct FakePublisher {messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,fail_users: HashSet<String>,@@ -123,6 +140,13 @@ fn document_entity() -> Entity<'static> {EntityType::Document.with_entity_string(DOCUMENT_ID.to_string())}+fn document_name(item: &SoupItem<()>) -> &str {+ let SoupItem::Document(document) = item else {+ panic!("expected document item")+ };+ &document.name+}+fn harness(users: Vec<MacroUserIdStr<'static>>,responses: HashMap<String, ReadResponse>,@@ -165,6 +189,62 @@ fn item_response()}+#[tokio::test]+async fn consumer_service_distributes_items_only_to_their_users() {+ let one = user("one");+ let two = user("two");+ let consumer = FakeRealtimeConsumer {+ messages: Mutex::new(VecDeque::from([+ Ok(SoupRealtimeMessage::new(+ one.clone(),+ document_item(DOCUMENT_ID, "For one", None),+ )),+ Ok(SoupRealtimeMessage::new(+ two.clone(),+ document_item(OTHER_DOCUMENT_ID, "For two", None),+ )),+ ])),+ };+ let service = Arc::new(SoupRealtimeConsumerService::new(consumer));+ let mut one_first = service.subscribe(one.clone());+ let mut one_second = service.subscribe(one);+ let mut two_receiver = service.subscribe(two);++ let run_error = tokio::spawn({+ let service = Arc::clone(&service);+ async move { service.run().await }+ })+ .await+ .expect("consumer task joins")+ .expect_err("fake consumer eventually stops");+ assert!(run_error.to_string().contains("failed to receive"));++ let one_first_item = tokio::time::timeout(Duration::from_secs(1), one_first.recv())+ .await+ .expect("first user subscription receives before timeout")+ .expect("first user subscription remains open");+ let one_second_item = tokio::time::timeout(Duration::from_secs(1), one_second.recv())+ .await+ .expect("second user subscription receives before timeout")+ .expect("second user subscription remains open");+ let two_item = tokio::time::timeout(Duration::from_secs(1), two_receiver.recv())+ .await+ .expect("other user subscription receives before timeout")+ .expect("other user subscription remains open");++ assert_eq!(document_name(&one_first_item), "For one");+ assert!(Arc::ptr_eq(&one_first_item, &one_second_item));+ assert_eq!(document_name(&two_item), "For two");+ assert!(matches!(+ one_first.try_recv(),+ Err(tokio::sync::mpsc::error::TryRecvError::Empty)+ ));+ assert!(matches!(+ two_receiver.try_recv(),+ Err(tokio::sync::mpsc::error::TryRecvError::Empty)+ ));+}+#[tokio::test]async fn zero_users_skips_reads_and_publications() {let harness = harness(Vec::new(), HashMap::new(), false, HashSet::new());diff --git a/crates/soup_realtime/src/inbound/mod.rs b/crates/soup_realtime/src/inbound/mod.rsindex e22ac579a2..1d73c01b15 100644--- a/crates/soup_realtime/src/inbound/mod.rs+++ b/crates/soup_realtime/src/inbound/mod.rs@@ -2,5 +2,3 @@/// Kafka consumer for document update events.pub mod kafka_consumer;-/// Independent Kafka consumer for recipient-targeted Soup messages.-pub mod soup_consumer;diff --git a/crates/soup_realtime/src/lib.rs b/crates/soup_realtime/src/lib.rsindex ab4794b1f9..4fe42ac425 100644--- a/crates/soup_realtime/src/lib.rs+++ b/crates/soup_realtime/src/lib.rs@@ -12,5 +12,5 @@ pub mod domain;pub mod inbound;/// Outbound infrastructure adapters.-#[cfg(feature = "outbound")]+#[cfg(any(feature = "consumer", feature = "outbound"))]pub mod outbound;diff --git a/crates/soup_realtime/src/outbound/mod.rs b/crates/soup_realtime/src/outbound/mod.rsindex 01d025d389..39fe268ff5 100644--- a/crates/soup_realtime/src/outbound/mod.rs+++ b/crates/soup_realtime/src/outbound/mod.rs@@ -1,8 +1,14 @@-//! Outbound adapters used by realtime Soup fan-out.+//! Outbound adapters used by realtime Soup services./// Current entity-access expansion adapter.+#[cfg(feature = "outbound")]pub mod entity_access;/// Kafka-backed realtime Soup publisher adapter.+#[cfg(feature = "outbound")]pub mod kafka_publisher;+/// Independent Kafka consumer for recipient-targeted Soup messages.+#[cfg(feature = "consumer")]+pub mod soup_consumer;/// User-scoped Soup item reader adapter.+#[cfg(feature = "outbound")]pub mod soup_item_reader;diff --git a/crates/soup_realtime/src/inbound/soup_consumer.rs b/crates/soup_realtime/src/outbound/soup_consumer.rssimilarity index 91%rename from crates/soup_realtime/src/inbound/soup_consumer.rsrename to crates/soup_realtime/src/outbound/soup_consumer.rsindex a0dca06f2a..6199e08029 100644--- a/crates/soup_realtime/src/inbound/soup_consumer.rs+++ b/crates/soup_realtime/src/outbound/soup_consumer.rs@@ -13,7 +13,10 @@ use kafka_util::{InitialOffset, KafkaEventConsumer, Ungrouped};use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};use rootcause::prelude::{Report, ResultExt as _};-use crate::domain::models::{SoupMacroEvent, SoupRealtimeMessage};+use crate::domain::{+ models::{SoupMacroEvent, SoupRealtimeMessage},+ ports::SoupRealtimeConsumer,+};/// Maximum time to wait for Soup topic metadata during partition assignment.const TOPIC_METADATA_TIMEOUT: Duration = Duration::from_secs(10);@@ -73,3 +76,9 @@ impl SoupTopicConsumer {}}}++impl SoupRealtimeConsumer for SoupTopicConsumer {+ async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {+ SoupTopicConsumer::recv(self).await+ }+}diff --git a/crates/soup_realtime/src/inbound/soup_consumer/test.rs b/crates/soup_realtime/src/outbound/soup_consumer/test.rssimilarity index 100%rename from crates/soup_realtime/src/inbound/soup_consumer/test.rsrename to crates/soup_realtime/src/outbound/soup_consumer/test.rsdiff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.tomlindex 39e586e27a..8651726f80 100644--- a/services/document_storage_service/Cargo.toml+++ b/services/document_storage_service/Cargo.toml@@ -27,7 +27,12 @@ default = ["location_check",]delete_document_worker = []-graphql = ["dep:async-graphql", "dep:async-graphql-axum", "dep:complete_graph"]+graphql = [+ "dep:async-graphql",+ "dep:async-graphql-axum",+ "dep:complete_graph",+ "dep:soup_realtime",+]local = ["local_auth"]local_auth = ["macro_authorization/local_auth"]location_check = []@@ -44,7 +49,12 @@ aws-sdk-s3 = { workspace = true }aws-sdk-secretsmanager = { workspace = true }aws-sdk-sns = { workspace = true }aws-sdk-sqs = { workspace = true }-axum = { workspace = true, features = ["macros", "multipart", "tower-log"] }+axum = { workspace = true, features = [+ "macros",+ "multipart",+ "tower-log",+ "ws",+] }axum-extra = { workspace = true }base64 = { workspace = true }bots = { path = "../../crates/bots" }@@ -168,6 +178,9 @@ soup = { path = "../../crates/soup", features = ["inbound","outbound",] }+soup_realtime = { path = "../../crates/soup_realtime", default-features = false, features = [+ "consumer",+], optional = true }sqlx = { workspace = true }sqs_client = { path = "../../crates/sqs_client", default-features = false, features = ["document",diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex f3402e6a51..1c6c1df6bf 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -95,6 +95,10 @@ use soup::{domain::service::SoupImpl, inbound::axum_router::SoupRouterState,outbound::pg_soup_repo::PgSoupRepo,};+#[cfg(feature = "graphql")]+use soup_realtime::{+ domain::service::SoupRealtimeConsumerService, outbound::soup_consumer::SoupTopicConsumer,+};use sqlx::PgPool;use std::sync::Arc;use sync_service_client::SyncServiceClient;@@ -145,12 +149,17 @@ pub(crate) type DssSoupService = SoupImpl<type DssSoupState =SoupRouterState<DssSoupService, DssEmailService, EntityAccessService, AuthorizationService>;+/// Realtime Soup consumer service used by GraphQL subscriptions.+#[cfg(feature = "graphql")]+pub(crate) type DssSoupRealtimeService = SoupRealtimeConsumerService<SoupTopicConsumer>;+/// GraphQL Soup schema wired to the DSS services; the `ApiContext` state/// parameter lets GraphQL resolvers run the same axum extractors as the REST/// routes, lazily, against the stored request parts.#[cfg(feature = "graphql")]pub(crate) type DssGraphqlSoupSchema = complete_graph::SharedSoupSchema<DssSoupService,+ DssSoupRealtimeService,DssEmailService,EntityAccessService,AuthorizationService,diff --git a/services/document_storage_service/src/api/graphql_soup.rs b/services/document_storage_service/src/api/graphql_soup.rsindex d1fb07f2e0..1a7fa28da0 100644--- a/services/document_storage_service/src/api/graphql_soup.rs+++ b/services/document_storage_service/src/api/graphql_soup.rs@@ -1,61 +1,96 @@use crate::api::context::{ApiContext, AuthorizationService};-use async_graphql::{ServerError, http::GraphiQLSource};-use async_graphql_axum::{GraphQLRequest, GraphQLResponse};+use async_graphql::{+ Data,+ http::{ALL_WEBSOCKET_PROTOCOLS, GraphiQLSource},+};+use async_graphql_axum::{GraphQLProtocol, GraphQLRequest, GraphQLResponse, GraphQLWebSocket};use axum::{Router,- extract::{FromRequest, FromRequestParts, OriginalUri, Request, State},+ extract::{State, WebSocketUpgrade},+ http::{StatusCode, request::Parts},response::{Html, IntoResponse, Response},routing::get,};use axum_extra::extract::Cached;-use complete_graph::GraphqlSoupRequestParts;+use complete_graph::GraphqlRequestParts;use macro_authorization::OptionalMacroAuthorizationExtractor;+use macro_user_id::user_id::MacroUserIdStr;++const GRAPHQL_PATH: &str = "/soup/graphql";+const GRAPHQL_SUBSCRIPTION_PATH: &str = "/soup/graphql/ws";pub(crate) fn router() -> Router<ApiContext> {- Router::new().route("/soup/graphql", get(graphiql).post(handler))+ Router::new()+ .route(GRAPHQL_PATH, get(graphiql).post(graphql_handler))+ .route(GRAPHQL_SUBSCRIPTION_PATH, get(subscription_handler))}-async fn graphiql(OriginalUri(uri): OriginalUri) -> Html<String> {- Html(GraphiQLSource::build().endpoint(uri.path()).finish())+async fn graphiql() -> Html<String> {+ Html(+ GraphiQLSource::build()+ .endpoint(GRAPHQL_PATH)+ .subscription_endpoint(GRAPHQL_SUBSCRIPTION_PATH)+ .finish(),+ )}-async fn handler(State(state): State<ApiContext>, req: Request) -> Response {- let (mut parts, body) = req.into_parts();+async fn graphql_handler(+ State(state): State<ApiContext>,+ Cached(auth): Cached<OptionalMacroAuthorizationExtractor<AuthorizationService>>,+ request_parts: Parts,+ request: GraphQLRequest,+) -> GraphQLResponse {+ let request = graphql_query_context_data(request.into_inner(), &state, auth.macro_user_id);+ state+ .graphql_soup_schema+ .execute(request.data(GraphqlRequestParts::new(request_parts)))+ .await+ .into()+}- // Authentication stays eager: it gates execution for non-introspection- // queries and primes the `Cached` entry that resolvers extract lazily.- let auth =- match Cached::<OptionalMacroAuthorizationExtractor<AuthorizationService>>::from_request_parts(- &mut parts, &state,+async fn subscription_handler(+ State(state): State<ApiContext>,+ Cached(auth): Cached<OptionalMacroAuthorizationExtractor<AuthorizationService>>,+ protocol: GraphQLProtocol,+ upgrade: WebSocketUpgrade,+) -> Response {+ let Some(macro_user_id) = auth.macro_user_id else {+ return (+ StatusCode::UNAUTHORIZED,+ "authentication required for GraphQL Soup subscriptions",)- .await- {- Ok(Cached(auth)) => auth,- Err(err) => return err.into_response(),- };+ .into_response();+ };- // `GraphQLRequest` consumes a whole request, but the original parts (with- // the middleware-populated extensions) must survive for the resolvers, so- // parse the body from a shallow copy of method/uri/headers.- let mut synthetic = Request::new(body);- *synthetic.method_mut() = parts.method.clone();- *synthetic.uri_mut() = parts.uri.clone();- *synthetic.headers_mut() = parts.headers.clone();+ let schema = state.graphql_soup_schema.clone();+ let data = graphql_subscription_context_data(state, macro_user_id);+ upgrade+ .protocols(ALL_WEBSOCKET_PROTOCOLS)+ .on_upgrade(move |socket| async move {+ GraphQLWebSocket::new(socket, schema, protocol)+ .with_data(data)+ .serve()+ .await;+ })+ .into_response()+}- let request = match <GraphQLRequest as FromRequest<()>>::from_request(synthetic, &()).await {- Ok(request) => request.into_inner(),- Err(rejection) => return rejection.into_response(),- };+fn graphql_subscription_context_data(state: ApiContext, user: MacroUserIdStr<'static>) -> Data {+ let mut data = Data::default();+ data.insert(state);+ data.insert(user);+ data+}- let Some(macro_user_id) = auth.macro_user_id else {- if is_introspection_query(&request.query) {- return GraphQLResponse::from(state.graphql_soup_schema.execute(request).await)- .into_response();- }- return GraphQLResponse::from(async_graphql::Response::from_errors(vec![- ServerError::new("authentication required for GraphQL Soup queries", None),- ]))- .into_response();+fn graphql_query_context_data(+ req: async_graphql::Request,+ state: &ApiContext,+ macro_user_id: Option<MacroUserIdStr<'static>>,+) -> async_graphql::Request {+ let req = req.data(state.clone());++ let Some(macro_user_id) = macro_user_id else {+ return req;};let property_reader = complete_graph::PropertiesEntityPropertyReader::new(@@ -71,9 +106,8 @@ async fn handler(State(state): State<ApiContext>, req: Request) -> Response {state.soup_router_state.email_service(),state.entity_access_service.clone(),);- let request = request- .data(GraphqlSoupRequestParts::new(parts))- .data(state.clone())++ req.data(macro_user_id.clone()).data(complete_graph::entity_properties_loader(macro_user_id.clone(),property_reader,@@ -86,11 +120,5 @@ async fn handler(State(state): State<ApiContext>, req: Request) -> Response {.data(complete_graph::entity_notifications_loader(macro_user_id,state.graphql_notification_reader.clone(),- ));-- GraphQLResponse::from(state.graphql_soup_schema.execute(request).await).into_response()-}--fn is_introspection_query(query: &str) -> bool {- query.contains("__schema") || query.contains("__type")+ ))}diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex 33278c3fdc..03ad704c0a 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -107,6 +107,10 @@ use soup::{domain::service::SoupImpl, inbound::axum_router::SoupRouterState,outbound::pg_soup_repo::PgSoupRepo,};+#[cfg(feature = "graphql")]+use soup_realtime::{+ domain::service::SoupRealtimeConsumerService, outbound::soup_consumer::SoupTopicConsumer,+};use sqlx::postgres::PgPoolOptions;use std::sync::Arc;use sync_service_client::SyncServiceClient;@@ -848,6 +852,28 @@ async fn main() -> anyhow::Result<()> {foreign_entity_service_for_soup,));+ #[cfg(feature = "graphql")]+ let soup_realtime_service = Arc::new(SoupRealtimeConsumerService::new(+ SoupTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {+ anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")+ })?,+ ));+ #[cfg(feature = "graphql")]+ tokio::spawn({+ let soup_realtime_service = Arc::clone(&soup_realtime_service);+ async move {+ loop {+ let _ = soup_realtime_service.run().await.inspect_err(|error| {+ tracing::error!(+ error = ?error,+ "realtime Soup subscription consumer stopped"+ );+ });+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;+ }+ }+ });+let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));let api_context = ApiContext {@@ -868,7 +894,10 @@ async fn main() -> anyhow::Result<()> {),favorites_service,#[cfg(feature = "graphql")]- graphql_soup_schema: complete_graph::build_schema_from_arc(soup_service),+ graphql_soup_schema: complete_graph::build_schema_from_arcs(+ soup_service,+ soup_realtime_service,+ ),#[cfg(feature = "graphql")]graphql_notification_reader,github_sync_service: Arc::new(github_sync_service_impl),diff --git a/static_assets/schema.graphql b/static_assets/schema.graphqlindex 2c5ce81bb9..e1ace525b5 100644--- a/static_assets/schema.graphql+++ b/static_assets/schema.graphql@@ -2620,6 +2620,16 @@ type SoupQueryRoot {user: GraphqlUser!}+"""+Root entry point for realtime Soup subscriptions.+"""+type SoupSubscriptionRoot {+ """+ Subscribe to realtime Soup updates for the authenticated user.+ """+ soupUpdates: GraphqlSoupItem!+}+"""Directs the executor to include this field or fragment only when the `if` argument is true."""@@ -2635,5 +2645,6 @@ directive @skip(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENTschema {query: SoupQueryRootmutation: PropertiesMutationRoot+ subscription: SoupSubscriptionRoot}