Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/feat/expose soup subscription (#5118)

Sean Ayec6754bbb4c79
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 05eff0dcde..0ddc1ffc31 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -927,6 +927,7 @@
"crates/attachment",
"crates/authentication_service_client",
"crates/bot_id",
+ "crates/broadcast",
"crates/call",
"crates/channel_sender",
"crates/channels",
@@ -938,9 +939,11 @@
"crates/cowlike",
"crates/crm",
"crates/document_sub_type",
+ "crates/documents",
"crates/email",
"crates/email_db_client",
"crates/email_utils",
+ "crates/embedding",
"crates/entity_access",
"crates/entity_access_db_utils",
"crates/entity_access_management",
@@ -956,14 +959,17 @@
"crates/item_filters",
"crates/kafka_util",
"crates/last_online_tracker",
+ "crates/lexical_client",
"crates/macro_auth",
"crates/macro_authorization",
"crates/macro_aws_config",
+ "crates/macro_entrypoint",
"crates/macro_env",
"crates/macro_env_var",
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_sync_service_jwt",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -978,23 +984,33 @@
"crates/models_email",
"crates/models_entity_access_management",
"crates/models_grouping",
+ "crates/models_opensearch",
"crates/models_pagination",
"crates/models_permissions",
"crates/models_properties",
+ "crates/models_search",
+ "crates/models_search_cursor",
"crates/models_soup",
"crates/non_empty",
"crates/notification",
+ "crates/opensearch_client",
+ "crates/opensearch_query_builder",
"crates/prompt",
"crates/properties",
"crates/rate_limit",
"crates/readonly_pool",
"crates/remote_env_var",
"crates/s3_key",
+ "crates/secretsmanager_client",
"crates/share_permission_db_utils",
"crates/soup",
+ "crates/soup_realtime",
"crates/sqs_client",
"crates/sqs_worker",
+ "crates/sync_service",
+ "crates/sync_service_client",
"crates/system_properties",
+ "crates/task_dedup",
"crates/unfurl",
"crates/workspace-hack"
],
@@ -1490,6 +1506,7 @@
"crates/authentication_service_client",
"crates/bot_id",
"crates/bots",
+ "crates/broadcast",
"crates/cal",
"crates/call",
"crates/channel_bots",
@@ -1597,6 +1614,7 @@
"crates/secretsmanager_client",
"crates/share_permission_db_utils",
"crates/soup",
+ "crates/soup_realtime",
"crates/sqs_client",
"crates/sqs_worker",
"crates/sync_service",
@@ -2767,6 +2785,7 @@
"crates/attachment",
"crates/authentication_service_client",
"crates/bot_id",
+ "crates/broadcast",
"crates/call",
"crates/channel_sender",
"crates/channels",
@@ -2777,9 +2796,11 @@
"crates/cowlike",
"crates/crm",
"crates/document_sub_type",
+ "crates/documents",
"crates/email",
"crates/email_db_client",
"crates/email_utils",
+ "crates/embedding",
"crates/entity_access",
"crates/entity_access_db_utils",
"crates/entity_access_management",
@@ -2792,14 +2813,17 @@
"crates/item_filters",
"crates/kafka_util",
"crates/last_online_tracker",
+ "crates/lexical_client",
"crates/macro_auth",
"crates/macro_authorization",
"crates/macro_aws_config",
+ "crates/macro_entrypoint",
"crates/macro_env",
"crates/macro_env_var",
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
+ "crates/macro_sync_service_jwt",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -2814,23 +2838,33 @@
"crates/models_email",
"crates/models_entity_access_management",
"crates/models_grouping",
+ "crates/models_opensearch",
"crates/models_pagination",
"crates/models_permissions",
"crates/models_properties",
+ "crates/models_search",
+ "crates/models_search_cursor",
"crates/models_soup",
"crates/non_empty",
"crates/notification",
+ "crates/opensearch_client",
+ "crates/opensearch_query_builder",
"crates/prompt",
"crates/properties",
"crates/rate_limit",
"crates/readonly_pool",
"crates/remote_env_var",
"crates/s3_key",
+ "crates/secretsmanager_client",
"crates/share_permission_db_utils",
"crates/soup",
+ "crates/soup_realtime",
"crates/sqs_client",
"crates/sqs_worker",
+ "crates/sync_service",
+ "crates/sync_service_client",
"crates/system_properties",
+ "crates/task_dedup",
"crates/unfurl",
"crates/workspace-hack"
],
@@ -5081,6 +5115,7 @@
"crates/attachment",
"crates/authentication_service_client",
"crates/bot_id",
+ "crates/broadcast",
"crates/call",
"crates/channel_sender",
"crates/channels",
diff --git a/Cargo.lock b/Cargo.lock
index 38e5eb3cc7..3cbc9982ac 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2897,6 +2897,7 @@ dependencies = [
"rootcause",
"serde",
"soup",
+ "soup_realtime",
"tokio",
"uuid",
"workspace-hack",
@@ -4317,6 +4318,7 @@ dependencies = [
"serde",
"serde_json",
"soup",
+ "soup_realtime",
"sqlx",
"sqs_client",
"sqs_worker",
@@ -6063,6 +6065,7 @@ name = "graphql_soup"
version = "0.1.0"
dependencies = [
"async-graphql",
+ "async-stream",
"axum",
"axum-extra",
"chrono",
@@ -6070,6 +6073,7 @@ dependencies = [
"email",
"entity_access",
"filter_ast",
+ "futures",
"graphql_common",
"item_filters",
"macro_authorization",
@@ -6082,6 +6086,7 @@ dependencies = [
"serde",
"serde_json",
"soup",
+ "soup_realtime",
"uuid",
"workspace-hack",
]
@@ -13467,6 +13472,7 @@ dependencies = [
name = "soup_realtime"
version = "0.1.0"
dependencies = [
+ "broadcast",
"chrono",
"documents",
"entity_access",
diff --git a/crates/complete_graph/Cargo.toml b/crates/complete_graph/Cargo.toml
index ffe1ee266b..9df69def8c 100644
--- a/crates/complete_graph/Cargo.toml
+++ b/crates/complete_graph/Cargo.toml
@@ -25,14 +25,17 @@ macro_authorization = { path = "../macro_authorization", features = [
"axum",
"outbound",
] }
+macro_user_id = { path = "../macro_user_id" }
model-entity = { path = "../model-entity" }
soup = { path = "../soup", default-features = false, features = ["ports"] }
+soup_realtime = { path = "../soup_realtime", default-features = false, features = [
+ "ports",
+] }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
apollo-compiler = { workspace = true }
-macro_user_id = { path = "../macro_user_id" }
model_user = { path = "../model_user" }
models_pagination = { path = "../models_pagination" }
models_properties = { path = "../models_properties" }
diff --git a/crates/complete_graph/src/lib.rs b/crates/complete_graph/src/lib.rs
index 777ceae15b..698ef07a01 100644
--- a/crates/complete_graph/src/lib.rs
+++ b/crates/complete_graph/src/lib.rs
@@ -12,7 +12,7 @@ mod schema;
mod sdl_test;
pub use edges::{SoupEdges, SoupEmailThreadEdges};
-pub use graphql_common::GraphqlSoupRequestParts;
+pub use graphql_common::GraphqlRequestParts;
pub use graphql_email::{
EmailContentKey, EmailContentLoad, EmailContentLoader, EmailServiceEmailContentReader,
NoOpSoupEmailContentEdgeReader, SoupEmailContentEdgeReader, email_content_loader,
@@ -27,5 +27,6 @@ pub use graphql_properties::{
};
pub use schema::{
SchemaOnlySoupSchema, SchemaOnlyState, SharedSoupSchema, SoupQueryRoot, SoupSchema,
- build_schema, build_schema_from_arc, build_schema_with_service,
+ SoupSubscriptionRoot, build_schema, build_schema_from_arc, build_schema_from_arcs,
+ build_schema_with_service, build_schema_with_services,
};
diff --git a/crates/complete_graph/src/schema.rs b/crates/complete_graph/src/schema.rs
index fd7d430498..71649b895c 100644
--- a/crates/complete_graph/src/schema.rs
+++ b/crates/complete_graph/src/schema.rs
@@ -3,7 +3,7 @@ mod test;
use std::{marker::PhantomData, sync::Arc};
-use async_graphql::{Context, EmptySubscription, Object, Schema};
+use async_graphql::{Context, Object, Schema, Subscription};
use axum::extract::FromRef;
use email::{
domain::ports::{EmailService, NoOpEmailService},
@@ -18,36 +18,43 @@ use graphql_properties::{
PropertiesMutationRoot,
};
use graphql_soup::{
- GroupedSoup, GroupedSoupInput, SoupInput, SoupPage, resolve_grouped_soup, resolve_soup,
+ GraphqlSoupItem, GroupedSoup, GroupedSoupInput, SoupInput, SoupPage, resolve_grouped_soup,
+ resolve_soup, resolve_soup_updates,
};
use macro_authorization::{
InternalAuthConfig, MacroAuthorizationService, MacroAuthorizationServiceImpl,
MacroAuthorizationState, NoopMacroAuthJwtValidator,
};
+use macro_user_id::user_id::MacroUserIdStr;
use soup::domain::ports::{NoOpSoupService, SoupService};
+use soup_realtime::domain::ports::{
+ NoOpSoupRealtimeSubscriptionService, SoupRealtimeSubscriptionService,
+};
use crate::SoupEdges;
/// GraphQL Soup schema type.
///
-/// `S` is the soup service, `E` the email service, `EAS` the entity access
-/// service, `Auth` the authorization service, `St` the embedding axum router
-/// state, `W` the property mutation writer, `NR` the notification edge reader,
-/// `PR` the property edge reader, and `ER` the email-content edge reader.
-pub type SoupSchema<S, E, EAS, Auth, St, W, NR, PR, ER> = Schema<
+/// `S` is the soup query service, `R` the realtime subscription service, `E`
+/// the email service, `EAS` the entity access service, `Auth` the authorization
+/// service, `St` the embedding axum router state, `W` the property mutation
+/// writer, `NR` the notification edge reader, `PR` the property edge reader,
+/// and `ER` the email-content edge reader.
+pub type SoupSchema<S, R, E, EAS, Auth, St, W, NR, PR, ER> = Schema<
SoupQueryRoot<S, E, EAS, Auth, St, NR, PR, ER>,
PropertiesMutationRoot<W>,
- EmptySubscription,
+ SoupSubscriptionRoot<R, Auth, St, NR, PR, ER>,
>;
-/// GraphQL Soup schema type backed by a shared soup service.
-pub type SharedSoupSchema<S, E, EAS, Auth, St, W, NR, PR, ER> =
- SoupSchema<Arc<S>, E, EAS, Auth, St, W, NR, PR, ER>;
+/// GraphQL Soup schema type backed by shared query and realtime services.
+pub type SharedSoupSchema<S, R, E, EAS, Auth, St, W, NR, PR, ER> =
+ SoupSchema<Arc<S>, Arc<R>, E, EAS, Auth, St, W, NR, PR, ER>;
/// GraphQL Soup schema type backed by the no-op services, used only for
/// SDL export or introspection.
pub type SchemaOnlySoupSchema = SoupSchema<
NoOpSoupService,
+ NoOpSoupRealtimeSubscriptionService,
NoOpEmailService,
NoOpEntityAccessService,
SchemaOnlyAuthorizationService,
@@ -113,6 +120,25 @@ impl<S, E, EAS, Auth, St, NR, PR, ER> SoupQueryRoot<S, E, EAS, Auth, St, NR, PR,
}
}
+/// Root GraphQL subscription object for realtime Soup updates.
+pub struct SoupSubscriptionRoot<R, Auth, St, NR, PR, ER> {
+ /// Realtime Soup service used by user-scoped subscriptions.
+ service: R,
+ /// Associates the root with authorization, state, and edge reader types.
+ #[allow(clippy::type_complexity)]
+ _marker: PhantomData<fn() -> (Auth, St, NR, PR, ER)>,
+}
+
+impl<R, Auth, St, NR, PR, ER> SoupSubscriptionRoot<R, Auth, St, NR, PR, ER> {
+ /// Creates a root GraphQL subscription object.
+ pub fn new(service: R) -> Self {
+ Self {
+ service,
+ _marker: PhantomData,
+ }
+ }
+}
+
/// The authenticated user (viewer). All user-scoped data hangs off this
/// object so clients (and their normalized caches) observe data ownership
/// structurally rather than implicitly through the session.
@@ -121,17 +147,19 @@ pub struct GraphqlUser<S, E, EAS, Auth, St, NR, PR, ER> {
service: S,
/// Associates the user object with its adapter and reader types.
_marker: ServicesMarker<E, EAS, Auth, St, NR, PR, ER>,
+ /// The [MacroUserIdStr] of the resolving user
+ user_id: MacroUserIdStr<'static>,
}
/// Build a GraphQL schema for Soup suitable for SDL export or introspection.
pub fn build_schema() -> SchemaOnlySoupSchema {
- build_schema_with_service(NoOpSoupService)
+ build_schema_with_services(NoOpSoupService, NoOpSoupRealtimeSubscriptionService)
}
-/// Build a GraphQL schema for Soup backed by the provided service.
+/// Build a GraphQL schema backed by a query service and no-op realtime service.
pub fn build_schema_with_service<S, E, EAS, Auth, St, W, NR, PR, ER>(
service: S,
-) -> SoupSchema<S, E, EAS, Auth, St, W, NR, PR, ER>
+) -> SoupSchema<S, NoOpSoupRealtimeSubscriptionService, E, EAS, Auth, St, W, NR, PR, ER>
where
S: SoupService + Clone,
E: EmailService,
@@ -145,19 +173,42 @@ where
NR: SoupNotificationEdgeReader,
PR: EntityPropertyReader,
ER: SoupEmailContentEdgeReader,
+{
+ build_schema_with_services(service, NoOpSoupRealtimeSubscriptionService)
+}
+
+/// Build a GraphQL schema backed by query and realtime Soup services.
+pub fn build_schema_with_services<S, R, E, EAS, Auth, St, W, NR, PR, ER>(
+ service: S,
+ realtime_service: R,
+) -> SoupSchema<S, R, E, EAS, Auth, St, W, NR, PR, ER>
+where
+ S: SoupService + Clone,
+ R: SoupRealtimeSubscriptionService + Clone,
+ E: EmailService,
+ EAS: EntityAccessService,
+ Auth: MacroAuthorizationService,
+ St: Clone + Send + Sync + 'static,
+ EmailRouterState<E>: FromRef<St>,
+ Arc<EAS>: FromRef<St>,
+ MacroAuthorizationState<Auth>: FromRef<St>,
+ W: EntityPropertyWriter,
+ NR: SoupNotificationEdgeReader,
+ PR: EntityPropertyReader,
+ ER: SoupEmailContentEdgeReader,
{
Schema::build(
SoupQueryRoot::new(service),
PropertiesMutationRoot::<W>::new(),
- EmptySubscription,
+ SoupSubscriptionRoot::new(realtime_service),
)
.finish()
}
-/// Build a GraphQL schema for Soup backed by an `Arc`-shared service.
+/// Build a GraphQL schema backed by an `Arc`-shared query service.
pub fn build_schema_from_arc<S, E, EAS, Auth, St, W, NR, PR, ER>(
service: Arc<S>,
-) -> SharedSoupSchema<S, E, EAS, Auth, St, W, NR, PR, ER>
+) -> SoupSchema<Arc<S>, NoOpSoupRealtimeSubscriptionService, E, EAS, Auth, St, W, NR, PR, ER>
where
S: SoupService,
E: EmailService,
@@ -175,6 +226,29 @@ where
build_schema_with_service(service)
}
+/// Build a GraphQL schema backed by `Arc`-shared query and realtime services.
+pub fn build_schema_from_arcs<S, R, E, EAS, Auth, St, W, NR, PR, ER>(
+ service: Arc<S>,
+ realtime_service: Arc<R>,
+) -> SharedSoupSchema<S, R, E, EAS, Auth, St, W, NR, PR, ER>
+where
+ S: SoupService,
+ R: SoupRealtimeSubscriptionService,
+ E: EmailService,
+ EAS: EntityAccessService,
+ Auth: MacroAuthorizationService,
+ St: Clone + Send + Sync + 'static,
+ EmailRouterState<E>: FromRef<St>,
+ Arc<EAS>: FromRef<St>,
+ MacroAuthorizationState<Auth>: FromRef<St>,
+ W: EntityPropertyWriter,
+ NR: SoupNotificationEdgeReader,
+ PR: EntityPropertyReader,
+ ER: SoupEmailContentEdgeReader,
+{
+ build_schema_with_services(service, realtime_service)
+}
+
/// Root entry point for the complete GraphQL API.
#[Object]
impl<S, E, EAS, Auth, St, NR, PR, ER> SoupQueryRoot<S, E, EAS, Auth, St, NR, PR, ER>
@@ -192,11 +266,40 @@ where
ER: SoupEmailContentEdgeReader,
{
/// The authenticated user.
- async fn user(&self) -> GraphqlUser<S, E, EAS, Auth, St, NR, PR, ER> {
- GraphqlUser {
+ async fn user(
+ &self,
+ ctx: &Context<'_>,
+ ) -> async_graphql::Result<GraphqlUser<S, E, EAS, Auth, St, NR, PR, ER>> {
+ let user_id = require_authorized_user::<Auth, St>(ctx).await?;
+
+ Ok(GraphqlUser {
service: self.service.clone(),
_marker: PhantomData,
- }
+ user_id,
+ })
+ }
+}
+
+/// Root entry point for realtime Soup subscriptions.
+#[Subscription]
+impl<R, Auth, St, NR, PR, ER> SoupSubscriptionRoot<R, Auth, St, NR, PR, ER>
+where
+ R: SoupRealtimeSubscriptionService,
+ Auth: MacroAuthorizationService,
+ St: Clone + Send + Sync + 'static,
+ MacroAuthorizationState<Auth>: FromRef<St>,
+ NR: SoupNotificationEdgeReader,
+ PR: EntityPropertyReader,
+ ER: SoupEmailContentEdgeReader,
+{
+ /// Subscribe to realtime Soup updates for the authenticated user.
+ async fn soup_updates(
+ &self,
+ ctx: &Context<'_>,
+ ) -> async_graphql::Result<
+ impl async_graphql::futures_util::Stream<Item = GraphqlSoupItem<SoupEdges<NR, PR, ER>>>,
+ > {
+ resolve_soup_updates::<R, Auth, St, SoupEdges<NR, PR, ER>>(&self.service, ctx).await
}
}
@@ -217,9 +320,8 @@ where
ER: SoupEmailContentEdgeReader,
{
/// Stable id of the authenticated user.
- async fn id(&self, ctx: &Context<'_>) -> async_graphql::Result<async_graphql::ID> {
- let macro_user_id = require_authorized_user::<Auth, St>(ctx).await?;
- Ok(async_graphql::ID(macro_user_id.to_string()))
+ async fn id(&self) -> async_graphql::ID {
+ async_graphql::ID(self.user_id.to_string())
}
/// Fetch Soup items nested into grouping bins.
diff --git a/crates/complete_graph/src/schema/test.rs b/crates/complete_graph/src/schema/test.rs
index f13c3bfb21..da88994568 100644
--- a/crates/complete_graph/src/schema/test.rs
+++ b/crates/complete_graph/src/schema/test.rs
@@ -1,3 +1,4 @@
+use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use axum::http::{Request as HttpRequest, header};
@@ -10,7 +11,7 @@ use entity_access::domain::models::{
AccessError, AccessLevel, BotId, CallChannelInfo, EditAccessLevel, EntityAccessReceipt,
EntityPermission, EntityType, RequiredPermission, TeamRole, UserTeamInfo, ViewAccessLevel,
};
-use graphql_common::GraphqlSoupRequestParts;
+use graphql_common::GraphqlRequestParts;
use macro_authorization::{
INTERNAL_API_KEY_HEADER, INTERNAL_MACRO_USER_ID_HEADER, InternalIdentityClaims,
MacroAuthorizationError, MacroAuthorizationService, MacroAuthorizationState,
@@ -31,6 +32,27 @@ const VALID_USER_ID: &str = "macro|[email protected]";
const INTERNAL_USER_ID: &str = "macro|[email protected]";
const VALID_INTERNAL_KEY: &str = "valid-internal-key";
+#[derive(Clone)]
+struct TestRealtimeSubscriptionService {
+ #[allow(clippy::type_complexity)]
+ receiver: Arc<Mutex<Option<tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>>>>>,
+ subscribed_user: Arc<Mutex<Option<MacroUserIdStr<'static>>>>,
+}
+
+impl SoupRealtimeSubscriptionService for TestRealtimeSubscriptionService {
+ fn subscribe(
+ &self,
+ user_id: MacroUserIdStr<'static>,
+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {
+ *self.subscribed_user.lock().expect("subscribed user lock") = Some(user_id);
+ self.receiver
+ .lock()
+ .expect("subscription receiver lock")
+ .take()
+ .expect("test subscription is opened once")
+ }
+}
+
#[derive(Clone, Default)]
struct CountingSoupService {
raw_calls: Arc<AtomicUsize>,
@@ -472,6 +494,7 @@ impl FromRef<TestState> for Arc<CountingEntityAccessService> {
struct TestHarness {
schema: SoupSchema<
CountingSoupService,
+ NoOpSoupRealtimeSubscriptionService,
CountingEmailService,
CountingEntityAccessService,
FakeAuthorizationService,
@@ -555,12 +578,66 @@ impl TestHarness {
parts: axum::http::request::Parts,
) -> async_graphql::Response {
let request = async_graphql::Request::new(query)
- .data(GraphqlSoupRequestParts::new(parts))
+ .data(GraphqlRequestParts::new(parts))
.data(self.state.clone());
self.schema.execute(request).await
}
}
+#[tokio::test]
+async fn soup_updates_subscribes_as_the_authenticated_user() {
+ use async_graphql::futures_util::{StreamExt as _, pin_mut};
+
+ let user_id = MacroUserIdStr::parse_from_str(VALID_USER_ID).unwrap();
+ let subscribed_user = Arc::new(Mutex::new(None));
+ let (sender, receiver) = tokio::sync::mpsc::channel(1);
+ let realtime = TestRealtimeSubscriptionService {
+ receiver: Arc::new(Mutex::new(Some(receiver))),
+ subscribed_user: Arc::clone(&subscribed_user),
+ };
+ let schema: SoupSchema<
+ NoOpSoupService,
+ TestRealtimeSubscriptionService,
+ NoOpEmailService,
+ NoOpEntityAccessService,
+ SchemaOnlyAuthorizationService,
+ SchemaOnlyState,
+ NoOpEntityPropertyWriter,
+ NoOpSoupNotificationEdgeReader,
+ NoOpEntityPropertyReader,
+ NoOpSoupEmailContentEdgeReader,
+ > = build_schema_with_services(NoOpSoupService, realtime);
+ let request = async_graphql::Request::new(
+ "subscription { soupUpdates { id entityType entity { __typename ... on GraphqlSoupDocument { id name } } } }",
+ )
+ .data(user_id.clone());
+ let responses = schema.execute_stream(request);
+ pin_mut!(responses);
+
+ let document_id = Uuid::from_u128(42);
+ sender
+ .send(Arc::new(grouped_document(document_id).map_extra(|_| ())))
+ .await
+ .expect("subscription remains open");
+ let response = responses.next().await.expect("one subscription response");
+
+ assert!(response.errors.is_empty(), "{:?}", response.errors);
+ let data = response.data.into_json().expect("response data is JSON");
+ assert_eq!(data["soupUpdates"]["id"], document_id.to_string());
+ assert_eq!(data["soupUpdates"]["entityType"], "DOCUMENT");
+ assert_eq!(
+ data["soupUpdates"]["entity"]["name"],
+ format!("Document {document_id}")
+ );
+ assert_eq!(
+ subscribed_user
+ .lock()
+ .expect("subscribed user lock")
+ .as_ref(),
+ Some(&user_id)
+ );
+}
+
#[tokio::test]
async fn user_id_resolves_without_touching_services() {
let harness = harness();
diff --git a/crates/complete_graph/src/sdl_test.rs b/crates/complete_graph/src/sdl_test.rs
index a232e557e8..b37a7e8ed0 100644
--- a/crates/complete_graph/src/sdl_test.rs
+++ b/crates/complete_graph/src/sdl_test.rs
@@ -34,6 +34,8 @@ fn soup_response_schema_exposes_frontend_fields() {
"type GraphqlSoupChat {",
"deletedAt: String",
"type GraphqlSoupProject {",
+ "type SoupSubscriptionRoot {",
+ "soupUpdates: GraphqlSoupItem!",
] {
assert_sdl_line(&sdl, expected);
}
diff --git a/crates/graphql_common/src/authorization.rs b/crates/graphql_common/src/authorization.rs
index 4812f9d478..ec8a487184 100644
--- a/crates/graphql_common/src/authorization.rs
+++ b/crates/graphql_common/src/authorization.rs
@@ -21,6 +21,10 @@ where
MacroAuthorizationState<Auth>: FromRef<St>,
St: Clone + Send + Sync + 'static,
{
+ if let Some(user_id) = ctx.data_opt::<MacroUserIdStr<'static>>() {
+ return Ok(user_id.clone());
+ }
+
let Cached(authorization) =
extract_part::<Cached<OptionalMacroAuthorizationExtractor<Auth>>, St>(ctx).await?;
diff --git a/crates/graphql_common/src/authorization/test.rs b/crates/graphql_common/src/authorization/test.rs
index c27815ad16..acb7080c6c 100644
--- a/crates/graphql_common/src/authorization/test.rs
+++ b/crates/graphql_common/src/authorization/test.rs
@@ -16,7 +16,7 @@ use model_user::UserContext;
use rootcause::Report;
use super::*;
-use crate::GraphqlSoupRequestParts;
+use crate::GraphqlRequestParts;
const VALID_USER_ID: &str = "macro|[email protected]";
@@ -105,7 +105,7 @@ async fn execute(
};
let schema = Schema::build(TestQuery, EmptyMutation, EmptySubscription).finish();
let request = async_graphql::Request::new(query)
- .data(GraphqlSoupRequestParts::new(parts))
+ .data(GraphqlRequestParts::new(parts))
.data(state);
schema.execute(request).await
diff --git a/crates/graphql_common/src/extract.rs b/crates/graphql_common/src/extract.rs
index 8de307a1aa..6ebfd2e166 100644
--- a/crates/graphql_common/src/extract.rs
+++ b/crates/graphql_common/src/extract.rs
@@ -1,7 +1,7 @@
use async_graphql::Context;
use axum::extract::FromRequestParts;
-use crate::request_context::GraphqlSoupRequestParts;
+use crate::request_context::GraphqlRequestParts;
/// Run an axum extractor against the request parts stored in the GraphQL
/// context, using the embedding router state `St` also stored there.
@@ -11,7 +11,7 @@ where
T::Rejection: std::fmt::Display,
St: Clone + Send + Sync + 'static,
{
- let parts = ctx.data::<GraphqlSoupRequestParts>()?;
+ let parts = ctx.data::<GraphqlRequestParts>()?;
let state = ctx.data::<St>()?;
parts
.extract_with_state::<T, St>(state)
diff --git a/crates/graphql_common/src/lib.rs b/crates/graphql_common/src/lib.rs
index eec3c05e39..3b6b013e29 100644
--- a/crates/graphql_common/src/lib.rs
+++ b/crates/graphql_common/src/lib.rs
@@ -28,4 +28,4 @@ pub use property_filter::{
GraphqlPropertiesBinaryExpr, GraphqlPropertiesExpr, GraphqlPropertiesLiteral,
GraphqlPropertyEntityType, GraphqlPropertyMatchValue,
};
-pub use request_context::GraphqlSoupRequestParts;
+pub use request_context::GraphqlRequestParts;
diff --git a/crates/graphql_common/src/request_context.rs b/crates/graphql_common/src/request_context.rs
index 1ebfe0093e..a9aae30de7 100644
--- a/crates/graphql_common/src/request_context.rs
+++ b/crates/graphql_common/src/request_context.rs
@@ -1,6 +1,6 @@
use axum::{extract::FromRequestParts, http::request::Parts};
-/// Request-scoped HTTP request parts for a Soup GraphQL query.
+/// Request-scoped HTTP request parts for a Soup GraphQL operation.
///
/// Resolvers run axum `FromRequestParts` extractors against these parts on
/// demand instead of the embedding service loading everything upfront. This
@@ -10,12 +10,12 @@ use axum::{extract::FromRequestParts, http::request::Parts};
/// Extraction results that should be computed once per request should be
/// wrapped in `axum_extra::extract::Cached`, which memoizes inside the parts'
/// extensions exactly like the REST routes do.
-pub struct GraphqlSoupRequestParts {
+pub struct GraphqlRequestParts {
/// Mutable HTTP request parts shared by lazy resolver extractors.
parts: tokio::sync::Mutex<Parts>,
}
-impl GraphqlSoupRequestParts {
+impl GraphqlRequestParts {
/// Wrap the parts of the incoming HTTP request.
pub fn new(parts: Parts) -> Self {
Self {
diff --git a/crates/graphql_soup/Cargo.toml b/crates/graphql_soup/Cargo.toml
index b8e6e82229..056d76310b 100644
--- a/crates/graphql_soup/Cargo.toml
+++ b/crates/graphql_soup/Cargo.toml
@@ -6,6 +6,8 @@ version = "0.1.0"
[dependencies]
async-graphql = { workspace = true, features = ["dataloader"] }
+async-stream = { workspace = true }
+futures = { workspace = true }
axum = { workspace = true }
axum-extra = { workspace = true }
chrono = { workspace = true }
@@ -31,5 +33,6 @@ models_soup = { path = "../models_soup" }
serde = { workspace = true }
serde_json = { workspace = true }
soup = { path = "../soup", default-features = false, features = ["ports"] }
+soup_realtime = { path = "../soup_realtime", default-features = false, features = ["ports"] }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/graphql_soup/src/lib.rs b/crates/graphql_soup/src/lib.rs
index 2dc1437b54..8c8560e498 100644
--- a/crates/graphql_soup/src/lib.rs
+++ b/crates/graphql_soup/src/lib.rs
@@ -13,7 +13,7 @@ mod objects;
/// Top-level Soup query resolver.
mod resolvers;
-pub use graphql_common::{GraphqlSoupEntityType, GraphqlSoupRequestParts};
+pub use graphql_common::{GraphqlRequestParts, GraphqlSoupEntityType};
pub use inputs::{GraphqlSimpleSortMethod, GroupedSoupInput, SoupInput};
pub use objects::{
GraphqlSoupBin, GraphqlSoupCall, GraphqlSoupChannel, GraphqlSoupChannelThread, GraphqlSoupChat,
@@ -21,4 +21,4 @@ pub use objects::{
GraphqlSoupEntity, GraphqlSoupForeignEntity, GraphqlSoupItem, GraphqlSoupProject, GroupedSoup,
SoupEntityEdges, SoupPage,
};
-pub use resolvers::{resolve_grouped_soup, resolve_soup};
+pub use resolvers::{resolve_grouped_soup, resolve_soup, resolve_soup_updates};
diff --git a/crates/graphql_soup/src/resolvers.rs b/crates/graphql_soup/src/resolvers.rs
index 6efaff51f0..ac8e903820 100644
--- a/crates/graphql_soup/src/resolvers.rs
+++ b/crates/graphql_soup/src/resolvers.rs
@@ -11,16 +11,40 @@ use entity_access::{
domain::{models::MemberTeamRole, ports::EntityAccessService},
inbound::axum_extractors::OptionalMacroUserTeamExtractorV2,
};
+use futures::Stream;
use graphql_common::{extract_part, require_authorized_user};
use macro_authorization::{MacroAuthorizationService, MacroAuthorizationState};
use models_pagination::TypeEraseCursor;
use soup::domain::{models::grouping::NestedSoupGroups, ports::SoupService};
+use soup_realtime::domain::ports::SoupRealtimeSubscriptionService;
use crate::{
inputs::{GroupedSoupInput, SoupInput},
- objects::{GroupedSoup, SoupEntityEdges, SoupPage},
+ objects::{GraphqlSoupItem, GroupedSoup, SoupEntityEdges, SoupPage},
};
+/// Subscribe to realtime Soup updates for the authenticated user.
+pub async fn resolve_soup_updates<R, Auth, St, Edges>(
+ service: &R,
+ ctx: &Context<'_>,
+) -> async_graphql::Result<impl Stream<Item = GraphqlSoupItem<Edges>> + Send + 'static>
+where
+ R: SoupRealtimeSubscriptionService,
+ Auth: MacroAuthorizationService,
+ MacroAuthorizationState<Auth>: FromRef<St>,
+ St: Clone + Send + Sync + 'static,
+ Edges: SoupEntityEdges,
+{
+ let macro_user_id = require_authorized_user::<Auth, St>(ctx).await?;
+ let mut receiver = service.subscribe(macro_user_id);
+
+ Ok(async_stream::stream! {
+ while let Some(item) = receiver.recv().await {
+ yield GraphqlSoupItem::from(item.as_ref().clone());
+ }
+ })
+}
+
/// Resolve Soup items nested into grouping bins for the authenticated user.
pub async fn resolve_grouped_soup<S, Auth, St, Edges>(
service: &S,
diff --git a/crates/models_soup/src/call_record.rs b/crates/models_soup/src/call_record.rs
index e5ab475cb8..ac464fc96a 100644
--- a/crates/models_soup/src/call_record.rs
+++ b/crates/models_soup/src/call_record.rs
@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
use uuid::Uuid;
/// A participant in a call record, as displayed in Soup.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[serde(rename_all = "camelCase")]
pub struct SoupCallRecordParticipant {
@@ -19,7 +19,7 @@ pub struct SoupCallRecordParticipant {
/// A call record as displayed in Soup. Excludes room_name, egress_id,
/// and transcript — fields that are irrelevant for the soup feed.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[serde(rename_all = "camelCase")]
pub struct SoupCallRecord<T = ()> {
diff --git a/crates/models_soup/src/comms.rs b/crates/models_soup/src/comms.rs
index c4a8de34a9..eeacaf66e8 100644
--- a/crates/models_soup/src/comms.rs
+++ b/crates/models_soup/src/comms.rs
@@ -105,7 +105,7 @@ pub struct LatestMessage {
}
/// A channel as displayed in Soup.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
pub struct SoupChannel {
/// Channel metadata and participants.
@@ -124,7 +124,7 @@ pub struct SoupChannel {
///
/// This mirrors the public channel-message API shape so soup consumers can
/// render a thread root the same way they render a channel timeline message.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
pub struct SoupChannelThread {
/// Message id.
diff --git a/crates/models_soup/src/crm_company.rs b/crates/models_soup/src/crm_company.rs
index 3bd0bcd001..bee5d664ae 100644
--- a/crates/models_soup/src/crm_company.rs
+++ b/crates/models_soup/src/crm_company.rs
@@ -5,7 +5,7 @@ use uuid::Uuid;
/// A CRM domain as displayed in Soup. Mirrors the crm crate's
/// [`CrmDomain`] with a stable wire shape that the FE can rely on.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[serde(rename_all = "camelCase")]
pub struct SoupCrmDomain {
@@ -33,7 +33,7 @@ impl From<CrmDomain> for SoupCrmDomain {
/// A CRM company as displayed in Soup. Carries the core company
/// fields plus display metadata resolved from `crm_domain_directory`
/// against the primary (earliest-created) domain.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[serde(rename_all = "camelCase")]
pub struct SoupCrmCompany<T = ()> {
diff --git a/crates/models_soup/src/email_thread.rs b/crates/models_soup/src/email_thread.rs
index c9d3aa4b8d..08cc2ed985 100644
--- a/crates/models_soup/src/email_thread.rs
+++ b/crates/models_soup/src/email_thread.rs
@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
use uuid::Uuid;
/// A contact participating in an email thread as displayed in Soup.
-#[derive(Debug, Doppleganger, Serialize, Deserialize)]
+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]
#[dg(backward = email::domain::models::Contact)]
@@ -24,7 +24,7 @@ pub struct SoupContact {
}
/// An email label as displayed in Soup.
-#[derive(Debug, Doppleganger, Serialize, Deserialize)]
+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]
#[dg(backward = email::domain::models::Label)]
@@ -49,7 +49,7 @@ pub struct SoupLabel {
}
/// Gmail-style message list visibility for a label.
-#[derive(Debug, Doppleganger, Serialize, Deserialize)]
+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]
#[dg(backward = email::domain::models::MessageListVisibility)]
@@ -62,7 +62,7 @@ pub enum SoupMessageListVisibility {
}
/// Gmail-style label list visibility.
-#[derive(Debug, Doppleganger, Serialize, Deserialize)]
+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]
#[dg(backward = email::domain::models::LabelListVisibility)]
@@ -77,7 +77,7 @@ pub enum SoupLabelListVisibility {
}
/// Whether an email label is provider-created or user-created.
-#[derive(Debug, Doppleganger, Serialize, Deserialize)]
+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]
#[dg(backward = email::domain::models::LabelType)]
@@ -90,7 +90,7 @@ pub enum SoupLabelType {
}
/// An email attachment as displayed in Soup.
-#[derive(Debug, Doppleganger, Serialize, Deserialize)]
+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]
#[dg(backward = email::domain::models::Attachment)]
@@ -115,7 +115,7 @@ pub struct SoupAttachment {
}
/// Email thread preview data as displayed in Soup.
-#[derive(Debug, Doppleganger, Serialize, Deserialize)]
+#[derive(Clone, Debug, Doppleganger, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "mock", derive(PartialEq, Eq))]
#[dg(backward = email::domain::models::EmailThreadPreview)]
@@ -159,7 +159,7 @@ pub struct SoupEmailThreadPreview {
}
/// Email thread preview enriched with related metadata for Soup.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[serde(rename_all = "camelCase")]
pub struct SoupEnrichedEmailThreadPreview<T = ()> {
diff --git a/crates/models_soup/src/foreign_entity.rs b/crates/models_soup/src/foreign_entity.rs
index ffd5b253d7..77655c7bc6 100644
--- a/crates/models_soup/src/foreign_entity.rs
+++ b/crates/models_soup/src/foreign_entity.rs
@@ -4,7 +4,7 @@ use serde_json::Value;
use uuid::Uuid;
/// A foreign entity record as displayed in Soup.
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
#[serde(rename_all = "camelCase")]
pub struct SoupForeignEntity {
diff --git a/crates/models_soup/src/item.rs b/crates/models_soup/src/item.rs
index fa9e60af2d..5cd58b747b 100644
--- a/crates/models_soup/src/item.rs
+++ b/crates/models_soup/src/item.rs
@@ -19,7 +19,7 @@ use uuid::Uuid;
mod test;
/// A single item in the Soup feed.
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Clone, Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase", tag = "tag", content = "data")]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
pub enum SoupItem<T = ()> {
diff --git a/crates/soup_realtime/Cargo.toml b/crates/soup_realtime/Cargo.toml
index a314e849e6..328f230c50 100644
--- a/crates/soup_realtime/Cargo.toml
+++ b/crates/soup_realtime/Cargo.toml
@@ -19,9 +19,16 @@ outbound = [
"dep:soup",
"ports",
]
-ports = ["dep:futures", "dep:rootcause", "dep:tracing"]
+ports = [
+ "dep:broadcast",
+ "dep:futures",
+ "dep:rootcause",
+ "dep:tokio",
+ "dep:tracing",
+]
[dependencies]
+broadcast = { path = "../broadcast", optional = true }
documents = { path = "../documents", default-features = false, optional = true }
entity_access = { path = "../entity_access", default-features = false, features = ["ports"], optional = true }
futures = { workspace = true, optional = true }
@@ -35,7 +42,7 @@ rdkafka = { workspace = true, optional = true }
rootcause = { workspace = true, optional = true }
serde = { workspace = true }
soup = { path = "../soup", default-features = false, features = ["ports"], optional = true }
-tokio = { workspace = true, features = ["time"], optional = true }
+tokio = { workspace = true, features = ["sync", "time"], optional = true }
tracing = { workspace = true, optional = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/soup_realtime/README.md b/crates/soup_realtime/README.md
index 2aa71b4ac3..d5c3e589b6 100644
--- a/crates/soup_realtime/README.md
+++ b/crates/soup_realtime/README.md
@@ -5,3 +5,5 @@ Hexagonal components for expanding `document.updated` Kafka events into one full
This crate intentionally does not start a consumer or wire itself into a service binary. A later composition root can combine the domain service with the entity-access, Soup repository, Kafka publisher, and Kafka consumer adapters exposed here.
Delivery is at least once. If publication partially succeeds and processing is retried, downstream consumers may receive duplicate messages.
+
+`SoupRealtimeConsumerService` can run a realtime consumer adapter and distribute received items through instance-local, user-keyed subscriptions. Subscription values are shared as `Arc<SoupItem<()>>` so multiple subscribers do not require cloning the underlying item.
diff --git a/crates/soup_realtime/src/domain/ports.rs b/crates/soup_realtime/src/domain/ports.rs
index a67f874ff2..5a43e5d60a 100644
--- a/crates/soup_realtime/src/domain/ports.rs
+++ b/crates/soup_realtime/src/domain/ports.rs
@@ -1,6 +1,6 @@
-//! Ports used by realtime Soup fan-out.
+//! Ports used by realtime Soup services.
-use std::future::Future;
+use std::{future::Future, sync::Arc};
use macro_user_id::user_id::MacroUserIdStr;
use model_entity::Entity;
@@ -18,6 +18,47 @@ pub trait SoupRealtimeService: Send + Sync + 'static {
) -> impl Future<Output = Result<(), Report>> + Send;
}
+/// Receives complete recipient-targeted Soup messages.
+pub trait SoupRealtimeConsumer: Send + Sync + 'static {
+ /// Waits for and returns the next realtime Soup message.
+ fn recv(&self) -> impl Future<Output = Result<SoupRealtimeMessage, Report>> + Send;
+}
+
+/// Provides user-scoped subscriptions to received realtime Soup items.
+pub trait SoupRealtimeSubscriptionService: Send + Sync + 'static {
+ /// Subscribes to realtime Soup items addressed to `user_id`.
+ fn subscribe(
+ &self,
+ user_id: MacroUserIdStr<'static>,
+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>>;
+}
+
+impl<S> SoupRealtimeSubscriptionService for Arc<S>
+where
+ S: SoupRealtimeSubscriptionService,
+{
+ fn subscribe(
+ &self,
+ user_id: MacroUserIdStr<'static>,
+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {
+ self.as_ref().subscribe(user_id)
+ }
+}
+
+/// No-op subscription service used when only the GraphQL schema shape is needed.
+#[derive(Clone, Copy, Debug, Default)]
+pub struct NoOpSoupRealtimeSubscriptionService;
+
+impl SoupRealtimeSubscriptionService for NoOpSoupRealtimeSubscriptionService {
+ fn subscribe(
+ &self,
+ _user_id: MacroUserIdStr<'static>,
+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {
+ let (_sender, receiver) = tokio::sync::mpsc::channel(1);
+ receiver
+ }
+}
+
/// Resolves the users who currently have access to an entity.
pub trait UserAccessExpander: Send + Sync + 'static {
/// Returns all current user accessors for `entity`.
diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rs
index 947b495bc1..cdf613cf0f 100644
--- a/crates/soup_realtime/src/domain/service.rs
+++ b/crates/soup_realtime/src/domain/service.rs
@@ -1,20 +1,99 @@
-//! Realtime Soup hydration and fan-out orchestration.
+//! Realtime Soup orchestration.
#[cfg(test)]
mod test;
-use std::collections::HashSet;
+use std::{collections::HashSet, num::NonZeroUsize, sync::Arc};
+use broadcast::{BroadcastManager, GlobalSpawner};
use futures::{StreamExt as _, stream};
+use macro_user_id::user_id::MacroUserIdStr;
use model_entity::Entity;
use models_soup::item::SoupItem;
use rootcause::prelude::{Report, ResultExt as _};
use super::{
models::SoupRealtimeMessage,
- ports::{SoupItemReader, SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},
+ ports::{
+ SoupItemReader, SoupRealtimeConsumer, SoupRealtimePublisher, SoupRealtimeService,
+ SoupRealtimeSubscriptionService, UserAccessExpander,
+ },
};
+/// Number of messages retained by each user-keyed broadcast channel.
+const BROADCAST_BUFFER_CAPACITY: NonZeroUsize = NonZeroUsize::new(64).unwrap();
+/// Number of messages buffered for each individual subscriber.
+const SUBSCRIBER_BUFFER_CAPACITY: NonZeroUsize = NonZeroUsize::new(16).unwrap();
+
+/// Service for distributing recipient-targeted realtime Soup messages.
+pub struct SoupRealtimeConsumerService<C>
+where
+ C: SoupRealtimeConsumer,
+{
+ consumer: C,
+ broadcasts: BroadcastManager<GlobalSpawner, MacroUserIdStr<'static>, Arc<SoupItem<()>>>,
+}
+
+impl<C> SoupRealtimeConsumerService<C>
+where
+ C: SoupRealtimeConsumer,
+{
+ /// Creates a realtime Soup consumer service backed by `consumer`.
+ pub fn new(consumer: C) -> Self {
+ Self {
+ consumer,
+ broadcasts: BroadcastManager::new(GlobalSpawner, BROADCAST_BUFFER_CAPACITY),
+ }
+ }
+
+ /// Subscribes to realtime Soup items addressed to `user_id`.
+ ///
+ /// The returned receiver is closed if its buffer fills, ensuring a slow
+ /// subscriber cannot delay the shared consumer or other subscribers.
+ #[must_use]
+ pub fn subscribe(
+ &self,
+ user_id: MacroUserIdStr<'static>,
+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {
+ self.broadcasts
+ .subscribe(user_id, SUBSCRIBER_BUFFER_CAPACITY)
+ }
+
+ /// Receives messages and distributes them to subscribers until reception fails.
+ ///
+ /// Callers should run this future in a supervised task. A message for a user
+ /// without active subscribers is intentionally dropped.
+ #[tracing::instrument(skip(self), err)]
+ pub async fn run(&self) -> Result<(), Report> {
+ loop {
+ let SoupRealtimeMessage { user_id, item } = self
+ .consumer
+ .recv()
+ .await
+ .context("failed to receive realtime Soup message")?;
+
+ match self.broadcasts.publish(&user_id, Arc::new(item)) {
+ Ok(subscriber_count) => {
+ tracing::trace!(subscriber_count, "distributed realtime Soup message")
+ }
+ Err(_) => tracing::trace!("dropping realtime Soup message without subscribers"),
+ }
+ }
+ }
+}
+
+impl<C> SoupRealtimeSubscriptionService for SoupRealtimeConsumerService<C>
+where
+ C: SoupRealtimeConsumer,
+{
+ fn subscribe(
+ &self,
+ user_id: MacroUserIdStr<'static>,
+ ) -> tokio::sync::mpsc::Receiver<Arc<SoupItem<()>>> {
+ SoupRealtimeConsumerService::subscribe(self, user_id)
+ }
+}
+
/// Maximum number of Kafka publications polled concurrently.
const PUBLISH_CONCURRENCY: usize = 16;
diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rs
index f9bf0e77c0..6829ef355c 100644
--- a/crates/soup_realtime/src/domain/service/test.rs
+++ b/crates/soup_realtime/src/domain/service/test.rs
@@ -1,6 +1,7 @@
-use std::collections::{HashMap, HashSet};
+use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
+use std::time::Duration;
use chrono::{DateTime, Utc};
use macro_user_id::user_id::MacroUserIdStr;
@@ -9,7 +10,9 @@ use models_soup::{document::SoupDocument, item::SoupItem};
use uuid::Uuid;
use super::*;
-use crate::domain::ports::{SoupItemReader, SoupRealtimePublisher, UserAccessExpander};
+use crate::domain::ports::{
+ SoupItemReader, SoupRealtimeConsumer, SoupRealtimePublisher, UserAccessExpander,
+};
const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";
const OTHER_DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000002";
@@ -66,6 +69,20 @@ impl SoupItemReader for FakeReader {
}
}
+struct FakeRealtimeConsumer {
+ messages: Mutex<VecDeque<Result<SoupRealtimeMessage, Report>>>,
+}
+
+impl SoupRealtimeConsumer for FakeRealtimeConsumer {
+ async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {
+ self.messages
+ .lock()
+ .expect("consumer messages lock")
+ .pop_front()
+ .unwrap_or_else(|| Err(rootcause::report!("consumer stopped")))
+ }
+}
+
struct FakePublisher {
messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,
fail_users: HashSet<String>,
@@ -123,6 +140,13 @@ fn document_entity() -> Entity<'static> {
EntityType::Document.with_entity_string(DOCUMENT_ID.to_string())
}
+fn document_name(item: &SoupItem<()>) -> &str {
+ let SoupItem::Document(document) = item else {
+ panic!("expected document item")
+ };
+ &document.name
+}
+
fn harness(
users: Vec<MacroUserIdStr<'static>>,
responses: HashMap<String, ReadResponse>,
@@ -165,6 +189,62 @@ fn item_response(
)
}
+#[tokio::test]
+async fn consumer_service_distributes_items_only_to_their_users() {
+ let one = user("one");
+ let two = user("two");
+ let consumer = FakeRealtimeConsumer {
+ messages: Mutex::new(VecDeque::from([
+ Ok(SoupRealtimeMessage::new(
+ one.clone(),
+ document_item(DOCUMENT_ID, "For one", None),
+ )),
+ Ok(SoupRealtimeMessage::new(
+ two.clone(),
+ document_item(OTHER_DOCUMENT_ID, "For two", None),
+ )),
+ ])),
+ };
+ let service = Arc::new(SoupRealtimeConsumerService::new(consumer));
+ let mut one_first = service.subscribe(one.clone());
+ let mut one_second = service.subscribe(one);
+ let mut two_receiver = service.subscribe(two);
+
+ let run_error = tokio::spawn({
+ let service = Arc::clone(&service);
+ async move { service.run().await }
+ })
+ .await
+ .expect("consumer task joins")
+ .expect_err("fake consumer eventually stops");
+ assert!(run_error.to_string().contains("failed to receive"));
+
+ let one_first_item = tokio::time::timeout(Duration::from_secs(1), one_first.recv())
+ .await
+ .expect("first user subscription receives before timeout")
+ .expect("first user subscription remains open");
+ let one_second_item = tokio::time::timeout(Duration::from_secs(1), one_second.recv())
+ .await
+ .expect("second user subscription receives before timeout")
+ .expect("second user subscription remains open");
+ let two_item = tokio::time::timeout(Duration::from_secs(1), two_receiver.recv())
+ .await
+ .expect("other user subscription receives before timeout")
+ .expect("other user subscription remains open");
+
+ assert_eq!(document_name(&one_first_item), "For one");
+ assert!(Arc::ptr_eq(&one_first_item, &one_second_item));
+ assert_eq!(document_name(&two_item), "For two");
+ assert!(matches!(
+ one_first.try_recv(),
+ Err(tokio::sync::mpsc::error::TryRecvError::Empty)
+ ));
+ assert!(matches!(
+ two_receiver.try_recv(),
+ Err(tokio::sync::mpsc::error::TryRecvError::Empty)
+ ));
+}
+
#[tokio::test]
async fn zero_users_skips_reads_and_publications() {
let harness = harness(Vec::new(), HashMap::new(), false, HashSet::new());
diff --git a/crates/soup_realtime/src/inbound/mod.rs b/crates/soup_realtime/src/inbound/mod.rs
index e22ac579a2..1d73c01b15 100644
--- a/crates/soup_realtime/src/inbound/mod.rs
+++ b/crates/soup_realtime/src/inbound/mod.rs
@@ -2,5 +2,3 @@
/// Kafka consumer for document update events.
pub mod kafka_consumer;
-/// Independent Kafka consumer for recipient-targeted Soup messages.
-pub mod soup_consumer;
diff --git a/crates/soup_realtime/src/lib.rs b/crates/soup_realtime/src/lib.rs
index ab4794b1f9..4fe42ac425 100644
--- a/crates/soup_realtime/src/lib.rs
+++ b/crates/soup_realtime/src/lib.rs
@@ -12,5 +12,5 @@ pub mod domain;
pub mod inbound;
/// Outbound infrastructure adapters.
-#[cfg(feature = "outbound")]
+#[cfg(any(feature = "consumer", feature = "outbound"))]
pub mod outbound;
diff --git a/crates/soup_realtime/src/outbound/mod.rs b/crates/soup_realtime/src/outbound/mod.rs
index 01d025d389..39fe268ff5 100644
--- a/crates/soup_realtime/src/outbound/mod.rs
+++ b/crates/soup_realtime/src/outbound/mod.rs
@@ -1,8 +1,14 @@
-//! Outbound adapters used by realtime Soup fan-out.
+//! Outbound adapters used by realtime Soup services.
/// Current entity-access expansion adapter.
+#[cfg(feature = "outbound")]
pub mod entity_access;
/// Kafka-backed realtime Soup publisher adapter.
+#[cfg(feature = "outbound")]
pub mod kafka_publisher;
+/// Independent Kafka consumer for recipient-targeted Soup messages.
+#[cfg(feature = "consumer")]
+pub mod soup_consumer;
/// User-scoped Soup item reader adapter.
+#[cfg(feature = "outbound")]
pub mod soup_item_reader;
diff --git a/crates/soup_realtime/src/inbound/soup_consumer.rs b/crates/soup_realtime/src/outbound/soup_consumer.rs
similarity index 91%
rename from crates/soup_realtime/src/inbound/soup_consumer.rs
rename to crates/soup_realtime/src/outbound/soup_consumer.rs
index a0dca06f2a..6199e08029 100644
--- a/crates/soup_realtime/src/inbound/soup_consumer.rs
+++ b/crates/soup_realtime/src/outbound/soup_consumer.rs
@@ -13,7 +13,10 @@ use kafka_util::{InitialOffset, KafkaEventConsumer, Ungrouped};
use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};
use rootcause::prelude::{Report, ResultExt as _};
-use crate::domain::models::{SoupMacroEvent, SoupRealtimeMessage};
+use crate::domain::{
+ models::{SoupMacroEvent, SoupRealtimeMessage},
+ ports::SoupRealtimeConsumer,
+};
/// Maximum time to wait for Soup topic metadata during partition assignment.
const TOPIC_METADATA_TIMEOUT: Duration = Duration::from_secs(10);
@@ -73,3 +76,9 @@ impl SoupTopicConsumer {
}
}
}
+
+impl SoupRealtimeConsumer for SoupTopicConsumer {
+ async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {
+ SoupTopicConsumer::recv(self).await
+ }
+}
diff --git a/crates/soup_realtime/src/inbound/soup_consumer/test.rs b/crates/soup_realtime/src/outbound/soup_consumer/test.rs
similarity index 100%
rename from crates/soup_realtime/src/inbound/soup_consumer/test.rs
rename to crates/soup_realtime/src/outbound/soup_consumer/test.rs
diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.toml
index 39e586e27a..8651726f80 100644
--- a/services/document_storage_service/Cargo.toml
+++ b/services/document_storage_service/Cargo.toml
@@ -27,7 +27,12 @@ default = [
"location_check",
]
delete_document_worker = []
-graphql = ["dep:async-graphql", "dep:async-graphql-axum", "dep:complete_graph"]
+graphql = [
+ "dep:async-graphql",
+ "dep:async-graphql-axum",
+ "dep:complete_graph",
+ "dep:soup_realtime",
+]
local = ["local_auth"]
local_auth = ["macro_authorization/local_auth"]
location_check = []
@@ -44,7 +49,12 @@ aws-sdk-s3 = { workspace = true }
aws-sdk-secretsmanager = { workspace = true }
aws-sdk-sns = { workspace = true }
aws-sdk-sqs = { workspace = true }
-axum = { workspace = true, features = ["macros", "multipart", "tower-log"] }
+axum = { workspace = true, features = [
+ "macros",
+ "multipart",
+ "tower-log",
+ "ws",
+] }
axum-extra = { workspace = true }
base64 = { workspace = true }
bots = { path = "../../crates/bots" }
@@ -168,6 +178,9 @@ soup = { path = "../../crates/soup", features = [
"inbound",
"outbound",
] }
+soup_realtime = { path = "../../crates/soup_realtime", default-features = false, features = [
+ "consumer",
+], optional = true }
sqlx = { workspace = true }
sqs_client = { path = "../../crates/sqs_client", default-features = false, features = [
"document",
diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rs
index f3402e6a51..1c6c1df6bf 100644
--- a/services/document_storage_service/src/api/context.rs
+++ b/services/document_storage_service/src/api/context.rs
@@ -95,6 +95,10 @@ use soup::{
domain::service::SoupImpl, inbound::axum_router::SoupRouterState,
outbound::pg_soup_repo::PgSoupRepo,
};
+#[cfg(feature = "graphql")]
+use soup_realtime::{
+ domain::service::SoupRealtimeConsumerService, outbound::soup_consumer::SoupTopicConsumer,
+};
use sqlx::PgPool;
use std::sync::Arc;
use sync_service_client::SyncServiceClient;
@@ -145,12 +149,17 @@ pub(crate) type DssSoupService = SoupImpl<
type DssSoupState =
SoupRouterState<DssSoupService, DssEmailService, EntityAccessService, AuthorizationService>;
+/// Realtime Soup consumer service used by GraphQL subscriptions.
+#[cfg(feature = "graphql")]
+pub(crate) type DssSoupRealtimeService = SoupRealtimeConsumerService<SoupTopicConsumer>;
+
/// GraphQL Soup schema wired to the DSS services; the `ApiContext` state
/// parameter lets GraphQL resolvers run the same axum extractors as the REST
/// routes, lazily, against the stored request parts.
#[cfg(feature = "graphql")]
pub(crate) type DssGraphqlSoupSchema = complete_graph::SharedSoupSchema<
DssSoupService,
+ DssSoupRealtimeService,
DssEmailService,
EntityAccessService,
AuthorizationService,
diff --git a/services/document_storage_service/src/api/graphql_soup.rs b/services/document_storage_service/src/api/graphql_soup.rs
index d1fb07f2e0..1a7fa28da0 100644
--- a/services/document_storage_service/src/api/graphql_soup.rs
+++ b/services/document_storage_service/src/api/graphql_soup.rs
@@ -1,61 +1,96 @@
use crate::api::context::{ApiContext, AuthorizationService};
-use async_graphql::{ServerError, http::GraphiQLSource};
-use async_graphql_axum::{GraphQLRequest, GraphQLResponse};
+use async_graphql::{
+ Data,
+ http::{ALL_WEBSOCKET_PROTOCOLS, GraphiQLSource},
+};
+use async_graphql_axum::{GraphQLProtocol, GraphQLRequest, GraphQLResponse, GraphQLWebSocket};
use axum::{
Router,
- extract::{FromRequest, FromRequestParts, OriginalUri, Request, State},
+ extract::{State, WebSocketUpgrade},
+ http::{StatusCode, request::Parts},
response::{Html, IntoResponse, Response},
routing::get,
};
use axum_extra::extract::Cached;
-use complete_graph::GraphqlSoupRequestParts;
+use complete_graph::GraphqlRequestParts;
use macro_authorization::OptionalMacroAuthorizationExtractor;
+use macro_user_id::user_id::MacroUserIdStr;
+
+const GRAPHQL_PATH: &str = "/soup/graphql";
+const GRAPHQL_SUBSCRIPTION_PATH: &str = "/soup/graphql/ws";
pub(crate) fn router() -> Router<ApiContext> {
- Router::new().route("/soup/graphql", get(graphiql).post(handler))
+ Router::new()
+ .route(GRAPHQL_PATH, get(graphiql).post(graphql_handler))
+ .route(GRAPHQL_SUBSCRIPTION_PATH, get(subscription_handler))
}
-async fn graphiql(OriginalUri(uri): OriginalUri) -> Html<String> {
- Html(GraphiQLSource::build().endpoint(uri.path()).finish())
+async fn graphiql() -> Html<String> {
+ Html(
+ GraphiQLSource::build()
+ .endpoint(GRAPHQL_PATH)
+ .subscription_endpoint(GRAPHQL_SUBSCRIPTION_PATH)
+ .finish(),
+ )
}
-async fn handler(State(state): State<ApiContext>, req: Request) -> Response {
- let (mut parts, body) = req.into_parts();
+async fn graphql_handler(
+ State(state): State<ApiContext>,
+ Cached(auth): Cached<OptionalMacroAuthorizationExtractor<AuthorizationService>>,
+ request_parts: Parts,
+ request: GraphQLRequest,
+) -> GraphQLResponse {
+ let request = graphql_query_context_data(request.into_inner(), &state, auth.macro_user_id);
+ state
+ .graphql_soup_schema
+ .execute(request.data(GraphqlRequestParts::new(request_parts)))
+ .await
+ .into()
+}
- // Authentication stays eager: it gates execution for non-introspection
- // queries and primes the `Cached` entry that resolvers extract lazily.
- let auth =
- match Cached::<OptionalMacroAuthorizationExtractor<AuthorizationService>>::from_request_parts(
- &mut parts, &state,
+async fn subscription_handler(
+ State(state): State<ApiContext>,
+ Cached(auth): Cached<OptionalMacroAuthorizationExtractor<AuthorizationService>>,
+ protocol: GraphQLProtocol,
+ upgrade: WebSocketUpgrade,
+) -> Response {
+ let Some(macro_user_id) = auth.macro_user_id else {
+ return (
+ StatusCode::UNAUTHORIZED,
+ "authentication required for GraphQL Soup subscriptions",
)
- .await
- {
- Ok(Cached(auth)) => auth,
- Err(err) => return err.into_response(),
- };
+ .into_response();
+ };
- // `GraphQLRequest` consumes a whole request, but the original parts (with
- // the middleware-populated extensions) must survive for the resolvers, so
- // parse the body from a shallow copy of method/uri/headers.
- let mut synthetic = Request::new(body);
- *synthetic.method_mut() = parts.method.clone();
- *synthetic.uri_mut() = parts.uri.clone();
- *synthetic.headers_mut() = parts.headers.clone();
+ let schema = state.graphql_soup_schema.clone();
+ let data = graphql_subscription_context_data(state, macro_user_id);
+ upgrade
+ .protocols(ALL_WEBSOCKET_PROTOCOLS)
+ .on_upgrade(move |socket| async move {
+ GraphQLWebSocket::new(socket, schema, protocol)
+ .with_data(data)
+ .serve()
+ .await;
+ })
+ .into_response()
+}
- let request = match <GraphQLRequest as FromRequest<()>>::from_request(synthetic, &()).await {
- Ok(request) => request.into_inner(),
- Err(rejection) => return rejection.into_response(),
- };
+fn graphql_subscription_context_data(state: ApiContext, user: MacroUserIdStr<'static>) -> Data {
+ let mut data = Data::default();
+ data.insert(state);
+ data.insert(user);
+ data
+}
- let Some(macro_user_id) = auth.macro_user_id else {
- if is_introspection_query(&request.query) {
- return GraphQLResponse::from(state.graphql_soup_schema.execute(request).await)
- .into_response();
- }
- return GraphQLResponse::from(async_graphql::Response::from_errors(vec![
- ServerError::new("authentication required for GraphQL Soup queries", None),
- ]))
- .into_response();
+fn graphql_query_context_data(
+ req: async_graphql::Request,
+ state: &ApiContext,
+ macro_user_id: Option<MacroUserIdStr<'static>>,
+) -> async_graphql::Request {
+ let req = req.data(state.clone());
+
+ let Some(macro_user_id) = macro_user_id else {
+ return req;
};
let property_reader = complete_graph::PropertiesEntityPropertyReader::new(
@@ -71,9 +106,8 @@ async fn handler(State(state): State<ApiContext>, req: Request) -> Response {
state.soup_router_state.email_service(),
state.entity_access_service.clone(),
);
- let request = request
- .data(GraphqlSoupRequestParts::new(parts))
- .data(state.clone())
+
+ req.data(macro_user_id.clone())
.data(complete_graph::entity_properties_loader(
macro_user_id.clone(),
property_reader,
@@ -86,11 +120,5 @@ async fn handler(State(state): State<ApiContext>, req: Request) -> Response {
.data(complete_graph::entity_notifications_loader(
macro_user_id,
state.graphql_notification_reader.clone(),
- ));
-
- GraphQLResponse::from(state.graphql_soup_schema.execute(request).await).into_response()
-}
-
-fn is_introspection_query(query: &str) -> bool {
- query.contains("__schema") || query.contains("__type")
+ ))
}
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index 33278c3fdc..03ad704c0a 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -107,6 +107,10 @@ use soup::{
domain::service::SoupImpl, inbound::axum_router::SoupRouterState,
outbound::pg_soup_repo::PgSoupRepo,
};
+#[cfg(feature = "graphql")]
+use soup_realtime::{
+ domain::service::SoupRealtimeConsumerService, outbound::soup_consumer::SoupTopicConsumer,
+};
use sqlx::postgres::PgPoolOptions;
use std::sync::Arc;
use sync_service_client::SyncServiceClient;
@@ -848,6 +852,28 @@ async fn main() -> anyhow::Result<()> {
foreign_entity_service_for_soup,
));
+ #[cfg(feature = "graphql")]
+ let soup_realtime_service = Arc::new(SoupRealtimeConsumerService::new(
+ SoupTopicConsumer::from_env(config.kafka_brokers.as_ref()).map_err(|error| {
+ anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")
+ })?,
+ ));
+ #[cfg(feature = "graphql")]
+ tokio::spawn({
+ let soup_realtime_service = Arc::clone(&soup_realtime_service);
+ async move {
+ loop {
+ let _ = soup_realtime_service.run().await.inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ "realtime Soup subscription consumer stopped"
+ );
+ });
+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+ }
+ }
+ });
+
let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));
let api_context = ApiContext {
@@ -868,7 +894,10 @@ async fn main() -> anyhow::Result<()> {
),
favorites_service,
#[cfg(feature = "graphql")]
- graphql_soup_schema: complete_graph::build_schema_from_arc(soup_service),
+ graphql_soup_schema: complete_graph::build_schema_from_arcs(
+ soup_service,
+ soup_realtime_service,
+ ),
#[cfg(feature = "graphql")]
graphql_notification_reader,
github_sync_service: Arc::new(github_sync_service_impl),
diff --git a/static_assets/schema.graphql b/static_assets/schema.graphql
index 2c5ce81bb9..e1ace525b5 100644
--- a/static_assets/schema.graphql
+++ b/static_assets/schema.graphql
@@ -2620,6 +2620,16 @@ type SoupQueryRoot {
user: GraphqlUser!
}
+"""
+Root entry point for realtime Soup subscriptions.
+"""
+type SoupSubscriptionRoot {
+ """
+ Subscribe to realtime Soup updates for the authenticated user.
+ """
+ soupUpdates: GraphqlSoupItem!
+}
+
"""
Directs the executor to include this field or fragment only when the `if` argument is true.
"""
@@ -2635,5 +2645,6 @@ directive @skip(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT
schema {
query: SoupQueryRoot
mutation: PropertiesMutationRoot
+ subscription: SoupSubscriptionRoot
}