Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(soup): hydrate realtime patches from the primary (#7962)
gbirman271cda201c16
diff --git a/crates/complete_graph/src/schema/test.rs b/crates/complete_graph/src/schema/test.rsindex c041320b0a..b0f8c34f07 100644--- a/crates/complete_graph/src/schema/test.rs+++ b/crates/complete_graph/src/schema/test.rs@@ -1298,7 +1298,10 @@ async fn soup_updates_subscribes_as_the_authenticated_user() {let document_id = Uuid::from_u128(42);let deleted_document_id = Uuid::from_u128(43);soup_service.set_raw_response(vec![soup_document(document_id)]);- let loader = graphql_soup::soup_item_loader(soup_service.clone(), Arc::new(NoOpEmailService));+ let loader = graphql_soup::SoupUpdateHydrationLoader(graphql_soup::soup_item_loader(+ soup_service.clone(),+ Arc::new(NoOpEmailService),+ ));let schema: SoupSchema<CountingSoupService,TestRealtimeSubscriptionService,diff --git a/crates/complete_graph/src/schema/test/soup_patches.rs b/crates/complete_graph/src/schema/test/soup_patches.rsindex cbc0b729c4..0f3608e793 100644--- a/crates/complete_graph/src/schema/test/soup_patches.rs+++ b/crates/complete_graph/src/schema/test/soup_patches.rs@@ -51,15 +51,28 @@ fn entity(id: u128) -> model_entity::Entity<'static> {async fn subscription_responses(soup: CountingSoupService,patches: Vec<Patch<model_entity::Entity<'static>>>,+) -> Vec<async_graphql::Response> {+ subscription_responses_from(soup.clone(), soup, patches).await+}++/// Runs `soupUpdates` with separate list (replica) and hydration (primary) readers.+async fn subscription_responses_from(+ replica: CountingSoupService,+ primary: CountingSoupService,+ patches: Vec<Patch<model_entity::Entity<'static>>>,) -> Vec<async_graphql::Response> {let (sender, receiver) = tokio::sync::mpsc::channel(16);for patch in patches {sender.send(patch).await.unwrap();}drop(sender);- let loader = graphql_soup::soup_item_loader(soup.clone(), Arc::new(NoOpEmailService));+ let list_loader = graphql_soup::soup_item_loader(replica.clone(), Arc::new(NoOpEmailService));+ let hydration_loader = graphql_soup::SoupUpdateHydrationLoader(graphql_soup::soup_item_loader(+ primary,+ Arc::new(NoOpEmailService),+ ));let schema: RealtimeSchema = build_schema_with_services(- soup,+ replica,TestRealtimeSubscriptionService {receiver: Arc::new(Mutex::new(Some(receiver))),subscribed_user: Arc::new(Mutex::new(None)),@@ -71,7 +84,8 @@ async fn subscription_responses("subscription {{ soupUpdates {{ {PATCH_SELECTION} }} }}")).data(MacroUserIdStr::parse_from_str(VALID_USER_ID).unwrap())- .data(loader);+ .data(list_loader)+ .data(hydration_loader);tokio::time::timeout(std::time::Duration::from_secs(5),schema.execute_stream(request).collect(),@@ -119,6 +133,27 @@ async fn batch_keeps_last_operation_per_entity_and_batches_hydration() {);}+#[tokio::test]+async fn updates_hydrate_from_the_primary_before_the_replica_replays_them() {+ let replica = CountingSoupService {+ return_empty_raw: true,+ ..CountingSoupService::default()+ };+ let replica_calls = replica.raw_calls.clone();+ let primary = CountingSoupService::default();+ primary.set_raw_response(vec![soup_document(Uuid::from_u128(1))]);+ let responses =+ subscription_responses_from(replica, primary, vec![Patch::Updated(entity(1))]).await;+ assert_eq!(responses.len(), 1);+ assert!(responses[0].errors.is_empty(), "{:?}", responses[0].errors);+ let data = responses[0].data.clone().into_json().unwrap();+ assert_eq!(+ data["soupUpdates"][0]["item"]["id"],+ Uuid::from_u128(1).to_string()+ );+ assert_eq!(replica_calls.load(Ordering::SeqCst), 0);+}+#[tokio::test]async fn later_batches_can_update_the_same_entity_again() {let soup = CountingSoupService::default();diff --git a/crates/graphql_soup/src/lib.rs b/crates/graphql_soup/src/lib.rsindex b19f2cce8c..0a105c746f 100644--- a/crates/graphql_soup/src/lib.rs+++ b/crates/graphql_soup/src/lib.rs@@ -26,8 +26,8 @@ pub use graphql_common::{GraphqlRequestParts, GraphqlSoupEntityType};pub use inputs::{GraphqlSimpleSortMethod, GroupedSoupInput, SoupInput};pub use loaders::{AgentSessionEntityLoader, EmailServiceInboxReader, SoupInboxReader, SoupItemDataLoader,- SoupItemLoader, SoupItemLoaderError, SoupItemLoaderKey, soup_item_loader,- soup_item_loader_with_team_access,+ SoupItemLoader, SoupItemLoaderError, SoupItemLoaderKey, SoupUpdateHydrationLoader,+ soup_item_loader, soup_item_loader_with_team_access,};pub use mutation_thread::{EmailMutationThreadLoader, EmailMutationThreadReader, email_mutation_thread_loader,diff --git a/crates/graphql_soup/src/loaders.rs b/crates/graphql_soup/src/loaders.rsindex e3af69f4cd..f0045f633c 100644--- a/crates/graphql_soup/src/loaders.rs+++ b/crates/graphql_soup/src/loaders.rs@@ -378,6 +378,13 @@ impl SoupItemDataLoader {#[derive(Clone)]pub struct AgentSessionEntityLoader(pub SoupItemDataLoader);+/// Primary-backed Soup reader for `soupUpdates` hydration. A patch announces a+/// write that committed milliseconds earlier, before a replica has replayed+/// it, so a replica read returns the pre-write row and the client caches it+/// as current. Lists keep the replica-backed reader.+#[derive(Clone)]+pub struct SoupUpdateHydrationLoader(pub SoupItemDataLoader);+/// Build the realtime Soup DataLoader from the existing Soup and email services.pub fn soup_item_loader<S, E>(soup_service: S, email_service: Arc<E>) -> SoupItemDataLoaderwherediff --git a/crates/graphql_soup/src/resolvers.rs b/crates/graphql_soup/src/resolvers.rsindex 797e2978b2..4c02c0121b 100644--- a/crates/graphql_soup/src/resolvers.rs+++ b/crates/graphql_soup/src/resolvers.rs@@ -23,7 +23,7 @@ use soup_realtime::domain::{models::Patch, ports::SoupRealtimeSubscriptionServicuse crate::{inputs::{GroupedSoupInput, SoupInput},- loaders::{AgentSessionEntityLoader, SoupItemDataLoader},+ loaders::{AgentSessionEntityLoader, SoupItemDataLoader, SoupUpdateHydrationLoader},objects::{GraphqlSoupAgentSession, GraphqlSoupEmailThread, GraphqlSoupEntity, GroupedSoup,SoupEntityEdges, SoupPage, SoupPatch,@@ -46,7 +46,9 @@ where{let macro_user_id = require_authorized_user::<Auth, St>(ctx).await?;let mut receiver = service.subscribe(macro_user_id.clone());- let loader = ctx.data_opt::<SoupItemDataLoader>().cloned();+ let loader = ctx+ .data_opt::<SoupUpdateHydrationLoader>()+ .map(|loader| loader.0.clone());const BUFFER_SIZE: usize = 10;let mut buf = Vec::with_capacity(BUFFER_SIZE);diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex 290f280bdb..80e4aae3a4 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -847,6 +847,7 @@ pub(crate) struct ApiContext {pub graphql_work_feed_context: complete_graph::WorkFeedGraphqlContext,pub graphql_initiative_entity_loader: graphql_initiative::InitiativeEntityLoader,pub graphql_agent_session_entity_loader: graphql_soup::AgentSessionEntityLoader,+ pub graphql_soup_update_hydration_loader: graphql_soup::SoupUpdateHydrationLoader,pub databases_state: DssDatabasesState,pub database_webhooks_state: DssDatabaseWebhooksState,pub forms_state: DssFormsState,diff --git a/services/document_storage_service/src/api/graphql_soup.rs b/services/document_storage_service/src/api/graphql_soup.rsindex fe9acd23e7..5afa99d1c7 100644--- a/services/document_storage_service/src/api/graphql_soup.rs+++ b/services/document_storage_service/src/api/graphql_soup.rs@@ -143,8 +143,8 @@ fn insert_graphql_context_data(state.soup_router_state.email_service(),state.entity_access_service.clone(),);- // Ordinary queries and subscription hydration use the replica-backed- // Soup reader. The mutation-only loader below retains the primary reader.+ // Ordinary queries use the replica-backed Soup reader. Subscription+ // hydration and the mutation-only loader below read the primary.let soup_item_loader = soup_item_loader_with_team_access(state.soup_router_state.service(),state.soup_router_state.email_service(),@@ -173,6 +173,7 @@ fn insert_graphql_context_data(data.insert(state.graphql_work_feed_context.clone());data.insert(state.graphql_initiative_entity_loader.clone());data.insert(state.graphql_agent_session_entity_loader.clone());+ data.insert(state.graphql_soup_update_hydration_loader.clone());data.insert(graphql_initiative::initiative_detail_loader(state.graphql_initiative_context.clone(),macro_user_id.clone(),diff --git a/services/document_storage_service/src/server.rs b/services/document_storage_service/src/server.rsindex e67f6352d5..5cce9a8065 100644--- a/services/document_storage_service/src/server.rs+++ b/services/document_storage_service/src/server.rs@@ -1512,9 +1512,10 @@ pub async fn run() -> anyhow::Result<()> {));// Individual initiative and agent-session reads preserve read-after-write- // consistency when a newly created entity opens immediately. Lists retain- // the replica reader. Reuse Soup hydration so detail and mutation metadata- // include viewer history.+ // consistency when a newly created entity opens immediately, as does+ // realtime patch hydration, which runs right after the write it announces.+ // Lists retain the replica reader. Reuse Soup hydration so detail and+ // mutation metadata include viewer history.let primary_entity_soup = Arc::new(SoupImpl::new(PgSoupRepo::new(readonly_pool::ReadOnlyPool(db.clone())),@@ -1540,9 +1541,9 @@ pub async fn run() -> anyhow::Result<()> {let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));// Keep the replica-backed Soup reader alongside the primary-backed email- // writer in SoupRouterState. REST, GraphQL lists, and realtime hydration- // share this reader; only email mutations and their reply loader use the- // writer service, so normal list traffic never switches to the primary.+ // writer in SoupRouterState. REST and GraphQL lists share this reader;+ // only email mutations and their reply loader use the writer service, so+ // normal list traffic never switches to the primary.let soup_service = Arc::new(SoupImpl::new(PgSoupRepo::new(readonly_pool::ReadOnlyPool(readonly_db.clone())),@@ -2040,7 +2041,17 @@ pub async fn run() -> anyhow::Result<()> {),),graphql_agent_session_entity_loader: graphql_soup::AgentSessionEntityLoader(- graphql_soup::soup_item_loader(primary_entity_soup, Arc::new(email_service.clone())),+ graphql_soup::soup_item_loader(+ primary_entity_soup.clone(),+ Arc::new(email_service.clone()),+ ),+ ),+ graphql_soup_update_hydration_loader: graphql_soup::SoupUpdateHydrationLoader(+ graphql_soup::soup_item_loader_with_team_access(+ primary_entity_soup,+ Arc::new(email_service.clone()),+ entity_access_service.as_ref().clone(),+ ),),graphql_crm_context: graphql_crm::CrmGraphqlContext::new(pipeline_service.clone()),graphql_initiative_context: graphql_initiative::InitiativeGraphqlContext::new(