Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix slow consumer (#5301)
Sean Ayeca265439a7fb
diff --git a/crates/macro_event_broker/src/domain/models.rs b/crates/macro_event_broker/src/domain/models.rsindex a5e6141bd8..6b1d51b0f8 100644--- a/crates/macro_event_broker/src/domain/models.rs+++ b/crates/macro_event_broker/src/domain/models.rs@@ -181,6 +181,7 @@ impl<T, M> MessageWrapper<T, M> {impl<T: MessageParts, M: MacroEventCollection> MessageWrapper<T, M> {/// Decodes the message into one of the declared event types.+ #[tracing::instrument(err, skip(self))]pub fn decode_payload(&self) -> Result<M, EventBrokerError> {M::decode(&self.inner)}diff --git a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rsindex 380669ccc4..90a9a3be95 100644--- a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs+++ b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs@@ -68,6 +68,7 @@ impl<T: GroupName, M> KafkaConsumerAdapter<T, M> {}/// Commits a message using the caller-selected commit mode.+ #[tracing::instrument(err, skip(self))]pub fn commit_message(&self,message: &BorrowedMessage<'_>,diff --git a/crates/soup_realtime/src/domain/ports.rs b/crates/soup_realtime/src/domain/ports.rsindex 452c03e777..4c7595b53c 100644--- a/crates/soup_realtime/src/domain/ports.rs+++ b/crates/soup_realtime/src/domain/ports.rs@@ -11,10 +11,7 @@ use super::models::{Patch, SoupRealtimeMessage, SoupRealtimePatch};/// Inbound use-case port driven by entity update transports.pub trait SoupRealtimeService: Send + Sync + 'static {/// Publishes one entity patch to every current accessor of its access source.- fn notify_users(- &self,- patch: SoupRealtimePatch,- ) -> impl Future<Output = Result<(), Report>> + Send;+ fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report>;}/// Receives recipient-targeted Soup patches.diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rsindex 0a96c8b49c..91d5aa1e78 100644--- a/crates/soup_realtime/src/domain/service.rs+++ b/crates/soup_realtime/src/domain/service.rs@@ -10,6 +10,7 @@ use futures::{StreamExt as _, stream};use macro_user_id::user_id::MacroUserIdStr;use model_entity::Entity;use rootcause::prelude::{Report, ResultExt as _};+use tokio::task::JoinHandle;use tokio_retry::{Retry, strategy::ExponentialBackoff};use super::{@@ -110,41 +111,29 @@ whereconst PUBLISH_CONCURRENCY: usize = 16;/// Domain service that expands entity access and fans out lightweight patches.-pub struct SoupRealtimeServiceImpl<A, P> {+pub struct SoupRealtimeServiceImpl {+ sender: tokio::sync::mpsc::Sender<SoupRealtimePatch>,+ _bg: JoinHandle<()>,+}++struct Worker<A, P> {access_expander: A,publisher: P,+ rx: tokio::sync::mpsc::Receiver<SoupRealtimePatch>,}-impl<A, P> SoupRealtimeServiceImpl<A, P> {- /// Creates a realtime Soup service from its outbound capabilities.- pub fn new(access_expander: A, publisher: P) -> Self {- Self {- access_expander,- publisher,+impl<A: UserAccessExpander, P: SoupRealtimePublisher> Worker<A, P> {+ async fn run(&mut self) -> () {+ while let Some(msg) = self.rx.recv().await {+ let _ = self.process_one(msg).await;}}-}-impl<A, P> SoupRealtimeService for SoupRealtimeServiceImpl<A, P>-where- A: UserAccessExpander,- P: SoupRealtimePublisher,-{- #[tracing::instrument(- skip(self),- fields(- entity_type = %patch.patch.value().entity_type,- entity_id = %patch.patch.value().entity_id,- access_source_type = %patch.access_source.entity_type,- access_source_id = %patch.access_source.entity_id,- recipient_count = tracing::field::Empty,- ),- err- )]- async fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {+ #[tracing::instrument(err, skip(self))]+ async fn process_one(&self, msg: SoupRealtimePatch) -> Result<(), Report> {let mut users = self.access_expander- .expand_user_access(&patch.access_source)+ .expand_user_access(&msg.access_source).await.context("failed to expand current user access")?;@@ -154,7 +143,7 @@ wherelet messages = users.into_iter()- .map(|user_id| SoupRealtimeMessage::new(user_id, patch.patch.clone()))+ .map(|user_id| SoupRealtimeMessage::new(user_id, msg.patch.clone())).collect::<Vec<_>>();let results = stream::iter(messages@@ -173,7 +162,40 @@ where)).into_dynamic());}-Ok(())}}++const BACKGROUND_CHANNEL_SIZE: usize = 256;++impl SoupRealtimeServiceImpl {+ /// Creates a realtime Soup service from its outbound capabilities.+ pub fn new<A: UserAccessExpander, P: SoupRealtimePublisher>(+ access_expander: A,+ publisher: P,+ ) -> Self {+ let (tx, rx) = tokio::sync::mpsc::channel(BACKGROUND_CHANNEL_SIZE);++ let task = tokio::spawn(async move {+ Worker {+ rx,+ access_expander,+ publisher,+ }+ .run()+ .await+ });++ Self {+ sender: tx,+ _bg: task,+ }+ }+}++impl SoupRealtimeService for SoupRealtimeServiceImpl {+ #[tracing::instrument(skip(self), err)]+ fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {+ Ok(self.sender.try_send(patch)?)+ }+}diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rsindex ed9bfa759b..922b6c59ff 100644--- a/crates/soup_realtime/src/domain/service/test.rs+++ b/crates/soup_realtime/src/domain/service/test.rs@@ -71,12 +71,22 @@ impl SoupRealtimePublisher for FakePublisher {}struct Harness {- service: SoupRealtimeServiceImpl<FakeAccessExpander, FakePublisher>,+ service: SoupRealtimeServiceImpl,access_calls: Arc<AtomicUsize>,access_entities: Arc<Mutex<Vec<Entity<'static>>>>,messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,}+async fn wait_until(mut condition: impl FnMut() -> bool) {+ tokio::time::timeout(Duration::from_secs(1), async {+ while !condition() {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("background worker completes");+}+fn user(local: &str) -> MacroUserIdStr<'static> {MacroUserIdStr::try_from(format!("macro|{local}@example.com")).expect("valid user id")}@@ -181,10 +191,9 @@ async fn zero_users_skips_publication() {harness.service.notify_users(SoupRealtimePatch::for_entity(updated_document()))- .await- .expect("zero recipients is successful");+ .expect("patch is queued");- assert_eq!(harness.access_calls.load(Ordering::SeqCst), 1);+ wait_until(|| harness.access_calls.load(Ordering::SeqCst) == 1).await;assert!(harness.messages.lock().expect("messages lock").is_empty());}@@ -197,8 +206,17 @@ async fn recipient_expansion_can_use_a_different_entity_from_the_patch() {harness.service.notify_users(SoupRealtimePatch::new(updated_document(), channel.clone()))- .await- .expect("fan-out succeeds");+ .expect("patch is queued");++ wait_until(|| {+ harness+ .access_entities+ .lock()+ .expect("access entities lock")+ .len()+ == 1+ })+ .await;assert_eq!(harness@@ -222,9 +240,9 @@ async fn updated_and_deleted_patches_are_published_unchanged() {harness.service.notify_users(SoupRealtimePatch::for_entity(patch.clone()))- .await- .expect("fan-out succeeds");+ .expect("patch is queued");+ wait_until(|| harness.messages.lock().expect("messages lock").len() == 1).await;let messages = harness.messages.lock().expect("messages lock");assert_eq!(messages.as_slice(),@@ -242,10 +260,9 @@ async fn duplicate_accessors_are_deduplicated() {harness.service.notify_users(SoupRealtimePatch::for_entity(updated_document()))- .await- .expect("fan-out succeeds");+ .expect("patch is queued");- assert_eq!(harness.messages.lock().expect("messages lock").len(), 2);+ wait_until(|| harness.messages.lock().expect("messages lock").len() == 2).await;}#[tokio::test]@@ -255,14 +272,14 @@ async fn access_failure_prevents_publication() {harness.service.notify_users(SoupRealtimePatch::for_entity(updated_document()))- .await- .expect_err("access failure propagates");+ .expect("patch is queued");+ wait_until(|| harness.access_calls.load(Ordering::SeqCst) == 1).await;assert!(harness.messages.lock().expect("messages lock").is_empty());}#[tokio::test]-async fn publisher_failure_is_returned_after_all_messages_are_attempted() {+async fn publisher_failure_does_not_block_other_messages() {let users = [user("one"), user("two"), user("three")];let harness = harness(users.to_vec(),@@ -273,8 +290,7 @@ async fn publisher_failure_is_returned_after_all_messages_are_attempted() {harness.service.notify_users(SoupRealtimePatch::for_entity(updated_document()))- .await- .expect_err("publication failure propagates");+ .expect("patch is queued");- assert_eq!(harness.messages.lock().expect("messages lock").len(), 3);+ wait_until(|| harness.messages.lock().expect("messages lock").len() == 3).await;}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rsindex 7d9b0c706e..69996feba6 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs@@ -7,8 +7,13 @@#[cfg(test)]mod test;-use std::{future::Future, time::Duration};+use std::future::Future;+use crate::domain::{+ models::{Patch, SoupRealtimePatch},+ ports::SoupRealtimeService,+ service::SoupRealtimeServiceImpl,+};use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};use chat::domain::events::{ChatMacroEvent, ChatTopicEvent};use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent, InteractionReason};@@ -24,13 +29,6 @@ use properties::domain::events::{PropertyMacroEvent, PropertyTopicEvent};use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};use rootcause::prelude::{Report, ResultExt as _};-use tokio_retry::{Retry, strategy::ExponentialBackoff};--use crate::domain::{- models::{Patch, SoupRealtimePatch},- ports::{SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},- service::SoupRealtimeServiceImpl,-};/// Consumer group used for Soup-affecting entity event offsets.struct SoupRealtimeConsumerGroup;@@ -53,17 +51,6 @@ macro_event_broker::declare_topics!(PropertyMacroEvent,);-/// Total service attempts before returning for supervisor-driven redelivery.-const MAX_SERVICE_ATTEMPTS: u32 = 5;-/// Delay before the first retry; each subsequent delay doubles.-const SERVICE_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);--fn service_retry_strategy() -> impl Iterator<Item = Duration> {- ExponentialBackoff::from_millis(2)- .factor(500)- .take((MAX_SERVICE_ATTEMPTS - 1) as usize)-}-fn entity(entity_type: EntityType, entity_id: impl ToString) -> Entity<'static> {entity_type.with_entity_string(entity_id.to_string())}@@ -397,12 +384,10 @@ enum EventOutcome {Ignored,}-#[tracing::instrument(skip(service, event), fields(partition, offset), err)]-async fn process_event<S: SoupRealtimeService>(+#[tracing::instrument(skip(service, event), err)]+fn process_event<S: SoupRealtimeService>(service: &S,event: &DeclaredMacroEvent,- partition: i32,- offset: i64,) -> Result<EventOutcome, Report> {let patches = patches_from_event(event);if patches.is_empty() {@@ -411,84 +396,16 @@ async fn process_event<S: SoupRealtimeService>(}for patch in patches {- notify_with_retry(service, patch, partition, offset).await?;+ service.notify_users(patch)?;}Ok(EventOutcome::Notified)}-#[tracing::instrument(- skip(service, patch),- fields(- entity_type = %patch.patch.value().entity_type,- entity_id = %patch.patch.value().entity_id,- access_source_type = %patch.access_source.entity_type,- access_source_id = %patch.access_source.entity_id,- partition,- offset,- ),- err-)]-async fn notify_with_retry<S: SoupRealtimeService>(- service: &S,- patch: SoupRealtimePatch,- partition: i32,- offset: i64,-) -> Result<(), Report> {- let mut attempt = 0u32;- Retry::start(service_retry_strategy(), || {- attempt += 1;- let patch = patch.clone();- async move {- tracing::trace!(attempt, "notifying realtime Soup recipients");- let result = service.notify_users(patch).await;- match &result {- Ok(()) => tracing::trace!(attempt, "realtime Soup recipients notified"),- Err(error) if attempt < MAX_SERVICE_ATTEMPTS => {- let delay = SERVICE_RETRY_BASE_DELAY * 2u32.pow(attempt - 1);- tracing::warn!(- error = ?error,- attempt,- delay_secs = delay.as_secs_f32(),- "realtime Soup fan-out failed, retrying"- );- }- Err(_) => {}- }- result- }- })- .await- .map_err(|error| {- error- .context(format!(- "realtime Soup fan-out failed after {MAX_SERVICE_ATTEMPTS} attempts \- (partition {partition} offset {offset})"- ))- .into_dynamic()- })-}-fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {- match consumer.inner().commit_message(message, CommitMode::Async) {- Ok(()) => tracing::trace!(- partition = message.partition(),- offset = message.offset(),- "committed realtime Soup input offset"- ),- Err(error) => tracing::error!(- error = ?error,- partition = message.partition(),- offset = message.offset(),- "failed to commit realtime Soup input offset"- ),- }+ let _ = consumer.inner().commit_message(message, CommitMode::Async);}-impl<A, P> SoupRealtimeServiceImpl<A, P>-where- A: UserAccessExpander,- P: SoupRealtimePublisher,-{+impl SoupRealtimeServiceImpl {/// Runs the Soup-affecting entity event consumer until `shutdown` resolves.////// The consumer subscribes to every existing entity topic with events that@@ -521,51 +438,24 @@ wherebreak;}result = consumer.recv() => {- let message = match result {- Ok(message) => message,- Err(error) => {- tracing::error!(error = ?error, "Kafka receive error");- continue;- }- };+ let Ok(message) = result else { continue; };let kafka_message = message.inner();+ let _message_span = tracing::info_span!(+ "realtime_soup_source_event",+ topic = kafka_message.topic(),+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),+ )+ .entered();let event = match message.decode_payload() {Ok(event) => event,- Err(error) => {- tracing::error!(- error = ?error,- topic = kafka_message.topic(),- partition = kafka_message.partition(),- offset = kafka_message.offset(),- "dropping malformed Soup source event"- );+ Err(_) => {commit_logged(&consumer, kafka_message);continue;}};- match process_event(- self,- &event,- kafka_message.partition(),- kafka_message.offset(),- )- .await- {- Ok(EventOutcome::Notified | EventOutcome::Ignored) => {}- Err(error) => {- tracing::error!(- error = ?error,- topic = kafka_message.topic(),- partition = kafka_message.partition(),- offset = kafka_message.offset(),- "realtime Soup fan-out retries exhausted; restarting consumer for redelivery"- );- return Err(error- .context("realtime Soup entity consumer requires restart for redelivery")- .into_dynamic());- }- }+ let _ = process_event(self,&event);commit_logged(&consumer, kafka_message);}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rsindex c823013078..7184a1a51d 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs@@ -79,7 +79,7 @@ struct FlakyService {}impl SoupRealtimeService for FlakyService {- async fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {+ fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {self.patches.lock().expect("patches lock").push(patch);let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;if attempt <= self.failures {@@ -390,8 +390,8 @@ fn deleting_a_root_channel_message_deletes_its_thread_patch() {assert_eq!(patches[1].access_source.entity_type, EntityType::Channel);}-#[tokio::test]-async fn updated_payload_maps_to_document_patch() {+#[test]+fn updated_payload_maps_to_document_patch() {let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(DOCUMENT_ID,updated_event(),@@ -399,9 +399,7 @@ async fn updated_payload_maps_to_document_patch() {let service = flaky_service(0);assert!(matches!(- process_event(&service, &event, 0, 0)- .await- .expect("processing succeeds"),+ process_event(&service, &event).expect("processing succeeds"),EventOutcome::Notified));@@ -425,34 +423,16 @@ fn malformed_and_unknown_events_are_rejected_by_the_declared_collection() {assert!(decode_payload(payload).is_err());}-#[tokio::test(start_paused = true)]-async fn transient_service_failures_retry_then_succeed() {- let service = flaky_service(2);- let patch = SoupRealtimePatch::for_entity(Patch::Updated(- EntityType::Document.with_entity_string(DOCUMENT_ID.to_string()),- ));-- notify_with_retry(&service, patch, 2, 17)- .await- .expect("eventual success");-- assert_eq!(service.attempts.load(Ordering::SeqCst), 3);- assert_eq!(service.patches.lock().expect("patches lock").len(), 3);-}--#[tokio::test(start_paused = true)]-async fn exhausted_retries_return_for_redelivery() {- let service = flaky_service(u32::MAX);- let patch = SoupRealtimePatch::for_entity(Patch::Updated(- EntityType::Document.with_entity_string(DOCUMENT_ID.to_string()),+#[test]+fn service_failure_is_returned_without_retry() {+ let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(+ DOCUMENT_ID,+ updated_event(),));+ let service = flaky_service(1);- notify_with_retry(&service, patch, 2, 17)- .await- .expect_err("persistent failure returns without a commit");+ assert!(process_event(&service, &event).is_err());- assert_eq!(- service.attempts.load(Ordering::SeqCst),- MAX_SERVICE_ATTEMPTS- );+ assert_eq!(service.attempts.load(Ordering::SeqCst), 1);+ assert_eq!(service.patches.lock().expect("patches lock").len(), 1);}diff --git a/crates/soup_realtime/src/outbound/entity_access.rs b/crates/soup_realtime/src/outbound/entity_access.rsindex abe77312b1..ed3a4b8275 100644--- a/crates/soup_realtime/src/outbound/entity_access.rs+++ b/crates/soup_realtime/src/outbound/entity_access.rs@@ -55,11 +55,7 @@ impl<S> UserAccessExpander for EntityAccessExpander<S>whereS: EntityAccessUserLookup,{- #[tracing::instrument(- skip(self),- fields(entity_type = %entity.entity_type, entity_id = %entity.entity_id),- err- )]+ #[tracing::instrument(skip(self), err)]async fn expand_user_access(&self,entity: &Entity<'static>,