Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix slow consumer (#5301)

Sean Ayeca265439a7fb
diff --git a/crates/macro_event_broker/src/domain/models.rs b/crates/macro_event_broker/src/domain/models.rs
index a5e6141bd8..6b1d51b0f8 100644
--- a/crates/macro_event_broker/src/domain/models.rs
+++ b/crates/macro_event_broker/src/domain/models.rs
@@ -181,6 +181,7 @@ impl<T, M> MessageWrapper<T, M> {
impl<T: MessageParts, M: MacroEventCollection> MessageWrapper<T, M> {
/// Decodes the message into one of the declared event types.
+ #[tracing::instrument(err, skip(self))]
pub fn decode_payload(&self) -> Result<M, EventBrokerError> {
M::decode(&self.inner)
}
diff --git a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs
index 380669ccc4..90a9a3be95 100644
--- a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs
+++ b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs
@@ -68,6 +68,7 @@ impl<T: GroupName, M> KafkaConsumerAdapter<T, M> {
}
/// Commits a message using the caller-selected commit mode.
+ #[tracing::instrument(err, skip(self))]
pub fn commit_message(
&self,
message: &BorrowedMessage<'_>,
diff --git a/crates/soup_realtime/src/domain/ports.rs b/crates/soup_realtime/src/domain/ports.rs
index 452c03e777..4c7595b53c 100644
--- a/crates/soup_realtime/src/domain/ports.rs
+++ b/crates/soup_realtime/src/domain/ports.rs
@@ -11,10 +11,7 @@ use super::models::{Patch, SoupRealtimeMessage, SoupRealtimePatch};
/// Inbound use-case port driven by entity update transports.
pub trait SoupRealtimeService: Send + Sync + 'static {
/// Publishes one entity patch to every current accessor of its access source.
- fn notify_users(
- &self,
- patch: SoupRealtimePatch,
- ) -> impl Future<Output = Result<(), Report>> + Send;
+ fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report>;
}
/// Receives recipient-targeted Soup patches.
diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rs
index 0a96c8b49c..91d5aa1e78 100644
--- a/crates/soup_realtime/src/domain/service.rs
+++ b/crates/soup_realtime/src/domain/service.rs
@@ -10,6 +10,7 @@ use futures::{StreamExt as _, stream};
use macro_user_id::user_id::MacroUserIdStr;
use model_entity::Entity;
use rootcause::prelude::{Report, ResultExt as _};
+use tokio::task::JoinHandle;
use tokio_retry::{Retry, strategy::ExponentialBackoff};
use super::{
@@ -110,41 +111,29 @@ where
const PUBLISH_CONCURRENCY: usize = 16;
/// Domain service that expands entity access and fans out lightweight patches.
-pub struct SoupRealtimeServiceImpl<A, P> {
+pub struct SoupRealtimeServiceImpl {
+ sender: tokio::sync::mpsc::Sender<SoupRealtimePatch>,
+ _bg: JoinHandle<()>,
+}
+
+struct Worker<A, P> {
access_expander: A,
publisher: P,
+ rx: tokio::sync::mpsc::Receiver<SoupRealtimePatch>,
}
-impl<A, P> SoupRealtimeServiceImpl<A, P> {
- /// Creates a realtime Soup service from its outbound capabilities.
- pub fn new(access_expander: A, publisher: P) -> Self {
- Self {
- access_expander,
- publisher,
+impl<A: UserAccessExpander, P: SoupRealtimePublisher> Worker<A, P> {
+ async fn run(&mut self) -> () {
+ while let Some(msg) = self.rx.recv().await {
+ let _ = self.process_one(msg).await;
}
}
-}
-impl<A, P> SoupRealtimeService for SoupRealtimeServiceImpl<A, P>
-where
- A: UserAccessExpander,
- P: SoupRealtimePublisher,
-{
- #[tracing::instrument(
- skip(self),
- fields(
- entity_type = %patch.patch.value().entity_type,
- entity_id = %patch.patch.value().entity_id,
- access_source_type = %patch.access_source.entity_type,
- access_source_id = %patch.access_source.entity_id,
- recipient_count = tracing::field::Empty,
- ),
- err
- )]
- async fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {
+ #[tracing::instrument(err, skip(self))]
+ async fn process_one(&self, msg: SoupRealtimePatch) -> Result<(), Report> {
let mut users = self
.access_expander
- .expand_user_access(&patch.access_source)
+ .expand_user_access(&msg.access_source)
.await
.context("failed to expand current user access")?;
@@ -154,7 +143,7 @@ where
let messages = users
.into_iter()
- .map(|user_id| SoupRealtimeMessage::new(user_id, patch.patch.clone()))
+ .map(|user_id| SoupRealtimeMessage::new(user_id, msg.patch.clone()))
.collect::<Vec<_>>();
let results = stream::iter(
messages
@@ -173,7 +162,40 @@ where
))
.into_dynamic());
}
-
Ok(())
}
}
+
+const BACKGROUND_CHANNEL_SIZE: usize = 256;
+
+impl SoupRealtimeServiceImpl {
+ /// Creates a realtime Soup service from its outbound capabilities.
+ pub fn new<A: UserAccessExpander, P: SoupRealtimePublisher>(
+ access_expander: A,
+ publisher: P,
+ ) -> Self {
+ let (tx, rx) = tokio::sync::mpsc::channel(BACKGROUND_CHANNEL_SIZE);
+
+ let task = tokio::spawn(async move {
+ Worker {
+ rx,
+ access_expander,
+ publisher,
+ }
+ .run()
+ .await
+ });
+
+ Self {
+ sender: tx,
+ _bg: task,
+ }
+ }
+}
+
+impl SoupRealtimeService for SoupRealtimeServiceImpl {
+ #[tracing::instrument(skip(self), err)]
+ fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {
+ Ok(self.sender.try_send(patch)?)
+ }
+}
diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rs
index ed9bfa759b..922b6c59ff 100644
--- a/crates/soup_realtime/src/domain/service/test.rs
+++ b/crates/soup_realtime/src/domain/service/test.rs
@@ -71,12 +71,22 @@ impl SoupRealtimePublisher for FakePublisher {
}
struct Harness {
- service: SoupRealtimeServiceImpl<FakeAccessExpander, FakePublisher>,
+ service: SoupRealtimeServiceImpl,
access_calls: Arc<AtomicUsize>,
access_entities: Arc<Mutex<Vec<Entity<'static>>>>,
messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,
}
+async fn wait_until(mut condition: impl FnMut() -> bool) {
+ tokio::time::timeout(Duration::from_secs(1), async {
+ while !condition() {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("background worker completes");
+}
+
fn user(local: &str) -> MacroUserIdStr<'static> {
MacroUserIdStr::try_from(format!("macro|{local}@example.com")).expect("valid user id")
}
@@ -181,10 +191,9 @@ async fn zero_users_skips_publication() {
harness
.service
.notify_users(SoupRealtimePatch::for_entity(updated_document()))
- .await
- .expect("zero recipients is successful");
+ .expect("patch is queued");
- assert_eq!(harness.access_calls.load(Ordering::SeqCst), 1);
+ wait_until(|| harness.access_calls.load(Ordering::SeqCst) == 1).await;
assert!(harness.messages.lock().expect("messages lock").is_empty());
}
@@ -197,8 +206,17 @@ async fn recipient_expansion_can_use_a_different_entity_from_the_patch() {
harness
.service
.notify_users(SoupRealtimePatch::new(updated_document(), channel.clone()))
- .await
- .expect("fan-out succeeds");
+ .expect("patch is queued");
+
+ wait_until(|| {
+ harness
+ .access_entities
+ .lock()
+ .expect("access entities lock")
+ .len()
+ == 1
+ })
+ .await;
assert_eq!(
harness
@@ -222,9 +240,9 @@ async fn updated_and_deleted_patches_are_published_unchanged() {
harness
.service
.notify_users(SoupRealtimePatch::for_entity(patch.clone()))
- .await
- .expect("fan-out succeeds");
+ .expect("patch is queued");
+ wait_until(|| harness.messages.lock().expect("messages lock").len() == 1).await;
let messages = harness.messages.lock().expect("messages lock");
assert_eq!(
messages.as_slice(),
@@ -242,10 +260,9 @@ async fn duplicate_accessors_are_deduplicated() {
harness
.service
.notify_users(SoupRealtimePatch::for_entity(updated_document()))
- .await
- .expect("fan-out succeeds");
+ .expect("patch is queued");
- assert_eq!(harness.messages.lock().expect("messages lock").len(), 2);
+ wait_until(|| harness.messages.lock().expect("messages lock").len() == 2).await;
}
#[tokio::test]
@@ -255,14 +272,14 @@ async fn access_failure_prevents_publication() {
harness
.service
.notify_users(SoupRealtimePatch::for_entity(updated_document()))
- .await
- .expect_err("access failure propagates");
+ .expect("patch is queued");
+ wait_until(|| harness.access_calls.load(Ordering::SeqCst) == 1).await;
assert!(harness.messages.lock().expect("messages lock").is_empty());
}
#[tokio::test]
-async fn publisher_failure_is_returned_after_all_messages_are_attempted() {
+async fn publisher_failure_does_not_block_other_messages() {
let users = [user("one"), user("two"), user("three")];
let harness = harness(
users.to_vec(),
@@ -273,8 +290,7 @@ async fn publisher_failure_is_returned_after_all_messages_are_attempted() {
harness
.service
.notify_users(SoupRealtimePatch::for_entity(updated_document()))
- .await
- .expect_err("publication failure propagates");
+ .expect("patch is queued");
- assert_eq!(harness.messages.lock().expect("messages lock").len(), 3);
+ wait_until(|| harness.messages.lock().expect("messages lock").len() == 3).await;
}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rs
index 7d9b0c706e..69996feba6 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs
@@ -7,8 +7,13 @@
#[cfg(test)]
mod test;
-use std::{future::Future, time::Duration};
+use std::future::Future;
+use crate::domain::{
+ models::{Patch, SoupRealtimePatch},
+ ports::SoupRealtimeService,
+ service::SoupRealtimeServiceImpl,
+};
use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};
use chat::domain::events::{ChatMacroEvent, ChatTopicEvent};
use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent, InteractionReason};
@@ -24,13 +29,6 @@ use properties::domain::events::{PropertyMacroEvent, PropertyTopicEvent};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use rootcause::prelude::{Report, ResultExt as _};
-use tokio_retry::{Retry, strategy::ExponentialBackoff};
-
-use crate::domain::{
- models::{Patch, SoupRealtimePatch},
- ports::{SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},
- service::SoupRealtimeServiceImpl,
-};
/// Consumer group used for Soup-affecting entity event offsets.
struct SoupRealtimeConsumerGroup;
@@ -53,17 +51,6 @@ macro_event_broker::declare_topics!(
PropertyMacroEvent,
);
-/// Total service attempts before returning for supervisor-driven redelivery.
-const MAX_SERVICE_ATTEMPTS: u32 = 5;
-/// Delay before the first retry; each subsequent delay doubles.
-const SERVICE_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
-
-fn service_retry_strategy() -> impl Iterator<Item = Duration> {
- ExponentialBackoff::from_millis(2)
- .factor(500)
- .take((MAX_SERVICE_ATTEMPTS - 1) as usize)
-}
-
fn entity(entity_type: EntityType, entity_id: impl ToString) -> Entity<'static> {
entity_type.with_entity_string(entity_id.to_string())
}
@@ -397,12 +384,10 @@ enum EventOutcome {
Ignored,
}
-#[tracing::instrument(skip(service, event), fields(partition, offset), err)]
-async fn process_event<S: SoupRealtimeService>(
+#[tracing::instrument(skip(service, event), err)]
+fn process_event<S: SoupRealtimeService>(
service: &S,
event: &DeclaredMacroEvent,
- partition: i32,
- offset: i64,
) -> Result<EventOutcome, Report> {
let patches = patches_from_event(event);
if patches.is_empty() {
@@ -411,84 +396,16 @@ async fn process_event<S: SoupRealtimeService>(
}
for patch in patches {
- notify_with_retry(service, patch, partition, offset).await?;
+ service.notify_users(patch)?;
}
Ok(EventOutcome::Notified)
}
-#[tracing::instrument(
- skip(service, patch),
- fields(
- entity_type = %patch.patch.value().entity_type,
- entity_id = %patch.patch.value().entity_id,
- access_source_type = %patch.access_source.entity_type,
- access_source_id = %patch.access_source.entity_id,
- partition,
- offset,
- ),
- err
-)]
-async fn notify_with_retry<S: SoupRealtimeService>(
- service: &S,
- patch: SoupRealtimePatch,
- partition: i32,
- offset: i64,
-) -> Result<(), Report> {
- let mut attempt = 0u32;
- Retry::start(service_retry_strategy(), || {
- attempt += 1;
- let patch = patch.clone();
- async move {
- tracing::trace!(attempt, "notifying realtime Soup recipients");
- let result = service.notify_users(patch).await;
- match &result {
- Ok(()) => tracing::trace!(attempt, "realtime Soup recipients notified"),
- Err(error) if attempt < MAX_SERVICE_ATTEMPTS => {
- let delay = SERVICE_RETRY_BASE_DELAY * 2u32.pow(attempt - 1);
- tracing::warn!(
- error = ?error,
- attempt,
- delay_secs = delay.as_secs_f32(),
- "realtime Soup fan-out failed, retrying"
- );
- }
- Err(_) => {}
- }
- result
- }
- })
- .await
- .map_err(|error| {
- error
- .context(format!(
- "realtime Soup fan-out failed after {MAX_SERVICE_ATTEMPTS} attempts \
- (partition {partition} offset {offset})"
- ))
- .into_dynamic()
- })
-}
-
fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {
- match consumer.inner().commit_message(message, CommitMode::Async) {
- Ok(()) => tracing::trace!(
- partition = message.partition(),
- offset = message.offset(),
- "committed realtime Soup input offset"
- ),
- Err(error) => tracing::error!(
- error = ?error,
- partition = message.partition(),
- offset = message.offset(),
- "failed to commit realtime Soup input offset"
- ),
- }
+ let _ = consumer.inner().commit_message(message, CommitMode::Async);
}
-impl<A, P> SoupRealtimeServiceImpl<A, P>
-where
- A: UserAccessExpander,
- P: SoupRealtimePublisher,
-{
+impl SoupRealtimeServiceImpl {
/// Runs the Soup-affecting entity event consumer until `shutdown` resolves.
///
/// The consumer subscribes to every existing entity topic with events that
@@ -521,51 +438,24 @@ where
break;
}
result = consumer.recv() => {
- let message = match result {
- Ok(message) => message,
- Err(error) => {
- tracing::error!(error = ?error, "Kafka receive error");
- continue;
- }
- };
+ let Ok(message) = result else { continue; };
let kafka_message = message.inner();
+ let _message_span = tracing::info_span!(
+ "realtime_soup_source_event",
+ topic = kafka_message.topic(),
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
+ )
+ .entered();
let event = match message.decode_payload() {
Ok(event) => event,
- Err(error) => {
- tracing::error!(
- error = ?error,
- topic = kafka_message.topic(),
- partition = kafka_message.partition(),
- offset = kafka_message.offset(),
- "dropping malformed Soup source event"
- );
+ Err(_) => {
commit_logged(&consumer, kafka_message);
continue;
}
};
- match process_event(
- self,
- &event,
- kafka_message.partition(),
- kafka_message.offset(),
- )
- .await
- {
- Ok(EventOutcome::Notified | EventOutcome::Ignored) => {}
- Err(error) => {
- tracing::error!(
- error = ?error,
- topic = kafka_message.topic(),
- partition = kafka_message.partition(),
- offset = kafka_message.offset(),
- "realtime Soup fan-out retries exhausted; restarting consumer for redelivery"
- );
- return Err(error
- .context("realtime Soup entity consumer requires restart for redelivery")
- .into_dynamic());
- }
- }
+ let _ = process_event(self,&event);
commit_logged(&consumer, kafka_message);
}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
index c823013078..7184a1a51d 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
@@ -79,7 +79,7 @@ struct FlakyService {
}
impl SoupRealtimeService for FlakyService {
- async fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {
+ fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {
self.patches.lock().expect("patches lock").push(patch);
let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;
if attempt <= self.failures {
@@ -390,8 +390,8 @@ fn deleting_a_root_channel_message_deletes_its_thread_patch() {
assert_eq!(patches[1].access_source.entity_type, EntityType::Channel);
}
-#[tokio::test]
-async fn updated_payload_maps_to_document_patch() {
+#[test]
+fn updated_payload_maps_to_document_patch() {
let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(
DOCUMENT_ID,
updated_event(),
@@ -399,9 +399,7 @@ async fn updated_payload_maps_to_document_patch() {
let service = flaky_service(0);
assert!(matches!(
- process_event(&service, &event, 0, 0)
- .await
- .expect("processing succeeds"),
+ process_event(&service, &event).expect("processing succeeds"),
EventOutcome::Notified
));
@@ -425,34 +423,16 @@ fn malformed_and_unknown_events_are_rejected_by_the_declared_collection() {
assert!(decode_payload(payload).is_err());
}
-#[tokio::test(start_paused = true)]
-async fn transient_service_failures_retry_then_succeed() {
- let service = flaky_service(2);
- let patch = SoupRealtimePatch::for_entity(Patch::Updated(
- EntityType::Document.with_entity_string(DOCUMENT_ID.to_string()),
- ));
-
- notify_with_retry(&service, patch, 2, 17)
- .await
- .expect("eventual success");
-
- assert_eq!(service.attempts.load(Ordering::SeqCst), 3);
- assert_eq!(service.patches.lock().expect("patches lock").len(), 3);
-}
-
-#[tokio::test(start_paused = true)]
-async fn exhausted_retries_return_for_redelivery() {
- let service = flaky_service(u32::MAX);
- let patch = SoupRealtimePatch::for_entity(Patch::Updated(
- EntityType::Document.with_entity_string(DOCUMENT_ID.to_string()),
+#[test]
+fn service_failure_is_returned_without_retry() {
+ let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(
+ DOCUMENT_ID,
+ updated_event(),
));
+ let service = flaky_service(1);
- notify_with_retry(&service, patch, 2, 17)
- .await
- .expect_err("persistent failure returns without a commit");
+ assert!(process_event(&service, &event).is_err());
- assert_eq!(
- service.attempts.load(Ordering::SeqCst),
- MAX_SERVICE_ATTEMPTS
- );
+ assert_eq!(service.attempts.load(Ordering::SeqCst), 1);
+ assert_eq!(service.patches.lock().expect("patches lock").len(), 1);
}
diff --git a/crates/soup_realtime/src/outbound/entity_access.rs b/crates/soup_realtime/src/outbound/entity_access.rs
index abe77312b1..ed3a4b8275 100644
--- a/crates/soup_realtime/src/outbound/entity_access.rs
+++ b/crates/soup_realtime/src/outbound/entity_access.rs
@@ -55,11 +55,7 @@ impl<S> UserAccessExpander for EntityAccessExpander<S>
where
S: EntityAccessUserLookup,
{
- #[tracing::instrument(
- skip(self),
- fields(entity_type = %entity.entity_type, entity_id = %entity.entity_id),
- err
- )]
+ #[tracing::instrument(skip(self), err)]
async fn expand_user_access(
&self,
entity: &Entity<'static>,