Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/fix/run the update consumer (#5135)
Sean Aye4a383f164076
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rsindex 8adff981d4..9a3962b5ff 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs@@ -19,7 +19,10 @@ use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};use rootcause::prelude::{Report, ResultExt as _};-use crate::domain::ports::SoupRealtimeService;+use crate::domain::{+ ports::{SoupItemReader, SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},+ service::SoupRealtimeServiceImpl,+};/// Consumer group used for document update fan-out offsets.struct SoupRealtimeConsumerGroup;@@ -155,96 +158,100 @@ fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage}}-/// Runs the document update consumer until `shutdown` resolves.-///-/// The consumer subscribes only to `macro.documents` under the-/// `soup-realtime` group. It commits malformed and recognized-but-ignored-/// events, and commits `document.updated` only after [`SoupRealtimeService`]-/// succeeds. Exhausted service retries return without committing so a future-/// supervisor restart can redeliver the record.-#[tracing::instrument(skip(service, shutdown), fields(brokers), err)]-pub async fn run_document_update_consumer<S>(- brokers: &str,- service: S,- shutdown: impl Future<Output = ()> + Send,-) -> Result<(), Report>+impl<A, R, P> SoupRealtimeServiceImpl<A, R, P>where- S: SoupRealtimeService,+ A: UserAccessExpander,+ R: SoupItemReader,+ P: SoupRealtimePublisher,{- let consumer = KafkaEventConsumer::<SoupRealtimeConsumerGroup>::from_env(brokers)?;- let consumer = KafkaConsumerAdapter::<SoupRealtimeConsumerGroup, ()>::new(consumer)- .subscribe::<DeclaredMacroEvent>()- .context("failed to subscribe to document update topic")?;- let consumer = SoupRealtimeKafkaConsumer::new(consumer);- tracing::info!(- topics = ?DeclaredMacroEvent::topics(),- group = SoupRealtimeConsumerGroup::GROUP_NAME,- "realtime Soup document consumer listening"- );-- let mut shutdown = std::pin::pin!(shutdown);- loop {- tokio::select! {- _ = &mut shutdown => {- tracing::info!("realtime Soup document consumer shutting down");- break;- }- result = consumer.recv() => {- let message = match result {- Ok(message) => message,- Err(error) => {- tracing::error!(error = ?error, "Kafka receive error");- continue;- }- };- let kafka_message = message.inner();- let event = match message.decode_payload() {- Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => event,- Err(error) => {- tracing::error!(- error = ?error,- topic = kafka_message.topic(),- partition = kafka_message.partition(),- offset = kafka_message.offset(),- "dropping malformed document event"- );- commit_logged(&consumer, kafka_message);- continue;- }- };-- match process_document_event(- &service,- &event,- kafka_message.partition(),- kafka_message.offset(),- )- .await- {- Ok(DocumentEventOutcome::Notified | DocumentEventOutcome::Ignored) => {}- Err(error) => {- tracing::error!(- error = ?error,- topic = kafka_message.topic(),- partition = kafka_message.partition(),- offset = kafka_message.offset(),- "realtime Soup fan-out retries exhausted; pausing partition for redelivery"- );- // Kafka commits are cumulative within a partition, so- // pause it before any later record can advance the- // committed offset past this failure.- consumer- .inner()- .pause_message_partition(kafka_message)- .context("failed to pause Kafka partition after fan-out failure")?;- continue;- }+ /// Runs the document update consumer until `shutdown` resolves.+ ///+ /// The consumer subscribes only to `macro.documents` under the+ /// `soup-realtime` group. It commits malformed and recognized-but-ignored+ /// events, and commits `document.updated` only after [`SoupRealtimeService`]+ /// succeeds. Exhausted service retries return without committing so a future+ /// supervisor restart can redeliver the record.+ #[tracing::instrument(skip(self, shutdown), fields(brokers), err)]+ pub async fn run_document_update_consumer(+ &self,+ brokers: &str,+ shutdown: impl Future<Output = ()> + Send,+ ) -> Result<(), Report> {+ let consumer = KafkaEventConsumer::<SoupRealtimeConsumerGroup>::from_env(brokers)?;+ let consumer = KafkaConsumerAdapter::<SoupRealtimeConsumerGroup, ()>::new(consumer)+ .subscribe::<DeclaredMacroEvent>()+ .context("failed to subscribe to document update topic")?;+ let consumer = SoupRealtimeKafkaConsumer::new(consumer);+ tracing::info!(+ topics = ?DeclaredMacroEvent::topics(),+ group = SoupRealtimeConsumerGroup::GROUP_NAME,+ "realtime Soup document consumer listening"+ );++ let mut shutdown = std::pin::pin!(shutdown);+ loop {+ tokio::select! {+ _ = &mut shutdown => {+ tracing::info!("realtime Soup document consumer shutting down");+ break;}+ result = consumer.recv() => {+ let message = match result {+ Ok(message) => message,+ Err(error) => {+ tracing::error!(error = ?error, "Kafka receive error");+ continue;+ }+ };+ let kafka_message = message.inner();+ let event = match message.decode_payload() {+ Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => event,+ Err(error) => {+ tracing::error!(+ error = ?error,+ topic = kafka_message.topic(),+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),+ "dropping malformed document event"+ );+ commit_logged(&consumer, kafka_message);+ continue;+ }+ };++ match process_document_event(+ self,+ &event,+ kafka_message.partition(),+ kafka_message.offset(),+ )+ .await+ {+ Ok(DocumentEventOutcome::Notified | DocumentEventOutcome::Ignored) => {}+ Err(error) => {+ tracing::error!(+ error = ?error,+ topic = kafka_message.topic(),+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),+ "realtime Soup fan-out retries exhausted; pausing partition for redelivery"+ );+ // Kafka commits are cumulative within a partition, so+ // pause it before any later record can advance the+ // committed offset past this failure.+ consumer+ .inner()+ .pause_message_partition(kafka_message)+ .context("failed to pause Kafka partition after fan-out failure")?;+ continue;+ }+ }- commit_logged(&consumer, kafka_message);+ commit_logged(&consumer, kafka_message);+ }}}- }- Ok(())+ Ok(())+ }}diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.tomlindex 65b3737b1e..a43ac39d05 100644--- a/services/document_storage_service/Cargo.toml+++ b/services/document_storage_service/Cargo.toml@@ -180,6 +180,7 @@ soup = { path = "../../crates/soup", features = [] }soup_realtime = { path = "../../crates/soup_realtime", default-features = false, features = ["consumer",+ "outbound",], optional = true }sqlx = { workspace = true }sqs_client = { path = "../../crates/sqs_client", default-features = false, features = [diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex 9a2d4c2816..1e315e13ad 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -113,7 +113,11 @@ use soup::{};#[cfg(feature = "graphql")]use soup_realtime::{- domain::service::SoupRealtimeConsumerService, outbound::soup_consumer::SoupTopicConsumer,+ domain::service::{SoupRealtimeConsumerService, SoupRealtimeServiceImpl},+ outbound::{+ entity_access::EntityAccessExpander, kafka_publisher::KafkaSoupRealtimePublisher,+ soup_consumer::SoupTopicConsumer, soup_item_reader::SoupRepoItemReader,+ },};use sqlx::postgres::PgPoolOptions;use std::sync::Arc;@@ -880,6 +884,37 @@ async fn main() -> anyhow::Result<()> {}});+ #[cfg(feature = "graphql")]+ tokio::spawn({+ let brokers = config.kafka_brokers.as_ref().to_string();+ let entity_access_service = entity_access_service.as_ref().clone();+ let soup_pool = readonly_pool::ReadOnlyPool(readonly_db.clone());+ let macro_event_broker = macro_event_broker.clone();+ async move {+ loop {+ let fanout_service = SoupRealtimeServiceImpl::new(+ EntityAccessExpander::new(entity_access_service.clone()),+ SoupRepoItemReader::new(PgSoupRepo::new(soup_pool.clone())),+ KafkaSoupRealtimePublisher::new(macro_event_broker.clone()),+ );+ tracing::info!("starting realtime Soup document consumer");+ let result = fanout_service+ .run_document_update_consumer(&brokers, std::future::pending::<()>())+ .await;+ match result {+ Ok(()) => {+ tracing::error!("realtime Soup document consumer exited unexpectedly")+ }+ Err(error) => tracing::error!(+ error = ?error,+ "realtime Soup document consumer exited unexpectedly"+ ),+ }+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ }+ }+ });+let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));let api_context = ApiContext {