Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/fix/run the update consumer (#5135)

Sean Aye4a383f164076
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rs
index 8adff981d4..9a3962b5ff 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs
@@ -19,7 +19,10 @@ use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use rootcause::prelude::{Report, ResultExt as _};
-use crate::domain::ports::SoupRealtimeService;
+use crate::domain::{
+ ports::{SoupItemReader, SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},
+ service::SoupRealtimeServiceImpl,
+};
/// Consumer group used for document update fan-out offsets.
struct SoupRealtimeConsumerGroup;
@@ -155,96 +158,100 @@ fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage
}
}
-/// Runs the document update consumer until `shutdown` resolves.
-///
-/// The consumer subscribes only to `macro.documents` under the
-/// `soup-realtime` group. It commits malformed and recognized-but-ignored
-/// events, and commits `document.updated` only after [`SoupRealtimeService`]
-/// succeeds. Exhausted service retries return without committing so a future
-/// supervisor restart can redeliver the record.
-#[tracing::instrument(skip(service, shutdown), fields(brokers), err)]
-pub async fn run_document_update_consumer<S>(
- brokers: &str,
- service: S,
- shutdown: impl Future<Output = ()> + Send,
-) -> Result<(), Report>
+impl<A, R, P> SoupRealtimeServiceImpl<A, R, P>
where
- S: SoupRealtimeService,
+ A: UserAccessExpander,
+ R: SoupItemReader,
+ P: SoupRealtimePublisher,
{
- let consumer = KafkaEventConsumer::<SoupRealtimeConsumerGroup>::from_env(brokers)?;
- let consumer = KafkaConsumerAdapter::<SoupRealtimeConsumerGroup, ()>::new(consumer)
- .subscribe::<DeclaredMacroEvent>()
- .context("failed to subscribe to document update topic")?;
- let consumer = SoupRealtimeKafkaConsumer::new(consumer);
- tracing::info!(
- topics = ?DeclaredMacroEvent::topics(),
- group = SoupRealtimeConsumerGroup::GROUP_NAME,
- "realtime Soup document consumer listening"
- );
-
- let mut shutdown = std::pin::pin!(shutdown);
- loop {
- tokio::select! {
- _ = &mut shutdown => {
- tracing::info!("realtime Soup document consumer shutting down");
- break;
- }
- result = consumer.recv() => {
- let message = match result {
- Ok(message) => message,
- Err(error) => {
- tracing::error!(error = ?error, "Kafka receive error");
- continue;
- }
- };
- let kafka_message = message.inner();
- let event = match message.decode_payload() {
- Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => event,
- Err(error) => {
- tracing::error!(
- error = ?error,
- topic = kafka_message.topic(),
- partition = kafka_message.partition(),
- offset = kafka_message.offset(),
- "dropping malformed document event"
- );
- commit_logged(&consumer, kafka_message);
- continue;
- }
- };
-
- match process_document_event(
- &service,
- &event,
- kafka_message.partition(),
- kafka_message.offset(),
- )
- .await
- {
- Ok(DocumentEventOutcome::Notified | DocumentEventOutcome::Ignored) => {}
- Err(error) => {
- tracing::error!(
- error = ?error,
- topic = kafka_message.topic(),
- partition = kafka_message.partition(),
- offset = kafka_message.offset(),
- "realtime Soup fan-out retries exhausted; pausing partition for redelivery"
- );
- // Kafka commits are cumulative within a partition, so
- // pause it before any later record can advance the
- // committed offset past this failure.
- consumer
- .inner()
- .pause_message_partition(kafka_message)
- .context("failed to pause Kafka partition after fan-out failure")?;
- continue;
- }
+ /// Runs the document update consumer until `shutdown` resolves.
+ ///
+ /// The consumer subscribes only to `macro.documents` under the
+ /// `soup-realtime` group. It commits malformed and recognized-but-ignored
+ /// events, and commits `document.updated` only after [`SoupRealtimeService`]
+ /// succeeds. Exhausted service retries return without committing so a future
+ /// supervisor restart can redeliver the record.
+ #[tracing::instrument(skip(self, shutdown), fields(brokers), err)]
+ pub async fn run_document_update_consumer(
+ &self,
+ brokers: &str,
+ shutdown: impl Future<Output = ()> + Send,
+ ) -> Result<(), Report> {
+ let consumer = KafkaEventConsumer::<SoupRealtimeConsumerGroup>::from_env(brokers)?;
+ let consumer = KafkaConsumerAdapter::<SoupRealtimeConsumerGroup, ()>::new(consumer)
+ .subscribe::<DeclaredMacroEvent>()
+ .context("failed to subscribe to document update topic")?;
+ let consumer = SoupRealtimeKafkaConsumer::new(consumer);
+ tracing::info!(
+ topics = ?DeclaredMacroEvent::topics(),
+ group = SoupRealtimeConsumerGroup::GROUP_NAME,
+ "realtime Soup document consumer listening"
+ );
+
+ let mut shutdown = std::pin::pin!(shutdown);
+ loop {
+ tokio::select! {
+ _ = &mut shutdown => {
+ tracing::info!("realtime Soup document consumer shutting down");
+ break;
}
+ result = consumer.recv() => {
+ let message = match result {
+ Ok(message) => message,
+ Err(error) => {
+ tracing::error!(error = ?error, "Kafka receive error");
+ continue;
+ }
+ };
+ let kafka_message = message.inner();
+ let event = match message.decode_payload() {
+ Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => event,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ topic = kafka_message.topic(),
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
+ "dropping malformed document event"
+ );
+ commit_logged(&consumer, kafka_message);
+ continue;
+ }
+ };
+
+ match process_document_event(
+ self,
+ &event,
+ kafka_message.partition(),
+ kafka_message.offset(),
+ )
+ .await
+ {
+ Ok(DocumentEventOutcome::Notified | DocumentEventOutcome::Ignored) => {}
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ topic = kafka_message.topic(),
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
+ "realtime Soup fan-out retries exhausted; pausing partition for redelivery"
+ );
+ // Kafka commits are cumulative within a partition, so
+ // pause it before any later record can advance the
+ // committed offset past this failure.
+ consumer
+ .inner()
+ .pause_message_partition(kafka_message)
+ .context("failed to pause Kafka partition after fan-out failure")?;
+ continue;
+ }
+ }
- commit_logged(&consumer, kafka_message);
+ commit_logged(&consumer, kafka_message);
+ }
}
}
- }
- Ok(())
+ Ok(())
+ }
}
diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.toml
index 65b3737b1e..a43ac39d05 100644
--- a/services/document_storage_service/Cargo.toml
+++ b/services/document_storage_service/Cargo.toml
@@ -180,6 +180,7 @@ soup = { path = "../../crates/soup", features = [
] }
soup_realtime = { path = "../../crates/soup_realtime", default-features = false, features = [
"consumer",
+ "outbound",
], optional = true }
sqlx = { workspace = true }
sqs_client = { path = "../../crates/sqs_client", default-features = false, features = [
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index 9a2d4c2816..1e315e13ad 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -113,7 +113,11 @@ use soup::{
};
#[cfg(feature = "graphql")]
use soup_realtime::{
- domain::service::SoupRealtimeConsumerService, outbound::soup_consumer::SoupTopicConsumer,
+ domain::service::{SoupRealtimeConsumerService, SoupRealtimeServiceImpl},
+ outbound::{
+ entity_access::EntityAccessExpander, kafka_publisher::KafkaSoupRealtimePublisher,
+ soup_consumer::SoupTopicConsumer, soup_item_reader::SoupRepoItemReader,
+ },
};
use sqlx::postgres::PgPoolOptions;
use std::sync::Arc;
@@ -880,6 +884,37 @@ async fn main() -> anyhow::Result<()> {
}
});
+ #[cfg(feature = "graphql")]
+ tokio::spawn({
+ let brokers = config.kafka_brokers.as_ref().to_string();
+ let entity_access_service = entity_access_service.as_ref().clone();
+ let soup_pool = readonly_pool::ReadOnlyPool(readonly_db.clone());
+ let macro_event_broker = macro_event_broker.clone();
+ async move {
+ loop {
+ let fanout_service = SoupRealtimeServiceImpl::new(
+ EntityAccessExpander::new(entity_access_service.clone()),
+ SoupRepoItemReader::new(PgSoupRepo::new(soup_pool.clone())),
+ KafkaSoupRealtimePublisher::new(macro_event_broker.clone()),
+ );
+ tracing::info!("starting realtime Soup document consumer");
+ let result = fanout_service
+ .run_document_update_consumer(&brokers, std::future::pending::<()>())
+ .await;
+ match result {
+ Ok(()) => {
+ tracing::error!("realtime Soup document consumer exited unexpectedly")
+ }
+ Err(error) => tracing::error!(
+ error = ?error,
+ "realtime Soup document consumer exited unexpectedly"
+ ),
+ }
+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ }
+ }
+ });
+
let favorites_service = Arc::new(FavoritesServiceImpl::new(PgFavoritesRepo::new(db.clone())));
let api_context = ApiContext {