Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/feat/strong consumer typing (#5092)
Sean Ayef222389a1f0d
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 6e08fc8a16..cd957cb0d2 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -58,6 +58,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -143,6 +144,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -220,6 +222,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -364,6 +367,7 @@"crates/invite_email","crates/ip_extractor","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/loops_client",@@ -495,6 +499,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -590,6 +595,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -673,6 +679,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -769,6 +776,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -866,6 +874,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -938,6 +947,7 @@"crates/graphql_soup","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -1142,6 +1152,7 @@"crates/invite_email","crates/ip_extractor","crates/item_filters",+ "crates/kafka_util","crates/lambda_client","crates/last_online_tracker","crates/macro_auth",@@ -1298,6 +1309,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -1382,6 +1394,7 @@"crates/invite_email","crates/ip_extractor","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -1507,7 +1520,7 @@"crates/invite_email","crates/ip_extractor","crates/item_filters",- "crates/kafka_consumer_util",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -1677,6 +1690,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",@@ -1760,6 +1774,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",@@ -1833,6 +1848,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/lambda_client","crates/last_online_tracker","crates/macro_auth",@@ -1910,6 +1926,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -2001,6 +2018,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -2062,6 +2080,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -2129,6 +2148,7 @@"crates/invite_email","crates/ip_extractor","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -2205,6 +2225,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -2259,6 +2280,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -2515,6 +2537,7 @@"crates/github","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",@@ -2619,6 +2642,7 @@"crates/graphql_email","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -2755,6 +2779,7 @@"crates/graphql_soup","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -2885,12 +2910,10 @@"crates/non_empty","crates/workspace-hack"],- "kafka_consumer_util": [- "crates/kafka_consumer_util",+ "kafka_util": [+ "crates/kafka_util","crates/macro_env","crates/macro_env_var",- "crates/macro_event_broker",- "crates/macro_event_topics","crates/workspace-hack"],"lambda_client": [@@ -2930,6 +2953,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",@@ -3003,6 +3027,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -3180,6 +3205,7 @@"crates/workspace-hack"],"macro_event_broker": [+ "crates/kafka_util","crates/macro_env","crates/macro_env_var","crates/macro_event_broker",@@ -3359,6 +3385,7 @@"crates/fusionauth","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -3462,6 +3489,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -3684,6 +3712,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -3764,6 +3793,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -3833,6 +3863,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -4054,6 +4085,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -4115,6 +4147,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -4177,6 +4210,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -4396,6 +4430,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -4495,6 +4530,7 @@"crates/invite_email","crates/ip_extractor","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",@@ -4581,6 +4617,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -4661,6 +4698,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -4729,6 +4767,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -4805,6 +4844,7 @@"crates/fusionauth","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",@@ -4922,6 +4962,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -4995,7 +5036,7 @@"crates/frecency","crates/invite_email","crates/item_filters",- "crates/kafka_consumer_util",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",@@ -5073,6 +5114,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -5257,6 +5299,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",@@ -5332,6 +5375,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -5413,6 +5457,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -5474,6 +5519,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_util","crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",@@ -5585,7 +5631,7 @@"crates/frecency","crates/invite_email","crates/item_filters",- "crates/kafka_consumer_util",+ "crates/kafka_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",diff --git a/Cargo.lock b/Cargo.lockindex f87c7393e8..dd357052a1 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2608,9 +2608,9 @@ dependencies = ["http-body-util","indexmap 2.14.0","item_filters",+ "kafka_util","macro_authorization","macro_db_migrator",- "macro_env","macro_env_var","macro_event_broker","macro_event_topics",@@ -4443,12 +4443,12 @@ dependencies = ["hex","http-body-util","jsonwebtoken 10.3.0",+ "kafka_util","lexical_client","macro_authorization","macro_aws_config","macro_db_migrator","macro_entrypoint",- "macro_env","macro_env_var","macro_event_broker","macro_event_topics",@@ -7540,14 +7540,18 @@ dependencies = [][[package]]-name = "kafka_consumer_util"+name = "kafka_util"version = "0.1.0"dependencies = [+ "aws-msk-iam-sasl-signer",+ "aws-types","either","macro_env",- "macro_event_broker",+ "macro_env_var","rdkafka","thiserror 2.0.18",+ "tokio",+ "tracing","uuid","workspace-hack",]@@ -8369,10 +8373,7 @@ dependencies = [name = "macro_event_broker"version = "0.1.0"dependencies = [- "aws-msk-iam-sasl-signer",- "aws-types",- "macro_env",- "macro_env_var",+ "kafka_util","macro_event_topics","rdkafka","rootcause",@@ -11045,10 +11046,10 @@ dependencies = ["futures","hex","http-body-util",+ "kafka_util","macro_authorization","macro_aws_config","macro_db_migrator",- "macro_env","macro_env_var","macro_event_broker","macro_event_topics",@@ -13407,7 +13408,7 @@ dependencies = ["documents","entity_access","futures",- "kafka_consumer_util",+ "kafka_util","macro_event_broker","macro_event_topics","macro_user_id",@@ -15555,7 +15556,7 @@ dependencies = ["futures","hex","hmac 0.12.1",- "kafka_consumer_util",+ "kafka_util","macro_authorization","macro_db_migrator","macro_event_broker",diff --git a/crates/bots/Cargo.toml b/crates/bots/Cargo.tomlindex 6c9ae8c6b7..5278a10216 100644--- a/crates/bots/Cargo.toml+++ b/crates/bots/Cargo.toml@@ -19,7 +19,7 @@ inbound = ["ports",]outbound = ["dep:sqlx", "dep:macro_uuid", "ports"]-ports = ["dep:tokio", "macro_event_broker/ports"]+ports = ["dep:tokio"][dependencies]anyhow = { workspace = true }diff --git a/crates/bots/src/domain/events/test.rs b/crates/bots/src/domain/events/test.rsindex c192d0104c..4cdec1cdfb 100644--- a/crates/bots/src/domain/events/test.rs+++ b/crates/bots/src/domain/events/test.rs@@ -1,6 +1,5 @@use chrono::{DateTime, Utc};use macro_event_broker::{Event, MacroEvent};-use macro_event_topics::Topic;use macro_user_id::user_id::MacroUserIdStr;use serde_json::{Value, json};use uuid::Uuid;@@ -165,7 +164,7 @@ fn constructors_use_bot_topic_bare_uuid_key_and_schema_version_one() {for (event, expected_event_type) in cases.into_iter().zip(expected_event_types) {assert_eq!(event.key(), BOT_ID);assert!(!event.key().starts_with("bot|"));- assert_eq!(event.topic().as_str(), "macro.bots");+ assert_eq!(event.topic(), "macro.bots");assert_eq!(event.event().schema_version, 1);assert_eq!(serde_json::to_value(event.event()).expect("serializable event")["event_type"],@@ -188,7 +187,7 @@ fn every_event_variant_round_trips() {assert_eq!(decoded.key(), BOT_ID);assert_eq!(decoded.event(), original.event());- assert_eq!(decoded.topic().as_str(), "macro.bots");+ assert_eq!(decoded.topic(), "macro.bots");}}diff --git a/crates/bots/src/outbound/pg_bots_repo/tests.rs b/crates/bots/src/outbound/pg_bots_repo/tests.rsindex 1ddbd7b913..dff2f252be 100644--- a/crates/bots/src/outbound/pg_bots_repo/tests.rs+++ b/crates/bots/src/outbound/pg_bots_repo/tests.rs@@ -8,9 +8,7 @@ use crate::domain::{service::BotServiceImpl,};use macro_db_migrator::MACRO_DB_MIGRATIONS;-use macro_event_broker::{- EventBrokerError, MacroEvent, MacroEventBroker, NoopMacroEventBroker, Topic as _,-};+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, NoopMacroEventBroker};use serde_json::{Value, json};use sqlx::PgPool;use std::sync::{Arc, Mutex};@@ -95,7 +93,7 @@ impl MacroEventBroker for RecordingEventBroker {.lock().expect("event lock poisoned").push(PublishedEvent {- topic: event.topic().as_str(),+ topic: event.topic(),key: event.key().to_string(),payload: serde_json::to_value(event.event())?,});diff --git a/crates/channels/Cargo.toml b/crates/channels/Cargo.tomlindex a5baf55dce..eb713994bd 100644--- a/crates/channels/Cargo.toml+++ b/crates/channels/Cargo.toml@@ -63,7 +63,7 @@ outbound = ["macro_user_id/sqlx","ports",]-ports = ["dep:tokio", "macro_event_broker/ports"]+ports = ["dep:tokio"][dependencies]anyhow = { workspace = true }@@ -120,8 +120,8 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]http-body-util = { workspace = true }+kafka_util = { path = "../kafka_util" }macro_db_migrator = { path = "../macro_db_migrator" }-macro_env = { path = "../macro_env" }macro_env_var = { path = "../macro_env_var" }# Examples need the Kafka adapter (outbound); the library itself only uses ports.macro_event_broker = { path = "../macro_event_broker" }diff --git a/crates/channels/examples/channels_consumer.rs b/crates/channels/examples/channels_consumer.rsindex f12dbbfd10..54667b9633 100644--- a/crates/channels/examples/channels_consumer.rs+++ b/crates/channels/examples/channels_consumer.rs@@ -3,8 +3,8 @@//! Connects with the same `KAFKA_BROKERS` env var and environment-driven//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),-//! subscribes to [`MacroChannelsTopic`], and prints every channel event it-//! receives.+//! subscribes to the [`ChannelMacroEvent`] topic and prints every channel event+//! it receives.//!//! The consumer follows the standard poll-loop / worker split: a Kafka poll//! loop decodes messages and hands them to a processing task over a bounded@@ -25,14 +25,12 @@use anyhow::Context as _;use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};-use macro_env::Environment;-use macro_event_broker::outbound::msk_iam::configure_sasl_iam;-use macro_event_broker::{EventBrokerError, MacroEvent, MskIamClientContext, Topic as _};-use macro_event_topics::MacroChannelsTopic;-use rdkafka::ClientConfig;-use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};-use rdkafka::error::KafkaResult;-use rdkafka::message::{BorrowedMessage, Message};+use kafka_util::{GroupName, KafkaEventConsumer};+use macro_event_broker::{+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,+};+use rdkafka::consumer::CommitMode;+use rdkafka::message::Message;use tokio::sync::mpsc;macro_env_var::env_var! {@@ -41,103 +39,28 @@ macro_env_var::env_var! {}}-/// Consumer group id for this example. Offsets are committed under this group,-/// so re-runs resume where the previous run left off.-const GROUP_ID: &str = "channels-consumer-example";+/// Consumer group for this example. Offsets are committed under this group, so+/// re-runs resume where the previous run left off.+struct ChannelsConsumerGroup;++impl GroupName for ChannelsConsumerGroup {+ const GROUP_NAME: &'static str = "channels-consumer-example";+}/// Bounded capacity of the channel between the poll loop and the processor.const CHANNEL_CAPACITY: usize = 128;-/// Consumer-specific event enum for a consumer subscribed to `macro.channels`,-/// following the pattern in `macro_event_broker/examples/example_event.rs`.-enum ChannelsConsumerEvent {- /// Event received on [`MacroChannelsTopic`].- Channels(ChannelMacroEvent),-}+macro_event_broker::declare_topics!(DeclaredMacroEvent: ChannelMacroEvent);-impl ChannelsConsumerEvent {- /// Decode one Kafka message into this consumer's event enum.- fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {- match topic {- topic if topic == MacroChannelsTopic.as_str() => {- Ok(Self::Channels(ChannelMacroEvent::decode(key, payload)?))- }- unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),- }- }-}+type ChannelsKafkaAdapter = KafkaConsumerAdapter<ChannelsConsumerGroup, DeclaredMacroEvent>;+type ChannelsConsumerService = MacroEventConsumerService<DeclaredMacroEvent, ChannelsKafkaAdapter>;/// A decoded event plus the Kafka coordinates it came from, as handed from the/// poll loop to the processor over the channel.struct ReceivedEvent {partition: i32,offset: i64,- event: ChannelsConsumerEvent,-}--/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).-enum ChannelsConsumer {- /// Unauthenticated plaintext connection (local docker broker).- Plaintext(StreamConsumer),- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).- MskIam(StreamConsumer<MskIamClientContext>),-}--impl ChannelsConsumer {- /// Build a consumer for the given brokers, choosing the transport from the- /// `ENVIRONMENT` variable exactly like the publisher does.- fn from_env(brokers: &str) -> anyhow::Result<Self> {- let mut config = ClientConfig::new();- config- .set("bootstrap.servers", brokers)- .set("group.id", GROUP_ID)- // Offsets are committed manually after a successful handoff.- .set("enable.auto.commit", "false")- // Start from the beginning of the topic on the first ever run.- .set("auto.offset.reset", "earliest");-- let consumer = match Environment::new_or_prod() {- Environment::Local => Self::Plaintext(- config- .create()- .context("failed to create plaintext kafka consumer")?,- ),- Environment::Develop | Environment::Production => {- let config = configure_sasl_iam(config);- Self::MskIam(- config- .create_with_context(MskIamClientContext::from_env())- .context("failed to create MSK IAM kafka consumer")?,- )- }- };-- Ok(consumer)- }-- fn subscribe(&self) -> KafkaResult<()> {- let topics = [MacroChannelsTopic.as_str()];- match self {- Self::Plaintext(consumer) => consumer.subscribe(&topics),- Self::MskIam(consumer) => consumer.subscribe(&topics),- }- }-- /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it- /// can sit in a `select!` without losing messages.- async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {- match self {- Self::Plaintext(consumer) => consumer.recv().await,- Self::MskIam(consumer) => consumer.recv().await,- }- }-- fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {- match self {- Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),- Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),- }- }+ event: ChannelMacroEvent,}fn event_type(event: &ChannelTopicEvent) -> &'static str {@@ -159,7 +82,7 @@ fn event_type(event: &ChannelTopicEvent) -> &'static str {/// the body for indexing, notifications, etc.async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {while let Some(received) = events.recv().await {- let ChannelsConsumerEvent::Channels(event) = received.event;+ let event = received.event;let envelope = event.event();println!(@@ -192,14 +115,16 @@ async fn main() -> anyhow::Result<()> {let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;- let consumer = ChannelsConsumer::from_env(env.kafka_brokers.as_ref())?;- consumer- .subscribe()- .context("failed to subscribe to channels topic")?;+ let consumer =+ KafkaEventConsumer::<ChannelsConsumerGroup>::from_env(env.kafka_brokers.as_ref())?;+ let consumer = KafkaConsumerAdapter::<ChannelsConsumerGroup, ()>::new(consumer)+ .subscribe::<DeclaredMacroEvent>()+ .map_err(|error| anyhow::anyhow!("failed to subscribe to channels topic: {error:?}"))?;+ let consumer = ChannelsConsumerService::new(consumer);println!("listening on topic={} group={} brokers={} (ctrl-c to stop)",- MacroChannelsTopic.as_str(),- GROUP_ID,+ DeclaredMacroEvent::topics()[0],+ ChannelsConsumerGroup::GROUP_NAME,env.kafka_brokers.as_ref(),);@@ -222,25 +147,13 @@ async fn main() -> anyhow::Result<()> {}};- let Some(payload) = message.payload() else {- eprintln!(- "skipping message with empty payload at partition {} offset {}",- message.partition(),- message.offset(),- );- let _ = consumer.commit(&message).inspect_err(|e| {- eprintln!("failed to commit offset {}: {e}", message.offset());- });- continue;- };- let key = String::from_utf8_lossy(message.key().unwrap_or_default());-- match ChannelsConsumerEvent::decode(message.topic(), key.as_ref(), payload) {- Ok(event) => {+ let kafka_message = message.inner();+ match message.decode_payload() {+ Ok(DeclaredMacroEvent::ChannelMacroEvent(event)) => {events_tx.send(ReceivedEvent {- partition: message.partition(),- offset: message.offset(),+ partition: kafka_message.partition(),+ offset: kafka_message.offset(),event,}).await@@ -250,16 +163,19 @@ async fn main() -> anyhow::Result<()> {// wedging the partition on a poison message.Err(e) => eprintln!("failed to decode message at partition {} offset {}: {e}",- message.partition(),- message.offset(),+ kafka_message.partition(),+ kafka_message.offset(),),}// Commit only after the event was handed to the processor:// at-least-once delivery (up to the channel buffer on a crash).- let _ = consumer.commit(&message).inspect_err(|e| {- eprintln!("failed to commit offset {}: {e}", message.offset());- });+ let _ = consumer+ .inner()+ .commit_message(kafka_message, CommitMode::Async)+ .inspect_err(|e| {+ eprintln!("failed to commit offset {}: {e:?}", kafka_message.offset());+ });}}}diff --git a/crates/channels/src/domain/side_effects/test.rs b/crates/channels/src/domain/side_effects/test.rsindex 34fa4a7348..79344c0884 100644--- a/crates/channels/src/domain/side_effects/test.rs+++ b/crates/channels/src/domain/side_effects/test.rs@@ -1053,9 +1053,8 @@ impl MacroEventBroker for TestEventBroker {tokio::task::JoinHandle<Result<(), macro_event_broker::EventBrokerError>>,macro_event_broker::EventBrokerError,> {- use macro_event_topics::Topic as _;self.published.lock().unwrap().push(PublishedEvent {- topic: event.topic().as_str().to_string(),+ topic: event.topic().to_string(),key: event.key().to_string(),envelope: serde_json::to_value(event.event())?,});diff --git a/crates/documents/Cargo.toml b/crates/documents/Cargo.tomlindex ae52bac348..224bed4553 100644--- a/crates/documents/Cargo.toml+++ b/crates/documents/Cargo.toml@@ -63,12 +63,7 @@ outbound = ["ports",]ports = ["dep:tokio"]-service = [- "dep:entity_access_management",- "dep:foreign_entity",- "macro_event_broker/ports",- "ports",-]+service = ["dep:entity_access_management", "dep:foreign_entity", "ports"][dependencies]# Core@@ -155,8 +150,8 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }dotenvy = { workspace = true }embedding = { path = "../embedding" }http-body-util = { workspace = true }+kafka_util = { path = "../kafka_util" }macro_db_migrator = { path = "../macro_db_migrator" }-macro_env = { path = "../macro_env" }# Examples need the Kafka adapter (outbound); the library itself only uses ports.macro_event_broker = { path = "../macro_event_broker" }mockall = { workspace = true }diff --git a/crates/documents/examples/documents_consumer.rs b/crates/documents/examples/documents_consumer.rsindex 66259c1317..2b9ad05229 100644--- a/crates/documents/examples/documents_consumer.rs+++ b/crates/documents/examples/documents_consumer.rs@@ -3,8 +3,8 @@//! Connects with the same `KAFKA_BROKERS` env var and environment-driven//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),-//! subscribes to [`MacroDocumentsTopic`], and prints every document lifecycle-//! event it receives.+//! subscribes to the [`DocumentMacroEvent`] topic and prints every document+//! lifecycle event it receives.//!//! The consumer follows the standard poll-loop / worker split: a Kafka poll//! loop decodes messages and hands them to a processing task over a bounded@@ -25,14 +25,12 @@use anyhow::Context as _;use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};-use macro_env::Environment;-use macro_event_broker::outbound::msk_iam::configure_sasl_iam;-use macro_event_broker::{EventBrokerError, MacroEvent, MskIamClientContext, Topic as _};-use macro_event_topics::MacroDocumentsTopic;-use rdkafka::ClientConfig;-use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};-use rdkafka::error::KafkaResult;-use rdkafka::message::{BorrowedMessage, Message};+use kafka_util::{GroupName, KafkaEventConsumer};+use macro_event_broker::{+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,+};+use rdkafka::consumer::CommitMode;+use rdkafka::message::Message;use tokio::sync::mpsc;macro_env_var::env_var! {@@ -41,103 +39,29 @@ macro_env_var::env_var! {}}-/// Consumer group id for this example. Offsets are committed under this group,-/// so re-runs resume where the previous run left off.-const GROUP_ID: &str = "documents-consumer-example";+/// Consumer group for this example. Offsets are committed under this group, so+/// re-runs resume where the previous run left off.+struct DocumentsConsumerGroup;++impl GroupName for DocumentsConsumerGroup {+ const GROUP_NAME: &'static str = "documents-consumer-example";+}/// Bounded capacity of the channel between the poll loop and the processor.const CHANNEL_CAPACITY: usize = 128;-/// Consumer-specific event enum for a consumer subscribed to `macro.documents`,-/// following the pattern in `macro_event_broker/examples/example_event.rs`.-enum DocumentsConsumerEvent {- /// Event received on [`MacroDocumentsTopic`].- Documents(DocumentMacroEvent),-}+macro_event_broker::declare_topics!(DeclaredMacroEvent: DocumentMacroEvent);-impl DocumentsConsumerEvent {- /// Decode one Kafka message into this consumer's event enum.- fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {- match topic {- topic if topic == MacroDocumentsTopic.as_str() => {- Ok(Self::Documents(DocumentMacroEvent::decode(key, payload)?))- }- unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),- }- }-}+type DocumentsKafkaAdapter = KafkaConsumerAdapter<DocumentsConsumerGroup, DeclaredMacroEvent>;+type DocumentsConsumerService =+ MacroEventConsumerService<DeclaredMacroEvent, DocumentsKafkaAdapter>;/// A decoded event plus the Kafka coordinates it came from, as handed from the/// poll loop to the processor over the channel.struct ReceivedEvent {partition: i32,offset: i64,- event: DocumentsConsumerEvent,-}--/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).-enum DocumentsConsumer {- /// Unauthenticated plaintext connection (local docker broker).- Plaintext(StreamConsumer),- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).- MskIam(StreamConsumer<MskIamClientContext>),-}--impl DocumentsConsumer {- /// Build a consumer for the given brokers, choosing the transport from the- /// `ENVIRONMENT` variable exactly like the publisher does.- fn from_env(brokers: &str) -> anyhow::Result<Self> {- let mut config = ClientConfig::new();- config- .set("bootstrap.servers", brokers)- .set("group.id", GROUP_ID)- // Offsets are committed manually after a successful handoff.- .set("enable.auto.commit", "false")- // Start from the beginning of the topic on the first ever run.- .set("auto.offset.reset", "earliest");-- let consumer = match Environment::new_or_prod() {- Environment::Local => Self::Plaintext(- config- .create()- .context("failed to create plaintext kafka consumer")?,- ),- Environment::Develop | Environment::Production => {- let config = configure_sasl_iam(config);- Self::MskIam(- config- .create_with_context(MskIamClientContext::from_env())- .context("failed to create MSK IAM kafka consumer")?,- )- }- };-- Ok(consumer)- }-- fn subscribe(&self) -> KafkaResult<()> {- let topics = [MacroDocumentsTopic.as_str()];- match self {- Self::Plaintext(consumer) => consumer.subscribe(&topics),- Self::MskIam(consumer) => consumer.subscribe(&topics),- }- }-- /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it- /// can sit in a `select!` without losing messages.- async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {- match self {- Self::Plaintext(consumer) => consumer.recv().await,- Self::MskIam(consumer) => consumer.recv().await,- }- }-- fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {- match self {- Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),- Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),- }- }+ event: DocumentMacroEvent,}fn event_type(event: &DocumentTopicEvent) -> &'static str {@@ -154,7 +78,7 @@ fn event_type(event: &DocumentTopicEvent) -> &'static str {/// the body for indexing, notifications, etc.async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {while let Some(received) = events.recv().await {- let DocumentsConsumerEvent::Documents(event) = received.event;+ let event = received.event;let envelope = event.event();println!(@@ -187,14 +111,16 @@ async fn main() -> anyhow::Result<()> {let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;- let consumer = DocumentsConsumer::from_env(env.kafka_brokers.as_ref())?;- consumer- .subscribe()- .context("failed to subscribe to documents topic")?;+ let consumer =+ KafkaEventConsumer::<DocumentsConsumerGroup>::from_env(env.kafka_brokers.as_ref())?;+ let consumer = KafkaConsumerAdapter::<DocumentsConsumerGroup, ()>::new(consumer)+ .subscribe::<DeclaredMacroEvent>()+ .map_err(|error| anyhow::anyhow!("failed to subscribe to documents topic: {error:?}"))?;+ let consumer = DocumentsConsumerService::new(consumer);println!("listening on topic={} group={} brokers={} (ctrl-c to stop)",- MacroDocumentsTopic.as_str(),- GROUP_ID,+ DeclaredMacroEvent::topics()[0],+ DocumentsConsumerGroup::GROUP_NAME,env.kafka_brokers.as_ref(),);@@ -217,25 +143,13 @@ async fn main() -> anyhow::Result<()> {}};- let Some(payload) = message.payload() else {- eprintln!(- "skipping message with empty payload at partition {} offset {}",- message.partition(),- message.offset(),- );- let _ = consumer.commit(&message).inspect_err(|e| {- eprintln!("failed to commit offset {}: {e}", message.offset());- });- continue;- };- let key = String::from_utf8_lossy(message.key().unwrap_or_default());-- match DocumentsConsumerEvent::decode(message.topic(), key.as_ref(), payload) {- Ok(event) => {+ let kafka_message = message.inner();+ match message.decode_payload() {+ Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => {events_tx.send(ReceivedEvent {- partition: message.partition(),- offset: message.offset(),+ partition: kafka_message.partition(),+ offset: kafka_message.offset(),event,}).await@@ -245,16 +159,19 @@ async fn main() -> anyhow::Result<()> {// wedging the partition on a poison message.Err(e) => eprintln!("failed to decode message at partition {} offset {}: {e}",- message.partition(),- message.offset(),+ kafka_message.partition(),+ kafka_message.offset(),),}// Commit only after the event was handed to the processor:// at-least-once delivery (up to the channel buffer on a crash).- let _ = consumer.commit(&message).inspect_err(|e| {- eprintln!("failed to commit offset {}: {e}", message.offset());- });+ let _ = consumer+ .inner()+ .commit_message(kafka_message, CommitMode::Async)+ .inspect_err(|e| {+ eprintln!("failed to commit offset {}: {e:?}", kafka_message.offset());+ });}}}diff --git a/crates/documents/src/domain/service/tests.rs b/crates/documents/src/domain/service/tests.rsindex cfd167332f..35c82e391d 100644--- a/crates/documents/src/domain/service/tests.rs+++ b/crates/documents/src/domain/service/tests.rs@@ -3,7 +3,7 @@ use foreign_entity::domain::models::{CreateForeignEntity, ForeignEntity, ForeignEntityError, PatchForeignEntity, SourceId,};use foreign_entity::domain::ports::{ForeignEntityListQuery, ForeignEntityService};-use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, Topic as _};+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};use macro_user_id::cowlike::CowLike;use model::document::DocumentMetadata;use std::sync::{Arc, Mutex};@@ -350,7 +350,7 @@ impl MacroEventBroker for TestEventBroker {event: &E,) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {self.published.lock().unwrap().push(PublishedEvent {- topic: event.topic().as_str(),+ topic: event.topic(),key: event.key().to_string(),payload: serde_json::to_value(event.event())?,});diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.tomlindex 0919f65269..a616219a97 100644--- a/crates/email/Cargo.toml+++ b/crates/email/Cargo.toml@@ -37,7 +37,7 @@ gmail_token = ["dep:authentication_service_client", "dep:redis", "ports"]http_client = ["dep:reqwest", "ports"]inbound = ["ports"]outbound = ["dep:sqlx", "dep:email_db_client", "ports"]-ports = ["crm/ports", "dep:tokio", "frecency/ports", "macro_event_broker/ports"]+ports = ["crm/ports", "dep:tokio", "frecency/ports"][dependencies]ai_toolset = { path = "../ai_toolset", optional = true }@@ -92,9 +92,6 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]crm = { path = "../crm", features = ["outbound"] }frecency = { path = "../frecency", features = ["outbound", "postgres"] }-macro_event_broker = { path = "../macro_event_broker", default-features = false, features = [- "ports",-] }macro_db_migrator = { path = "../macro_db_migrator" }serde_json = { workspace = true }tracing-subscriber = { workspace = true, features = ["env-filter"] }diff --git a/crates/kafka_consumer_util/Cargo.toml b/crates/kafka_util/Cargo.tomlsimilarity index 59%rename from crates/kafka_consumer_util/Cargo.tomlrename to crates/kafka_util/Cargo.tomlindex dbdf5d7356..882a4f17b9 100644--- a/crates/kafka_consumer_util/Cargo.toml+++ b/crates/kafka_util/Cargo.toml@@ -1,16 +1,18 @@[package]edition = "2024"-name = "kafka_consumer_util"+name = "kafka_util"publish = falseversion = "0.1.0"[dependencies]+aws-msk-iam-sasl-signer = { workspace = true }+aws-types = { workspace = true }either = { workspace = true }macro_env = { path = "../macro_env" }-macro_event_broker = { path = "../macro_event_broker", default-features = false, features = [- "kafka",-] }+macro_env_var = { path = "../macro_env_var" }rdkafka = { workspace = true }thiserror = { workspace = true }+tokio = { workspace = true }+tracing = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/kafka_consumer_util/README.md b/crates/kafka_util/README.mdsimilarity index 55%rename from crates/kafka_consumer_util/README.mdrename to crates/kafka_util/README.mdindex 6373185896..f571e2f866 100644--- a/crates/kafka_consumer_util/README.md+++ b/crates/kafka_util/README.md@@ -1,10 +1,10 @@-# kafka_consumer_util+# kafka_util-Shared environment-aware Kafka consumer transport for backend inbound adapters.+Shared environment-aware Kafka producer and consumer transports for backend adapters.-The crate centralizes plaintext-local versus MSK-IAM construction and exposes two type-safe consumption modes:+The crate centralizes plaintext-local versus MSK-IAM construction, provides a delivery-confirmed event producer, and exposes two type-safe consumption modes:- Named consumer groups implement `GroupName` and may subscribe, pause, receive, and commit.- `Ungrouped` consumers use manual partition assignment and may pause and receive, but cannot subscribe or commit.-Ungrouped consumers assign every current partition at an explicit earliest or latest offset. Application adapters remain responsible for decoding, retries, poison-message handling, and refreshing assignments if topic partition counts change.+Ungrouped consumers assign every current partition at an explicit earliest or latest offset. Application adapters remain responsible for domain mapping, decoding, retries, poison-message handling, and refreshing assignments if topic partition counts change.diff --git a/crates/kafka_consumer_util/src/lib.rs b/crates/kafka_util/src/lib.rssimilarity index 71%rename from crates/kafka_consumer_util/src/lib.rsrename to crates/kafka_util/src/lib.rsindex 1331329a39..2db0ad1b58 100644--- a/crates/kafka_consumer_util/src/lib.rs+++ b/crates/kafka_util/src/lib.rs@@ -1,7 +1,7 @@#![deny(missing_docs)]-//! Environment-aware Kafka consumer transport.+//! Environment-aware Kafka producer and consumer transports.//!-//! This adapter centralizes the repository's plaintext-local versus MSK-IAM+//! These adapters centralize the repository's plaintext-local versus MSK-IAM//! transport setup. Application inbound adapters retain responsibility for//! decoding, retries, poison-message handling, and deciding when to commit.@@ -16,12 +16,17 @@ use macro_env::Environment;use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};use rdkafka::error::{KafkaError, KafkaResult};use rdkafka::message::{BorrowedMessage, Message as _};+use rdkafka::producer::{FutureProducer, FutureRecord};use rdkafka::{ClientConfig, Offset, TopicPartitionList};use uuid::Uuid;-use macro_event_broker::kafka::msk_iam::{MskIamClientContext, configure_sasl_iam};+pub use msk_iam::{MskIamClientContext, configure_sasl_iam};++mod msk_iam;const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";+const MESSAGE_TIMEOUT_MS: &str = "5000";+const SEND_TIMEOUT: Duration = Duration::from_secs(5);/// Failure to construct an environment-specific Kafka consumer.#[derive(Debug, thiserror::Error)]@@ -34,9 +39,24 @@ pub enum KafkaConsumerError {MskIam(#[source] KafkaError),}+/// Failure to construct an environment-specific Kafka producer.+#[derive(Debug, thiserror::Error)]+pub enum KafkaProducerError {+ /// Failed to create the unauthenticated local producer.+ #[error("failed to create plaintext Kafka producer")]+ Plaintext(#[source] KafkaError),+ /// Failed to create the TLS and MSK-IAM authenticated producer.+ #[error("failed to create MSK IAM Kafka producer")]+ MskIam(#[source] KafkaError),+}+/// Underlying Kafka consumer transport selected from the runtime environment.struct ConsumerTransport(Either<StreamConsumer, StreamConsumer<MskIamClientContext>>);+/// Underlying Kafka producer transport selected from the runtime environment.+#[derive(Clone)]+struct ProducerTransport(Either<FutureProducer, FutureProducer<MskIamClientContext>>);+/// Type-level name for a durable Kafka consumer group.////// Defining group names on marker types keeps group identities centralized and@@ -81,16 +101,32 @@ pub struct KafkaEventConsumer<T> {marker: PhantomData<T>,}+/// Shared Kafka producer with environment-aware transport.+#[derive(Clone)]+pub struct KafkaEventProducer {+ producer: ProducerTransport,+}+fn base_config(brokers: &str) -> ClientConfig {let mut config = ClientConfig::new();+ config.set("bootstrap.servers", brokers);config- .set("bootstrap.servers", brokers)- .set("enable.auto.commit", "false");+}++fn consumer_config(brokers: &str) -> ClientConfig {+ let mut config = base_config(brokers);+ config.set("enable.auto.commit", "false");config}-fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {+fn producer_config(brokers: &str) -> ClientConfig {let mut config = base_config(brokers);+ config.set("message.timeout.ms", MESSAGE_TIMEOUT_MS);+ config+}++fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {+ let mut config = consumer_config(brokers);config.set("group.id", T::GROUP_NAME).set("auto.offset.reset", "earliest");@@ -98,7 +134,7 @@ fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {}fn ungrouped_config(brokers: &str) -> ClientConfig {- let mut config = base_config(brokers);+ let mut config = consumer_config(brokers);let group_id = format!("{UNGROUPED_GROUP_PREFIX}-{}", Uuid::new_v4());config.set("group.id", group_id)@@ -106,7 +142,9 @@ fn ungrouped_config(brokers: &str) -> ClientConfig {config}-fn create_from_env<T>(config: ClientConfig) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {+fn create_consumer_from_env<T>(+ config: ClientConfig,+) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {let consumer = match Environment::new_or_prod() {Environment::Local => Either::Left(config.create().map_err(KafkaConsumerError::Plaintext)?),Environment::Develop | Environment::Production => {@@ -125,6 +163,26 @@ fn create_from_env<T>(config: ClientConfig) -> Result<KafkaEventConsumer<T>, Kaf})}+fn create_producer_from_env(+ config: ClientConfig,+) -> Result<KafkaEventProducer, KafkaProducerError> {+ let producer = match Environment::new_or_prod() {+ Environment::Local => Either::Left(config.create().map_err(KafkaProducerError::Plaintext)?),+ Environment::Develop | Environment::Production => {+ let config = configure_sasl_iam(config);+ Either::Right(+ config+ .create_with_context(MskIamClientContext::from_env())+ .map_err(KafkaProducerError::MskIam)?,+ )+ }+ };++ Ok(KafkaEventProducer {+ producer: ProducerTransport(producer),+ })+}+fn build_assignment<C, T>(consumer: &T,topics: &[&str],@@ -178,6 +236,25 @@ whereOk(assignment)}+impl KafkaEventProducer {+ /// Creates a producer, selecting plaintext or MSK IAM transport from the runtime environment.+ ///+ /// Producer creation is lazy: no broker connection or IAM token is created+ /// until a message is sent.+ pub fn from_env(brokers: &str) -> Result<Self, KafkaProducerError> {+ create_producer_from_env(producer_config(brokers))+ }++ /// Sends a keyed payload to `topic` and waits for delivery confirmation.+ #[tracing::instrument(err, skip(self, payload), fields(topic, key))]+ pub async fn send(&self, topic: &str, key: &str, payload: &[u8]) -> KafkaResult<()> {+ let record = FutureRecord::to(topic).key(key).payload(payload);+ either::for_both!(&self.producer.0, producer => producer.send(record, SEND_TIMEOUT).await)+ .map(|_| ())+ .map_err(|(error, _)| error)+ }+}+impl<T> KafkaEventConsumer<T> {/// Receives the next Kafka message.///@@ -202,7 +279,7 @@ impl<T: GroupName> KafkaEventConsumer<T> {/// Creates a named-group consumer, selecting plaintext or MSK IAM transport/// from the runtime environment.pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {- create_from_env(grouped_config::<T>(brokers))+ create_consumer_from_env(grouped_config::<T>(brokers))}/// Subscribes the consumer to exactly the provided topics.@@ -226,7 +303,7 @@ impl KafkaEventConsumer<Ungrouped> {////// Call [`Self::assign_topics`] before receiving messages.pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {- create_from_env(ungrouped_config(brokers))+ create_consumer_from_env(ungrouped_config(brokers))}/// Manually assigns every current partition of `topics` at `initial_offset`.diff --git a/crates/macro_event_broker/src/kafka/msk_iam.rs b/crates/kafka_util/src/msk_iam.rssimilarity index 62%rename from crates/macro_event_broker/src/kafka/msk_iam.rsrename to crates/kafka_util/src/msk_iam.rsindex de1d18e2f8..4beb4e67c5 100644--- a/crates/macro_event_broker/src/kafka/msk_iam.rs+++ b/crates/kafka_util/src/msk_iam.rs@@ -15,8 +15,7 @@ macro_env_var::maybe_env_var! {struct AwsRegion;}-/// Apply the TLS + SASL/OAUTHBEARER settings required to authenticate to an-/// AWS MSK cluster with IAM auth (the `:9098` bootstrap listener).+/// Applies the TLS and SASL/OAUTHBEARER settings required for MSK IAM auth.pub fn configure_sasl_iam(mut config: ClientConfig) -> ClientConfig {config.set("security.protocol", "SASL_SSL")@@ -24,18 +23,15 @@ pub fn configure_sasl_iam(mut config: ClientConfig) -> ClientConfig {config}-/// Client context that supplies AWS MSK IAM (SASL/OAUTHBEARER) auth tokens,-/// signing with the ambient AWS credentials (e.g. the ECS task role).+/// Client context that supplies AWS MSK IAM authentication tokens.///-/// librdkafka invokes [`ClientContext::generate_oauth_token`] when the-/// connection is first established and again before each token expires.+/// Tokens are signed with ambient AWS credentials such as an ECS task role.pub struct MskIamClientContext {region: Region,}impl MskIamClientContext {- /// Build a context that signs tokens for the region in `AWS_REGION`,- /// falling back to `us-east-1`.+ /// Builds a context using `AWS_REGION`, falling back to `us-east-1`.pub fn from_env() -> Self {let region = AwsRegion::new().and_then(|region| region.value().map(str::to_string))@@ -56,12 +52,9 @@ impl ClientContext for MskIamClientContext {) -> Result<OAuthToken, Box<dyn std::error::Error>> {let region = self.region.clone();- // librdkafka may invoke this callback on a tokio runtime thread (for a- // `StreamConsumer`, it fires while `recv()` is polled from async code),- // where blocking on the ambient runtime panics. Sign on a dedicated- // thread with its own single-threaded runtime instead, so the callback- // is safe from any calling context. Tokens live ~15 minutes, so the- // per-refresh thread + runtime cost is negligible.+ // librdkafka may invoke this callback on a Tokio runtime thread. Sign+ // on a dedicated thread with its own runtime so blocking is safe from+ // every calling context.let signer = std::thread::spawn(move || -> Result<(String, i64), Box<dyn std::error::Error + Send + Sync>> {let runtime = tokio::runtime::Builder::new_current_thread()@@ -74,9 +67,9 @@ impl ClientContext for MskIamClientContext {let (token, expiration_time_ms) = signer.join().map_err(|_| "MSK IAM token signer thread panicked")?- .map_err(|e| -> Box<dyn std::error::Error> { e })- .inspect_err(|e| {- tracing::error!(error = %e, "failed to sign MSK IAM auth token");+ .map_err(|error| -> Box<dyn std::error::Error> { error })+ .inspect_err(|error| {+ tracing::error!(error = %error, "failed to sign MSK IAM auth token");})?;tracing::info!(expiration_time_ms, "signed MSK IAM auth token");@@ -89,6 +82,4 @@ impl ClientContext for MskIamClientContext {}}-/// Lets consumers (e.g. `StreamConsumer<MskIamClientContext>`) use this-/// context; all behavior is the trait's defaults.impl ConsumerContext for MskIamClientContext {}diff --git a/crates/kafka_consumer_util/src/test.rs b/crates/kafka_util/src/test.rssimilarity index 79%rename from crates/kafka_consumer_util/src/test.rsrename to crates/kafka_util/src/test.rsindex ca3ed674af..fe85196334 100644--- a/crates/kafka_consumer_util/src/test.rs+++ b/crates/kafka_util/src/test.rs@@ -6,6 +6,18 @@ impl GroupName for TestConsumerGroup {const GROUP_NAME: &'static str = "consumer-group";}+#[test]+fn producer_config_uses_brokers_and_message_timeout() {+ let config = producer_config("broker-a:9092,broker-b:9092");++ assert_eq!(+ config.get("bootstrap.servers"),+ Some("broker-a:9092,broker-b:9092")+ );+ assert_eq!(config.get("message.timeout.ms"), Some(MESSAGE_TIMEOUT_MS));+ assert_eq!(config.get("enable.auto.commit"), None);+}+#[test]fn grouped_config_uses_named_group_manual_commits_and_earliest_offsets() {let config = grouped_config::<TestConsumerGroup>("broker-a:9092,broker-b:9092");diff --git a/crates/macro_event_broker/Cargo.toml b/crates/macro_event_broker/Cargo.tomlindex 397b42481c..78cab6a390 100644--- a/crates/macro_event_broker/Cargo.toml+++ b/crates/macro_event_broker/Cargo.toml@@ -5,29 +5,17 @@ publish = falseversion = "0.1.0"[features]-default = ["outbound", "ports"]-kafka = [- "dep:aws-msk-iam-sasl-signer",- "dep:aws-types",- "dep:macro_env",- "dep:macro_env_var",- "dep:rdkafka",- "dep:tokio",-]-outbound = ["kafka", "ports"]-ports = ["dep:tokio"]+default = ["outbound"]+outbound = ["dep:kafka_util", "dep:rdkafka"][dependencies]-aws-msk-iam-sasl-signer = { workspace = true, optional = true }-aws-types = { workspace = true, optional = true }-macro_env = { path = "../macro_env", optional = true }-macro_env_var = { path = "../macro_env_var", optional = true }+kafka_util = { path = "../kafka_util", optional = true }rootcause = { workspace = true }rdkafka = { workspace = true, optional = true }serde = { workspace = true }serde_json = { workspace = true }thiserror = { workspace = true }-tokio = { workspace = true, features = ["macros", "time"], optional = true }+tokio = { workspace = true, features = ["macros", "time"] }tracing = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/macro_event_broker/examples/example_event.rs b/crates/macro_event_broker/examples/example_event.rsindex 3f001ae6db..942be611c6 100644--- a/crates/macro_event_broker/examples/example_event.rs+++ b/crates/macro_event_broker/examples/example_event.rs@@ -9,15 +9,10 @@ use serde::{Deserialize, Serialize};pub struct ExampleEventPublisher;impl EventPublisher for ExampleEventPublisher {- async fn publish<T: Topic>(- &self,- topic: T,- key: &str,- payload: &[u8],- ) -> Result<(), EventBrokerError> {+ async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {println!("publishing topic={} key={} payload={}",- topic.as_str(),+ T::TOPIC_STR,key,String::from_utf8_lossy(payload));@@ -118,7 +113,7 @@ impl ExampleConsumerEvent {/// Decode one Kafka message into this consumer's event enum.pub fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {match topic {- topic if topic == MacroExampleTopic.as_str() => {+ topic if topic == MacroExampleTopic::TOPIC_STR => {Ok(Self::Example(ExampleMacroEvent::decode(key, payload)?))}unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),diff --git a/crates/macro_event_broker/src/domain/mod.rs b/crates/macro_event_broker/src/domain/mod.rsindex 1e30fc3ae2..04c1260629 100644--- a/crates/macro_event_broker/src/domain/mod.rs+++ b/crates/macro_event_broker/src/domain/mod.rs@@ -2,9 +2,7 @@pub mod models;/// Port traits: the inbound and outbound boundaries of the broker.-#[cfg(feature = "ports")]pub mod ports;-/// Service orchestration for publishing events.-#[cfg(feature = "ports")]+/// Service orchestration for producing and consuming typed events.pub mod service;diff --git a/crates/macro_event_broker/src/domain/models.rs b/crates/macro_event_broker/src/domain/models.rsindex 9ac7dd6af4..b84fcf0af1 100644--- a/crates/macro_event_broker/src/domain/models.rs+++ b/crates/macro_event_broker/src/domain/models.rs@@ -1,9 +1,13 @@//! Domain models for the event broker.+use std::marker::PhantomData;+use macro_event_topics::Topic;use serde::{Deserialize, Serialize, de::DeserializeOwned};use uuid::Uuid;+use crate::domain::ports::{MacroEventCollection, MessageParts};+/// Event payload enum for a single [`Topic`].////// Implement this trait for each topic-specific event enum. The enum should use@@ -64,9 +68,9 @@ impl<E: TopicEvent> Event<E> {Ok(serde_json::from_slice(payload)?)}- /// Kafka topic this event belongs to.- pub fn topic(&self) -> E::Topic {- E::Topic::default()+ /// Broker topic this event belongs to.+ pub fn topic(&self) -> &'static str {+ E::Topic::TOPIC_STR}}@@ -79,9 +83,9 @@ pub trait MacroEvent: Send + Sync {/// Topic-specific event enum carried by this macro event.type EventPayload: TopicEvent;- /// Kafka topic this event should be published to.- fn topic(&self) -> <Self::EventPayload as TopicEvent>::Topic {- <<Self as MacroEvent>::EventPayload as TopicEvent>::Topic::default()+ /// Broker topic this event should be published to.+ fn topic(&self) -> &'static str {+ <Self::EventPayload as TopicEvent>::Topic::TOPIC_STR}/// Kafka message key used for partitioning and compaction.@@ -110,9 +114,25 @@ pub enum EventBrokerError {/// The event payload could not be serialized to or deserialized from JSON.#[error("failed to serialize or deserialize event payload")]Serialization(#[from] serde_json::Error),+ /// The consumed broker message did not include a key.+ #[error("event broker message is missing a key")]+ MissingMessageKey,+ /// The consumed broker message did not include a payload.+ #[error("event broker message is missing a payload")]+ MissingMessagePayload,/// The Kafka topic name is not handled by a consumer-specific event enum.#[error("unknown event topic: {0}")]UnknownTopic(String),+ /// The event envelope schema version is not supported by its typed payload.+ #[error("unsupported schema version {actual} for topic {topic}; expected {expected}")]+ UnsupportedSchemaVersion {+ /// Topic carrying the event.+ topic: &'static str,+ /// Schema version declared by the typed event payload.+ expected: u8,+ /// Schema version found in the decoded envelope.+ actual: u8,+ },/// The broker rejected or failed to deliver the message.#[error("failed to publish event: {0}")]Publish(String),@@ -133,5 +153,105 @@ impl From<rootcause::Report> for EventBrokerError {}}+/// A broker message paired with the type of event collection it can decode.+pub struct MessageWrapper<T, M> {+ inner: T,+ marker: PhantomData<M>,+}++impl<T, M> MessageWrapper<T, M> {+ /// Wraps the transport-specific message.+ pub fn new(inner: T) -> Self {+ Self {+ inner,+ marker: PhantomData,+ }+ }++ /// Borrows the transport-specific message.+ pub fn inner(&self) -> &T {+ &self.inner+ }++ /// Returns the transport-specific message.+ pub fn into_inner(self) -> T {+ self.inner+ }+}++impl<T: MessageParts, M: MacroEventCollection> MessageWrapper<T, M> {+ /// Decodes the message into one of the declared event types.+ pub fn decode_payload(&self) -> Result<M, EventBrokerError> {+ M::decode(&self.inner)+ }+}++/// Declares the [`MacroEvent`] types accepted by a consumer.+///+/// The first identifier names the generated event collection enum. Each+/// identifier after the colon becomes a variant and must implement+/// [`MacroEvent`]. The generated enum implements topic-based decoding.+///+/// ```text+/// declare_topics!(ConsumerEvents: DocumentMacroEvent, ChannelMacroEvent);+/// ```+#[macro_export]+macro_rules! declare_topics {+ ($collection:ident: $($event:ident),+ $(,)?) => {+ #[doc = concat!(+ "An event decoded from one of the topics declared for `",+ stringify!($collection),+ "`."+ )]+ pub enum $collection {+ $(+ #[doc = concat!("A decoded `", stringify!($event), "` event.")]+ $event($event),+ )++ }++ impl $crate::MacroEventCollection for $collection {+ fn decode<T: $crate::MessageParts>(+ message: &T,+ ) -> Result<Self, $crate::EventBrokerError> {+ $(+ let expected_topic =+ <<<$event as $crate::MacroEvent>::EventPayload as $crate::TopicEvent>::Topic as $crate::Topic>::TOPIC_STR;+ if message.topic() == expected_topic {+ let key = message+ .key()+ .ok_or($crate::EventBrokerError::MissingMessageKey)?;+ let payload = message+ .payload()+ .ok_or($crate::EventBrokerError::MissingMessagePayload)?;+ let event = <$event as $crate::MacroEvent>::decode(key, payload)?;+ let envelope = <$event as $crate::MacroEvent>::event(&event);+ let expected = $crate::TopicEvent::schema_version(&envelope.event);+ let actual = envelope.schema_version;+ if actual != expected {+ return Err($crate::EventBrokerError::UnsupportedSchemaVersion {+ topic: expected_topic,+ expected,+ actual,+ });+ }+ return Ok($collection::$event(event));+ }+ )+++ Err($crate::EventBrokerError::UnknownTopic(message.topic().to_owned()))+ }++ fn topics() -> &'static [&'static str] {+ &[+ $(+ <<<$event as $crate::MacroEvent>::EventPayload as $crate::TopicEvent>::Topic as $crate::Topic>::TOPIC_STR,+ )++ ]+ }+ }+ };+}+#[cfg(test)]mod test;diff --git a/crates/macro_event_broker/src/domain/models/test.rs b/crates/macro_event_broker/src/domain/models/test.rsindex 95e2c942b1..197213f11d 100644--- a/crates/macro_event_broker/src/domain/models/test.rs+++ b/crates/macro_event_broker/src/domain/models/test.rs@@ -1,19 +1,20 @@-use macro_event_topics::{MacroExampleTopic, Topic};+use macro_event_topics::{MacroDocumentsTopic, MacroExampleTopic, Topic};use serde::{Deserialize, Serialize};use serde_json::json;use uuid::Uuid;use super::*;+use crate::{MacroEventCollection, MessageParts};#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]-struct ExampleCreatedMetadata {+pub struct ExampleCreatedMetadata {name: String,count: u32,}#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]#[serde(tag = "event_type", content = "metadata")]-enum ExampleTopicEvent {+pub enum ExampleTopicEvent {#[serde(rename = "example.created")]Created(ExampleCreatedMetadata),}@@ -27,7 +28,7 @@ impl TopicEvent for ExampleTopicEvent {}#[derive(Debug, Clone, PartialEq, Eq)]-struct ExampleMacroEvent {+pub struct ExampleMacroEvent {key: String,event: Event<ExampleTopicEvent>,}@@ -64,7 +65,7 @@ enum ExampleConsumerEvent {impl ExampleConsumerEvent {fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {match topic {- topic if topic == MacroExampleTopic.as_str() => {+ topic if topic == MacroExampleTopic::TOPIC_STR => {Ok(Self::Example(ExampleMacroEvent::decode(key, payload)?))}unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),@@ -116,9 +117,172 @@ fn consumer_specific_enum_decodes_by_topic() {let event = example_event();let payload = serde_json::to_vec(&event).unwrap();- let decoded = ExampleConsumerEvent::decode(MacroExampleTopic.as_str(), "msg-123", &payload)+ let decoded = ExampleConsumerEvent::decode(MacroExampleTopic::TOPIC_STR, "msg-123", &payload).expect("topic should decode");let ExampleConsumerEvent::Example(decoded) = decoded;assert_eq!(decoded.key(), "msg-123");}++#[derive(Debug, Serialize, Deserialize)]+pub struct DocumentsTopicEvent;++impl TopicEvent for DocumentsTopicEvent {+ type Topic = MacroDocumentsTopic;++ fn schema_version(&self) -> u8 {+ 1+ }+}++pub struct DocumentsMacroEvent {+ key: String,+ event: Event<DocumentsTopicEvent>,+}++impl MacroEvent for DocumentsMacroEvent {+ type EventPayload = DocumentsTopicEvent;++ fn key(&self) -> &str {+ &self.key+ }++ fn event(&self) -> &Event<Self::EventPayload> {+ &self.event+ }++ fn from_event(key: String, event: Event<Self::EventPayload>) -> Self {+ Self { key, event }+ }+}++crate::declare_topics!(TestMacroEvents: ExampleMacroEvent, DocumentsMacroEvent,);++type DeclaredMacroEvent = TestMacroEvents;++struct TestMessage {+ topic: &'static str,+ key: Option<&'static str>,+ payload: Option<Vec<u8>>,+}++impl MessageParts for TestMessage {+ fn key(&self) -> Option<&str> {+ self.key+ }++ fn payload(&self) -> Option<&[u8]> {+ self.payload.as_deref()+ }++ fn topic(&self) -> &str {+ self.topic+ }+}++#[test]+fn exposes_each_declared_topic_in_order() {+ assert_eq!(+ DeclaredMacroEvent::topics(),+ [MacroExampleTopic::TOPIC_STR, MacroDocumentsTopic::TOPIC_STR,]+ );+}++#[test]+fn decodes_each_declared_topic_into_its_enum_variant() {+ let example_payload = serde_json::to_vec(&example_event()).unwrap();+ let example_message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {+ topic: MacroExampleTopic::TOPIC_STR,+ key: Some("example-key"),+ payload: Some(example_payload),+ });++ match example_message.decode_payload().unwrap() {+ DeclaredMacroEvent::ExampleMacroEvent(event) => {+ assert_eq!(event.key(), "example-key");+ }+ DeclaredMacroEvent::DocumentsMacroEvent(_) => panic!("decoded the wrong event type"),+ }++ let documents_payload = serde_json::to_vec(&Event::new(DocumentsTopicEvent)).unwrap();+ let documents_message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {+ topic: MacroDocumentsTopic::TOPIC_STR,+ key: Some("documents-key"),+ payload: Some(documents_payload),+ });++ match documents_message.decode_payload().unwrap() {+ DeclaredMacroEvent::DocumentsMacroEvent(event) => {+ assert_eq!(event.key(), "documents-key");+ }+ DeclaredMacroEvent::ExampleMacroEvent(_) => panic!("decoded the wrong event type"),+ }+}++#[test]+fn rejects_topics_not_declared_by_the_macro() {+ let message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {+ topic: "macro.unknown",+ key: None,+ payload: None,+ });++ assert!(matches!(+ message.decode_payload(),+ Err(EventBrokerError::UnknownTopic(topic)) if topic == "macro.unknown"+ ));+}++#[test]+fn rejects_a_missing_message_key() {+ let message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {+ topic: MacroExampleTopic::TOPIC_STR,+ key: None,+ payload: Some(serde_json::to_vec(&example_event()).unwrap()),+ });++ assert!(matches!(+ message.decode_payload(),+ Err(EventBrokerError::MissingMessageKey)+ ));+}++#[test]+fn rejects_a_missing_message_payload() {+ let message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {+ topic: MacroExampleTopic::TOPIC_STR,+ key: Some("example-key"),+ payload: None,+ });++ assert!(matches!(+ message.decode_payload(),+ Err(EventBrokerError::MissingMessagePayload)+ ));+}++#[test]+fn rejects_an_unsupported_schema_version() {+ let event = Event::with_event_id_and_schema_version(+ Uuid::from_u128(1),+ 2,+ ExampleTopicEvent::Created(ExampleCreatedMetadata {+ name: "hello".to_string(),+ count: 7,+ }),+ );+ let message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {+ topic: MacroExampleTopic::TOPIC_STR,+ key: Some("example-key"),+ payload: Some(serde_json::to_vec(&event).unwrap()),+ });++ assert!(matches!(+ message.decode_payload(),+ Err(EventBrokerError::UnsupportedSchemaVersion {+ topic,+ expected: 1,+ actual: 2,+ }) if topic == MacroExampleTopic::TOPIC_STR+ ));+}diff --git a/crates/macro_event_broker/src/domain/ports.rs b/crates/macro_event_broker/src/domain/ports.rsindex 285b27f1a9..f9886ed7b4 100644--- a/crates/macro_event_broker/src/domain/ports.rs+++ b/crates/macro_event_broker/src/domain/ports.rs@@ -1,10 +1,8 @@//! Port traits defining the broker's inbound and outbound boundaries.-use std::future::Future;-use macro_event_topics::Topic;-use crate::domain::models::{EventBrokerError, MacroEvent};+use crate::domain::models::{EventBrokerError, MacroEvent, MessageWrapper};/// Inbound port: the public API for sending events through the broker.///@@ -28,11 +26,46 @@ pub trait MacroEventBroker: Send + Sync + 'static {/// Kept byte-oriented so payload serialization stays the service's concern and the/// port is trivial to mock or stub in tests.pub trait EventPublisher: Send + Sync + 'static {- /// Publish a raw `payload` to `topic`, keyed by `key`.+ /// Publishes a raw `payload` to [`T::TOPIC_STR`](Topic::TOPIC_STR), keyed by `key`.fn publish<T: Topic>(&self,- topic: T,key: &str,payload: &[u8],) -> impl Future<Output = Result<(), EventBrokerError>> + Send;}++/// A collection of [`MacroEvent`] types that can decode a broker message.+pub trait MacroEventCollection: Sized {+ /// Decodes an event from the supplied broker message.+ fn decode<T: MessageParts>(message: &T) -> Result<Self, EventBrokerError>;++ /// Lists the statically known topics in this event collection.+ fn topics() -> &'static [&'static str];+}++/// Broker message fields needed to decode a [`MacroEvent`].+pub trait MessageParts {+ /// Returns the broker message key.+ fn key(&self) -> Option<&str>;++ /// Returns the serialized event payload.+ fn payload(&self) -> Option<&[u8]>;++ /// Returns the broker topic name.+ fn topic(&self) -> &str;+}++/// Inbound boundary for receiving messages from the event broker.+pub trait EventConsumer<M: MacroEventCollection>: Send + Sync + 'static {+ /// The concrete transport-specific message type.+ type MessageType<'a>: MessageParts+ where+ Self: 'a;++ /// Waits for and returns the next broker message.+ fn recv<'a>(+ &'a self,+ ) -> impl Future<Output = Result<MessageWrapper<Self::MessageType<'a>, M>, rootcause::Report>>+ + Send+ + 'a;+}diff --git a/crates/macro_event_broker/src/domain/service.rs b/crates/macro_event_broker/src/domain/service.rsindex 65521aa0d2..748f3fe3b1 100644--- a/crates/macro_event_broker/src/domain/service.rs+++ b/crates/macro_event_broker/src/domain/service.rs@@ -1,19 +1,62 @@-//! The concrete event broker service.+//! Typed producer and consumer services.#[cfg(test)]mod test;+use std::marker::PhantomData;use std::sync::Arc;use std::time::Duration;-use macro_event_topics::Topic as _;use tracing::Instrument as _;-use crate::domain::models::{EventBrokerError, MacroEvent};-use crate::domain::ports::{EventPublisher, MacroEventBroker};+use crate::MessageWrapper;+use crate::domain::models::{EventBrokerError, MacroEvent, TopicEvent};+use crate::domain::ports::{EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection};const PUBLISH_TIMEOUT: Duration = Duration::from_secs(6);+/// Receives transport messages associated with a declared macro event collection.+pub struct MacroEventConsumerService<M, C>+where+ M: MacroEventCollection,+ C: EventConsumer<M>,+{+ consumer: C,+ marker: PhantomData<fn() -> M>,+}++impl<M, C> MacroEventConsumerService<M, C>+where+ M: MacroEventCollection,+ C: EventConsumer<M>,+{+ /// Creates a typed consumer service backed by `consumer`.+ pub const fn new(consumer: C) -> Self {+ Self {+ consumer,+ marker: PhantomData,+ }+ }++ /// Receives the next message from the underlying consumer.+ ///+ /// Call [`MessageWrapper::decode_payload`] to decode its declared event.+ #[tracing::instrument(skip(self), err)]+ pub async fn recv<'a>(+ &'a self,+ ) -> Result<MessageWrapper<C::MessageType<'a>, M>, rootcause::Report> {+ self.consumer.recv().await+ }++ /// Returns the underlying consumer.+ ///+ /// This provides access to transport-specific delivery operations after a+ /// message has been received through this service.+ pub const fn inner(&self) -> &C {+ &self.consumer+ }+}+/// Orchestrates serializing events and handing them to an [`EventPublisher`].pub struct MacroEventBrokerService<P: EventPublisher> {publisher: Arc<P>,@@ -37,7 +80,7 @@ impl<P: EventPublisher> MacroEventBrokerService<P> {}impl<P: EventPublisher> MacroEventBroker for MacroEventBrokerService<P> {- #[tracing::instrument(err, skip(self, event), fields(topic = %event.topic().as_str(), key = %event.key()))]+ #[tracing::instrument(err, skip(self, event), fields(topic = event.topic(), key = %event.key()))]fn send_event<E: MacroEvent + ?Sized>(&self,event: &E,@@ -50,20 +93,25 @@ impl<P: EventPublisher> MacroEventBroker for MacroEventBrokerService<P> {let handle = tokio::spawn(async move {- tokio::time::timeout(PUBLISH_TIMEOUT, publisher.publish(topic, &key, &payload))- .await- .map_err(|_| EventBrokerError::PublishTimeout {- timeout: PUBLISH_TIMEOUT,- })- .and_then(std::convert::identity)- .inspect_err(|error| {- tracing::error!(- error = ?error,- topic = topic.as_str(),- key = %key,- "failed to publish event",- );- })+ tokio::time::timeout(+ PUBLISH_TIMEOUT,+ publisher.publish::<<<E as MacroEvent>::EventPayload as TopicEvent>::Topic>(+ &key, &payload,+ ),+ )+ .await+ .map_err(|_| EventBrokerError::PublishTimeout {+ timeout: PUBLISH_TIMEOUT,+ })+ .and_then(std::convert::identity)+ .inspect_err(|error| {+ tracing::error!(+ error = ?error,+ topic,+ key = %key,+ "failed to publish event",+ );+ })}.instrument(span),);diff --git a/crates/macro_event_broker/src/domain/service/test.rs b/crates/macro_event_broker/src/domain/service/test.rsindex ad42172c8e..5938c0dc58 100644--- a/crates/macro_event_broker/src/domain/service/test.rs+++ b/crates/macro_event_broker/src/domain/service/test.rs@@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize};use uuid::Uuid;use super::*;-use crate::domain::models::{Event, EventBrokerError, MacroEvent, TopicEvent};-use crate::domain::ports::{EventPublisher, MacroEventBroker};+use crate::domain::models::{Event, EventBrokerError, MacroEvent, MessageWrapper, TopicEvent};+use crate::domain::ports::{EventConsumer, EventPublisher, MacroEventBroker, MessageParts};/// A captured publish call: topic, key, and raw payload bytes.#[derive(Debug, Clone, PartialEq, Eq)]@@ -28,14 +28,9 @@ struct RecordingPublisher {}impl EventPublisher for RecordingPublisher {- async fn publish<T: Topic>(- &self,- topic: T,- key: &str,- payload: &[u8],- ) -> Result<(), EventBrokerError> {+ async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {self.calls.lock().unwrap().push(Published {- topic: topic.as_str().to_string(),+ topic: T::TOPIC_STR.to_string(),key: key.to_string(),payload: payload.to_vec(),});@@ -48,12 +43,7 @@ struct PendingPublisher {}impl EventPublisher for PendingPublisher {- async fn publish<T: Topic>(- &self,- _topic: T,- _key: &str,- _payload: &[u8],- ) -> Result<(), EventBrokerError> {+ async fn publish<T: Topic>(&self, _key: &str, _payload: &[u8]) -> Result<(), EventBrokerError> {self.started.store(true, Ordering::SeqCst);pending().await}@@ -75,12 +65,7 @@ struct HangingPublisher {}impl EventPublisher for HangingPublisher {- async fn publish<T: Topic>(- &self,- _topic: T,- _key: &str,- _payload: &[u8],- ) -> Result<(), EventBrokerError> {+ async fn publish<T: Topic>(&self, _key: &str, _payload: &[u8]) -> Result<(), EventBrokerError> {self.started.store(true, Ordering::SeqCst);let _drop_guard = PublishDropGuard {dropped: Arc::clone(&self.dropped),@@ -94,26 +79,21 @@ struct FailingPublisher {}impl EventPublisher for FailingPublisher {- async fn publish<T: Topic>(- &self,- _topic: T,- _key: &str,- _payload: &[u8],- ) -> Result<(), EventBrokerError> {+ async fn publish<T: Topic>(&self, _key: &str, _payload: &[u8]) -> Result<(), EventBrokerError> {self.attempted.store(true, Ordering::SeqCst);Err(EventBrokerError::Publish("test failure".to_string()))}}#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]-struct ExampleCreatedMetadata {+pub struct ExampleCreatedMetadata {name: String,count: u32,}#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]#[serde(tag = "event_type", content = "metadata")]-enum ExampleTopicEvent {+pub enum ExampleTopicEvent {#[serde(rename = "example.created")]Created(ExampleCreatedMetadata),}@@ -126,7 +106,7 @@ impl TopicEvent for ExampleTopicEvent {}}-struct ExampleMacroEvent {+pub struct ExampleMacroEvent {key: String,event: Event<ExampleTopicEvent>,}@@ -156,6 +136,48 @@ impl MacroEvent for ExampleMacroEvent {}}+crate::declare_topics!(DeclaredMacroEvent: ExampleMacroEvent);++struct TestMessage {+ topic: &'static str,+ key: &'static str,+ payload: Vec<u8>,+}++impl MessageParts for TestMessage {+ fn key(&self) -> Option<&str> {+ Some(self.key)+ }++ fn payload(&self) -> Option<&[u8]> {+ Some(&self.payload)+ }++ fn topic(&self) -> &str {+ self.topic+ }+}++struct TestEventConsumer {+ topic: &'static str,+ key: &'static str,+ payload: Vec<u8>,+}++impl EventConsumer<DeclaredMacroEvent> for TestEventConsumer {+ type MessageType<'a> = TestMessage;++ async fn recv<'a>(+ &'a self,+ ) -> Result<MessageWrapper<Self::MessageType<'a>, DeclaredMacroEvent>, rootcause::Report> {+ Ok(MessageWrapper::new(TestMessage {+ topic: self.topic,+ key: self.key,+ payload: self.payload.clone(),+ }))+ }+}+fn example_event() -> ExampleMacroEvent {ExampleMacroEvent::with_event("msg-123",@@ -169,6 +191,23 @@ fn example_event() -> ExampleMacroEvent {)}+#[tokio::test]+async fn consumer_service_receives_and_decodes_typed_event() {+ let event = example_event();+ let consumer = TestEventConsumer {+ topic: MacroExampleTopic::TOPIC_STR,+ key: "msg-123",+ payload: serde_json::to_vec(event.event()).expect("event serializes"),+ };+ let service = MacroEventConsumerService::<DeclaredMacroEvent, _>::new(consumer);++ let message = service.recv().await.expect("message is received");+ let decoded = message.decode_payload().expect("associated topic decodes");+ let DeclaredMacroEvent::ExampleMacroEvent(decoded) = decoded;+ assert_eq!(decoded.key(), "msg-123");+ assert_eq!(decoded.event(), event.event());+}+#[derive(Debug, Deserialize)]struct UnserializableTopicEvent;@@ -225,7 +264,7 @@ async fn dispatch_serializes_and_routes() {let calls = service.publisher.calls.lock().unwrap();assert_eq!(calls.len(), 1);let call = &calls[0];- assert_eq!(call.topic, MacroExampleTopic.as_str());+ assert_eq!(call.topic, MacroExampleTopic::TOPIC_STR);assert_eq!(call.key, "msg-123");assert_eq!(call.payload, expected_payload);}diff --git a/crates/macro_event_broker/src/kafka/mod.rs b/crates/macro_event_broker/src/kafka/mod.rsdeleted file mode 100644index a7074dedfc..0000000000--- a/crates/macro_event_broker/src/kafka/mod.rs+++ /dev/null@@ -1,4 +0,0 @@-//! Kafka transport support shared by inbound and outbound adapters.--/// AWS MSK IAM authentication support for Kafka clients.-pub mod msk_iam;diff --git a/crates/macro_event_broker/src/lib.rs b/crates/macro_event_broker/src/lib.rsindex bd60384a92..6b2ffc2f16 100644--- a/crates/macro_event_broker/src/lib.rs+++ b/crates/macro_event_broker/src/lib.rs@@ -5,32 +5,34 @@//! [`TopicEvent`](domain::models::TopicEvent) enums, the//! [`MacroEvent`](domain::models::MacroEvent) event abstraction, the inbound//! [`MacroEventBroker`](domain::ports::MacroEventBroker) API, the outbound-//! [`EventPublisher`](domain::ports::EventPublisher) port, and the-//! [`MacroEventBrokerService`](domain::service::MacroEventBrokerService) that ties-//! them together. Kafka topic definitions live in the `macro_event_topics` crate.-//! The [`outbound`] layer provides the Kafka publisher adapter.+//! [`EventPublisher`](domain::ports::EventPublisher) port, the producing+//! [`MacroEventBrokerService`](domain::service::MacroEventBrokerService), and the+//! typed [`MacroEventConsumerService`](domain::service::MacroEventConsumerService),+//! which receives messages through the [`EventConsumer`](domain::ports::EventConsumer) port.+//! Kafka topic definitions live in the `macro_event_topics` crate. Shared Kafka+//! producer and consumer transports live in `kafka_util`, while the [`outbound`]+//! layer adapts its producer to [`EventPublisher`](domain::ports::EventPublisher)./// Domain layer: models, ports, and service.pub mod domain;-pub use domain::models::{Event, EventBrokerError, MacroEvent, TopicEvent};+pub use domain::models::{Event, EventBrokerError, MacroEvent, MessageWrapper, TopicEvent};pub use macro_event_topics::{MacroChannelsTopic, MacroDocumentsTopic, MacroEmailTopic, MacroExampleTopic,MacroProjectsTopic, Topic,};-#[cfg(feature = "ports")]-pub use domain::ports::{EventPublisher, MacroEventBroker};-#[cfg(feature = "ports")]-pub use domain::service::{MacroEventBrokerService, NoopMacroEventBroker};-#[cfg(feature = "kafka")]-pub use kafka::msk_iam::MskIamClientContext;+pub use domain::ports::{+ EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection, MessageParts,+};+pub use domain::service::{+ MacroEventBrokerService, MacroEventConsumerService, NoopMacroEventBroker,+};#[cfg(feature = "outbound")]-pub use outbound::kafka_event_publisher::KafkaEventPublisher;+pub use outbound::{+ kafka_event_consumer::KafkaConsumerAdapter, kafka_event_publisher::KafkaEventPublisher,+};-/// Kafka transport support shared by inbound and outbound adapters.-#[cfg(feature = "kafka")]-pub mod kafka;-/// Outbound layer: Kafka publishing adapters.+/// Outbound adapters for the macro event broker's required ports.#[cfg(feature = "outbound")]pub mod outbound;diff --git a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rsnew file mode 100644index 0000000000..380669ccc4--- /dev/null+++ b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs@@ -0,0 +1,119 @@+//! Kafka adapters for configuring topic-aware grouped and ungrouped consumers+//! and implementing the [`EventConsumer`] port.++use std::{marker::PhantomData, time::Duration};++use kafka_util::{GroupName, InitialOffset, KafkaEventConsumer, Ungrouped};+use rdkafka::{+ consumer::CommitMode,+ message::{BorrowedMessage, Message as _},+};++use crate::{EventConsumer, MacroEventCollection, MessageWrapper};++/// Topic-aware wrapper around a [`KafkaEventConsumer`].+///+/// `T` selects grouped or [`Ungrouped`] consumption. `M` records the+/// [`MacroEventCollection`] whose topics were assigned or subscribed, so the+/// configured topic set remains visible in the adapter's type.+pub struct KafkaConsumerAdapter<T, M> {+ inner: KafkaEventConsumer<T>,+ topics: PhantomData<fn() -> M>,+}++impl<M: MacroEventCollection> KafkaConsumerAdapter<Ungrouped, M> {+ /// Creates an ungrouped adapter and manually assigns every topic in `M`.+ ///+ /// Every current partition starts at `initial_offset`. Topic metadata must+ /// be fetched within `metadata_timeout`; failures are returned to the+ /// caller.+ pub fn new(+ consumer: KafkaEventConsumer<Ungrouped>,+ initial_offset: InitialOffset,+ metadata_timeout: Duration,+ ) -> Result<Self, rootcause::Report> {+ consumer.assign_topics(M::topics(), initial_offset, metadata_timeout)?;+ Ok(KafkaConsumerAdapter {+ inner: consumer,+ topics: PhantomData,+ })+ }+}++impl<T: GroupName> KafkaConsumerAdapter<T, ()> {+ /// Wraps a grouped consumer before its event topics are subscribed.+ pub fn new(consumer: KafkaEventConsumer<T>) -> Self {+ KafkaConsumerAdapter {+ inner: consumer,+ topics: PhantomData,+ }+ }+}++impl<T: GroupName, M> KafkaConsumerAdapter<T, M> {+ /// Subscribes the grouped consumer to every topic declared by `M2`.+ ///+ /// Kafka subscription is synchronous, replaces the previous subscription,+ /// and causes the group coordinator to rebalance asynchronously while the+ /// consumer is polled. The returned adapter records `M2` in its type.+ pub fn subscribe<M2: MacroEventCollection>(+ self,+ ) -> Result<KafkaConsumerAdapter<T, M2>, rootcause::Report> {+ let KafkaConsumerAdapter { inner, .. } = self;+ inner.subscribe(M2::topics())?;+ Ok(KafkaConsumerAdapter {+ inner,+ topics: PhantomData,+ })+ }++ /// Commits a message using the caller-selected commit mode.+ pub fn commit_message(+ &self,+ message: &BorrowedMessage<'_>,+ mode: CommitMode,+ ) -> Result<(), rootcause::Report> {+ Ok(self.inner.commit_message(message, mode)?)+ }+}++impl<T, M: MacroEventCollection> KafkaConsumerAdapter<T, M> {+ /// Pauses the partition containing `message`.+ ///+ /// Grouped consumers can use this to prevent a later cumulative commit+ /// from advancing past a failed record. Ungrouped consumers can use it to+ /// stop additional delivery from a failed partition.+ pub fn pause_message_partition(+ &self,+ message: &BorrowedMessage<'_>,+ ) -> Result<(), rootcause::Report> {+ Ok(self.inner.pause_message_partition(message)?)+ }+}++impl<T, M> EventConsumer<M> for KafkaConsumerAdapter<T, M>+where+ T: Send + Sync + 'static,+ M: MacroEventCollection + 'static,+{+ type MessageType<'a> = BorrowedMessage<'a>;++ async fn recv<'a>(+ &'a self,+ ) -> Result<MessageWrapper<Self::MessageType<'a>, M>, rootcause::Report> {+ let message = self.inner.recv().await?;++ tracing::trace!(+ topic = message.topic(),+ partition = message.partition(),+ offset = message.offset(),+ payload_len = message.payload().map_or(0, <[u8]>::len),+ "received Kafka event message"+ );++ Ok(MessageWrapper::new(message))+ }+}++#[cfg(test)]+mod test;diff --git a/crates/macro_event_broker/src/outbound/kafka_event_consumer/test.rs b/crates/macro_event_broker/src/outbound/kafka_event_consumer/test.rsnew file mode 100644index 0000000000..a139d7b16b--- /dev/null+++ b/crates/macro_event_broker/src/outbound/kafka_event_consumer/test.rs@@ -0,0 +1,23 @@+use kafka_util::Ungrouped;++use super::*;+use crate::{EventBrokerError, MessageParts};++struct TestEvents;++impl MacroEventCollection for TestEvents {+ fn decode<T: MessageParts>(_message: &T) -> Result<Self, EventBrokerError> {+ unreachable!("compile-time adapter assertion does not decode messages")+ }++ fn topics() -> &'static [&'static str] {+ &[]+ }+}++fn assert_event_consumer<T: EventConsumer<TestEvents>>() {}++#[test]+fn kafka_adapter_implements_event_consumer_port() {+ assert_event_consumer::<KafkaConsumerAdapter<Ungrouped, TestEvents>>();+}diff --git a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rsindex 9e46870d93..432bfbb0df 100644--- a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs+++ b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs@@ -1,99 +1,51 @@//! Kafka adapter for the [`EventPublisher`] port.//!-//! Wraps an `rdkafka` [`FutureProducer`] to publish events. The broker address-//! is supplied by the caller (typically from environment configuration at the-//! consuming service); the transport is chosen here from the `ENVIRONMENT`-//! variable: `Local` uses a plaintext connection (the docker-compose broker),-//! while `Develop` / `Production` use TLS + SASL/OAUTHBEARER with AWS MSK IAM-//! auth via [`MskIamClientContext`].+//! The environment-aware Kafka producer transport lives in `kafka_util`; this+//! adapter maps that transport onto the macro event broker's domain port.-use std::time::Duration;--use macro_env::Environment;+use kafka_util::{KafkaEventProducer, KafkaProducerError};use macro_event_topics::Topic;-use rdkafka::ClientConfig;-use rdkafka::producer::{FutureProducer, FutureRecord};+use rdkafka::Message;+use crate::MessageParts;use crate::domain::models::EventBrokerError;use crate::domain::ports::EventPublisher;-use crate::kafka::msk_iam::{MskIamClientContext, configure_sasl_iam};--/// How long a record may sit in the producer queue before delivery is considered failed.-const MESSAGE_TIMEOUT_MS: &str = "5000";--/// How long [`EventPublisher::publish`] waits for delivery confirmation before timing out.-const SEND_TIMEOUT: Duration = Duration::from_secs(5);--/// The underlying producer, split by transport.-#[derive(Clone)]-enum Producer {- /// Unauthenticated plaintext connection (local docker broker).- Plaintext(FutureProducer),- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).- MskIam(FutureProducer<MskIamClientContext>),-}/// Kafka-backed implementation of [`EventPublisher`].#[derive(Clone)]pub struct KafkaEventPublisher {- producer: Producer,-}--fn base_config(brokers: &str) -> ClientConfig {- let mut config = ClientConfig::new();- config- .set("bootstrap.servers", brokers)- .set("message.timeout.ms", MESSAGE_TIMEOUT_MS);- config-}--fn create_error(e: impl std::fmt::Display) -> EventBrokerError {- EventBrokerError::Publish(format!("failed to create producer: {e}"))+ producer: KafkaEventProducer,}impl KafkaEventPublisher {- /// Build a producer connected to the given comma-separated `brokers` list,- /// choosing the transport from the `ENVIRONMENT` variable (see the module- /// docs).- ///- /// Producer creation is lazy: nothing connects to Kafka (and no IAM token- /// is signed) until an event is published.- pub fn new(brokers: &str) -> Result<Self, EventBrokerError> {- let producer = match Environment::new_or_prod() {- Environment::Local => {- Producer::Plaintext(base_config(brokers).create().map_err(create_error)?)- }- Environment::Develop | Environment::Production => {- let config = configure_sasl_iam(base_config(brokers));-- Producer::MskIam(- config- .create_with_context(MskIamClientContext::from_env())- .map_err(create_error)?,- )- }- };-- Ok(Self { producer })+ /// Builds an environment-aware Kafka event publisher.+ pub fn new(brokers: &str) -> Result<Self, KafkaProducerError> {+ Ok(Self {+ producer: KafkaEventProducer::from_env(brokers)?,+ })}}impl EventPublisher for KafkaEventPublisher {- #[tracing::instrument(err, skip(self, payload), fields(topic = %topic.as_str(), key = %key))]- async fn publish<T: Topic>(- &self,- topic: T,- key: &str,- payload: &[u8],- ) -> Result<(), EventBrokerError> {- let record = FutureRecord::to(topic.as_str()).key(key).payload(payload);+ #[tracing::instrument(err, skip(self, payload), fields(topic = T::TOPIC_STR, key = %key))]+ async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {+ self.producer+ .send(T::TOPIC_STR, key, payload)+ .await+ .map_err(|error| EventBrokerError::Publish(error.to_string()))+ }+}++impl<T: Message> MessageParts for T {+ fn key(&self) -> Option<&str> {+ Message::key(self).and_then(|key| std::str::from_utf8(key).ok())+ }- match &self.producer {- Producer::Plaintext(producer) => producer.send(record, SEND_TIMEOUT).await,- Producer::MskIam(producer) => producer.send(record, SEND_TIMEOUT).await,- }- .map_err(|(e, _)| EventBrokerError::Publish(e.to_string()))?;+ fn payload(&self) -> Option<&[u8]> {+ Message::payload(self)+ }- Ok(())+ fn topic(&self) -> &str {+ Message::topic(self)}}diff --git a/crates/macro_event_broker/src/outbound/mod.rs b/crates/macro_event_broker/src/outbound/mod.rsindex 66675e0544..4fef313236 100644--- a/crates/macro_event_broker/src/outbound/mod.rs+++ b/crates/macro_event_broker/src/outbound/mod.rs@@ -1,5 +1,6 @@+/// Kafka adapter implementing the [`EventConsumer`](crate::domain::ports::EventConsumer) port.+#[cfg(feature = "outbound")]+pub mod kafka_event_consumer;/// Kafka adapter implementing the [`EventPublisher`](crate::domain::ports::EventPublisher) port.#[cfg(feature = "outbound")]pub mod kafka_event_publisher;-/// AWS MSK IAM (SASL/OAUTHBEARER) auth support shared by producers and consumers.-pub use crate::kafka::msk_iam;diff --git a/crates/macro_event_topics/src/lib.rs b/crates/macro_event_topics/src/lib.rsindex bbbc542250..17adf90de5 100644--- a/crates/macro_event_topics/src/lib.rs+++ b/crates/macro_event_topics/src/lib.rs@@ -17,9 +17,9 @@ pub enum TopicError {/// A Topic is mapped to a Kafka topic that events can be published to.#[sealed]-pub trait Topic: Default + Copy + Send + Sync + 'static {- /// The kafka topic name as a string.- fn as_str(&self) -> &'static str;+pub trait Topic: Copy + Send + Sync + 'static {+ /// the statically known string name of this topic+ const TOPIC_STR: &'static str;}/// Defines each topic struct with its `Topic` impl, plus [`all_topic_names`]@@ -33,15 +33,13 @@ macro_rules! topics {#[sealed]impl Topic for $name {- fn as_str(&self) -> &'static str {- $topic- }+ const TOPIC_STR: &'static str = $topic;})*/// The names of all Kafka topics defined in this crate.- pub fn all_topic_names() -> Vec<&'static str> {- vec![$($name.as_str()),*]+ pub fn all_topic_names() -> &'static [&'static str] {+ &[$($name::TOPIC_STR),*]}};}diff --git a/crates/macro_event_topics/src/test.rs b/crates/macro_event_topics/src/test.rsindex a5e62663ee..39b00a8999 100644--- a/crates/macro_event_topics/src/test.rs+++ b/crates/macro_event_topics/src/test.rs@@ -17,12 +17,12 @@ fn all_topic_names_is_non_empty_and_unique() {#[test]fn all_topic_names_includes_declared_topics() {- assert!(all_topic_names().contains(&MacroExampleTopic.as_str()));- assert!(all_topic_names().contains(&MacroDocumentsTopic.as_str()));- assert!(all_topic_names().contains(&MacroSoupRealtimeTopic.as_str()));- assert!(all_topic_names().contains(&MacroProjectsTopic.as_str()));- assert!(all_topic_names().contains(&MacroTeamsTopic.as_str()));- assert!(all_topic_names().contains(&MacroChannelsTopic.as_str()));- assert!(all_topic_names().contains(&MacroBotsTopic.as_str()));- assert!(all_topic_names().contains(&MacroWebhooksTopic.as_str()));+ assert!(all_topic_names().contains(&MacroExampleTopic::TOPIC_STR));+ assert!(all_topic_names().contains(&MacroDocumentsTopic::TOPIC_STR));+ assert!(all_topic_names().contains(&MacroSoupRealtimeTopic::TOPIC_STR));+ assert!(all_topic_names().contains(&MacroProjectsTopic::TOPIC_STR));+ assert!(all_topic_names().contains(&MacroTeamsTopic::TOPIC_STR));+ assert!(all_topic_names().contains(&MacroChannelsTopic::TOPIC_STR));+ assert!(all_topic_names().contains(&MacroBotsTopic::TOPIC_STR));+ assert!(all_topic_names().contains(&MacroWebhooksTopic::TOPIC_STR));}diff --git a/crates/projects/Cargo.toml b/crates/projects/Cargo.tomlindex 4dae5b8b20..f16bc46c62 100644--- a/crates/projects/Cargo.toml+++ b/crates/projects/Cargo.toml@@ -30,11 +30,7 @@ outbound = ["ports",]ports = ["dep:tokio"]-service = [- "dep:entity_access_management",- "macro_event_broker/ports",- "ports",-]+service = ["dep:entity_access_management", "ports"][dependencies]# Core@@ -95,8 +91,8 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]dotenvy = { workspace = true }http-body-util = { workspace = true }+kafka_util = { path = "../kafka_util" }macro_db_migrator = { path = "../macro_db_migrator" }-macro_env = { path = "../macro_env" }# Examples need the Kafka adapter (outbound); the library itself only uses ports.macro_event_broker = { path = "../macro_event_broker" }mockall = { workspace = true }diff --git a/crates/projects/examples/projects_consumer.rs b/crates/projects/examples/projects_consumer.rsindex e94972efb9..c8a389ea2e 100644--- a/crates/projects/examples/projects_consumer.rs+++ b/crates/projects/examples/projects_consumer.rs@@ -3,8 +3,8 @@//! Connects with the same `KAFKA_BROKERS` env var and environment-driven//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),-//! subscribes to [`MacroProjectsTopic`], and prints every project lifecycle-//! event it receives.+//! subscribes to the [`ProjectMacroEvent`] topic and prints every project+//! lifecycle event it receives.//!//! The consumer follows the standard poll-loop / worker split: a Kafka poll//! loop decodes messages and hands them to a processing task over a bounded@@ -24,15 +24,13 @@//! ```use anyhow::Context as _;-use macro_env::Environment;-use macro_event_broker::outbound::msk_iam::configure_sasl_iam;-use macro_event_broker::{EventBrokerError, MacroEvent, MskIamClientContext, Topic as _};-use macro_event_topics::MacroProjectsTopic;+use kafka_util::{GroupName, KafkaEventConsumer};+use macro_event_broker::{+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,+};use projects::domain::events::{ProjectMacroEvent, ProjectTopicEvent};-use rdkafka::ClientConfig;-use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};-use rdkafka::error::KafkaResult;-use rdkafka::message::{BorrowedMessage, Message};+use rdkafka::consumer::CommitMode;+use rdkafka::message::Message;use tokio::sync::mpsc;macro_env_var::env_var! {@@ -41,103 +39,28 @@ macro_env_var::env_var! {}}-/// Consumer group id for this example. Offsets are committed under this group,-/// so re-runs resume where the previous run left off.-const GROUP_ID: &str = "projects-consumer-example";+/// Consumer group for this example. Offsets are committed under this group, so+/// re-runs resume where the previous run left off.+struct ProjectsConsumerGroup;++impl GroupName for ProjectsConsumerGroup {+ const GROUP_NAME: &'static str = "projects-consumer-example";+}/// Bounded capacity of the channel between the poll loop and the processor.const CHANNEL_CAPACITY: usize = 128;-/// Consumer-specific event enum for a consumer subscribed to `macro.projects`,-/// following the pattern in `macro_event_broker/examples/example_event.rs`.-enum ProjectsConsumerEvent {- /// Event received on [`MacroProjectsTopic`].- Projects(ProjectMacroEvent),-}+macro_event_broker::declare_topics!(DeclaredMacroEvent: ProjectMacroEvent);-impl ProjectsConsumerEvent {- /// Decode one Kafka message into this consumer's event enum.- fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {- match topic {- topic if topic == MacroProjectsTopic.as_str() => {- Ok(Self::Projects(ProjectMacroEvent::decode(key, payload)?))- }- unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),- }- }-}+type ProjectsKafkaAdapter = KafkaConsumerAdapter<ProjectsConsumerGroup, DeclaredMacroEvent>;+type ProjectsConsumerService = MacroEventConsumerService<DeclaredMacroEvent, ProjectsKafkaAdapter>;/// A decoded event plus the Kafka coordinates it came from, as handed from the/// poll loop to the processor over the channel.struct ReceivedEvent {partition: i32,offset: i64,- event: ProjectsConsumerEvent,-}--/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).-enum ProjectsConsumer {- /// Unauthenticated plaintext connection (local docker broker).- Plaintext(StreamConsumer),- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).- MskIam(StreamConsumer<MskIamClientContext>),-}--impl ProjectsConsumer {- /// Build a consumer for the given brokers, choosing the transport from the- /// `ENVIRONMENT` variable exactly like the publisher does.- fn from_env(brokers: &str) -> anyhow::Result<Self> {- let mut config = ClientConfig::new();- config- .set("bootstrap.servers", brokers)- .set("group.id", GROUP_ID)- // Offsets are committed manually after a successful handoff.- .set("enable.auto.commit", "false")- // Start from the beginning of the topic on the first ever run.- .set("auto.offset.reset", "earliest");-- let consumer = match Environment::new_or_prod() {- Environment::Local => Self::Plaintext(- config- .create()- .context("failed to create plaintext kafka consumer")?,- ),- Environment::Develop | Environment::Production => {- let config = configure_sasl_iam(config);- Self::MskIam(- config- .create_with_context(MskIamClientContext::from_env())- .context("failed to create MSK IAM kafka consumer")?,- )- }- };-- Ok(consumer)- }-- fn subscribe(&self) -> KafkaResult<()> {- let topics = [MacroProjectsTopic.as_str()];- match self {- Self::Plaintext(consumer) => consumer.subscribe(&topics),- Self::MskIam(consumer) => consumer.subscribe(&topics),- }- }-- /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it- /// can sit in a `select!` without losing messages.- async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {- match self {- Self::Plaintext(consumer) => consumer.recv().await,- Self::MskIam(consumer) => consumer.recv().await,- }- }-- fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {- match self {- Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),- Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),- }- }+ event: ProjectMacroEvent,}fn event_type(event: &ProjectTopicEvent) -> &'static str {@@ -155,7 +78,7 @@ fn event_type(event: &ProjectTopicEvent) -> &'static str {/// the body for indexing, notifications, etc.async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {while let Some(received) = events.recv().await {- let ProjectsConsumerEvent::Projects(event) = received.event;+ let event = received.event;let envelope = event.event();println!(@@ -188,14 +111,16 @@ async fn main() -> anyhow::Result<()> {let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;- let consumer = ProjectsConsumer::from_env(env.kafka_brokers.as_ref())?;- consumer- .subscribe()- .context("failed to subscribe to projects topic")?;+ let consumer =+ KafkaEventConsumer::<ProjectsConsumerGroup>::from_env(env.kafka_brokers.as_ref())?;+ let consumer = KafkaConsumerAdapter::<ProjectsConsumerGroup, ()>::new(consumer)+ .subscribe::<DeclaredMacroEvent>()+ .map_err(|error| anyhow::anyhow!("failed to subscribe to projects topic: {error:?}"))?;+ let consumer = ProjectsConsumerService::new(consumer);println!("listening on topic={} group={} brokers={} (ctrl-c to stop)",- MacroProjectsTopic.as_str(),- GROUP_ID,+ DeclaredMacroEvent::topics()[0],+ ProjectsConsumerGroup::GROUP_NAME,env.kafka_brokers.as_ref(),);@@ -218,25 +143,13 @@ async fn main() -> anyhow::Result<()> {}};- let Some(payload) = message.payload() else {- eprintln!(- "skipping message with empty payload at partition {} offset {}",- message.partition(),- message.offset(),- );- let _ = consumer.commit(&message).inspect_err(|e| {- eprintln!("failed to commit offset {}: {e}", message.offset());- });- continue;- };- let key = String::from_utf8_lossy(message.key().unwrap_or_default());-- match ProjectsConsumerEvent::decode(message.topic(), key.as_ref(), payload) {- Ok(event) => {+ let kafka_message = message.inner();+ match message.decode_payload() {+ Ok(DeclaredMacroEvent::ProjectMacroEvent(event)) => {events_tx.send(ReceivedEvent {- partition: message.partition(),- offset: message.offset(),+ partition: kafka_message.partition(),+ offset: kafka_message.offset(),event,}).await@@ -246,16 +159,19 @@ async fn main() -> anyhow::Result<()> {// wedging the partition on a poison message.Err(e) => eprintln!("failed to decode message at partition {} offset {}: {e}",- message.partition(),- message.offset(),+ kafka_message.partition(),+ kafka_message.offset(),),}// Commit only after the event was handed to the processor:// at-least-once delivery (up to the channel buffer on a crash).- let _ = consumer.commit(&message).inspect_err(|e| {- eprintln!("failed to commit offset {}: {e}", message.offset());- });+ let _ = consumer+ .inner()+ .commit_message(kafka_message, CommitMode::Async)+ .inspect_err(|e| {+ eprintln!("failed to commit offset {}: {e:?}", kafka_message.offset());+ });}}}diff --git a/crates/projects/src/domain/service/tests.rs b/crates/projects/src/domain/service/tests.rsindex 907ea85acf..c13154610b 100644--- a/crates/projects/src/domain/service/tests.rs+++ b/crates/projects/src/domain/service/tests.rs@@ -6,7 +6,7 @@ use entity_access::domain::models::{};use entity_access_management::domain::models::EntityAccessManagementError;use entity_access_management::domain::ports::EntityAccessManagementService;-use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, Topic as _};+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};use macro_user_id::user_id::MacroUserIdStr;use model::document::{ContentType, DocumentMetadata, FileType};use model::folder::{@@ -69,7 +69,7 @@ impl MacroEventBroker for TestEventBroker {}self.published.lock().unwrap().push(PublishedEvent {- topic: event.topic().as_str(),+ topic: event.topic(),key: event.key().to_string(),payload: serde_json::to_value(event.event())?,});diff --git a/crates/soup_realtime/Cargo.toml b/crates/soup_realtime/Cargo.tomlindex 2cfe4c4827..a314e849e6 100644--- a/crates/soup_realtime/Cargo.toml+++ b/crates/soup_realtime/Cargo.toml@@ -7,20 +7,16 @@ version = "0.1.0"[features]consumer = ["dep:documents",- "dep:kafka_consumer_util",- "dep:macro_event_broker",- "dep:macro_event_topics",+ "dep:kafka_util","dep:rdkafka","dep:tokio",+ "macro_event_broker/outbound","ports",]default = ["consumer", "outbound", "ports"]outbound = ["dep:entity_access",- "dep:macro_event_topics",- "dep:serde_json","dep:soup",- "macro_event_broker/ports","ports",]ports = ["dep:futures", "dep:rootcause", "dep:tracing"]@@ -29,16 +25,15 @@ ports = ["dep:futures", "dep:rootcause", "dep:tracing"]documents = { path = "../documents", default-features = false, optional = true }entity_access = { path = "../entity_access", default-features = false, features = ["ports"], optional = true }futures = { workspace = true, optional = true }-kafka_consumer_util = { path = "../kafka_consumer_util", optional = true }-macro_event_broker = { path = "../macro_event_broker", default-features = false, optional = true }-macro_event_topics = { path = "../macro_event_topics", optional = true }+kafka_util = { path = "../kafka_util", optional = true }+macro_event_broker = { path = "../macro_event_broker", default-features = false }+macro_event_topics = { path = "../macro_event_topics" }macro_user_id = { path = "../macro_user_id" }model-entity = { path = "../model-entity" }models_soup = { path = "../models_soup" }rdkafka = { workspace = true, optional = true }rootcause = { workspace = true, optional = true }serde = { workspace = true }-serde_json = { workspace = true, optional = true }soup = { path = "../soup", default-features = false, features = ["ports"], optional = true }tokio = { workspace = true, features = ["time"], optional = true }tracing = { workspace = true, optional = true }diff --git a/crates/soup_realtime/src/domain/models.rs b/crates/soup_realtime/src/domain/models.rsindex af1b0e96fa..d060c319eb 100644--- a/crates/soup_realtime/src/domain/models.rs+++ b/crates/soup_realtime/src/domain/models.rs@@ -1,14 +1,14 @@-//! Versioned realtime Soup output models.+//! Realtime Soup message and broker event models.+use macro_event_broker::{Event, MacroEvent, TopicEvent};+use macro_event_topics::MacroSoupRealtimeTopic;use macro_user_id::user_id::MacroUserIdStr;use models_soup::item::SoupItem;use serde::{Deserialize, Serialize};-/// One full Soup item published to a recipient for realtime delivery.+/// One full Soup item addressed to a recipient for realtime delivery.#[derive(Debug, Serialize, Deserialize)]pub struct SoupRealtimeMessage {- /// Version of this message contract.- pub schema_version: u8,/// User to whom the Soup item is addressed.pub user_id: MacroUserIdStr<'static>,/// Complete Soup item with transient user-specific fields normalized.@@ -16,15 +16,70 @@ pub struct SoupRealtimeMessage {}impl SoupRealtimeMessage {- /// Current schema version for realtime Soup messages.- pub const SCHEMA_VERSION: u8 = 1;-- /// Creates a version-one message addressed to a recipient.+ /// Creates a message addressed to a recipient.pub fn new(user_id: MacroUserIdStr<'static>, item: SoupItem<()>) -> Self {+ Self { user_id, item }+ }+}++/// Events published to [`MacroSoupRealtimeTopic`].+#[derive(Debug, Serialize, Deserialize)]+#[serde(tag = "event_type", content = "metadata")]+pub enum SoupTopicEvent {+ /// A complete Soup item changed and should be delivered to its recipient.+ #[serde(rename = "soup.item.updated")]+ ItemUpdated(SoupRealtimeMessage),+}++impl TopicEvent for SoupTopicEvent {+ type Topic = MacroSoupRealtimeTopic;++ fn schema_version(&self) -> u8 {+ 1+ }+}++/// Publishable realtime Soup event keyed by recipient user ID.+pub struct SoupMacroEvent {+ key: String,+ event: Event<SoupTopicEvent>,+}++impl SoupMacroEvent {+ /// Creates a recipient-keyed item update event.+ pub fn item_updated(message: SoupRealtimeMessage) -> Self {+ let key = message.user_id.as_ref().to_string();+ Self::with_event(key, Event::new(SoupTopicEvent::ItemUpdated(message)))+ }++ /// Builds an event from its Kafka key and pre-built envelope.+ pub fn with_event(key: impl Into<String>, event: Event<SoupTopicEvent>) -> Self {Self {- schema_version: Self::SCHEMA_VERSION,- user_id,- item,+ key: key.into(),+ event,}}++ /// Returns the recipient-targeted message carried by this event.+ pub fn into_message(self) -> SoupRealtimeMessage {+ match self.event.event {+ SoupTopicEvent::ItemUpdated(message) => message,+ }+ }+}++impl MacroEvent for SoupMacroEvent {+ type EventPayload = SoupTopicEvent;++ fn key(&self) -> &str {+ &self.key+ }++ fn event(&self) -> &Event<Self::EventPayload> {+ &self.event+ }++ fn from_event(key: String, event: Event<Self::EventPayload>) -> Self {+ Self::with_event(key, event)+ }}diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rsindex 66e43e9342..f9bf0e77c0 100644--- a/crates/soup_realtime/src/domain/service/test.rs+++ b/crates/soup_realtime/src/domain/service/test.rs@@ -181,7 +181,7 @@ async fn zero_users_skips_reads_and_publications() {}#[tokio::test]-async fn one_user_receives_one_full_versioned_message() {+async fn one_user_receives_one_full_message() {let recipient = user("one");let viewed_at = Some(timestamp(3));let responses = HashMap::from([item_response(@@ -199,7 +199,6 @@ async fn one_user_receives_one_full_versioned_message() {let mut messages = harness.messages.lock().expect("messages lock");let message = messages.pop().expect("one message");assert!(messages.is_empty());- assert_eq!(message.schema_version, 1);assert_eq!(message.user_id, recipient);match message.item {SoupItem::Document(document) => {diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rsindex 7659820494..8adff981d4 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs@@ -9,10 +9,11 @@ mod test;use std::{future::Future, time::Duration};-use documents::domain::events::DocumentTopicEvent;-use kafka_consumer_util::{GroupName, KafkaEventConsumer};-use macro_event_broker::{Event, EventBrokerError, Topic as _};-use macro_event_topics::MacroDocumentsTopic;+use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};+use kafka_util::{GroupName, KafkaEventConsumer};+use macro_event_broker::{+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,+};use model_entity::{Entity, EntityType};use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};@@ -27,21 +28,21 @@ impl GroupName for SoupRealtimeConsumerGroup {const GROUP_NAME: &'static str = "soup-realtime";}-type SoupRealtimeKafkaConsumer = KafkaEventConsumer<SoupRealtimeConsumerGroup>;+type SoupRealtimeKafkaAdapter = KafkaConsumerAdapter<SoupRealtimeConsumerGroup, DeclaredMacroEvent>;+type SoupRealtimeKafkaConsumer =+ MacroEventConsumerService<DeclaredMacroEvent, SoupRealtimeKafkaAdapter>;++macro_event_broker::declare_topics!(DeclaredMacroEvent: DocumentMacroEvent);/// Total service attempts before returning for supervisor-driven redelivery.const MAX_SERVICE_ATTEMPTS: u32 = 5;/// Delay before the first retry; each subsequent delay doubles.const SERVICE_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);-fn subscribed_topics() -> [&'static str; 1] {- [MacroDocumentsTopic.as_str()]-}--fn entity_from_document_event(event: DocumentTopicEvent) -> Option<Entity<'static>> {- match event {+fn entity_from_document_event(event: &DocumentMacroEvent) -> Option<Entity<'static>> {+ match &event.event().event {DocumentTopicEvent::Updated(metadata) => {- Some(EntityType::Document.with_entity_string(metadata.document_id))+ Some(EntityType::Document.with_entity_string(metadata.document_id.clone()))}DocumentTopicEvent::Created(_) => {tracing::trace!(event_type = "document.created", "ignoring document event");@@ -55,38 +56,37 @@ fn entity_from_document_event(event: DocumentTopicEvent) -> Option<Entity<'statitracing::trace!(event_type = "document.copied", "ignoring document event");None}+ DocumentTopicEvent::Interaction(_) => {+ tracing::trace!(+ event_type = "document.interaction",+ "ignoring document event"+ );+ None+ }}}-fn decode_updated_entity(payload: &[u8]) -> Result<Option<Entity<'static>>, EventBrokerError> {- let event = Event::<DocumentTopicEvent>::decode(payload)?;- Ok(entity_from_document_event(event.event))-}--/// Commit-safe outcome after processing one non-empty document payload.-enum DocumentPayloadOutcome {+/// Commit-safe outcome after processing one document event.+enum DocumentEventOutcome {/// An update was successfully sent through the domain service.Notified,/// A recognized document event is outside the current scope.Ignored,- /// The payload could not be decoded and should be treated as poison.- Malformed(EventBrokerError),}-#[tracing::instrument(skip(service, payload), fields(partition, offset, payload_len = payload.len()), err)]-async fn process_document_payload<S: SoupRealtimeService>(+#[tracing::instrument(skip(service, event), fields(partition, offset), err)]+async fn process_document_event<S: SoupRealtimeService>(service: &S,- payload: &[u8],+ event: &DocumentMacroEvent,partition: i32,offset: i64,-) -> Result<DocumentPayloadOutcome, Report> {- match decode_updated_entity(payload) {- Ok(Some(entity)) => {+) -> Result<DocumentEventOutcome, Report> {+ match entity_from_document_event(event) {+ Some(entity) => {notify_with_retry(service, entity, partition, offset).await?;- Ok(DocumentPayloadOutcome::Notified)+ Ok(DocumentEventOutcome::Notified)}- Ok(None) => Ok(DocumentPayloadOutcome::Ignored),- Err(error) => Ok(DocumentPayloadOutcome::Malformed(error)),+ None => Ok(DocumentEventOutcome::Ignored),}}@@ -140,7 +140,7 @@ async fn notify_with_retry<S: SoupRealtimeService>(}fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {- match consumer.commit_message(message, CommitMode::Async) {+ match consumer.inner().commit_message(message, CommitMode::Async) {Ok(()) => tracing::trace!(partition = message.partition(),offset = message.offset(),@@ -171,12 +171,13 @@ pub async fn run_document_update_consumer<S>(whereS: SoupRealtimeService,{- let consumer = SoupRealtimeKafkaConsumer::from_env(brokers)?;- consumer- .subscribe(&subscribed_topics())+ let consumer = KafkaEventConsumer::<SoupRealtimeConsumerGroup>::from_env(brokers)?;+ let consumer = KafkaConsumerAdapter::<SoupRealtimeConsumerGroup, ()>::new(consumer)+ .subscribe::<DeclaredMacroEvent>().context("failed to subscribe to document update topic")?;+ let consumer = SoupRealtimeKafkaConsumer::new(consumer);tracing::info!(- topics = ?subscribed_topics(),+ topics = ?DeclaredMacroEvent::topics(),group = SoupRealtimeConsumerGroup::GROUP_NAME,"realtime Soup document consumer listening");@@ -196,63 +197,51 @@ wherecontinue;}};- tracing::trace!(- topic = message.topic(),- partition = message.partition(),- offset = message.offset(),- payload_len = message.payload().map_or(0, <[u8]>::len),- "received realtime Soup input"- );-- let Some(payload) = message.payload().filter(|payload| !payload.is_empty()) else {- tracing::warn!(- partition = message.partition(),- offset = message.offset(),- "skipping document event with empty payload"- );- commit_logged(&consumer, &message);- continue;+ let kafka_message = message.inner();+ let event = match message.decode_payload() {+ Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => event,+ Err(error) => {+ tracing::error!(+ error = ?error,+ topic = kafka_message.topic(),+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),+ "dropping malformed document event"+ );+ commit_logged(&consumer, kafka_message);+ continue;+ }};- let outcome = match process_document_payload(+ match process_document_event(&service,- payload,- message.partition(),- message.offset(),+ &event,+ kafka_message.partition(),+ kafka_message.offset(),).await{- Ok(outcome) => outcome,+ Ok(DocumentEventOutcome::Notified | DocumentEventOutcome::Ignored) => {}Err(error) => {tracing::error!(error = ?error,- topic = message.topic(),- partition = message.partition(),- offset = message.offset(),+ topic = kafka_message.topic(),+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),"realtime Soup fan-out retries exhausted; pausing partition for redelivery");// Kafka commits are cumulative within a partition, so// pause it before any later record can advance the// committed offset past this failure.consumer- .pause_message_partition(&message)+ .inner()+ .pause_message_partition(kafka_message).context("failed to pause Kafka partition after fan-out failure")?;continue;}- };-- match outcome {- DocumentPayloadOutcome::Notified | DocumentPayloadOutcome::Ignored => {}- DocumentPayloadOutcome::Malformed(error) => tracing::error!(- error = ?error,- topic = message.topic(),- partition = message.partition(),- offset = message.offset(),- "dropping malformed document event"- ),}- commit_logged(&consumer, &message);+ commit_logged(&consumer, kafka_message);}}}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rsindex a00258b697..85c2d787c0 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs@@ -3,12 +3,35 @@ use std::sync::{Arc, Mutex};use documents::domain::events::{DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,- DocumentUpdatedMetadata,+ DocumentInteractionMetadata, DocumentUpdatedMetadata, InteractionReason,};+use macro_event_broker::{Event, EventBrokerError, MacroEventCollection as _, MessageParts};use macro_user_id::user_id::MacroUserIdStr;use super::*;+struct TestMessage {+ payload: Vec<u8>,+}++impl MessageParts for TestMessage {+ fn key(&self) -> Option<&str> {+ Some(DOCUMENT_ID)+ }++ fn payload(&self) -> Option<&[u8]> {+ Some(&self.payload)+ }++ fn topic(&self) -> &str {+ "macro.documents"+ }+}++fn decode_payload(payload: Vec<u8>) -> Result<DeclaredMacroEvent, EventBrokerError> {+ DeclaredMacroEvent::decode(&TestMessage { payload })+}+const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";fn user() -> MacroUserIdStr<'static> {@@ -54,6 +77,10 @@ fn ignored_events() -> Vec<DocumentTopicEvent> {project_id: None,sub_type: None,}),+ DocumentTopicEvent::Interaction(DocumentInteractionMetadata {+ document_id: DOCUMENT_ID.to_string(),+ reason: InteractionReason::Edited,+ }),]}@@ -86,19 +113,19 @@ fn flaky_service(failures: u32) -> FlakyService {#[test]fn subscribes_only_to_documents() {- assert_eq!(subscribed_topics(), ["macro.documents"]);+ assert_eq!(DeclaredMacroEvent::topics(), ["macro.documents"]);}#[tokio::test]async fn updated_payload_maps_to_document_entity() {- let payload = serde_json::to_vec(&updated_event()).expect("serializable event");+ let event = DocumentMacroEvent::with_event(DOCUMENT_ID, updated_event());let service = flaky_service(0);assert!(matches!(- process_document_payload(&service, &payload, 0, 0)+ process_document_event(&service, &event, 0, 0).await.expect("processing succeeds"),- DocumentPayloadOutcome::Notified+ DocumentEventOutcome::Notified));let entities = service.entities.lock().expect("entities lock");@@ -108,29 +135,23 @@ async fn updated_payload_maps_to_document_entity() {}#[tokio::test]-async fn created_deleted_and_copied_events_are_ignored() {+async fn non_update_events_are_ignored() {let service = flaky_service(0);for event in ignored_events() {- let payload = serde_json::to_vec(&Event::new(event)).expect("serializable event");+ let event = DocumentMacroEvent::with_event(DOCUMENT_ID, Event::new(event));assert!(matches!(- process_document_payload(&service, &payload, 0, 0)+ process_document_event(&service, &event, 0, 0).await.expect("processing succeeds"),- DocumentPayloadOutcome::Ignored+ DocumentEventOutcome::Ignored));}assert_eq!(service.attempts.load(Ordering::SeqCst), 0);}-#[tokio::test]-async fn malformed_and_unknown_events_are_rejected_without_notifying() {- let service = flaky_service(0);- assert!(matches!(- process_document_payload(&service, b"not json", 0, 0)- .await- .expect("poison payload is commit-safe"),- DocumentPayloadOutcome::Malformed(_)- ));+#[test]+fn malformed_and_unknown_events_are_rejected_by_the_declared_collection() {+ assert!(decode_payload(b"not json".to_vec()).is_err());let unknown = serde_json::json!({"event_id": "00000000-0000-0000-0000-000000000003",@@ -139,13 +160,7 @@ async fn malformed_and_unknown_events_are_rejected_without_notifying() {"metadata": { "document_id": DOCUMENT_ID }});let payload = serde_json::to_vec(&unknown).expect("serializable JSON");- assert!(matches!(- process_document_payload(&service, &payload, 0, 0)- .await- .expect("poison payload is commit-safe"),- DocumentPayloadOutcome::Malformed(_)- ));- assert_eq!(service.attempts.load(Ordering::SeqCst), 0);+ assert!(decode_payload(payload).is_err());}#[tokio::test(start_paused = true)]diff --git a/crates/soup_realtime/src/inbound/mod.rs b/crates/soup_realtime/src/inbound/mod.rsindex 1d73c01b15..e22ac579a2 100644--- a/crates/soup_realtime/src/inbound/mod.rs+++ b/crates/soup_realtime/src/inbound/mod.rs@@ -2,3 +2,5 @@/// Kafka consumer for document update events.pub mod kafka_consumer;+/// Independent Kafka consumer for recipient-targeted Soup messages.+pub mod soup_consumer;diff --git a/crates/soup_realtime/src/inbound/soup_consumer.rs b/crates/soup_realtime/src/inbound/soup_consumer.rsnew file mode 100644index 0000000000..a0dca06f2a--- /dev/null+++ b/crates/soup_realtime/src/inbound/soup_consumer.rs@@ -0,0 +1,75 @@+//! Independent Kafka consumer for recipient-targeted Soup messages.+//!+//! Every process receives every message published after it starts because this+//! adapter manually assigns all `macro.soup` partitions without joining a+//! durable consumer group. It does not commit offsets.++#[cfg(test)]+mod test;++use std::time::Duration;++use kafka_util::{InitialOffset, KafkaEventConsumer, Ungrouped};+use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};+use rootcause::prelude::{Report, ResultExt as _};++use crate::domain::models::{SoupMacroEvent, SoupRealtimeMessage};++/// Maximum time to wait for Soup topic metadata during partition assignment.+const TOPIC_METADATA_TIMEOUT: Duration = Duration::from_secs(10);++type IndependentKafkaConsumer = KafkaConsumerAdapter<Ungrouped, DeclaredMacroEvent>;+type SoupEventConsumer = MacroEventConsumerService<DeclaredMacroEvent, IndependentKafkaConsumer>;++macro_event_broker::declare_topics!(DeclaredMacroEvent: SoupMacroEvent);++/// Independent consumer of recipient-targeted Soup messages.+///+/// This consumer starts at the end of every current `macro.soup` partition, so+/// it receives only messages published after construction. It does not join a+/// durable consumer group or persist offsets. Partitions added after+/// construction require a new consumer so they can be assigned.+pub struct SoupTopicConsumer {+ consumer: SoupEventConsumer,+}++impl SoupTopicConsumer {+ /// Creates a consumer and assigns every current Soup topic partition.+ #[tracing::instrument(fields(brokers), err)]+ pub fn from_env(brokers: &str) -> Result<Self, Report> {+ let consumer = KafkaEventConsumer::<Ungrouped>::from_env(brokers)+ .context("failed to create independent realtime Soup consumer")?;+ let consumer =+ IndependentKafkaConsumer::new(consumer, InitialOffset::Latest, TOPIC_METADATA_TIMEOUT)+ .context("failed to assign realtime Soup topic partitions")?;++ tracing::info!(+ topics = ?DeclaredMacroEvent::topics(),+ "independent realtime Soup consumer listening"+ );++ Ok(Self {+ consumer: SoupEventConsumer::new(consumer),+ })+ }++ /// Receives and decodes the next typed realtime Soup event.+ ///+ /// This operation is cancel-safe. Missing keys and malformed or unsupported+ /// payloads are returned as errors; because the consumer is ungrouped, a+ /// subsequent call proceeds to the next locally assigned record.+ pub async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {+ let message = self+ .consumer+ .recv()+ .await+ .context("failed to receive realtime Soup event")?;+ let event = message+ .decode_payload()+ .context("failed to decode realtime Soup event")?;++ match event {+ DeclaredMacroEvent::SoupMacroEvent(event) => Ok(event.into_message()),+ }+ }+}diff --git a/crates/soup_realtime/src/inbound/soup_consumer/test.rs b/crates/soup_realtime/src/inbound/soup_consumer/test.rsnew file mode 100644index 0000000000..2d5e577f53--- /dev/null+++ b/crates/soup_realtime/src/inbound/soup_consumer/test.rs@@ -0,0 +1,8 @@+use macro_event_broker::MacroEventCollection as _;++use super::DeclaredMacroEvent;++#[test]+fn assigns_only_the_typed_soup_topic() {+ assert_eq!(DeclaredMacroEvent::topics(), ["macro.soup"]);+}diff --git a/crates/soup_realtime/src/outbound/kafka_publisher.rs b/crates/soup_realtime/src/outbound/kafka_publisher.rsindex 5b1a8987da..2f59c1904b 100644--- a/crates/soup_realtime/src/outbound/kafka_publisher.rs+++ b/crates/soup_realtime/src/outbound/kafka_publisher.rs@@ -1,29 +1,31 @@-//! Kafka publisher for versioned realtime Soup messages.+//! Macro event broker publisher for realtime Soup events.#[cfg(test)]mod test;-use macro_event_broker::EventPublisher;-use macro_event_topics::MacroSoupRealtimeTopic;+use macro_event_broker::MacroEventBroker;use rootcause::prelude::{Report, ResultExt as _};-use crate::domain::{models::SoupRealtimeMessage, ports::SoupRealtimePublisher};+use crate::domain::{+ models::{SoupMacroEvent, SoupRealtimeMessage},+ ports::SoupRealtimePublisher,+};-/// Realtime Soup publisher backed by a byte-oriented event publisher.-pub struct KafkaSoupRealtimePublisher<P> {- publisher: P,+/// Realtime Soup publisher backed by the typed macro event broker service.+pub struct KafkaSoupRealtimePublisher<B> {+ broker: B,}-impl<P> KafkaSoupRealtimePublisher<P> {- /// Creates a realtime Soup Kafka publisher.- pub fn new(publisher: P) -> Self {- Self { publisher }+impl<B> KafkaSoupRealtimePublisher<B> {+ /// Creates a realtime Soup publisher from a macro event broker service.+ pub fn new(broker: B) -> Self {+ Self { broker }}}-impl<P> SoupRealtimePublisher for KafkaSoupRealtimePublisher<P>+impl<B> SoupRealtimePublisher for KafkaSoupRealtimePublisher<B>where- P: EventPublisher,+ B: MacroEventBroker,{#[tracing::instrument(skip(self, message),@@ -31,13 +33,16 @@ whereerr)]async fn publish(&self, message: SoupRealtimeMessage) -> Result<(), Report> {- let payload =- serde_json::to_vec(&message).context("failed to serialize realtime Soup message")?;+ let event = SoupMacroEvent::item_updated(message);+ let publish = self+ .broker+ .send_event(&event)+ .context("failed to dispatch realtime Soup event")?;- self.publisher- .publish(MacroSoupRealtimeTopic, message.user_id.as_ref(), &payload)+ publish.await- .context("failed to publish realtime Soup message")?;+ .context("realtime Soup publish task failed")?+ .context("failed to publish realtime Soup event")?;Ok(())}diff --git a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs b/crates/soup_realtime/src/outbound/kafka_publisher/test.rsindex a16489484c..a585ef2010 100644--- a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs+++ b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs@@ -1,7 +1,9 @@use std::sync::{Arc, Mutex};use chrono::{DateTime, Utc};-use macro_event_broker::{EventBrokerError, Topic};+use macro_event_broker::{+ EventBrokerError, EventPublisher, MacroEvent, MacroEventBrokerService, Topic,+};use macro_user_id::user_id::MacroUserIdStr;use models_soup::{document::SoupDocument, item::SoupItem};use uuid::Uuid;@@ -21,17 +23,12 @@ struct RecordingPublisher {}impl EventPublisher for RecordingPublisher {- async fn publish<T: Topic>(- &self,- topic: T,- key: &str,- payload: &[u8],- ) -> Result<(), EventBrokerError> {+ async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {self.records.lock().expect("records lock").push(PublishedRecord {- topic: topic.as_str().to_string(),+ topic: T::TOPIC_STR.to_string(),key: key.to_string(),payload: payload.to_vec(),});@@ -76,12 +73,13 @@ fn message() -> SoupRealtimeMessage {}#[tokio::test]-async fn publishes_user_keyed_full_message_to_realtime_topic() {+async fn publishes_typed_recipient_keyed_event_to_soup_topic() {let records = Arc::new(Mutex::new(Vec::new()));- let adapter = KafkaSoupRealtimePublisher::new(RecordingPublisher {+ let broker = MacroEventBrokerService::new(RecordingPublisher {records: records.clone(),fail: false,});+ let adapter = KafkaSoupRealtimePublisher::new(broker);adapter.publish(message()).await.expect("publish succeeds");@@ -92,13 +90,14 @@ async fn publishes_user_keyed_full_message_to_realtime_topic() {assert_eq!(record.key, "macro|[email protected]");let json: serde_json::Value = serde_json::from_slice(&record.payload).expect("payload is JSON");+ assert!(json["event_id"].is_string());assert_eq!(json["schema_version"], 1);- assert_eq!(json["user_id"], "macro|[email protected]");- assert!(json.get("event_id").is_none());+ assert_eq!(json["event_type"], "soup.item.updated");+ assert_eq!(json["metadata"]["user_id"], "macro|[email protected]");- let decoded: SoupRealtimeMessage =- serde_json::from_slice(&record.payload).expect("payload round-trips");- assert_eq!(decoded.schema_version, 1);+ let decoded = SoupMacroEvent::decode(record.key, &record.payload).expect("event round-trips");+ assert_eq!(decoded.event().schema_version, 1);+ let decoded = decoded.into_message();assert_eq!(decoded.user_id.as_ref(), "macro|[email protected]");match decoded.item {SoupItem::Document(document) => {@@ -111,11 +110,12 @@ async fn publishes_user_keyed_full_message_to_realtime_topic() {}#[tokio::test]-async fn propagates_delivery_failures() {- let adapter = KafkaSoupRealtimePublisher::new(RecordingPublisher {+async fn propagates_delivery_failures_from_event_broker_service() {+ let broker = MacroEventBrokerService::new(RecordingPublisher {records: Arc::new(Mutex::new(Vec::new())),fail: true,});+ let adapter = KafkaSoupRealtimePublisher::new(broker);adapter.publish(message())diff --git a/crates/teams/Cargo.toml b/crates/teams/Cargo.tomlindex 8f45d130fb..0b25e0af47 100644--- a/crates/teams/Cargo.toml+++ b/crates/teams/Cargo.toml@@ -38,7 +38,6 @@ ports = ["dep:model_notifications","dep:notification","dep:tokio",- "macro_event_broker/ports",][dependencies]diff --git a/crates/teams/src/domain/events/test.rs b/crates/teams/src/domain/events/test.rsindex 3416f2d825..ba05f11c5d 100644--- a/crates/teams/src/domain/events/test.rs+++ b/crates/teams/src/domain/events/test.rs@@ -1,5 +1,4 @@use macro_event_broker::{Event, MacroEvent};-use macro_event_topics::Topic;use macro_user_id::user_id::MacroUserIdStr;use serde_json::{Value, json};use uuid::Uuid;@@ -261,7 +260,7 @@ fn every_variant_round_trips() {assert_eq!(decoded.key(), TEAM_ID);assert_eq!(decoded.event(), original.event());- assert_eq!(decoded.topic().as_str(), "macro.teams");+ assert_eq!(decoded.topic(), "macro.teams");}}@@ -270,7 +269,7 @@ fn constructors_use_teams_topic_bare_uuid_key_and_schema_version_one() {for event in macro_events() {assert_eq!(event.key(), TEAM_ID);assert!(!event.key().starts_with("team|"));- assert_eq!(event.topic().as_str(), "macro.teams");+ assert_eq!(event.topic(), "macro.teams");assert_eq!(event.event().schema_version, 1);}}diff --git a/crates/teams/src/domain/team_service/test.rs b/crates/teams/src/domain/team_service/test.rsindex f74e3485ac..4f6bb36b1b 100644--- a/crates/teams/src/domain/team_service/test.rs+++ b/crates/teams/src/domain/team_service/test.rs@@ -22,7 +22,7 @@ use entity_access::domain::models::{AdminTeamRole, EntityAccessReceipt, EntityType, MemberTeamRole, OwnerTeamRole,RequiredPermission,};-use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, Topic as _};+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};use macro_user_id::{email::Email, lowercased::Lowercase, user_id::MacroUserIdStr};use models_pagination::{CreatedAt, Query};use notification::domain::{@@ -1219,7 +1219,7 @@ impl MacroEventBroker for RecordingEventBroker {}self.events.lock().unwrap().push(PublishedTeamEvent {- topic: event.topic().as_str(),+ topic: event.topic(),key: event.key().to_string(),envelope: serde_json::to_value(event.event())?,});diff --git a/crates/webhook/Cargo.toml b/crates/webhook/Cargo.tomlindex c5f68ced18..104b3b8ad8 100644--- a/crates/webhook/Cargo.toml+++ b/crates/webhook/Cargo.toml@@ -6,10 +6,11 @@ version = "0.1.0"[features]consumer = [- "dep:kafka_consumer_util",+ "dep:kafka_util","dep:rdkafka","dep:tokio","ingestion",+ "macro_event_broker/outbound",]default = ["consumer", "inbound", "outbound", "ports"]inbound = [@@ -44,7 +45,7 @@ outbound = ["dep:url","ports",]-ports = ["dep:serde_json", "dep:url", "macro_event_broker/ports"]+ports = ["dep:serde_json", "dep:url"]worker = ["dep:tokio", "ports"][dependencies]@@ -60,7 +61,7 @@ entity_access = { path = "../entity_access", default-features = false, featuresfutures = { workspace = true, optional = true }hex = { workspace = true, optional = true }hmac = { workspace = true, optional = true }-kafka_consumer_util = { path = "../kafka_consumer_util", optional = true }+kafka_util = { path = "../kafka_util", optional = true }macro_authorization = { path = "../macro_authorization", optional = true, default-features = false, features = ["axum",] }diff --git a/crates/webhook/src/domain/events/test.rs b/crates/webhook/src/domain/events/test.rsindex d64141fd07..09fc241d56 100644--- a/crates/webhook/src/domain/events/test.rs+++ b/crates/webhook/src/domain/events/test.rs@@ -1,6 +1,5 @@use chrono::{DateTime, Utc};use macro_event_broker::{Event, MacroEvent};-use macro_event_topics::Topic;use macro_user_id::user_id::MacroUserIdStr;use serde_json::json;use uuid::Uuid;@@ -254,7 +253,7 @@ fn constructors_key_events_by_subject_webhook_id() {for (event, expected_event_type) in cases.into_iter().zip(expected_event_types) {assert_eq!(event.key(), WEBHOOK_ID);- assert_eq!(event.topic().as_str(), "macro.webhooks");+ assert_eq!(event.topic(), "macro.webhooks");assert_eq!(event.event().schema_version, 1);assert_eq!(serde_json::to_value(event.event()).expect("serializable event")["event_type"],@@ -290,5 +289,5 @@ fn macro_event_round_trips_without_exposing_secrets_or_header_values() {let decoded = WebhookMacroEvent::decode(original.key(), &payload).expect("decodable event");assert_eq!(decoded.key(), WEBHOOK_ID);assert_eq!(decoded.event(), original.event());- assert_eq!(decoded.topic().as_str(), "macro.webhooks");+ assert_eq!(decoded.topic(), "macro.webhooks");}diff --git a/crates/webhook/src/domain/service_test.rs b/crates/webhook/src/domain/service_test.rsindex 43b0f6273e..686bb68744 100644--- a/crates/webhook/src/domain/service_test.rs+++ b/crates/webhook/src/domain/service_test.rs@@ -8,9 +8,7 @@ use super::{service::WebhookServiceImpl,};use chrono::Utc;-use macro_event_broker::{- EventBrokerError, MacroEvent, MacroEventBroker, NoopMacroEventBroker, Topic as _,-};+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, NoopMacroEventBroker};use macro_user_id::user_id::MacroUserIdStr;use serde_json::json;use std::{@@ -266,7 +264,7 @@ impl MacroEventBroker for TestEventBroker {}self.published.lock().unwrap().push(PublishedEvent {- topic: event.topic().as_str(),+ topic: event.topic(),key: event.key().to_string(),payload: serde_json::to_value(event.event())?,});diff --git a/crates/webhook/src/inbound/kafka_consumer.rs b/crates/webhook/src/inbound/kafka_consumer.rsindex 8327fee250..d2bdb6d455 100644--- a/crates/webhook/src/inbound/kafka_consumer.rs+++ b/crates/webhook/src/inbound/kafka_consumer.rs@@ -1,7 +1,7 @@//! Kafka consumer that feeds broker events into webhook ingestion.//!-//! Subscribes to [`MacroDocumentsTopic`], [`MacroChannelsTopic`], and-//! [`MacroWebhooksTopic`] and hands every decoded event envelope to a+//! Subscribes to [`DocumentMacroEvent`], [`ChannelMacroEvent`], and+//! [`WebhookMacroEvent`] topics and hands every decoded event envelope to a//! [`WebhookEventIngestionService`].//!//! Delivery is at-least-once: an event's offset is committed only after the@@ -21,13 +21,14 @@#[cfg(test)]mod test;-use crate::domain::{events::WebhookTopicEvent, ingestion::WebhookEventIngestionService};+use crate::domain::{events::WebhookMacroEvent, ingestion::WebhookEventIngestionService};use anyhow::Context as _;-use channels::domain::broker_events::ChannelTopicEvent;-use documents::domain::events::DocumentTopicEvent;-use kafka_consumer_util::{GroupName, KafkaEventConsumer};-use macro_event_broker::{Event, EventBrokerError, Topic as _};-use macro_event_topics::{MacroChannelsTopic, MacroDocumentsTopic, MacroWebhooksTopic};+use channels::domain::broker_events::ChannelMacroEvent;+use documents::domain::events::DocumentMacroEvent;+use kafka_util::{GroupName, KafkaEventConsumer};+use macro_event_broker::{+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,+};use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message};use std::future::Future;@@ -41,7 +42,15 @@ impl GroupName for WebhookEventIngestionConsumerGroup {const GROUP_NAME: &'static str = "webhook-event-ingestion";}-type WebhookKafkaConsumer = KafkaEventConsumer<WebhookEventIngestionConsumerGroup>;+type WebhookKafkaAdapter =+ KafkaConsumerAdapter<WebhookEventIngestionConsumerGroup, DeclaredMacroEvent>;+type WebhookKafkaConsumer = MacroEventConsumerService<DeclaredMacroEvent, WebhookKafkaAdapter>;++macro_event_broker::declare_topics!(+ DeclaredMacroEvent: DocumentMacroEvent,+ ChannelMacroEvent,+ WebhookMacroEvent,+);/// Maximum in-process ingestion attempts per event before the consumer bails/// out and lets a restart redeliver from the last committed offset.@@ -53,49 +62,9 @@ const MAX_INGEST_ATTEMPTS: u32 = 5;/// consumer from its group.const INGEST_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);-fn subscribed_topics() -> [&'static str; 3] {- [- MacroDocumentsTopic.as_str(),- MacroChannelsTopic.as_str(),- MacroWebhooksTopic.as_str(),- ]-}--/// One decoded event envelope from a topic this consumer subscribes to.-///-/// The Kafka message key is not carried: every event's metadata already-/// contains the entity ids the ingestion service needs.-#[derive(Debug)]-pub enum WebhookConsumerEvent {- /// Event received on [`MacroDocumentsTopic`].- Documents(Event<DocumentTopicEvent>),- /// Event received on [`MacroChannelsTopic`].- Channels(Event<ChannelTopicEvent>),- /// Event received on [`MacroWebhooksTopic`].- Webhooks(Event<WebhookTopicEvent>),-}--impl WebhookConsumerEvent {- /// Decode one Kafka message into this consumer's event enum.- pub fn decode(topic: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {- match topic {- topic if topic == MacroDocumentsTopic.as_str() => {- Ok(Self::Documents(Event::decode(payload)?))- }- topic if topic == MacroChannelsTopic.as_str() => {- Ok(Self::Channels(Event::decode(payload)?))- }- topic if topic == MacroWebhooksTopic.as_str() => {- Ok(Self::Webhooks(Event::decode(payload)?))- }- unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),- }- }-}-/// Commit `message`'s offset, logging the outcome.fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>) {- match consumer.commit_message(message, CommitMode::Async) {+ match consumer.inner().commit_message(message, CommitMode::Async) {Ok(()) => tracing::trace!(partition = message.partition(),offset = message.offset(),@@ -118,7 +87,7 @@ fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>)/// the caller must exit without committing so the event is redelivered.async fn ingest_with_retry<S: WebhookEventIngestionService>(service: &S,- event: &WebhookConsumerEvent,+ event: &DeclaredMacroEvent,partition: i32,offset: i64,) -> anyhow::Result<()> {@@ -127,14 +96,14 @@ async fn ingest_with_retry<S: WebhookEventIngestionService>(loop {tracing::trace!(partition, offset, attempt, "ingesting broker event");let result = match event {- WebhookConsumerEvent::Documents(event) => {- service.ingest_document_event(event.clone()).await+ DeclaredMacroEvent::DocumentMacroEvent(event) => {+ service.ingest_document_event(event.event().clone()).await}- WebhookConsumerEvent::Channels(event) => {- service.ingest_channel_event(event.clone()).await+ DeclaredMacroEvent::ChannelMacroEvent(event) => {+ service.ingest_channel_event(event.event().clone()).await}- WebhookConsumerEvent::Webhooks(event) => {- service.ingest_webhook_event(event.clone()).await+ DeclaredMacroEvent::WebhookMacroEvent(event) => {+ service.ingest_webhook_event(event.event().clone()).await}};match result {@@ -179,9 +148,9 @@ async fn ingest_with_retry<S: WebhookEventIngestionService>(/// Run the webhook event consumer until `shutdown` resolves.///-/// Connects to `brokers` and subscribes to [`MacroDocumentsTopic`],-/// [`MacroChannelsTopic`], and [`MacroWebhooksTopic`] under the-/// `webhook-event-ingestion` consumer group. Every decoded event is fed to+/// Connects to `brokers` and subscribes to the topics declared by+/// [`DocumentMacroEvent`], [`ChannelMacroEvent`], and [`WebhookMacroEvent`]+/// under the `webhook-event-ingestion` consumer group. Every decoded event is fed to/// `service`, committing each offset only/// after ingestion succeeds (see `ingest_with_retry` for the retry policy)./// Returns an error when the consumer cannot be created or subscribed, or when@@ -197,12 +166,15 @@ pub async fn run_webhook_event_consumer<S>(whereS: WebhookEventIngestionService,{- let consumer = WebhookKafkaConsumer::from_env(brokers)?;- consumer- .subscribe(&subscribed_topics())- .context("failed to subscribe to webhook event topics")?;+ let consumer = KafkaEventConsumer::<WebhookEventIngestionConsumerGroup>::from_env(brokers)?;+ let consumer = KafkaConsumerAdapter::<WebhookEventIngestionConsumerGroup, ()>::new(consumer)+ .subscribe::<DeclaredMacroEvent>()+ .map_err(|error| {+ anyhow::anyhow!("failed to subscribe to webhook event topics: {error:?}")+ })?;+ let consumer = WebhookKafkaConsumer::new(consumer);tracing::info!(- topics = ?subscribed_topics(),+ topics = ?DeclaredMacroEvent::topics(),group = WebhookEventIngestionConsumerGroup::GROUP_NAME,"webhook event consumer listening");@@ -222,36 +194,19 @@ wherecontinue;}};- tracing::trace!(- topic = message.topic(),- partition = message.partition(),- offset = message.offset(),- payload_len = message.payload().map_or(0, <[u8]>::len),- "received kafka message"- );-- let Some(payload) = message.payload() else {- tracing::warn!(- partition = message.partition(),- offset = message.offset(),- "skipping message with empty payload"- );- commit_logged(&consumer, &message);- continue;- };-- match WebhookConsumerEvent::decode(message.topic(), payload) {+ let kafka_message = message.inner();+ match message.decode_payload() {Ok(event) => {tracing::trace!(- partition = message.partition(),- offset = message.offset(),+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),"decoded broker event");ingest_with_retry(&service,&event,- message.partition(),- message.offset(),+ kafka_message.partition(),+ kafka_message.offset(),).await?;}@@ -259,16 +214,16 @@ where// wedging the partition on a poison message.Err(e) => tracing::error!(error = ?e,- topic = message.topic(),- partition = message.partition(),- offset = message.offset(),+ topic = kafka_message.topic(),+ partition = kafka_message.partition(),+ offset = kafka_message.offset(),"failed to decode broker event"),}// Commit only after the event was ingested or permanently// rejected: at-least-once, retried across restarts.- commit_logged(&consumer, &message);+ commit_logged(&consumer, kafka_message);}}}diff --git a/crates/webhook/src/inbound/kafka_consumer/test.rs b/crates/webhook/src/inbound/kafka_consumer/test.rsindex 861f11315c..1e456c670f 100644--- a/crates/webhook/src/inbound/kafka_consumer/test.rs+++ b/crates/webhook/src/inbound/kafka_consumer/test.rs@@ -4,13 +4,39 @@ use crate::domain::{ingestion::WebhookEventIngestionError,};use channel_sender::ChannelSender;-use channels::domain::broker_events::ChannelDeletedMetadata;-use documents::domain::events::DocumentDeletedMetadata;+use channels::domain::broker_events::{ChannelDeletedMetadata, ChannelTopicEvent};+use documents::domain::events::{DocumentDeletedMetadata, DocumentTopicEvent};+use macro_event_broker::{+ Event, EventBrokerError, MacroEvent as _, MacroEventCollection as _, MessageParts,+};use macro_user_id::user_id::MacroUserIdStr;use std::sync::Arc;use std::sync::atomic::{AtomicU32, Ordering};use uuid::Uuid;+struct TestMessage<'a> {+ topic: &'a str,+ payload: &'a [u8],+}++impl MessageParts for TestMessage<'_> {+ fn key(&self) -> Option<&str> {+ Some("event-key")+ }++ fn payload(&self) -> Option<&[u8]> {+ Some(self.payload)+ }++ fn topic(&self) -> &str {+ self.topic+ }+}++fn decode_message(topic: &str, payload: &[u8]) -> Result<DeclaredMacroEvent, EventBrokerError> {+ DeclaredMacroEvent::decode(&TestMessage { topic, payload })+}+fn document_event() -> Event<DocumentTopicEvent> {Event::new(DocumentTopicEvent::Deleted(DocumentDeletedMetadata {document_id: "doc_1".to_string(),@@ -28,10 +54,14 @@ fn webhook_event() -> Event<WebhookTopicEvent> {}))}+fn declared_webhook_event() -> DeclaredMacroEvent {+ DeclaredMacroEvent::WebhookMacroEvent(WebhookMacroEvent::with_event("wh_1", webhook_event()))+}+#[test]fn subscribes_to_all_ingestion_topics() {assert_eq!(- subscribed_topics(),+ DeclaredMacroEvent::topics(),["macro.documents", "macro.channels", "macro.webhooks"]);}@@ -41,12 +71,11 @@ fn decodes_document_events() {let event = document_event();let payload = serde_json::to_vec(&event).expect("serializable");- let decoded =- WebhookConsumerEvent::decode(MacroDocumentsTopic.as_str(), &payload).expect("decodable");+ let decoded = decode_message("macro.documents", &payload).expect("decodable");match decoded {- WebhookConsumerEvent::Documents(decoded) => assert_eq!(decoded, event),- WebhookConsumerEvent::Channels(_) | WebhookConsumerEvent::Webhooks(_) => {+ DeclaredMacroEvent::DocumentMacroEvent(decoded) => assert_eq!(decoded.event(), &event),+ DeclaredMacroEvent::ChannelMacroEvent(_) | DeclaredMacroEvent::WebhookMacroEvent(_) => {panic!("decoded into the wrong topic variant")}}@@ -61,12 +90,11 @@ fn decodes_channel_events() {}));let payload = serde_json::to_vec(&event).expect("serializable");- let decoded =- WebhookConsumerEvent::decode(MacroChannelsTopic.as_str(), &payload).expect("decodable");+ let decoded = decode_message("macro.channels", &payload).expect("decodable");match decoded {- WebhookConsumerEvent::Channels(decoded) => assert_eq!(decoded, event),- WebhookConsumerEvent::Documents(_) | WebhookConsumerEvent::Webhooks(_) => {+ DeclaredMacroEvent::ChannelMacroEvent(decoded) => assert_eq!(decoded.event(), &event),+ DeclaredMacroEvent::DocumentMacroEvent(_) | DeclaredMacroEvent::WebhookMacroEvent(_) => {panic!("decoded into the wrong topic variant")}}@@ -77,12 +105,11 @@ fn decodes_webhook_events() {let event = webhook_event();let payload = serde_json::to_vec(&event).expect("serializable");- let decoded =- WebhookConsumerEvent::decode(MacroWebhooksTopic.as_str(), &payload).expect("decodable");+ let decoded = decode_message("macro.webhooks", &payload).expect("decodable");match decoded {- WebhookConsumerEvent::Webhooks(decoded) => assert_eq!(decoded, event),- WebhookConsumerEvent::Documents(_) | WebhookConsumerEvent::Channels(_) => {+ DeclaredMacroEvent::WebhookMacroEvent(decoded) => assert_eq!(decoded.event(), &event),+ DeclaredMacroEvent::DocumentMacroEvent(_) | DeclaredMacroEvent::ChannelMacroEvent(_) => {panic!("decoded into the wrong topic variant")}}@@ -90,14 +117,17 @@ fn decodes_webhook_events() {#[test]fn rejects_unknown_topics() {- let err = WebhookConsumerEvent::decode("macro.example", b"{}").expect_err("unknown topic");+ let err = decode_message("macro.example", b"{}")+ .err()+ .expect("unknown topic");assert!(matches!(err, EventBrokerError::UnknownTopic(topic) if topic == "macro.example"));}#[test]fn rejects_malformed_webhook_payloads() {- let err = WebhookConsumerEvent::decode(MacroWebhooksTopic.as_str(), b"not json")- .expect_err("malformed payload");+ let err = decode_message("macro.webhooks", b"not json")+ .err()+ .expect("malformed payload");assert!(matches!(err, EventBrokerError::Serialization(_)));}@@ -169,14 +199,9 @@ fn flaky_service(failures: u32, transient: bool) -> (FlakyIngestionService, Arc<async fn retries_transient_webhook_failures_until_success() {let (service, attempts) = flaky_service(2, true);- ingest_with_retry(- &service,- &WebhookConsumerEvent::Webhooks(webhook_event()),- 0,- 0,- )- .await- .expect("succeeds once the transient failure clears");+ ingest_with_retry(&service, &declared_webhook_event(), 0, 0)+ .await+ .expect("succeeds once the transient failure clears");assert_eq!(attempts.load(Ordering::SeqCst), 3);}@@ -185,14 +210,9 @@ async fn retries_transient_webhook_failures_until_success() {async fn exhausted_transient_retries_bubble_up_for_redelivery() {let (service, attempts) = flaky_service(u32::MAX, true);- ingest_with_retry(- &service,- &WebhookConsumerEvent::Webhooks(webhook_event()),- 0,- 0,- )- .await- .expect_err("persistent transient failure aborts the consumer without committing");+ ingest_with_retry(&service, &declared_webhook_event(), 0, 0)+ .await+ .expect_err("persistent transient failure aborts the consumer without committing");assert_eq!(attempts.load(Ordering::SeqCst), MAX_INGEST_ATTEMPTS);}@@ -201,14 +221,9 @@ async fn exhausted_transient_retries_bubble_up_for_redelivery() {async fn permanent_webhook_failures_are_commit_safe_without_retry() {let (service, attempts) = flaky_service(u32::MAX, false);- ingest_with_retry(- &service,- &WebhookConsumerEvent::Webhooks(webhook_event()),- 0,- 0,- )- .await- .expect("permanent failures are skipped so the offset commits");+ ingest_with_retry(&service, &declared_webhook_event(), 0, 0)+ .await+ .expect("permanent failures are skipped so the offset commits");assert_eq!(attempts.load(Ordering::SeqCst), 1);}diff --git a/tooling/xtask/crates/xtask_kafka_topics/src/main.rs b/tooling/xtask/crates/xtask_kafka_topics/src/main.rsindex b6f6d57dfd..b2b6db9777 100644--- a/tooling/xtask/crates/xtask_kafka_topics/src/main.rs+++ b/tooling/xtask/crates/xtask_kafka_topics/src/main.rs@@ -27,8 +27,10 @@ fn run(check: bool) -> Result<()> {// Sorted for a deterministic file regardless of declaration order. A bare// array (no disclaimer wrapper) so infra can parse the file directly.- let topics: BTreeSet<&'static str> =- macro_event_topics::all_topic_names().into_iter().collect();+ let topics: BTreeSet<&'static str> = macro_event_topics::all_topic_names()+ .iter()+ .copied()+ .collect();let new_contents =serde_json::to_string_pretty(&topics).context("serializing kafka topics")? + "\n";