Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/feat/strong consumer typing (#5092)

Sean Ayef222389a1f0d
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 6e08fc8a16..cd957cb0d2 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -58,6 +58,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -143,6 +144,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -220,6 +222,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -364,6 +367,7 @@
"crates/invite_email",
"crates/ip_extractor",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/loops_client",
@@ -495,6 +499,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -590,6 +595,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -673,6 +679,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -769,6 +776,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -866,6 +874,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -938,6 +947,7 @@
"crates/graphql_soup",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -1142,6 +1152,7 @@
"crates/invite_email",
"crates/ip_extractor",
"crates/item_filters",
+ "crates/kafka_util",
"crates/lambda_client",
"crates/last_online_tracker",
"crates/macro_auth",
@@ -1298,6 +1309,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -1382,6 +1394,7 @@
"crates/invite_email",
"crates/ip_extractor",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -1507,7 +1520,7 @@
"crates/invite_email",
"crates/ip_extractor",
"crates/item_filters",
- "crates/kafka_consumer_util",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -1677,6 +1690,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
@@ -1760,6 +1774,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
@@ -1833,6 +1848,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/lambda_client",
"crates/last_online_tracker",
"crates/macro_auth",
@@ -1910,6 +1926,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -2001,6 +2018,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -2062,6 +2080,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -2129,6 +2148,7 @@
"crates/invite_email",
"crates/ip_extractor",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -2205,6 +2225,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -2259,6 +2280,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -2515,6 +2537,7 @@
"crates/github",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
@@ -2619,6 +2642,7 @@
"crates/graphql_email",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -2755,6 +2779,7 @@
"crates/graphql_soup",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -2885,12 +2910,10 @@
"crates/non_empty",
"crates/workspace-hack"
],
- "kafka_consumer_util": [
- "crates/kafka_consumer_util",
+ "kafka_util": [
+ "crates/kafka_util",
"crates/macro_env",
"crates/macro_env_var",
- "crates/macro_event_broker",
- "crates/macro_event_topics",
"crates/workspace-hack"
],
"lambda_client": [
@@ -2930,6 +2953,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
@@ -3003,6 +3027,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -3180,6 +3205,7 @@
"crates/workspace-hack"
],
"macro_event_broker": [
+ "crates/kafka_util",
"crates/macro_env",
"crates/macro_env_var",
"crates/macro_event_broker",
@@ -3359,6 +3385,7 @@
"crates/fusionauth",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -3462,6 +3489,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -3684,6 +3712,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -3764,6 +3793,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -3833,6 +3863,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -4054,6 +4085,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -4115,6 +4147,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -4177,6 +4210,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -4396,6 +4430,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -4495,6 +4530,7 @@
"crates/invite_email",
"crates/ip_extractor",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
@@ -4581,6 +4617,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -4661,6 +4698,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -4729,6 +4767,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -4805,6 +4844,7 @@
"crates/fusionauth",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
@@ -4922,6 +4962,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -4995,7 +5036,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
- "crates/kafka_consumer_util",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
@@ -5073,6 +5114,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -5257,6 +5299,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
@@ -5332,6 +5375,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -5413,6 +5457,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -5474,6 +5519,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
@@ -5585,7 +5631,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
- "crates/kafka_consumer_util",
+ "crates/kafka_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
diff --git a/Cargo.lock b/Cargo.lock
index f87c7393e8..dd357052a1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2608,9 +2608,9 @@ dependencies = [
"http-body-util",
"indexmap 2.14.0",
"item_filters",
+ "kafka_util",
"macro_authorization",
"macro_db_migrator",
- "macro_env",
"macro_env_var",
"macro_event_broker",
"macro_event_topics",
@@ -4443,12 +4443,12 @@ dependencies = [
"hex",
"http-body-util",
"jsonwebtoken 10.3.0",
+ "kafka_util",
"lexical_client",
"macro_authorization",
"macro_aws_config",
"macro_db_migrator",
"macro_entrypoint",
- "macro_env",
"macro_env_var",
"macro_event_broker",
"macro_event_topics",
@@ -7540,14 +7540,18 @@ dependencies = [
]
[[package]]
-name = "kafka_consumer_util"
+name = "kafka_util"
version = "0.1.0"
dependencies = [
+ "aws-msk-iam-sasl-signer",
+ "aws-types",
"either",
"macro_env",
- "macro_event_broker",
+ "macro_env_var",
"rdkafka",
"thiserror 2.0.18",
+ "tokio",
+ "tracing",
"uuid",
"workspace-hack",
]
@@ -8369,10 +8373,7 @@ dependencies = [
name = "macro_event_broker"
version = "0.1.0"
dependencies = [
- "aws-msk-iam-sasl-signer",
- "aws-types",
- "macro_env",
- "macro_env_var",
+ "kafka_util",
"macro_event_topics",
"rdkafka",
"rootcause",
@@ -11045,10 +11046,10 @@ dependencies = [
"futures",
"hex",
"http-body-util",
+ "kafka_util",
"macro_authorization",
"macro_aws_config",
"macro_db_migrator",
- "macro_env",
"macro_env_var",
"macro_event_broker",
"macro_event_topics",
@@ -13407,7 +13408,7 @@ dependencies = [
"documents",
"entity_access",
"futures",
- "kafka_consumer_util",
+ "kafka_util",
"macro_event_broker",
"macro_event_topics",
"macro_user_id",
@@ -15555,7 +15556,7 @@ dependencies = [
"futures",
"hex",
"hmac 0.12.1",
- "kafka_consumer_util",
+ "kafka_util",
"macro_authorization",
"macro_db_migrator",
"macro_event_broker",
diff --git a/crates/bots/Cargo.toml b/crates/bots/Cargo.toml
index 6c9ae8c6b7..5278a10216 100644
--- a/crates/bots/Cargo.toml
+++ b/crates/bots/Cargo.toml
@@ -19,7 +19,7 @@ inbound = [
"ports",
]
outbound = ["dep:sqlx", "dep:macro_uuid", "ports"]
-ports = ["dep:tokio", "macro_event_broker/ports"]
+ports = ["dep:tokio"]
[dependencies]
anyhow = { workspace = true }
diff --git a/crates/bots/src/domain/events/test.rs b/crates/bots/src/domain/events/test.rs
index c192d0104c..4cdec1cdfb 100644
--- a/crates/bots/src/domain/events/test.rs
+++ b/crates/bots/src/domain/events/test.rs
@@ -1,6 +1,5 @@
use chrono::{DateTime, Utc};
use macro_event_broker::{Event, MacroEvent};
-use macro_event_topics::Topic;
use macro_user_id::user_id::MacroUserIdStr;
use serde_json::{Value, json};
use uuid::Uuid;
@@ -165,7 +164,7 @@ fn constructors_use_bot_topic_bare_uuid_key_and_schema_version_one() {
for (event, expected_event_type) in cases.into_iter().zip(expected_event_types) {
assert_eq!(event.key(), BOT_ID);
assert!(!event.key().starts_with("bot|"));
- assert_eq!(event.topic().as_str(), "macro.bots");
+ assert_eq!(event.topic(), "macro.bots");
assert_eq!(event.event().schema_version, 1);
assert_eq!(
serde_json::to_value(event.event()).expect("serializable event")["event_type"],
@@ -188,7 +187,7 @@ fn every_event_variant_round_trips() {
assert_eq!(decoded.key(), BOT_ID);
assert_eq!(decoded.event(), original.event());
- assert_eq!(decoded.topic().as_str(), "macro.bots");
+ assert_eq!(decoded.topic(), "macro.bots");
}
}
diff --git a/crates/bots/src/outbound/pg_bots_repo/tests.rs b/crates/bots/src/outbound/pg_bots_repo/tests.rs
index 1ddbd7b913..dff2f252be 100644
--- a/crates/bots/src/outbound/pg_bots_repo/tests.rs
+++ b/crates/bots/src/outbound/pg_bots_repo/tests.rs
@@ -8,9 +8,7 @@ use crate::domain::{
service::BotServiceImpl,
};
use macro_db_migrator::MACRO_DB_MIGRATIONS;
-use macro_event_broker::{
- EventBrokerError, MacroEvent, MacroEventBroker, NoopMacroEventBroker, Topic as _,
-};
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, NoopMacroEventBroker};
use serde_json::{Value, json};
use sqlx::PgPool;
use std::sync::{Arc, Mutex};
@@ -95,7 +93,7 @@ impl MacroEventBroker for RecordingEventBroker {
.lock()
.expect("event lock poisoned")
.push(PublishedEvent {
- topic: event.topic().as_str(),
+ topic: event.topic(),
key: event.key().to_string(),
payload: serde_json::to_value(event.event())?,
});
diff --git a/crates/channels/Cargo.toml b/crates/channels/Cargo.toml
index a5baf55dce..eb713994bd 100644
--- a/crates/channels/Cargo.toml
+++ b/crates/channels/Cargo.toml
@@ -63,7 +63,7 @@ outbound = [
"macro_user_id/sqlx",
"ports",
]
-ports = ["dep:tokio", "macro_event_broker/ports"]
+ports = ["dep:tokio"]
[dependencies]
anyhow = { workspace = true }
@@ -120,8 +120,8 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
http-body-util = { workspace = true }
+kafka_util = { path = "../kafka_util" }
macro_db_migrator = { path = "../macro_db_migrator" }
-macro_env = { path = "../macro_env" }
macro_env_var = { path = "../macro_env_var" }
# Examples need the Kafka adapter (outbound); the library itself only uses ports.
macro_event_broker = { path = "../macro_event_broker" }
diff --git a/crates/channels/examples/channels_consumer.rs b/crates/channels/examples/channels_consumer.rs
index f12dbbfd10..54667b9633 100644
--- a/crates/channels/examples/channels_consumer.rs
+++ b/crates/channels/examples/channels_consumer.rs
@@ -3,8 +3,8 @@
//! Connects with the same `KAFKA_BROKERS` env var and environment-driven
//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for
//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),
-//! subscribes to [`MacroChannelsTopic`], and prints every channel event it
-//! receives.
+//! subscribes to the [`ChannelMacroEvent`] topic and prints every channel event
+//! it receives.
//!
//! The consumer follows the standard poll-loop / worker split: a Kafka poll
//! loop decodes messages and hands them to a processing task over a bounded
@@ -25,14 +25,12 @@
use anyhow::Context as _;
use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};
-use macro_env::Environment;
-use macro_event_broker::outbound::msk_iam::configure_sasl_iam;
-use macro_event_broker::{EventBrokerError, MacroEvent, MskIamClientContext, Topic as _};
-use macro_event_topics::MacroChannelsTopic;
-use rdkafka::ClientConfig;
-use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};
-use rdkafka::error::KafkaResult;
-use rdkafka::message::{BorrowedMessage, Message};
+use kafka_util::{GroupName, KafkaEventConsumer};
+use macro_event_broker::{
+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
+};
+use rdkafka::consumer::CommitMode;
+use rdkafka::message::Message;
use tokio::sync::mpsc;
macro_env_var::env_var! {
@@ -41,103 +39,28 @@ macro_env_var::env_var! {
}
}
-/// Consumer group id for this example. Offsets are committed under this group,
-/// so re-runs resume where the previous run left off.
-const GROUP_ID: &str = "channels-consumer-example";
+/// Consumer group for this example. Offsets are committed under this group, so
+/// re-runs resume where the previous run left off.
+struct ChannelsConsumerGroup;
+
+impl GroupName for ChannelsConsumerGroup {
+ const GROUP_NAME: &'static str = "channels-consumer-example";
+}
/// Bounded capacity of the channel between the poll loop and the processor.
const CHANNEL_CAPACITY: usize = 128;
-/// Consumer-specific event enum for a consumer subscribed to `macro.channels`,
-/// following the pattern in `macro_event_broker/examples/example_event.rs`.
-enum ChannelsConsumerEvent {
- /// Event received on [`MacroChannelsTopic`].
- Channels(ChannelMacroEvent),
-}
+macro_event_broker::declare_topics!(DeclaredMacroEvent: ChannelMacroEvent);
-impl ChannelsConsumerEvent {
- /// Decode one Kafka message into this consumer's event enum.
- fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {
- match topic {
- topic if topic == MacroChannelsTopic.as_str() => {
- Ok(Self::Channels(ChannelMacroEvent::decode(key, payload)?))
- }
- unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),
- }
- }
-}
+type ChannelsKafkaAdapter = KafkaConsumerAdapter<ChannelsConsumerGroup, DeclaredMacroEvent>;
+type ChannelsConsumerService = MacroEventConsumerService<DeclaredMacroEvent, ChannelsKafkaAdapter>;
/// A decoded event plus the Kafka coordinates it came from, as handed from the
/// poll loop to the processor over the channel.
struct ReceivedEvent {
partition: i32,
offset: i64,
- event: ChannelsConsumerEvent,
-}
-
-/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).
-enum ChannelsConsumer {
- /// Unauthenticated plaintext connection (local docker broker).
- Plaintext(StreamConsumer),
- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).
- MskIam(StreamConsumer<MskIamClientContext>),
-}
-
-impl ChannelsConsumer {
- /// Build a consumer for the given brokers, choosing the transport from the
- /// `ENVIRONMENT` variable exactly like the publisher does.
- fn from_env(brokers: &str) -> anyhow::Result<Self> {
- let mut config = ClientConfig::new();
- config
- .set("bootstrap.servers", brokers)
- .set("group.id", GROUP_ID)
- // Offsets are committed manually after a successful handoff.
- .set("enable.auto.commit", "false")
- // Start from the beginning of the topic on the first ever run.
- .set("auto.offset.reset", "earliest");
-
- let consumer = match Environment::new_or_prod() {
- Environment::Local => Self::Plaintext(
- config
- .create()
- .context("failed to create plaintext kafka consumer")?,
- ),
- Environment::Develop | Environment::Production => {
- let config = configure_sasl_iam(config);
- Self::MskIam(
- config
- .create_with_context(MskIamClientContext::from_env())
- .context("failed to create MSK IAM kafka consumer")?,
- )
- }
- };
-
- Ok(consumer)
- }
-
- fn subscribe(&self) -> KafkaResult<()> {
- let topics = [MacroChannelsTopic.as_str()];
- match self {
- Self::Plaintext(consumer) => consumer.subscribe(&topics),
- Self::MskIam(consumer) => consumer.subscribe(&topics),
- }
- }
-
- /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it
- /// can sit in a `select!` without losing messages.
- async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {
- match self {
- Self::Plaintext(consumer) => consumer.recv().await,
- Self::MskIam(consumer) => consumer.recv().await,
- }
- }
-
- fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {
- match self {
- Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),
- Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),
- }
- }
+ event: ChannelMacroEvent,
}
fn event_type(event: &ChannelTopicEvent) -> &'static str {
@@ -159,7 +82,7 @@ fn event_type(event: &ChannelTopicEvent) -> &'static str {
/// the body for indexing, notifications, etc.
async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {
while let Some(received) = events.recv().await {
- let ChannelsConsumerEvent::Channels(event) = received.event;
+ let event = received.event;
let envelope = event.event();
println!(
@@ -192,14 +115,16 @@ async fn main() -> anyhow::Result<()> {
let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;
- let consumer = ChannelsConsumer::from_env(env.kafka_brokers.as_ref())?;
- consumer
- .subscribe()
- .context("failed to subscribe to channels topic")?;
+ let consumer =
+ KafkaEventConsumer::<ChannelsConsumerGroup>::from_env(env.kafka_brokers.as_ref())?;
+ let consumer = KafkaConsumerAdapter::<ChannelsConsumerGroup, ()>::new(consumer)
+ .subscribe::<DeclaredMacroEvent>()
+ .map_err(|error| anyhow::anyhow!("failed to subscribe to channels topic: {error:?}"))?;
+ let consumer = ChannelsConsumerService::new(consumer);
println!(
"listening on topic={} group={} brokers={} (ctrl-c to stop)",
- MacroChannelsTopic.as_str(),
- GROUP_ID,
+ DeclaredMacroEvent::topics()[0],
+ ChannelsConsumerGroup::GROUP_NAME,
env.kafka_brokers.as_ref(),
);
@@ -222,25 +147,13 @@ async fn main() -> anyhow::Result<()> {
}
};
- let Some(payload) = message.payload() else {
- eprintln!(
- "skipping message with empty payload at partition {} offset {}",
- message.partition(),
- message.offset(),
- );
- let _ = consumer.commit(&message).inspect_err(|e| {
- eprintln!("failed to commit offset {}: {e}", message.offset());
- });
- continue;
- };
- let key = String::from_utf8_lossy(message.key().unwrap_or_default());
-
- match ChannelsConsumerEvent::decode(message.topic(), key.as_ref(), payload) {
- Ok(event) => {
+ let kafka_message = message.inner();
+ match message.decode_payload() {
+ Ok(DeclaredMacroEvent::ChannelMacroEvent(event)) => {
events_tx
.send(ReceivedEvent {
- partition: message.partition(),
- offset: message.offset(),
+ partition: kafka_message.partition(),
+ offset: kafka_message.offset(),
event,
})
.await
@@ -250,16 +163,19 @@ async fn main() -> anyhow::Result<()> {
// wedging the partition on a poison message.
Err(e) => eprintln!(
"failed to decode message at partition {} offset {}: {e}",
- message.partition(),
- message.offset(),
+ kafka_message.partition(),
+ kafka_message.offset(),
),
}
// Commit only after the event was handed to the processor:
// at-least-once delivery (up to the channel buffer on a crash).
- let _ = consumer.commit(&message).inspect_err(|e| {
- eprintln!("failed to commit offset {}: {e}", message.offset());
- });
+ let _ = consumer
+ .inner()
+ .commit_message(kafka_message, CommitMode::Async)
+ .inspect_err(|e| {
+ eprintln!("failed to commit offset {}: {e:?}", kafka_message.offset());
+ });
}
}
}
diff --git a/crates/channels/src/domain/side_effects/test.rs b/crates/channels/src/domain/side_effects/test.rs
index 34fa4a7348..79344c0884 100644
--- a/crates/channels/src/domain/side_effects/test.rs
+++ b/crates/channels/src/domain/side_effects/test.rs
@@ -1053,9 +1053,8 @@ impl MacroEventBroker for TestEventBroker {
tokio::task::JoinHandle<Result<(), macro_event_broker::EventBrokerError>>,
macro_event_broker::EventBrokerError,
> {
- use macro_event_topics::Topic as _;
self.published.lock().unwrap().push(PublishedEvent {
- topic: event.topic().as_str().to_string(),
+ topic: event.topic().to_string(),
key: event.key().to_string(),
envelope: serde_json::to_value(event.event())?,
});
diff --git a/crates/documents/Cargo.toml b/crates/documents/Cargo.toml
index ae52bac348..224bed4553 100644
--- a/crates/documents/Cargo.toml
+++ b/crates/documents/Cargo.toml
@@ -63,12 +63,7 @@ outbound = [
"ports",
]
ports = ["dep:tokio"]
-service = [
- "dep:entity_access_management",
- "dep:foreign_entity",
- "macro_event_broker/ports",
- "ports",
-]
+service = ["dep:entity_access_management", "dep:foreign_entity", "ports"]
[dependencies]
# Core
@@ -155,8 +150,8 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }
dotenvy = { workspace = true }
embedding = { path = "../embedding" }
http-body-util = { workspace = true }
+kafka_util = { path = "../kafka_util" }
macro_db_migrator = { path = "../macro_db_migrator" }
-macro_env = { path = "../macro_env" }
# Examples need the Kafka adapter (outbound); the library itself only uses ports.
macro_event_broker = { path = "../macro_event_broker" }
mockall = { workspace = true }
diff --git a/crates/documents/examples/documents_consumer.rs b/crates/documents/examples/documents_consumer.rs
index 66259c1317..2b9ad05229 100644
--- a/crates/documents/examples/documents_consumer.rs
+++ b/crates/documents/examples/documents_consumer.rs
@@ -3,8 +3,8 @@
//! Connects with the same `KAFKA_BROKERS` env var and environment-driven
//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for
//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),
-//! subscribes to [`MacroDocumentsTopic`], and prints every document lifecycle
-//! event it receives.
+//! subscribes to the [`DocumentMacroEvent`] topic and prints every document
+//! lifecycle event it receives.
//!
//! The consumer follows the standard poll-loop / worker split: a Kafka poll
//! loop decodes messages and hands them to a processing task over a bounded
@@ -25,14 +25,12 @@
use anyhow::Context as _;
use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};
-use macro_env::Environment;
-use macro_event_broker::outbound::msk_iam::configure_sasl_iam;
-use macro_event_broker::{EventBrokerError, MacroEvent, MskIamClientContext, Topic as _};
-use macro_event_topics::MacroDocumentsTopic;
-use rdkafka::ClientConfig;
-use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};
-use rdkafka::error::KafkaResult;
-use rdkafka::message::{BorrowedMessage, Message};
+use kafka_util::{GroupName, KafkaEventConsumer};
+use macro_event_broker::{
+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
+};
+use rdkafka::consumer::CommitMode;
+use rdkafka::message::Message;
use tokio::sync::mpsc;
macro_env_var::env_var! {
@@ -41,103 +39,29 @@ macro_env_var::env_var! {
}
}
-/// Consumer group id for this example. Offsets are committed under this group,
-/// so re-runs resume where the previous run left off.
-const GROUP_ID: &str = "documents-consumer-example";
+/// Consumer group for this example. Offsets are committed under this group, so
+/// re-runs resume where the previous run left off.
+struct DocumentsConsumerGroup;
+
+impl GroupName for DocumentsConsumerGroup {
+ const GROUP_NAME: &'static str = "documents-consumer-example";
+}
/// Bounded capacity of the channel between the poll loop and the processor.
const CHANNEL_CAPACITY: usize = 128;
-/// Consumer-specific event enum for a consumer subscribed to `macro.documents`,
-/// following the pattern in `macro_event_broker/examples/example_event.rs`.
-enum DocumentsConsumerEvent {
- /// Event received on [`MacroDocumentsTopic`].
- Documents(DocumentMacroEvent),
-}
+macro_event_broker::declare_topics!(DeclaredMacroEvent: DocumentMacroEvent);
-impl DocumentsConsumerEvent {
- /// Decode one Kafka message into this consumer's event enum.
- fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {
- match topic {
- topic if topic == MacroDocumentsTopic.as_str() => {
- Ok(Self::Documents(DocumentMacroEvent::decode(key, payload)?))
- }
- unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),
- }
- }
-}
+type DocumentsKafkaAdapter = KafkaConsumerAdapter<DocumentsConsumerGroup, DeclaredMacroEvent>;
+type DocumentsConsumerService =
+ MacroEventConsumerService<DeclaredMacroEvent, DocumentsKafkaAdapter>;
/// A decoded event plus the Kafka coordinates it came from, as handed from the
/// poll loop to the processor over the channel.
struct ReceivedEvent {
partition: i32,
offset: i64,
- event: DocumentsConsumerEvent,
-}
-
-/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).
-enum DocumentsConsumer {
- /// Unauthenticated plaintext connection (local docker broker).
- Plaintext(StreamConsumer),
- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).
- MskIam(StreamConsumer<MskIamClientContext>),
-}
-
-impl DocumentsConsumer {
- /// Build a consumer for the given brokers, choosing the transport from the
- /// `ENVIRONMENT` variable exactly like the publisher does.
- fn from_env(brokers: &str) -> anyhow::Result<Self> {
- let mut config = ClientConfig::new();
- config
- .set("bootstrap.servers", brokers)
- .set("group.id", GROUP_ID)
- // Offsets are committed manually after a successful handoff.
- .set("enable.auto.commit", "false")
- // Start from the beginning of the topic on the first ever run.
- .set("auto.offset.reset", "earliest");
-
- let consumer = match Environment::new_or_prod() {
- Environment::Local => Self::Plaintext(
- config
- .create()
- .context("failed to create plaintext kafka consumer")?,
- ),
- Environment::Develop | Environment::Production => {
- let config = configure_sasl_iam(config);
- Self::MskIam(
- config
- .create_with_context(MskIamClientContext::from_env())
- .context("failed to create MSK IAM kafka consumer")?,
- )
- }
- };
-
- Ok(consumer)
- }
-
- fn subscribe(&self) -> KafkaResult<()> {
- let topics = [MacroDocumentsTopic.as_str()];
- match self {
- Self::Plaintext(consumer) => consumer.subscribe(&topics),
- Self::MskIam(consumer) => consumer.subscribe(&topics),
- }
- }
-
- /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it
- /// can sit in a `select!` without losing messages.
- async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {
- match self {
- Self::Plaintext(consumer) => consumer.recv().await,
- Self::MskIam(consumer) => consumer.recv().await,
- }
- }
-
- fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {
- match self {
- Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),
- Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),
- }
- }
+ event: DocumentMacroEvent,
}
fn event_type(event: &DocumentTopicEvent) -> &'static str {
@@ -154,7 +78,7 @@ fn event_type(event: &DocumentTopicEvent) -> &'static str {
/// the body for indexing, notifications, etc.
async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {
while let Some(received) = events.recv().await {
- let DocumentsConsumerEvent::Documents(event) = received.event;
+ let event = received.event;
let envelope = event.event();
println!(
@@ -187,14 +111,16 @@ async fn main() -> anyhow::Result<()> {
let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;
- let consumer = DocumentsConsumer::from_env(env.kafka_brokers.as_ref())?;
- consumer
- .subscribe()
- .context("failed to subscribe to documents topic")?;
+ let consumer =
+ KafkaEventConsumer::<DocumentsConsumerGroup>::from_env(env.kafka_brokers.as_ref())?;
+ let consumer = KafkaConsumerAdapter::<DocumentsConsumerGroup, ()>::new(consumer)
+ .subscribe::<DeclaredMacroEvent>()
+ .map_err(|error| anyhow::anyhow!("failed to subscribe to documents topic: {error:?}"))?;
+ let consumer = DocumentsConsumerService::new(consumer);
println!(
"listening on topic={} group={} brokers={} (ctrl-c to stop)",
- MacroDocumentsTopic.as_str(),
- GROUP_ID,
+ DeclaredMacroEvent::topics()[0],
+ DocumentsConsumerGroup::GROUP_NAME,
env.kafka_brokers.as_ref(),
);
@@ -217,25 +143,13 @@ async fn main() -> anyhow::Result<()> {
}
};
- let Some(payload) = message.payload() else {
- eprintln!(
- "skipping message with empty payload at partition {} offset {}",
- message.partition(),
- message.offset(),
- );
- let _ = consumer.commit(&message).inspect_err(|e| {
- eprintln!("failed to commit offset {}: {e}", message.offset());
- });
- continue;
- };
- let key = String::from_utf8_lossy(message.key().unwrap_or_default());
-
- match DocumentsConsumerEvent::decode(message.topic(), key.as_ref(), payload) {
- Ok(event) => {
+ let kafka_message = message.inner();
+ match message.decode_payload() {
+ Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => {
events_tx
.send(ReceivedEvent {
- partition: message.partition(),
- offset: message.offset(),
+ partition: kafka_message.partition(),
+ offset: kafka_message.offset(),
event,
})
.await
@@ -245,16 +159,19 @@ async fn main() -> anyhow::Result<()> {
// wedging the partition on a poison message.
Err(e) => eprintln!(
"failed to decode message at partition {} offset {}: {e}",
- message.partition(),
- message.offset(),
+ kafka_message.partition(),
+ kafka_message.offset(),
),
}
// Commit only after the event was handed to the processor:
// at-least-once delivery (up to the channel buffer on a crash).
- let _ = consumer.commit(&message).inspect_err(|e| {
- eprintln!("failed to commit offset {}: {e}", message.offset());
- });
+ let _ = consumer
+ .inner()
+ .commit_message(kafka_message, CommitMode::Async)
+ .inspect_err(|e| {
+ eprintln!("failed to commit offset {}: {e:?}", kafka_message.offset());
+ });
}
}
}
diff --git a/crates/documents/src/domain/service/tests.rs b/crates/documents/src/domain/service/tests.rs
index cfd167332f..35c82e391d 100644
--- a/crates/documents/src/domain/service/tests.rs
+++ b/crates/documents/src/domain/service/tests.rs
@@ -3,7 +3,7 @@ use foreign_entity::domain::models::{
CreateForeignEntity, ForeignEntity, ForeignEntityError, PatchForeignEntity, SourceId,
};
use foreign_entity::domain::ports::{ForeignEntityListQuery, ForeignEntityService};
-use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, Topic as _};
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
use macro_user_id::cowlike::CowLike;
use model::document::DocumentMetadata;
use std::sync::{Arc, Mutex};
@@ -350,7 +350,7 @@ impl MacroEventBroker for TestEventBroker {
event: &E,
) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {
self.published.lock().unwrap().push(PublishedEvent {
- topic: event.topic().as_str(),
+ topic: event.topic(),
key: event.key().to_string(),
payload: serde_json::to_value(event.event())?,
});
diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.toml
index 0919f65269..a616219a97 100644
--- a/crates/email/Cargo.toml
+++ b/crates/email/Cargo.toml
@@ -37,7 +37,7 @@ gmail_token = ["dep:authentication_service_client", "dep:redis", "ports"]
http_client = ["dep:reqwest", "ports"]
inbound = ["ports"]
outbound = ["dep:sqlx", "dep:email_db_client", "ports"]
-ports = ["crm/ports", "dep:tokio", "frecency/ports", "macro_event_broker/ports"]
+ports = ["crm/ports", "dep:tokio", "frecency/ports"]
[dependencies]
ai_toolset = { path = "../ai_toolset", optional = true }
@@ -92,9 +92,6 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
crm = { path = "../crm", features = ["outbound"] }
frecency = { path = "../frecency", features = ["outbound", "postgres"] }
-macro_event_broker = { path = "../macro_event_broker", default-features = false, features = [
- "ports",
-] }
macro_db_migrator = { path = "../macro_db_migrator" }
serde_json = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter"] }
diff --git a/crates/kafka_consumer_util/Cargo.toml b/crates/kafka_util/Cargo.toml
similarity index 59%
rename from crates/kafka_consumer_util/Cargo.toml
rename to crates/kafka_util/Cargo.toml
index dbdf5d7356..882a4f17b9 100644
--- a/crates/kafka_consumer_util/Cargo.toml
+++ b/crates/kafka_util/Cargo.toml
@@ -1,16 +1,18 @@
[package]
edition = "2024"
-name = "kafka_consumer_util"
+name = "kafka_util"
publish = false
version = "0.1.0"
[dependencies]
+aws-msk-iam-sasl-signer = { workspace = true }
+aws-types = { workspace = true }
either = { workspace = true }
macro_env = { path = "../macro_env" }
-macro_event_broker = { path = "../macro_event_broker", default-features = false, features = [
- "kafka",
-] }
+macro_env_var = { path = "../macro_env_var" }
rdkafka = { workspace = true }
thiserror = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/kafka_consumer_util/README.md b/crates/kafka_util/README.md
similarity index 55%
rename from crates/kafka_consumer_util/README.md
rename to crates/kafka_util/README.md
index 6373185896..f571e2f866 100644
--- a/crates/kafka_consumer_util/README.md
+++ b/crates/kafka_util/README.md
@@ -1,10 +1,10 @@
-# kafka_consumer_util
+# kafka_util
-Shared environment-aware Kafka consumer transport for backend inbound adapters.
+Shared environment-aware Kafka producer and consumer transports for backend adapters.
-The crate centralizes plaintext-local versus MSK-IAM construction and exposes two type-safe consumption modes:
+The crate centralizes plaintext-local versus MSK-IAM construction, provides a delivery-confirmed event producer, and exposes two type-safe consumption modes:
- Named consumer groups implement `GroupName` and may subscribe, pause, receive, and commit.
- `Ungrouped` consumers use manual partition assignment and may pause and receive, but cannot subscribe or commit.
-Ungrouped consumers assign every current partition at an explicit earliest or latest offset. Application adapters remain responsible for decoding, retries, poison-message handling, and refreshing assignments if topic partition counts change.
+Ungrouped consumers assign every current partition at an explicit earliest or latest offset. Application adapters remain responsible for domain mapping, decoding, retries, poison-message handling, and refreshing assignments if topic partition counts change.
diff --git a/crates/kafka_consumer_util/src/lib.rs b/crates/kafka_util/src/lib.rs
similarity index 71%
rename from crates/kafka_consumer_util/src/lib.rs
rename to crates/kafka_util/src/lib.rs
index 1331329a39..2db0ad1b58 100644
--- a/crates/kafka_consumer_util/src/lib.rs
+++ b/crates/kafka_util/src/lib.rs
@@ -1,7 +1,7 @@
#![deny(missing_docs)]
-//! Environment-aware Kafka consumer transport.
+//! Environment-aware Kafka producer and consumer transports.
//!
-//! This adapter centralizes the repository's plaintext-local versus MSK-IAM
+//! These adapters centralize the repository's plaintext-local versus MSK-IAM
//! transport setup. Application inbound adapters retain responsibility for
//! decoding, retries, poison-message handling, and deciding when to commit.
@@ -16,12 +16,17 @@ use macro_env::Environment;
use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};
use rdkafka::error::{KafkaError, KafkaResult};
use rdkafka::message::{BorrowedMessage, Message as _};
+use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::{ClientConfig, Offset, TopicPartitionList};
use uuid::Uuid;
-use macro_event_broker::kafka::msk_iam::{MskIamClientContext, configure_sasl_iam};
+pub use msk_iam::{MskIamClientContext, configure_sasl_iam};
+
+mod msk_iam;
const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";
+const MESSAGE_TIMEOUT_MS: &str = "5000";
+const SEND_TIMEOUT: Duration = Duration::from_secs(5);
/// Failure to construct an environment-specific Kafka consumer.
#[derive(Debug, thiserror::Error)]
@@ -34,9 +39,24 @@ pub enum KafkaConsumerError {
MskIam(#[source] KafkaError),
}
+/// Failure to construct an environment-specific Kafka producer.
+#[derive(Debug, thiserror::Error)]
+pub enum KafkaProducerError {
+ /// Failed to create the unauthenticated local producer.
+ #[error("failed to create plaintext Kafka producer")]
+ Plaintext(#[source] KafkaError),
+ /// Failed to create the TLS and MSK-IAM authenticated producer.
+ #[error("failed to create MSK IAM Kafka producer")]
+ MskIam(#[source] KafkaError),
+}
+
/// Underlying Kafka consumer transport selected from the runtime environment.
struct ConsumerTransport(Either<StreamConsumer, StreamConsumer<MskIamClientContext>>);
+/// Underlying Kafka producer transport selected from the runtime environment.
+#[derive(Clone)]
+struct ProducerTransport(Either<FutureProducer, FutureProducer<MskIamClientContext>>);
+
/// Type-level name for a durable Kafka consumer group.
///
/// Defining group names on marker types keeps group identities centralized and
@@ -81,16 +101,32 @@ pub struct KafkaEventConsumer<T> {
marker: PhantomData<T>,
}
+/// Shared Kafka producer with environment-aware transport.
+#[derive(Clone)]
+pub struct KafkaEventProducer {
+ producer: ProducerTransport,
+}
+
fn base_config(brokers: &str) -> ClientConfig {
let mut config = ClientConfig::new();
+ config.set("bootstrap.servers", brokers);
config
- .set("bootstrap.servers", brokers)
- .set("enable.auto.commit", "false");
+}
+
+fn consumer_config(brokers: &str) -> ClientConfig {
+ let mut config = base_config(brokers);
+ config.set("enable.auto.commit", "false");
config
}
-fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {
+fn producer_config(brokers: &str) -> ClientConfig {
let mut config = base_config(brokers);
+ config.set("message.timeout.ms", MESSAGE_TIMEOUT_MS);
+ config
+}
+
+fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {
+ let mut config = consumer_config(brokers);
config
.set("group.id", T::GROUP_NAME)
.set("auto.offset.reset", "earliest");
@@ -98,7 +134,7 @@ fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {
}
fn ungrouped_config(brokers: &str) -> ClientConfig {
- let mut config = base_config(brokers);
+ let mut config = consumer_config(brokers);
let group_id = format!("{UNGROUPED_GROUP_PREFIX}-{}", Uuid::new_v4());
config
.set("group.id", group_id)
@@ -106,7 +142,9 @@ fn ungrouped_config(brokers: &str) -> ClientConfig {
config
}
-fn create_from_env<T>(config: ClientConfig) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {
+fn create_consumer_from_env<T>(
+ config: ClientConfig,
+) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {
let consumer = match Environment::new_or_prod() {
Environment::Local => Either::Left(config.create().map_err(KafkaConsumerError::Plaintext)?),
Environment::Develop | Environment::Production => {
@@ -125,6 +163,26 @@ fn create_from_env<T>(config: ClientConfig) -> Result<KafkaEventConsumer<T>, Kaf
})
}
+fn create_producer_from_env(
+ config: ClientConfig,
+) -> Result<KafkaEventProducer, KafkaProducerError> {
+ let producer = match Environment::new_or_prod() {
+ Environment::Local => Either::Left(config.create().map_err(KafkaProducerError::Plaintext)?),
+ Environment::Develop | Environment::Production => {
+ let config = configure_sasl_iam(config);
+ Either::Right(
+ config
+ .create_with_context(MskIamClientContext::from_env())
+ .map_err(KafkaProducerError::MskIam)?,
+ )
+ }
+ };
+
+ Ok(KafkaEventProducer {
+ producer: ProducerTransport(producer),
+ })
+}
+
fn build_assignment<C, T>(
consumer: &T,
topics: &[&str],
@@ -178,6 +236,25 @@ where
Ok(assignment)
}
+impl KafkaEventProducer {
+ /// Creates a producer, selecting plaintext or MSK IAM transport from the runtime environment.
+ ///
+ /// Producer creation is lazy: no broker connection or IAM token is created
+ /// until a message is sent.
+ pub fn from_env(brokers: &str) -> Result<Self, KafkaProducerError> {
+ create_producer_from_env(producer_config(brokers))
+ }
+
+ /// Sends a keyed payload to `topic` and waits for delivery confirmation.
+ #[tracing::instrument(err, skip(self, payload), fields(topic, key))]
+ pub async fn send(&self, topic: &str, key: &str, payload: &[u8]) -> KafkaResult<()> {
+ let record = FutureRecord::to(topic).key(key).payload(payload);
+ either::for_both!(&self.producer.0, producer => producer.send(record, SEND_TIMEOUT).await)
+ .map(|_| ())
+ .map_err(|(error, _)| error)
+ }
+}
+
impl<T> KafkaEventConsumer<T> {
/// Receives the next Kafka message.
///
@@ -202,7 +279,7 @@ impl<T: GroupName> KafkaEventConsumer<T> {
/// Creates a named-group consumer, selecting plaintext or MSK IAM transport
/// from the runtime environment.
pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {
- create_from_env(grouped_config::<T>(brokers))
+ create_consumer_from_env(grouped_config::<T>(brokers))
}
/// Subscribes the consumer to exactly the provided topics.
@@ -226,7 +303,7 @@ impl KafkaEventConsumer<Ungrouped> {
///
/// Call [`Self::assign_topics`] before receiving messages.
pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {
- create_from_env(ungrouped_config(brokers))
+ create_consumer_from_env(ungrouped_config(brokers))
}
/// Manually assigns every current partition of `topics` at `initial_offset`.
diff --git a/crates/macro_event_broker/src/kafka/msk_iam.rs b/crates/kafka_util/src/msk_iam.rs
similarity index 62%
rename from crates/macro_event_broker/src/kafka/msk_iam.rs
rename to crates/kafka_util/src/msk_iam.rs
index de1d18e2f8..4beb4e67c5 100644
--- a/crates/macro_event_broker/src/kafka/msk_iam.rs
+++ b/crates/kafka_util/src/msk_iam.rs
@@ -15,8 +15,7 @@ macro_env_var::maybe_env_var! {
struct AwsRegion;
}
-/// Apply the TLS + SASL/OAUTHBEARER settings required to authenticate to an
-/// AWS MSK cluster with IAM auth (the `:9098` bootstrap listener).
+/// Applies the TLS and SASL/OAUTHBEARER settings required for MSK IAM auth.
pub fn configure_sasl_iam(mut config: ClientConfig) -> ClientConfig {
config
.set("security.protocol", "SASL_SSL")
@@ -24,18 +23,15 @@ pub fn configure_sasl_iam(mut config: ClientConfig) -> ClientConfig {
config
}
-/// Client context that supplies AWS MSK IAM (SASL/OAUTHBEARER) auth tokens,
-/// signing with the ambient AWS credentials (e.g. the ECS task role).
+/// Client context that supplies AWS MSK IAM authentication tokens.
///
-/// librdkafka invokes [`ClientContext::generate_oauth_token`] when the
-/// connection is first established and again before each token expires.
+/// Tokens are signed with ambient AWS credentials such as an ECS task role.
pub struct MskIamClientContext {
region: Region,
}
impl MskIamClientContext {
- /// Build a context that signs tokens for the region in `AWS_REGION`,
- /// falling back to `us-east-1`.
+ /// Builds a context using `AWS_REGION`, falling back to `us-east-1`.
pub fn from_env() -> Self {
let region = AwsRegion::new()
.and_then(|region| region.value().map(str::to_string))
@@ -56,12 +52,9 @@ impl ClientContext for MskIamClientContext {
) -> Result<OAuthToken, Box<dyn std::error::Error>> {
let region = self.region.clone();
- // librdkafka may invoke this callback on a tokio runtime thread (for a
- // `StreamConsumer`, it fires while `recv()` is polled from async code),
- // where blocking on the ambient runtime panics. Sign on a dedicated
- // thread with its own single-threaded runtime instead, so the callback
- // is safe from any calling context. Tokens live ~15 minutes, so the
- // per-refresh thread + runtime cost is negligible.
+ // librdkafka may invoke this callback on a Tokio runtime thread. Sign
+ // on a dedicated thread with its own runtime so blocking is safe from
+ // every calling context.
let signer = std::thread::spawn(
move || -> Result<(String, i64), Box<dyn std::error::Error + Send + Sync>> {
let runtime = tokio::runtime::Builder::new_current_thread()
@@ -74,9 +67,9 @@ impl ClientContext for MskIamClientContext {
let (token, expiration_time_ms) = signer
.join()
.map_err(|_| "MSK IAM token signer thread panicked")?
- .map_err(|e| -> Box<dyn std::error::Error> { e })
- .inspect_err(|e| {
- tracing::error!(error = %e, "failed to sign MSK IAM auth token");
+ .map_err(|error| -> Box<dyn std::error::Error> { error })
+ .inspect_err(|error| {
+ tracing::error!(error = %error, "failed to sign MSK IAM auth token");
})?;
tracing::info!(expiration_time_ms, "signed MSK IAM auth token");
@@ -89,6 +82,4 @@ impl ClientContext for MskIamClientContext {
}
}
-/// Lets consumers (e.g. `StreamConsumer<MskIamClientContext>`) use this
-/// context; all behavior is the trait's defaults.
impl ConsumerContext for MskIamClientContext {}
diff --git a/crates/kafka_consumer_util/src/test.rs b/crates/kafka_util/src/test.rs
similarity index 79%
rename from crates/kafka_consumer_util/src/test.rs
rename to crates/kafka_util/src/test.rs
index ca3ed674af..fe85196334 100644
--- a/crates/kafka_consumer_util/src/test.rs
+++ b/crates/kafka_util/src/test.rs
@@ -6,6 +6,18 @@ impl GroupName for TestConsumerGroup {
const GROUP_NAME: &'static str = "consumer-group";
}
+#[test]
+fn producer_config_uses_brokers_and_message_timeout() {
+ let config = producer_config("broker-a:9092,broker-b:9092");
+
+ assert_eq!(
+ config.get("bootstrap.servers"),
+ Some("broker-a:9092,broker-b:9092")
+ );
+ assert_eq!(config.get("message.timeout.ms"), Some(MESSAGE_TIMEOUT_MS));
+ assert_eq!(config.get("enable.auto.commit"), None);
+}
+
#[test]
fn grouped_config_uses_named_group_manual_commits_and_earliest_offsets() {
let config = grouped_config::<TestConsumerGroup>("broker-a:9092,broker-b:9092");
diff --git a/crates/macro_event_broker/Cargo.toml b/crates/macro_event_broker/Cargo.toml
index 397b42481c..78cab6a390 100644
--- a/crates/macro_event_broker/Cargo.toml
+++ b/crates/macro_event_broker/Cargo.toml
@@ -5,29 +5,17 @@ publish = false
version = "0.1.0"
[features]
-default = ["outbound", "ports"]
-kafka = [
- "dep:aws-msk-iam-sasl-signer",
- "dep:aws-types",
- "dep:macro_env",
- "dep:macro_env_var",
- "dep:rdkafka",
- "dep:tokio",
-]
-outbound = ["kafka", "ports"]
-ports = ["dep:tokio"]
+default = ["outbound"]
+outbound = ["dep:kafka_util", "dep:rdkafka"]
[dependencies]
-aws-msk-iam-sasl-signer = { workspace = true, optional = true }
-aws-types = { workspace = true, optional = true }
-macro_env = { path = "../macro_env", optional = true }
-macro_env_var = { path = "../macro_env_var", optional = true }
+kafka_util = { path = "../kafka_util", optional = true }
rootcause = { workspace = true }
rdkafka = { workspace = true, optional = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
-tokio = { workspace = true, features = ["macros", "time"], optional = true }
+tokio = { workspace = true, features = ["macros", "time"] }
tracing = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/macro_event_broker/examples/example_event.rs b/crates/macro_event_broker/examples/example_event.rs
index 3f001ae6db..942be611c6 100644
--- a/crates/macro_event_broker/examples/example_event.rs
+++ b/crates/macro_event_broker/examples/example_event.rs
@@ -9,15 +9,10 @@ use serde::{Deserialize, Serialize};
pub struct ExampleEventPublisher;
impl EventPublisher for ExampleEventPublisher {
- async fn publish<T: Topic>(
- &self,
- topic: T,
- key: &str,
- payload: &[u8],
- ) -> Result<(), EventBrokerError> {
+ async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {
println!(
"publishing topic={} key={} payload={}",
- topic.as_str(),
+ T::TOPIC_STR,
key,
String::from_utf8_lossy(payload)
);
@@ -118,7 +113,7 @@ impl ExampleConsumerEvent {
/// Decode one Kafka message into this consumer's event enum.
pub fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {
match topic {
- topic if topic == MacroExampleTopic.as_str() => {
+ topic if topic == MacroExampleTopic::TOPIC_STR => {
Ok(Self::Example(ExampleMacroEvent::decode(key, payload)?))
}
unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),
diff --git a/crates/macro_event_broker/src/domain/mod.rs b/crates/macro_event_broker/src/domain/mod.rs
index 1e30fc3ae2..04c1260629 100644
--- a/crates/macro_event_broker/src/domain/mod.rs
+++ b/crates/macro_event_broker/src/domain/mod.rs
@@ -2,9 +2,7 @@
pub mod models;
/// Port traits: the inbound and outbound boundaries of the broker.
-#[cfg(feature = "ports")]
pub mod ports;
-/// Service orchestration for publishing events.
-#[cfg(feature = "ports")]
+/// Service orchestration for producing and consuming typed events.
pub mod service;
diff --git a/crates/macro_event_broker/src/domain/models.rs b/crates/macro_event_broker/src/domain/models.rs
index 9ac7dd6af4..b84fcf0af1 100644
--- a/crates/macro_event_broker/src/domain/models.rs
+++ b/crates/macro_event_broker/src/domain/models.rs
@@ -1,9 +1,13 @@
//! Domain models for the event broker.
+use std::marker::PhantomData;
+
use macro_event_topics::Topic;
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use uuid::Uuid;
+use crate::domain::ports::{MacroEventCollection, MessageParts};
+
/// Event payload enum for a single [`Topic`].
///
/// Implement this trait for each topic-specific event enum. The enum should use
@@ -64,9 +68,9 @@ impl<E: TopicEvent> Event<E> {
Ok(serde_json::from_slice(payload)?)
}
- /// Kafka topic this event belongs to.
- pub fn topic(&self) -> E::Topic {
- E::Topic::default()
+ /// Broker topic this event belongs to.
+ pub fn topic(&self) -> &'static str {
+ E::Topic::TOPIC_STR
}
}
@@ -79,9 +83,9 @@ pub trait MacroEvent: Send + Sync {
/// Topic-specific event enum carried by this macro event.
type EventPayload: TopicEvent;
- /// Kafka topic this event should be published to.
- fn topic(&self) -> <Self::EventPayload as TopicEvent>::Topic {
- <<Self as MacroEvent>::EventPayload as TopicEvent>::Topic::default()
+ /// Broker topic this event should be published to.
+ fn topic(&self) -> &'static str {
+ <Self::EventPayload as TopicEvent>::Topic::TOPIC_STR
}
/// Kafka message key used for partitioning and compaction.
@@ -110,9 +114,25 @@ pub enum EventBrokerError {
/// The event payload could not be serialized to or deserialized from JSON.
#[error("failed to serialize or deserialize event payload")]
Serialization(#[from] serde_json::Error),
+ /// The consumed broker message did not include a key.
+ #[error("event broker message is missing a key")]
+ MissingMessageKey,
+ /// The consumed broker message did not include a payload.
+ #[error("event broker message is missing a payload")]
+ MissingMessagePayload,
/// The Kafka topic name is not handled by a consumer-specific event enum.
#[error("unknown event topic: {0}")]
UnknownTopic(String),
+ /// The event envelope schema version is not supported by its typed payload.
+ #[error("unsupported schema version {actual} for topic {topic}; expected {expected}")]
+ UnsupportedSchemaVersion {
+ /// Topic carrying the event.
+ topic: &'static str,
+ /// Schema version declared by the typed event payload.
+ expected: u8,
+ /// Schema version found in the decoded envelope.
+ actual: u8,
+ },
/// The broker rejected or failed to deliver the message.
#[error("failed to publish event: {0}")]
Publish(String),
@@ -133,5 +153,105 @@ impl From<rootcause::Report> for EventBrokerError {
}
}
+/// A broker message paired with the type of event collection it can decode.
+pub struct MessageWrapper<T, M> {
+ inner: T,
+ marker: PhantomData<M>,
+}
+
+impl<T, M> MessageWrapper<T, M> {
+ /// Wraps the transport-specific message.
+ pub fn new(inner: T) -> Self {
+ Self {
+ inner,
+ marker: PhantomData,
+ }
+ }
+
+ /// Borrows the transport-specific message.
+ pub fn inner(&self) -> &T {
+ &self.inner
+ }
+
+ /// Returns the transport-specific message.
+ pub fn into_inner(self) -> T {
+ self.inner
+ }
+}
+
+impl<T: MessageParts, M: MacroEventCollection> MessageWrapper<T, M> {
+ /// Decodes the message into one of the declared event types.
+ pub fn decode_payload(&self) -> Result<M, EventBrokerError> {
+ M::decode(&self.inner)
+ }
+}
+
+/// Declares the [`MacroEvent`] types accepted by a consumer.
+///
+/// The first identifier names the generated event collection enum. Each
+/// identifier after the colon becomes a variant and must implement
+/// [`MacroEvent`]. The generated enum implements topic-based decoding.
+///
+/// ```text
+/// declare_topics!(ConsumerEvents: DocumentMacroEvent, ChannelMacroEvent);
+/// ```
+#[macro_export]
+macro_rules! declare_topics {
+ ($collection:ident: $($event:ident),+ $(,)?) => {
+ #[doc = concat!(
+ "An event decoded from one of the topics declared for `",
+ stringify!($collection),
+ "`."
+ )]
+ pub enum $collection {
+ $(
+ #[doc = concat!("A decoded `", stringify!($event), "` event.")]
+ $event($event),
+ )+
+ }
+
+ impl $crate::MacroEventCollection for $collection {
+ fn decode<T: $crate::MessageParts>(
+ message: &T,
+ ) -> Result<Self, $crate::EventBrokerError> {
+ $(
+ let expected_topic =
+ <<<$event as $crate::MacroEvent>::EventPayload as $crate::TopicEvent>::Topic as $crate::Topic>::TOPIC_STR;
+ if message.topic() == expected_topic {
+ let key = message
+ .key()
+ .ok_or($crate::EventBrokerError::MissingMessageKey)?;
+ let payload = message
+ .payload()
+ .ok_or($crate::EventBrokerError::MissingMessagePayload)?;
+ let event = <$event as $crate::MacroEvent>::decode(key, payload)?;
+ let envelope = <$event as $crate::MacroEvent>::event(&event);
+ let expected = $crate::TopicEvent::schema_version(&envelope.event);
+ let actual = envelope.schema_version;
+ if actual != expected {
+ return Err($crate::EventBrokerError::UnsupportedSchemaVersion {
+ topic: expected_topic,
+ expected,
+ actual,
+ });
+ }
+ return Ok($collection::$event(event));
+ }
+ )+
+
+ Err($crate::EventBrokerError::UnknownTopic(message.topic().to_owned()))
+ }
+
+ fn topics() -> &'static [&'static str] {
+ &[
+ $(
+ <<<$event as $crate::MacroEvent>::EventPayload as $crate::TopicEvent>::Topic as $crate::Topic>::TOPIC_STR,
+ )+
+ ]
+ }
+ }
+ };
+}
+
#[cfg(test)]
mod test;
diff --git a/crates/macro_event_broker/src/domain/models/test.rs b/crates/macro_event_broker/src/domain/models/test.rs
index 95e2c942b1..197213f11d 100644
--- a/crates/macro_event_broker/src/domain/models/test.rs
+++ b/crates/macro_event_broker/src/domain/models/test.rs
@@ -1,19 +1,20 @@
-use macro_event_topics::{MacroExampleTopic, Topic};
+use macro_event_topics::{MacroDocumentsTopic, MacroExampleTopic, Topic};
use serde::{Deserialize, Serialize};
use serde_json::json;
use uuid::Uuid;
use super::*;
+use crate::{MacroEventCollection, MessageParts};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-struct ExampleCreatedMetadata {
+pub struct ExampleCreatedMetadata {
name: String,
count: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "event_type", content = "metadata")]
-enum ExampleTopicEvent {
+pub enum ExampleTopicEvent {
#[serde(rename = "example.created")]
Created(ExampleCreatedMetadata),
}
@@ -27,7 +28,7 @@ impl TopicEvent for ExampleTopicEvent {
}
#[derive(Debug, Clone, PartialEq, Eq)]
-struct ExampleMacroEvent {
+pub struct ExampleMacroEvent {
key: String,
event: Event<ExampleTopicEvent>,
}
@@ -64,7 +65,7 @@ enum ExampleConsumerEvent {
impl ExampleConsumerEvent {
fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {
match topic {
- topic if topic == MacroExampleTopic.as_str() => {
+ topic if topic == MacroExampleTopic::TOPIC_STR => {
Ok(Self::Example(ExampleMacroEvent::decode(key, payload)?))
}
unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),
@@ -116,9 +117,172 @@ fn consumer_specific_enum_decodes_by_topic() {
let event = example_event();
let payload = serde_json::to_vec(&event).unwrap();
- let decoded = ExampleConsumerEvent::decode(MacroExampleTopic.as_str(), "msg-123", &payload)
+ let decoded = ExampleConsumerEvent::decode(MacroExampleTopic::TOPIC_STR, "msg-123", &payload)
.expect("topic should decode");
let ExampleConsumerEvent::Example(decoded) = decoded;
assert_eq!(decoded.key(), "msg-123");
}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct DocumentsTopicEvent;
+
+impl TopicEvent for DocumentsTopicEvent {
+ type Topic = MacroDocumentsTopic;
+
+ fn schema_version(&self) -> u8 {
+ 1
+ }
+}
+
+pub struct DocumentsMacroEvent {
+ key: String,
+ event: Event<DocumentsTopicEvent>,
+}
+
+impl MacroEvent for DocumentsMacroEvent {
+ type EventPayload = DocumentsTopicEvent;
+
+ fn key(&self) -> &str {
+ &self.key
+ }
+
+ fn event(&self) -> &Event<Self::EventPayload> {
+ &self.event
+ }
+
+ fn from_event(key: String, event: Event<Self::EventPayload>) -> Self {
+ Self { key, event }
+ }
+}
+
+crate::declare_topics!(TestMacroEvents: ExampleMacroEvent, DocumentsMacroEvent,);
+
+type DeclaredMacroEvent = TestMacroEvents;
+
+struct TestMessage {
+ topic: &'static str,
+ key: Option<&'static str>,
+ payload: Option<Vec<u8>>,
+}
+
+impl MessageParts for TestMessage {
+ fn key(&self) -> Option<&str> {
+ self.key
+ }
+
+ fn payload(&self) -> Option<&[u8]> {
+ self.payload.as_deref()
+ }
+
+ fn topic(&self) -> &str {
+ self.topic
+ }
+}
+
+#[test]
+fn exposes_each_declared_topic_in_order() {
+ assert_eq!(
+ DeclaredMacroEvent::topics(),
+ [MacroExampleTopic::TOPIC_STR, MacroDocumentsTopic::TOPIC_STR,]
+ );
+}
+
+#[test]
+fn decodes_each_declared_topic_into_its_enum_variant() {
+ let example_payload = serde_json::to_vec(&example_event()).unwrap();
+ let example_message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {
+ topic: MacroExampleTopic::TOPIC_STR,
+ key: Some("example-key"),
+ payload: Some(example_payload),
+ });
+
+ match example_message.decode_payload().unwrap() {
+ DeclaredMacroEvent::ExampleMacroEvent(event) => {
+ assert_eq!(event.key(), "example-key");
+ }
+ DeclaredMacroEvent::DocumentsMacroEvent(_) => panic!("decoded the wrong event type"),
+ }
+
+ let documents_payload = serde_json::to_vec(&Event::new(DocumentsTopicEvent)).unwrap();
+ let documents_message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {
+ topic: MacroDocumentsTopic::TOPIC_STR,
+ key: Some("documents-key"),
+ payload: Some(documents_payload),
+ });
+
+ match documents_message.decode_payload().unwrap() {
+ DeclaredMacroEvent::DocumentsMacroEvent(event) => {
+ assert_eq!(event.key(), "documents-key");
+ }
+ DeclaredMacroEvent::ExampleMacroEvent(_) => panic!("decoded the wrong event type"),
+ }
+}
+
+#[test]
+fn rejects_topics_not_declared_by_the_macro() {
+ let message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {
+ topic: "macro.unknown",
+ key: None,
+ payload: None,
+ });
+
+ assert!(matches!(
+ message.decode_payload(),
+ Err(EventBrokerError::UnknownTopic(topic)) if topic == "macro.unknown"
+ ));
+}
+
+#[test]
+fn rejects_a_missing_message_key() {
+ let message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {
+ topic: MacroExampleTopic::TOPIC_STR,
+ key: None,
+ payload: Some(serde_json::to_vec(&example_event()).unwrap()),
+ });
+
+ assert!(matches!(
+ message.decode_payload(),
+ Err(EventBrokerError::MissingMessageKey)
+ ));
+}
+
+#[test]
+fn rejects_a_missing_message_payload() {
+ let message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {
+ topic: MacroExampleTopic::TOPIC_STR,
+ key: Some("example-key"),
+ payload: None,
+ });
+
+ assert!(matches!(
+ message.decode_payload(),
+ Err(EventBrokerError::MissingMessagePayload)
+ ));
+}
+
+#[test]
+fn rejects_an_unsupported_schema_version() {
+ let event = Event::with_event_id_and_schema_version(
+ Uuid::from_u128(1),
+ 2,
+ ExampleTopicEvent::Created(ExampleCreatedMetadata {
+ name: "hello".to_string(),
+ count: 7,
+ }),
+ );
+ let message = MessageWrapper::<_, DeclaredMacroEvent>::new(TestMessage {
+ topic: MacroExampleTopic::TOPIC_STR,
+ key: Some("example-key"),
+ payload: Some(serde_json::to_vec(&event).unwrap()),
+ });
+
+ assert!(matches!(
+ message.decode_payload(),
+ Err(EventBrokerError::UnsupportedSchemaVersion {
+ topic,
+ expected: 1,
+ actual: 2,
+ }) if topic == MacroExampleTopic::TOPIC_STR
+ ));
+}
diff --git a/crates/macro_event_broker/src/domain/ports.rs b/crates/macro_event_broker/src/domain/ports.rs
index 285b27f1a9..f9886ed7b4 100644
--- a/crates/macro_event_broker/src/domain/ports.rs
+++ b/crates/macro_event_broker/src/domain/ports.rs
@@ -1,10 +1,8 @@
//! Port traits defining the broker's inbound and outbound boundaries.
-use std::future::Future;
-
use macro_event_topics::Topic;
-use crate::domain::models::{EventBrokerError, MacroEvent};
+use crate::domain::models::{EventBrokerError, MacroEvent, MessageWrapper};
/// Inbound port: the public API for sending events through the broker.
///
@@ -28,11 +26,46 @@ pub trait MacroEventBroker: Send + Sync + 'static {
/// Kept byte-oriented so payload serialization stays the service's concern and the
/// port is trivial to mock or stub in tests.
pub trait EventPublisher: Send + Sync + 'static {
- /// Publish a raw `payload` to `topic`, keyed by `key`.
+ /// Publishes a raw `payload` to [`T::TOPIC_STR`](Topic::TOPIC_STR), keyed by `key`.
fn publish<T: Topic>(
&self,
- topic: T,
key: &str,
payload: &[u8],
) -> impl Future<Output = Result<(), EventBrokerError>> + Send;
}
+
+/// A collection of [`MacroEvent`] types that can decode a broker message.
+pub trait MacroEventCollection: Sized {
+ /// Decodes an event from the supplied broker message.
+ fn decode<T: MessageParts>(message: &T) -> Result<Self, EventBrokerError>;
+
+ /// Lists the statically known topics in this event collection.
+ fn topics() -> &'static [&'static str];
+}
+
+/// Broker message fields needed to decode a [`MacroEvent`].
+pub trait MessageParts {
+ /// Returns the broker message key.
+ fn key(&self) -> Option<&str>;
+
+ /// Returns the serialized event payload.
+ fn payload(&self) -> Option<&[u8]>;
+
+ /// Returns the broker topic name.
+ fn topic(&self) -> &str;
+}
+
+/// Inbound boundary for receiving messages from the event broker.
+pub trait EventConsumer<M: MacroEventCollection>: Send + Sync + 'static {
+ /// The concrete transport-specific message type.
+ type MessageType<'a>: MessageParts
+ where
+ Self: 'a;
+
+ /// Waits for and returns the next broker message.
+ fn recv<'a>(
+ &'a self,
+ ) -> impl Future<Output = Result<MessageWrapper<Self::MessageType<'a>, M>, rootcause::Report>>
+ + Send
+ + 'a;
+}
diff --git a/crates/macro_event_broker/src/domain/service.rs b/crates/macro_event_broker/src/domain/service.rs
index 65521aa0d2..748f3fe3b1 100644
--- a/crates/macro_event_broker/src/domain/service.rs
+++ b/crates/macro_event_broker/src/domain/service.rs
@@ -1,19 +1,62 @@
-//! The concrete event broker service.
+//! Typed producer and consumer services.
#[cfg(test)]
mod test;
+use std::marker::PhantomData;
use std::sync::Arc;
use std::time::Duration;
-use macro_event_topics::Topic as _;
use tracing::Instrument as _;
-use crate::domain::models::{EventBrokerError, MacroEvent};
-use crate::domain::ports::{EventPublisher, MacroEventBroker};
+use crate::MessageWrapper;
+use crate::domain::models::{EventBrokerError, MacroEvent, TopicEvent};
+use crate::domain::ports::{EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection};
const PUBLISH_TIMEOUT: Duration = Duration::from_secs(6);
+/// Receives transport messages associated with a declared macro event collection.
+pub struct MacroEventConsumerService<M, C>
+where
+ M: MacroEventCollection,
+ C: EventConsumer<M>,
+{
+ consumer: C,
+ marker: PhantomData<fn() -> M>,
+}
+
+impl<M, C> MacroEventConsumerService<M, C>
+where
+ M: MacroEventCollection,
+ C: EventConsumer<M>,
+{
+ /// Creates a typed consumer service backed by `consumer`.
+ pub const fn new(consumer: C) -> Self {
+ Self {
+ consumer,
+ marker: PhantomData,
+ }
+ }
+
+ /// Receives the next message from the underlying consumer.
+ ///
+ /// Call [`MessageWrapper::decode_payload`] to decode its declared event.
+ #[tracing::instrument(skip(self), err)]
+ pub async fn recv<'a>(
+ &'a self,
+ ) -> Result<MessageWrapper<C::MessageType<'a>, M>, rootcause::Report> {
+ self.consumer.recv().await
+ }
+
+ /// Returns the underlying consumer.
+ ///
+ /// This provides access to transport-specific delivery operations after a
+ /// message has been received through this service.
+ pub const fn inner(&self) -> &C {
+ &self.consumer
+ }
+}
+
/// Orchestrates serializing events and handing them to an [`EventPublisher`].
pub struct MacroEventBrokerService<P: EventPublisher> {
publisher: Arc<P>,
@@ -37,7 +80,7 @@ impl<P: EventPublisher> MacroEventBrokerService<P> {
}
impl<P: EventPublisher> MacroEventBroker for MacroEventBrokerService<P> {
- #[tracing::instrument(err, skip(self, event), fields(topic = %event.topic().as_str(), key = %event.key()))]
+ #[tracing::instrument(err, skip(self, event), fields(topic = event.topic(), key = %event.key()))]
fn send_event<E: MacroEvent + ?Sized>(
&self,
event: &E,
@@ -50,20 +93,25 @@ impl<P: EventPublisher> MacroEventBroker for MacroEventBrokerService<P> {
let handle = tokio::spawn(
async move {
- tokio::time::timeout(PUBLISH_TIMEOUT, publisher.publish(topic, &key, &payload))
- .await
- .map_err(|_| EventBrokerError::PublishTimeout {
- timeout: PUBLISH_TIMEOUT,
- })
- .and_then(std::convert::identity)
- .inspect_err(|error| {
- tracing::error!(
- error = ?error,
- topic = topic.as_str(),
- key = %key,
- "failed to publish event",
- );
- })
+ tokio::time::timeout(
+ PUBLISH_TIMEOUT,
+ publisher.publish::<<<E as MacroEvent>::EventPayload as TopicEvent>::Topic>(
+ &key, &payload,
+ ),
+ )
+ .await
+ .map_err(|_| EventBrokerError::PublishTimeout {
+ timeout: PUBLISH_TIMEOUT,
+ })
+ .and_then(std::convert::identity)
+ .inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ topic,
+ key = %key,
+ "failed to publish event",
+ );
+ })
}
.instrument(span),
);
diff --git a/crates/macro_event_broker/src/domain/service/test.rs b/crates/macro_event_broker/src/domain/service/test.rs
index ad42172c8e..5938c0dc58 100644
--- a/crates/macro_event_broker/src/domain/service/test.rs
+++ b/crates/macro_event_broker/src/domain/service/test.rs
@@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize};
use uuid::Uuid;
use super::*;
-use crate::domain::models::{Event, EventBrokerError, MacroEvent, TopicEvent};
-use crate::domain::ports::{EventPublisher, MacroEventBroker};
+use crate::domain::models::{Event, EventBrokerError, MacroEvent, MessageWrapper, TopicEvent};
+use crate::domain::ports::{EventConsumer, EventPublisher, MacroEventBroker, MessageParts};
/// A captured publish call: topic, key, and raw payload bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -28,14 +28,9 @@ struct RecordingPublisher {
}
impl EventPublisher for RecordingPublisher {
- async fn publish<T: Topic>(
- &self,
- topic: T,
- key: &str,
- payload: &[u8],
- ) -> Result<(), EventBrokerError> {
+ async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {
self.calls.lock().unwrap().push(Published {
- topic: topic.as_str().to_string(),
+ topic: T::TOPIC_STR.to_string(),
key: key.to_string(),
payload: payload.to_vec(),
});
@@ -48,12 +43,7 @@ struct PendingPublisher {
}
impl EventPublisher for PendingPublisher {
- async fn publish<T: Topic>(
- &self,
- _topic: T,
- _key: &str,
- _payload: &[u8],
- ) -> Result<(), EventBrokerError> {
+ async fn publish<T: Topic>(&self, _key: &str, _payload: &[u8]) -> Result<(), EventBrokerError> {
self.started.store(true, Ordering::SeqCst);
pending().await
}
@@ -75,12 +65,7 @@ struct HangingPublisher {
}
impl EventPublisher for HangingPublisher {
- async fn publish<T: Topic>(
- &self,
- _topic: T,
- _key: &str,
- _payload: &[u8],
- ) -> Result<(), EventBrokerError> {
+ async fn publish<T: Topic>(&self, _key: &str, _payload: &[u8]) -> Result<(), EventBrokerError> {
self.started.store(true, Ordering::SeqCst);
let _drop_guard = PublishDropGuard {
dropped: Arc::clone(&self.dropped),
@@ -94,26 +79,21 @@ struct FailingPublisher {
}
impl EventPublisher for FailingPublisher {
- async fn publish<T: Topic>(
- &self,
- _topic: T,
- _key: &str,
- _payload: &[u8],
- ) -> Result<(), EventBrokerError> {
+ async fn publish<T: Topic>(&self, _key: &str, _payload: &[u8]) -> Result<(), EventBrokerError> {
self.attempted.store(true, Ordering::SeqCst);
Err(EventBrokerError::Publish("test failure".to_string()))
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-struct ExampleCreatedMetadata {
+pub struct ExampleCreatedMetadata {
name: String,
count: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "event_type", content = "metadata")]
-enum ExampleTopicEvent {
+pub enum ExampleTopicEvent {
#[serde(rename = "example.created")]
Created(ExampleCreatedMetadata),
}
@@ -126,7 +106,7 @@ impl TopicEvent for ExampleTopicEvent {
}
}
-struct ExampleMacroEvent {
+pub struct ExampleMacroEvent {
key: String,
event: Event<ExampleTopicEvent>,
}
@@ -156,6 +136,48 @@ impl MacroEvent for ExampleMacroEvent {
}
}
+crate::declare_topics!(DeclaredMacroEvent: ExampleMacroEvent);
+
+struct TestMessage {
+ topic: &'static str,
+ key: &'static str,
+ payload: Vec<u8>,
+}
+
+impl MessageParts for TestMessage {
+ fn key(&self) -> Option<&str> {
+ Some(self.key)
+ }
+
+ fn payload(&self) -> Option<&[u8]> {
+ Some(&self.payload)
+ }
+
+ fn topic(&self) -> &str {
+ self.topic
+ }
+}
+
+struct TestEventConsumer {
+ topic: &'static str,
+ key: &'static str,
+ payload: Vec<u8>,
+}
+
+impl EventConsumer<DeclaredMacroEvent> for TestEventConsumer {
+ type MessageType<'a> = TestMessage;
+
+ async fn recv<'a>(
+ &'a self,
+ ) -> Result<MessageWrapper<Self::MessageType<'a>, DeclaredMacroEvent>, rootcause::Report> {
+ Ok(MessageWrapper::new(TestMessage {
+ topic: self.topic,
+ key: self.key,
+ payload: self.payload.clone(),
+ }))
+ }
+}
+
fn example_event() -> ExampleMacroEvent {
ExampleMacroEvent::with_event(
"msg-123",
@@ -169,6 +191,23 @@ fn example_event() -> ExampleMacroEvent {
)
}
+#[tokio::test]
+async fn consumer_service_receives_and_decodes_typed_event() {
+ let event = example_event();
+ let consumer = TestEventConsumer {
+ topic: MacroExampleTopic::TOPIC_STR,
+ key: "msg-123",
+ payload: serde_json::to_vec(event.event()).expect("event serializes"),
+ };
+ let service = MacroEventConsumerService::<DeclaredMacroEvent, _>::new(consumer);
+
+ let message = service.recv().await.expect("message is received");
+ let decoded = message.decode_payload().expect("associated topic decodes");
+ let DeclaredMacroEvent::ExampleMacroEvent(decoded) = decoded;
+ assert_eq!(decoded.key(), "msg-123");
+ assert_eq!(decoded.event(), event.event());
+}
+
#[derive(Debug, Deserialize)]
struct UnserializableTopicEvent;
@@ -225,7 +264,7 @@ async fn dispatch_serializes_and_routes() {
let calls = service.publisher.calls.lock().unwrap();
assert_eq!(calls.len(), 1);
let call = &calls[0];
- assert_eq!(call.topic, MacroExampleTopic.as_str());
+ assert_eq!(call.topic, MacroExampleTopic::TOPIC_STR);
assert_eq!(call.key, "msg-123");
assert_eq!(call.payload, expected_payload);
}
diff --git a/crates/macro_event_broker/src/kafka/mod.rs b/crates/macro_event_broker/src/kafka/mod.rs
deleted file mode 100644
index a7074dedfc..0000000000
--- a/crates/macro_event_broker/src/kafka/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-//! Kafka transport support shared by inbound and outbound adapters.
-
-/// AWS MSK IAM authentication support for Kafka clients.
-pub mod msk_iam;
diff --git a/crates/macro_event_broker/src/lib.rs b/crates/macro_event_broker/src/lib.rs
index bd60384a92..6b2ffc2f16 100644
--- a/crates/macro_event_broker/src/lib.rs
+++ b/crates/macro_event_broker/src/lib.rs
@@ -5,32 +5,34 @@
//! [`TopicEvent`](domain::models::TopicEvent) enums, the
//! [`MacroEvent`](domain::models::MacroEvent) event abstraction, the inbound
//! [`MacroEventBroker`](domain::ports::MacroEventBroker) API, the outbound
-//! [`EventPublisher`](domain::ports::EventPublisher) port, and the
-//! [`MacroEventBrokerService`](domain::service::MacroEventBrokerService) that ties
-//! them together. Kafka topic definitions live in the `macro_event_topics` crate.
-//! The [`outbound`] layer provides the Kafka publisher adapter.
+//! [`EventPublisher`](domain::ports::EventPublisher) port, the producing
+//! [`MacroEventBrokerService`](domain::service::MacroEventBrokerService), and the
+//! typed [`MacroEventConsumerService`](domain::service::MacroEventConsumerService),
+//! which receives messages through the [`EventConsumer`](domain::ports::EventConsumer) port.
+//! Kafka topic definitions live in the `macro_event_topics` crate. Shared Kafka
+//! producer and consumer transports live in `kafka_util`, while the [`outbound`]
+//! layer adapts its producer to [`EventPublisher`](domain::ports::EventPublisher).
/// Domain layer: models, ports, and service.
pub mod domain;
-pub use domain::models::{Event, EventBrokerError, MacroEvent, TopicEvent};
+pub use domain::models::{Event, EventBrokerError, MacroEvent, MessageWrapper, TopicEvent};
pub use macro_event_topics::{
MacroChannelsTopic, MacroDocumentsTopic, MacroEmailTopic, MacroExampleTopic,
MacroProjectsTopic, Topic,
};
-#[cfg(feature = "ports")]
-pub use domain::ports::{EventPublisher, MacroEventBroker};
-#[cfg(feature = "ports")]
-pub use domain::service::{MacroEventBrokerService, NoopMacroEventBroker};
-#[cfg(feature = "kafka")]
-pub use kafka::msk_iam::MskIamClientContext;
+pub use domain::ports::{
+ EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection, MessageParts,
+};
+pub use domain::service::{
+ MacroEventBrokerService, MacroEventConsumerService, NoopMacroEventBroker,
+};
#[cfg(feature = "outbound")]
-pub use outbound::kafka_event_publisher::KafkaEventPublisher;
+pub use outbound::{
+ kafka_event_consumer::KafkaConsumerAdapter, kafka_event_publisher::KafkaEventPublisher,
+};
-/// Kafka transport support shared by inbound and outbound adapters.
-#[cfg(feature = "kafka")]
-pub mod kafka;
-/// Outbound layer: Kafka publishing adapters.
+/// Outbound adapters for the macro event broker's required ports.
#[cfg(feature = "outbound")]
pub mod outbound;
diff --git a/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs
new file mode 100644
index 0000000000..380669ccc4
--- /dev/null
+++ b/crates/macro_event_broker/src/outbound/kafka_event_consumer.rs
@@ -0,0 +1,119 @@
+//! Kafka adapters for configuring topic-aware grouped and ungrouped consumers
+//! and implementing the [`EventConsumer`] port.
+
+use std::{marker::PhantomData, time::Duration};
+
+use kafka_util::{GroupName, InitialOffset, KafkaEventConsumer, Ungrouped};
+use rdkafka::{
+ consumer::CommitMode,
+ message::{BorrowedMessage, Message as _},
+};
+
+use crate::{EventConsumer, MacroEventCollection, MessageWrapper};
+
+/// Topic-aware wrapper around a [`KafkaEventConsumer`].
+///
+/// `T` selects grouped or [`Ungrouped`] consumption. `M` records the
+/// [`MacroEventCollection`] whose topics were assigned or subscribed, so the
+/// configured topic set remains visible in the adapter's type.
+pub struct KafkaConsumerAdapter<T, M> {
+ inner: KafkaEventConsumer<T>,
+ topics: PhantomData<fn() -> M>,
+}
+
+impl<M: MacroEventCollection> KafkaConsumerAdapter<Ungrouped, M> {
+ /// Creates an ungrouped adapter and manually assigns every topic in `M`.
+ ///
+ /// Every current partition starts at `initial_offset`. Topic metadata must
+ /// be fetched within `metadata_timeout`; failures are returned to the
+ /// caller.
+ pub fn new(
+ consumer: KafkaEventConsumer<Ungrouped>,
+ initial_offset: InitialOffset,
+ metadata_timeout: Duration,
+ ) -> Result<Self, rootcause::Report> {
+ consumer.assign_topics(M::topics(), initial_offset, metadata_timeout)?;
+ Ok(KafkaConsumerAdapter {
+ inner: consumer,
+ topics: PhantomData,
+ })
+ }
+}
+
+impl<T: GroupName> KafkaConsumerAdapter<T, ()> {
+ /// Wraps a grouped consumer before its event topics are subscribed.
+ pub fn new(consumer: KafkaEventConsumer<T>) -> Self {
+ KafkaConsumerAdapter {
+ inner: consumer,
+ topics: PhantomData,
+ }
+ }
+}
+
+impl<T: GroupName, M> KafkaConsumerAdapter<T, M> {
+ /// Subscribes the grouped consumer to every topic declared by `M2`.
+ ///
+ /// Kafka subscription is synchronous, replaces the previous subscription,
+ /// and causes the group coordinator to rebalance asynchronously while the
+ /// consumer is polled. The returned adapter records `M2` in its type.
+ pub fn subscribe<M2: MacroEventCollection>(
+ self,
+ ) -> Result<KafkaConsumerAdapter<T, M2>, rootcause::Report> {
+ let KafkaConsumerAdapter { inner, .. } = self;
+ inner.subscribe(M2::topics())?;
+ Ok(KafkaConsumerAdapter {
+ inner,
+ topics: PhantomData,
+ })
+ }
+
+ /// Commits a message using the caller-selected commit mode.
+ pub fn commit_message(
+ &self,
+ message: &BorrowedMessage<'_>,
+ mode: CommitMode,
+ ) -> Result<(), rootcause::Report> {
+ Ok(self.inner.commit_message(message, mode)?)
+ }
+}
+
+impl<T, M: MacroEventCollection> KafkaConsumerAdapter<T, M> {
+ /// Pauses the partition containing `message`.
+ ///
+ /// Grouped consumers can use this to prevent a later cumulative commit
+ /// from advancing past a failed record. Ungrouped consumers can use it to
+ /// stop additional delivery from a failed partition.
+ pub fn pause_message_partition(
+ &self,
+ message: &BorrowedMessage<'_>,
+ ) -> Result<(), rootcause::Report> {
+ Ok(self.inner.pause_message_partition(message)?)
+ }
+}
+
+impl<T, M> EventConsumer<M> for KafkaConsumerAdapter<T, M>
+where
+ T: Send + Sync + 'static,
+ M: MacroEventCollection + 'static,
+{
+ type MessageType<'a> = BorrowedMessage<'a>;
+
+ async fn recv<'a>(
+ &'a self,
+ ) -> Result<MessageWrapper<Self::MessageType<'a>, M>, rootcause::Report> {
+ let message = self.inner.recv().await?;
+
+ tracing::trace!(
+ topic = message.topic(),
+ partition = message.partition(),
+ offset = message.offset(),
+ payload_len = message.payload().map_or(0, <[u8]>::len),
+ "received Kafka event message"
+ );
+
+ Ok(MessageWrapper::new(message))
+ }
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/macro_event_broker/src/outbound/kafka_event_consumer/test.rs b/crates/macro_event_broker/src/outbound/kafka_event_consumer/test.rs
new file mode 100644
index 0000000000..a139d7b16b
--- /dev/null
+++ b/crates/macro_event_broker/src/outbound/kafka_event_consumer/test.rs
@@ -0,0 +1,23 @@
+use kafka_util::Ungrouped;
+
+use super::*;
+use crate::{EventBrokerError, MessageParts};
+
+struct TestEvents;
+
+impl MacroEventCollection for TestEvents {
+ fn decode<T: MessageParts>(_message: &T) -> Result<Self, EventBrokerError> {
+ unreachable!("compile-time adapter assertion does not decode messages")
+ }
+
+ fn topics() -> &'static [&'static str] {
+ &[]
+ }
+}
+
+fn assert_event_consumer<T: EventConsumer<TestEvents>>() {}
+
+#[test]
+fn kafka_adapter_implements_event_consumer_port() {
+ assert_event_consumer::<KafkaConsumerAdapter<Ungrouped, TestEvents>>();
+}
diff --git a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
index 9e46870d93..432bfbb0df 100644
--- a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
+++ b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
@@ -1,99 +1,51 @@
//! Kafka adapter for the [`EventPublisher`] port.
//!
-//! Wraps an `rdkafka` [`FutureProducer`] to publish events. The broker address
-//! is supplied by the caller (typically from environment configuration at the
-//! consuming service); the transport is chosen here from the `ENVIRONMENT`
-//! variable: `Local` uses a plaintext connection (the docker-compose broker),
-//! while `Develop` / `Production` use TLS + SASL/OAUTHBEARER with AWS MSK IAM
-//! auth via [`MskIamClientContext`].
+//! The environment-aware Kafka producer transport lives in `kafka_util`; this
+//! adapter maps that transport onto the macro event broker's domain port.
-use std::time::Duration;
-
-use macro_env::Environment;
+use kafka_util::{KafkaEventProducer, KafkaProducerError};
use macro_event_topics::Topic;
-use rdkafka::ClientConfig;
-use rdkafka::producer::{FutureProducer, FutureRecord};
+use rdkafka::Message;
+use crate::MessageParts;
use crate::domain::models::EventBrokerError;
use crate::domain::ports::EventPublisher;
-use crate::kafka::msk_iam::{MskIamClientContext, configure_sasl_iam};
-
-/// How long a record may sit in the producer queue before delivery is considered failed.
-const MESSAGE_TIMEOUT_MS: &str = "5000";
-
-/// How long [`EventPublisher::publish`] waits for delivery confirmation before timing out.
-const SEND_TIMEOUT: Duration = Duration::from_secs(5);
-
-/// The underlying producer, split by transport.
-#[derive(Clone)]
-enum Producer {
- /// Unauthenticated plaintext connection (local docker broker).
- Plaintext(FutureProducer),
- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).
- MskIam(FutureProducer<MskIamClientContext>),
-}
/// Kafka-backed implementation of [`EventPublisher`].
#[derive(Clone)]
pub struct KafkaEventPublisher {
- producer: Producer,
-}
-
-fn base_config(brokers: &str) -> ClientConfig {
- let mut config = ClientConfig::new();
- config
- .set("bootstrap.servers", brokers)
- .set("message.timeout.ms", MESSAGE_TIMEOUT_MS);
- config
-}
-
-fn create_error(e: impl std::fmt::Display) -> EventBrokerError {
- EventBrokerError::Publish(format!("failed to create producer: {e}"))
+ producer: KafkaEventProducer,
}
impl KafkaEventPublisher {
- /// Build a producer connected to the given comma-separated `brokers` list,
- /// choosing the transport from the `ENVIRONMENT` variable (see the module
- /// docs).
- ///
- /// Producer creation is lazy: nothing connects to Kafka (and no IAM token
- /// is signed) until an event is published.
- pub fn new(brokers: &str) -> Result<Self, EventBrokerError> {
- let producer = match Environment::new_or_prod() {
- Environment::Local => {
- Producer::Plaintext(base_config(brokers).create().map_err(create_error)?)
- }
- Environment::Develop | Environment::Production => {
- let config = configure_sasl_iam(base_config(brokers));
-
- Producer::MskIam(
- config
- .create_with_context(MskIamClientContext::from_env())
- .map_err(create_error)?,
- )
- }
- };
-
- Ok(Self { producer })
+ /// Builds an environment-aware Kafka event publisher.
+ pub fn new(brokers: &str) -> Result<Self, KafkaProducerError> {
+ Ok(Self {
+ producer: KafkaEventProducer::from_env(brokers)?,
+ })
}
}
impl EventPublisher for KafkaEventPublisher {
- #[tracing::instrument(err, skip(self, payload), fields(topic = %topic.as_str(), key = %key))]
- async fn publish<T: Topic>(
- &self,
- topic: T,
- key: &str,
- payload: &[u8],
- ) -> Result<(), EventBrokerError> {
- let record = FutureRecord::to(topic.as_str()).key(key).payload(payload);
+ #[tracing::instrument(err, skip(self, payload), fields(topic = T::TOPIC_STR, key = %key))]
+ async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {
+ self.producer
+ .send(T::TOPIC_STR, key, payload)
+ .await
+ .map_err(|error| EventBrokerError::Publish(error.to_string()))
+ }
+}
+
+impl<T: Message> MessageParts for T {
+ fn key(&self) -> Option<&str> {
+ Message::key(self).and_then(|key| std::str::from_utf8(key).ok())
+ }
- match &self.producer {
- Producer::Plaintext(producer) => producer.send(record, SEND_TIMEOUT).await,
- Producer::MskIam(producer) => producer.send(record, SEND_TIMEOUT).await,
- }
- .map_err(|(e, _)| EventBrokerError::Publish(e.to_string()))?;
+ fn payload(&self) -> Option<&[u8]> {
+ Message::payload(self)
+ }
- Ok(())
+ fn topic(&self) -> &str {
+ Message::topic(self)
}
}
diff --git a/crates/macro_event_broker/src/outbound/mod.rs b/crates/macro_event_broker/src/outbound/mod.rs
index 66675e0544..4fef313236 100644
--- a/crates/macro_event_broker/src/outbound/mod.rs
+++ b/crates/macro_event_broker/src/outbound/mod.rs
@@ -1,5 +1,6 @@
+/// Kafka adapter implementing the [`EventConsumer`](crate::domain::ports::EventConsumer) port.
+#[cfg(feature = "outbound")]
+pub mod kafka_event_consumer;
/// Kafka adapter implementing the [`EventPublisher`](crate::domain::ports::EventPublisher) port.
#[cfg(feature = "outbound")]
pub mod kafka_event_publisher;
-/// AWS MSK IAM (SASL/OAUTHBEARER) auth support shared by producers and consumers.
-pub use crate::kafka::msk_iam;
diff --git a/crates/macro_event_topics/src/lib.rs b/crates/macro_event_topics/src/lib.rs
index bbbc542250..17adf90de5 100644
--- a/crates/macro_event_topics/src/lib.rs
+++ b/crates/macro_event_topics/src/lib.rs
@@ -17,9 +17,9 @@ pub enum TopicError {
/// A Topic is mapped to a Kafka topic that events can be published to.
#[sealed]
-pub trait Topic: Default + Copy + Send + Sync + 'static {
- /// The kafka topic name as a string.
- fn as_str(&self) -> &'static str;
+pub trait Topic: Copy + Send + Sync + 'static {
+ /// the statically known string name of this topic
+ const TOPIC_STR: &'static str;
}
/// Defines each topic struct with its `Topic` impl, plus [`all_topic_names`]
@@ -33,15 +33,13 @@ macro_rules! topics {
#[sealed]
impl Topic for $name {
- fn as_str(&self) -> &'static str {
- $topic
- }
+ const TOPIC_STR: &'static str = $topic;
}
)*
/// The names of all Kafka topics defined in this crate.
- pub fn all_topic_names() -> Vec<&'static str> {
- vec![$($name.as_str()),*]
+ pub fn all_topic_names() -> &'static [&'static str] {
+ &[$($name::TOPIC_STR),*]
}
};
}
diff --git a/crates/macro_event_topics/src/test.rs b/crates/macro_event_topics/src/test.rs
index a5e62663ee..39b00a8999 100644
--- a/crates/macro_event_topics/src/test.rs
+++ b/crates/macro_event_topics/src/test.rs
@@ -17,12 +17,12 @@ fn all_topic_names_is_non_empty_and_unique() {
#[test]
fn all_topic_names_includes_declared_topics() {
- assert!(all_topic_names().contains(&MacroExampleTopic.as_str()));
- assert!(all_topic_names().contains(&MacroDocumentsTopic.as_str()));
- assert!(all_topic_names().contains(&MacroSoupRealtimeTopic.as_str()));
- assert!(all_topic_names().contains(&MacroProjectsTopic.as_str()));
- assert!(all_topic_names().contains(&MacroTeamsTopic.as_str()));
- assert!(all_topic_names().contains(&MacroChannelsTopic.as_str()));
- assert!(all_topic_names().contains(&MacroBotsTopic.as_str()));
- assert!(all_topic_names().contains(&MacroWebhooksTopic.as_str()));
+ assert!(all_topic_names().contains(&MacroExampleTopic::TOPIC_STR));
+ assert!(all_topic_names().contains(&MacroDocumentsTopic::TOPIC_STR));
+ assert!(all_topic_names().contains(&MacroSoupRealtimeTopic::TOPIC_STR));
+ assert!(all_topic_names().contains(&MacroProjectsTopic::TOPIC_STR));
+ assert!(all_topic_names().contains(&MacroTeamsTopic::TOPIC_STR));
+ assert!(all_topic_names().contains(&MacroChannelsTopic::TOPIC_STR));
+ assert!(all_topic_names().contains(&MacroBotsTopic::TOPIC_STR));
+ assert!(all_topic_names().contains(&MacroWebhooksTopic::TOPIC_STR));
}
diff --git a/crates/projects/Cargo.toml b/crates/projects/Cargo.toml
index 4dae5b8b20..f16bc46c62 100644
--- a/crates/projects/Cargo.toml
+++ b/crates/projects/Cargo.toml
@@ -30,11 +30,7 @@ outbound = [
"ports",
]
ports = ["dep:tokio"]
-service = [
- "dep:entity_access_management",
- "macro_event_broker/ports",
- "ports",
-]
+service = ["dep:entity_access_management", "ports"]
[dependencies]
# Core
@@ -95,8 +91,8 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
dotenvy = { workspace = true }
http-body-util = { workspace = true }
+kafka_util = { path = "../kafka_util" }
macro_db_migrator = { path = "../macro_db_migrator" }
-macro_env = { path = "../macro_env" }
# Examples need the Kafka adapter (outbound); the library itself only uses ports.
macro_event_broker = { path = "../macro_event_broker" }
mockall = { workspace = true }
diff --git a/crates/projects/examples/projects_consumer.rs b/crates/projects/examples/projects_consumer.rs
index e94972efb9..c8a389ea2e 100644
--- a/crates/projects/examples/projects_consumer.rs
+++ b/crates/projects/examples/projects_consumer.rs
@@ -3,8 +3,8 @@
//! Connects with the same `KAFKA_BROKERS` env var and environment-driven
//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for
//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),
-//! subscribes to [`MacroProjectsTopic`], and prints every project lifecycle
-//! event it receives.
+//! subscribes to the [`ProjectMacroEvent`] topic and prints every project
+//! lifecycle event it receives.
//!
//! The consumer follows the standard poll-loop / worker split: a Kafka poll
//! loop decodes messages and hands them to a processing task over a bounded
@@ -24,15 +24,13 @@
//! ```
use anyhow::Context as _;
-use macro_env::Environment;
-use macro_event_broker::outbound::msk_iam::configure_sasl_iam;
-use macro_event_broker::{EventBrokerError, MacroEvent, MskIamClientContext, Topic as _};
-use macro_event_topics::MacroProjectsTopic;
+use kafka_util::{GroupName, KafkaEventConsumer};
+use macro_event_broker::{
+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
+};
use projects::domain::events::{ProjectMacroEvent, ProjectTopicEvent};
-use rdkafka::ClientConfig;
-use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};
-use rdkafka::error::KafkaResult;
-use rdkafka::message::{BorrowedMessage, Message};
+use rdkafka::consumer::CommitMode;
+use rdkafka::message::Message;
use tokio::sync::mpsc;
macro_env_var::env_var! {
@@ -41,103 +39,28 @@ macro_env_var::env_var! {
}
}
-/// Consumer group id for this example. Offsets are committed under this group,
-/// so re-runs resume where the previous run left off.
-const GROUP_ID: &str = "projects-consumer-example";
+/// Consumer group for this example. Offsets are committed under this group, so
+/// re-runs resume where the previous run left off.
+struct ProjectsConsumerGroup;
+
+impl GroupName for ProjectsConsumerGroup {
+ const GROUP_NAME: &'static str = "projects-consumer-example";
+}
/// Bounded capacity of the channel between the poll loop and the processor.
const CHANNEL_CAPACITY: usize = 128;
-/// Consumer-specific event enum for a consumer subscribed to `macro.projects`,
-/// following the pattern in `macro_event_broker/examples/example_event.rs`.
-enum ProjectsConsumerEvent {
- /// Event received on [`MacroProjectsTopic`].
- Projects(ProjectMacroEvent),
-}
+macro_event_broker::declare_topics!(DeclaredMacroEvent: ProjectMacroEvent);
-impl ProjectsConsumerEvent {
- /// Decode one Kafka message into this consumer's event enum.
- fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {
- match topic {
- topic if topic == MacroProjectsTopic.as_str() => {
- Ok(Self::Projects(ProjectMacroEvent::decode(key, payload)?))
- }
- unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),
- }
- }
-}
+type ProjectsKafkaAdapter = KafkaConsumerAdapter<ProjectsConsumerGroup, DeclaredMacroEvent>;
+type ProjectsConsumerService = MacroEventConsumerService<DeclaredMacroEvent, ProjectsKafkaAdapter>;
/// A decoded event plus the Kafka coordinates it came from, as handed from the
/// poll loop to the processor over the channel.
struct ReceivedEvent {
partition: i32,
offset: i64,
- event: ProjectsConsumerEvent,
-}
-
-/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).
-enum ProjectsConsumer {
- /// Unauthenticated plaintext connection (local docker broker).
- Plaintext(StreamConsumer),
- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).
- MskIam(StreamConsumer<MskIamClientContext>),
-}
-
-impl ProjectsConsumer {
- /// Build a consumer for the given brokers, choosing the transport from the
- /// `ENVIRONMENT` variable exactly like the publisher does.
- fn from_env(brokers: &str) -> anyhow::Result<Self> {
- let mut config = ClientConfig::new();
- config
- .set("bootstrap.servers", brokers)
- .set("group.id", GROUP_ID)
- // Offsets are committed manually after a successful handoff.
- .set("enable.auto.commit", "false")
- // Start from the beginning of the topic on the first ever run.
- .set("auto.offset.reset", "earliest");
-
- let consumer = match Environment::new_or_prod() {
- Environment::Local => Self::Plaintext(
- config
- .create()
- .context("failed to create plaintext kafka consumer")?,
- ),
- Environment::Develop | Environment::Production => {
- let config = configure_sasl_iam(config);
- Self::MskIam(
- config
- .create_with_context(MskIamClientContext::from_env())
- .context("failed to create MSK IAM kafka consumer")?,
- )
- }
- };
-
- Ok(consumer)
- }
-
- fn subscribe(&self) -> KafkaResult<()> {
- let topics = [MacroProjectsTopic.as_str()];
- match self {
- Self::Plaintext(consumer) => consumer.subscribe(&topics),
- Self::MskIam(consumer) => consumer.subscribe(&topics),
- }
- }
-
- /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it
- /// can sit in a `select!` without losing messages.
- async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {
- match self {
- Self::Plaintext(consumer) => consumer.recv().await,
- Self::MskIam(consumer) => consumer.recv().await,
- }
- }
-
- fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {
- match self {
- Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),
- Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),
- }
- }
+ event: ProjectMacroEvent,
}
fn event_type(event: &ProjectTopicEvent) -> &'static str {
@@ -155,7 +78,7 @@ fn event_type(event: &ProjectTopicEvent) -> &'static str {
/// the body for indexing, notifications, etc.
async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {
while let Some(received) = events.recv().await {
- let ProjectsConsumerEvent::Projects(event) = received.event;
+ let event = received.event;
let envelope = event.event();
println!(
@@ -188,14 +111,16 @@ async fn main() -> anyhow::Result<()> {
let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;
- let consumer = ProjectsConsumer::from_env(env.kafka_brokers.as_ref())?;
- consumer
- .subscribe()
- .context("failed to subscribe to projects topic")?;
+ let consumer =
+ KafkaEventConsumer::<ProjectsConsumerGroup>::from_env(env.kafka_brokers.as_ref())?;
+ let consumer = KafkaConsumerAdapter::<ProjectsConsumerGroup, ()>::new(consumer)
+ .subscribe::<DeclaredMacroEvent>()
+ .map_err(|error| anyhow::anyhow!("failed to subscribe to projects topic: {error:?}"))?;
+ let consumer = ProjectsConsumerService::new(consumer);
println!(
"listening on topic={} group={} brokers={} (ctrl-c to stop)",
- MacroProjectsTopic.as_str(),
- GROUP_ID,
+ DeclaredMacroEvent::topics()[0],
+ ProjectsConsumerGroup::GROUP_NAME,
env.kafka_brokers.as_ref(),
);
@@ -218,25 +143,13 @@ async fn main() -> anyhow::Result<()> {
}
};
- let Some(payload) = message.payload() else {
- eprintln!(
- "skipping message with empty payload at partition {} offset {}",
- message.partition(),
- message.offset(),
- );
- let _ = consumer.commit(&message).inspect_err(|e| {
- eprintln!("failed to commit offset {}: {e}", message.offset());
- });
- continue;
- };
- let key = String::from_utf8_lossy(message.key().unwrap_or_default());
-
- match ProjectsConsumerEvent::decode(message.topic(), key.as_ref(), payload) {
- Ok(event) => {
+ let kafka_message = message.inner();
+ match message.decode_payload() {
+ Ok(DeclaredMacroEvent::ProjectMacroEvent(event)) => {
events_tx
.send(ReceivedEvent {
- partition: message.partition(),
- offset: message.offset(),
+ partition: kafka_message.partition(),
+ offset: kafka_message.offset(),
event,
})
.await
@@ -246,16 +159,19 @@ async fn main() -> anyhow::Result<()> {
// wedging the partition on a poison message.
Err(e) => eprintln!(
"failed to decode message at partition {} offset {}: {e}",
- message.partition(),
- message.offset(),
+ kafka_message.partition(),
+ kafka_message.offset(),
),
}
// Commit only after the event was handed to the processor:
// at-least-once delivery (up to the channel buffer on a crash).
- let _ = consumer.commit(&message).inspect_err(|e| {
- eprintln!("failed to commit offset {}: {e}", message.offset());
- });
+ let _ = consumer
+ .inner()
+ .commit_message(kafka_message, CommitMode::Async)
+ .inspect_err(|e| {
+ eprintln!("failed to commit offset {}: {e:?}", kafka_message.offset());
+ });
}
}
}
diff --git a/crates/projects/src/domain/service/tests.rs b/crates/projects/src/domain/service/tests.rs
index 907ea85acf..c13154610b 100644
--- a/crates/projects/src/domain/service/tests.rs
+++ b/crates/projects/src/domain/service/tests.rs
@@ -6,7 +6,7 @@ use entity_access::domain::models::{
};
use entity_access_management::domain::models::EntityAccessManagementError;
use entity_access_management::domain::ports::EntityAccessManagementService;
-use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, Topic as _};
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
use macro_user_id::user_id::MacroUserIdStr;
use model::document::{ContentType, DocumentMetadata, FileType};
use model::folder::{
@@ -69,7 +69,7 @@ impl MacroEventBroker for TestEventBroker {
}
self.published.lock().unwrap().push(PublishedEvent {
- topic: event.topic().as_str(),
+ topic: event.topic(),
key: event.key().to_string(),
payload: serde_json::to_value(event.event())?,
});
diff --git a/crates/soup_realtime/Cargo.toml b/crates/soup_realtime/Cargo.toml
index 2cfe4c4827..a314e849e6 100644
--- a/crates/soup_realtime/Cargo.toml
+++ b/crates/soup_realtime/Cargo.toml
@@ -7,20 +7,16 @@ version = "0.1.0"
[features]
consumer = [
"dep:documents",
- "dep:kafka_consumer_util",
- "dep:macro_event_broker",
- "dep:macro_event_topics",
+ "dep:kafka_util",
"dep:rdkafka",
"dep:tokio",
+ "macro_event_broker/outbound",
"ports",
]
default = ["consumer", "outbound", "ports"]
outbound = [
"dep:entity_access",
- "dep:macro_event_topics",
- "dep:serde_json",
"dep:soup",
- "macro_event_broker/ports",
"ports",
]
ports = ["dep:futures", "dep:rootcause", "dep:tracing"]
@@ -29,16 +25,15 @@ ports = ["dep:futures", "dep:rootcause", "dep:tracing"]
documents = { path = "../documents", default-features = false, optional = true }
entity_access = { path = "../entity_access", default-features = false, features = ["ports"], optional = true }
futures = { workspace = true, optional = true }
-kafka_consumer_util = { path = "../kafka_consumer_util", optional = true }
-macro_event_broker = { path = "../macro_event_broker", default-features = false, optional = true }
-macro_event_topics = { path = "../macro_event_topics", optional = true }
+kafka_util = { path = "../kafka_util", optional = true }
+macro_event_broker = { path = "../macro_event_broker", default-features = false }
+macro_event_topics = { path = "../macro_event_topics" }
macro_user_id = { path = "../macro_user_id" }
model-entity = { path = "../model-entity" }
models_soup = { path = "../models_soup" }
rdkafka = { workspace = true, optional = true }
rootcause = { workspace = true, optional = true }
serde = { workspace = true }
-serde_json = { workspace = true, optional = true }
soup = { path = "../soup", default-features = false, features = ["ports"], optional = true }
tokio = { workspace = true, features = ["time"], optional = true }
tracing = { workspace = true, optional = true }
diff --git a/crates/soup_realtime/src/domain/models.rs b/crates/soup_realtime/src/domain/models.rs
index af1b0e96fa..d060c319eb 100644
--- a/crates/soup_realtime/src/domain/models.rs
+++ b/crates/soup_realtime/src/domain/models.rs
@@ -1,14 +1,14 @@
-//! Versioned realtime Soup output models.
+//! Realtime Soup message and broker event models.
+use macro_event_broker::{Event, MacroEvent, TopicEvent};
+use macro_event_topics::MacroSoupRealtimeTopic;
use macro_user_id::user_id::MacroUserIdStr;
use models_soup::item::SoupItem;
use serde::{Deserialize, Serialize};
-/// One full Soup item published to a recipient for realtime delivery.
+/// One full Soup item addressed to a recipient for realtime delivery.
#[derive(Debug, Serialize, Deserialize)]
pub struct SoupRealtimeMessage {
- /// Version of this message contract.
- pub schema_version: u8,
/// User to whom the Soup item is addressed.
pub user_id: MacroUserIdStr<'static>,
/// Complete Soup item with transient user-specific fields normalized.
@@ -16,15 +16,70 @@ pub struct SoupRealtimeMessage {
}
impl SoupRealtimeMessage {
- /// Current schema version for realtime Soup messages.
- pub const SCHEMA_VERSION: u8 = 1;
-
- /// Creates a version-one message addressed to a recipient.
+ /// Creates a message addressed to a recipient.
pub fn new(user_id: MacroUserIdStr<'static>, item: SoupItem<()>) -> Self {
+ Self { user_id, item }
+ }
+}
+
+/// Events published to [`MacroSoupRealtimeTopic`].
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "event_type", content = "metadata")]
+pub enum SoupTopicEvent {
+ /// A complete Soup item changed and should be delivered to its recipient.
+ #[serde(rename = "soup.item.updated")]
+ ItemUpdated(SoupRealtimeMessage),
+}
+
+impl TopicEvent for SoupTopicEvent {
+ type Topic = MacroSoupRealtimeTopic;
+
+ fn schema_version(&self) -> u8 {
+ 1
+ }
+}
+
+/// Publishable realtime Soup event keyed by recipient user ID.
+pub struct SoupMacroEvent {
+ key: String,
+ event: Event<SoupTopicEvent>,
+}
+
+impl SoupMacroEvent {
+ /// Creates a recipient-keyed item update event.
+ pub fn item_updated(message: SoupRealtimeMessage) -> Self {
+ let key = message.user_id.as_ref().to_string();
+ Self::with_event(key, Event::new(SoupTopicEvent::ItemUpdated(message)))
+ }
+
+ /// Builds an event from its Kafka key and pre-built envelope.
+ pub fn with_event(key: impl Into<String>, event: Event<SoupTopicEvent>) -> Self {
Self {
- schema_version: Self::SCHEMA_VERSION,
- user_id,
- item,
+ key: key.into(),
+ event,
}
}
+
+ /// Returns the recipient-targeted message carried by this event.
+ pub fn into_message(self) -> SoupRealtimeMessage {
+ match self.event.event {
+ SoupTopicEvent::ItemUpdated(message) => message,
+ }
+ }
+}
+
+impl MacroEvent for SoupMacroEvent {
+ type EventPayload = SoupTopicEvent;
+
+ fn key(&self) -> &str {
+ &self.key
+ }
+
+ fn event(&self) -> &Event<Self::EventPayload> {
+ &self.event
+ }
+
+ fn from_event(key: String, event: Event<Self::EventPayload>) -> Self {
+ Self::with_event(key, event)
+ }
}
diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rs
index 66e43e9342..f9bf0e77c0 100644
--- a/crates/soup_realtime/src/domain/service/test.rs
+++ b/crates/soup_realtime/src/domain/service/test.rs
@@ -181,7 +181,7 @@ async fn zero_users_skips_reads_and_publications() {
}
#[tokio::test]
-async fn one_user_receives_one_full_versioned_message() {
+async fn one_user_receives_one_full_message() {
let recipient = user("one");
let viewed_at = Some(timestamp(3));
let responses = HashMap::from([item_response(
@@ -199,7 +199,6 @@ async fn one_user_receives_one_full_versioned_message() {
let mut messages = harness.messages.lock().expect("messages lock");
let message = messages.pop().expect("one message");
assert!(messages.is_empty());
- assert_eq!(message.schema_version, 1);
assert_eq!(message.user_id, recipient);
match message.item {
SoupItem::Document(document) => {
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rs
index 7659820494..8adff981d4 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs
@@ -9,10 +9,11 @@ mod test;
use std::{future::Future, time::Duration};
-use documents::domain::events::DocumentTopicEvent;
-use kafka_consumer_util::{GroupName, KafkaEventConsumer};
-use macro_event_broker::{Event, EventBrokerError, Topic as _};
-use macro_event_topics::MacroDocumentsTopic;
+use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};
+use kafka_util::{GroupName, KafkaEventConsumer};
+use macro_event_broker::{
+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
+};
use model_entity::{Entity, EntityType};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
@@ -27,21 +28,21 @@ impl GroupName for SoupRealtimeConsumerGroup {
const GROUP_NAME: &'static str = "soup-realtime";
}
-type SoupRealtimeKafkaConsumer = KafkaEventConsumer<SoupRealtimeConsumerGroup>;
+type SoupRealtimeKafkaAdapter = KafkaConsumerAdapter<SoupRealtimeConsumerGroup, DeclaredMacroEvent>;
+type SoupRealtimeKafkaConsumer =
+ MacroEventConsumerService<DeclaredMacroEvent, SoupRealtimeKafkaAdapter>;
+
+macro_event_broker::declare_topics!(DeclaredMacroEvent: DocumentMacroEvent);
/// Total service attempts before returning for supervisor-driven redelivery.
const MAX_SERVICE_ATTEMPTS: u32 = 5;
/// Delay before the first retry; each subsequent delay doubles.
const SERVICE_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
-fn subscribed_topics() -> [&'static str; 1] {
- [MacroDocumentsTopic.as_str()]
-}
-
-fn entity_from_document_event(event: DocumentTopicEvent) -> Option<Entity<'static>> {
- match event {
+fn entity_from_document_event(event: &DocumentMacroEvent) -> Option<Entity<'static>> {
+ match &event.event().event {
DocumentTopicEvent::Updated(metadata) => {
- Some(EntityType::Document.with_entity_string(metadata.document_id))
+ Some(EntityType::Document.with_entity_string(metadata.document_id.clone()))
}
DocumentTopicEvent::Created(_) => {
tracing::trace!(event_type = "document.created", "ignoring document event");
@@ -55,38 +56,37 @@ fn entity_from_document_event(event: DocumentTopicEvent) -> Option<Entity<'stati
tracing::trace!(event_type = "document.copied", "ignoring document event");
None
}
+ DocumentTopicEvent::Interaction(_) => {
+ tracing::trace!(
+ event_type = "document.interaction",
+ "ignoring document event"
+ );
+ None
+ }
}
}
-fn decode_updated_entity(payload: &[u8]) -> Result<Option<Entity<'static>>, EventBrokerError> {
- let event = Event::<DocumentTopicEvent>::decode(payload)?;
- Ok(entity_from_document_event(event.event))
-}
-
-/// Commit-safe outcome after processing one non-empty document payload.
-enum DocumentPayloadOutcome {
+/// Commit-safe outcome after processing one document event.
+enum DocumentEventOutcome {
/// An update was successfully sent through the domain service.
Notified,
/// A recognized document event is outside the current scope.
Ignored,
- /// The payload could not be decoded and should be treated as poison.
- Malformed(EventBrokerError),
}
-#[tracing::instrument(skip(service, payload), fields(partition, offset, payload_len = payload.len()), err)]
-async fn process_document_payload<S: SoupRealtimeService>(
+#[tracing::instrument(skip(service, event), fields(partition, offset), err)]
+async fn process_document_event<S: SoupRealtimeService>(
service: &S,
- payload: &[u8],
+ event: &DocumentMacroEvent,
partition: i32,
offset: i64,
-) -> Result<DocumentPayloadOutcome, Report> {
- match decode_updated_entity(payload) {
- Ok(Some(entity)) => {
+) -> Result<DocumentEventOutcome, Report> {
+ match entity_from_document_event(event) {
+ Some(entity) => {
notify_with_retry(service, entity, partition, offset).await?;
- Ok(DocumentPayloadOutcome::Notified)
+ Ok(DocumentEventOutcome::Notified)
}
- Ok(None) => Ok(DocumentPayloadOutcome::Ignored),
- Err(error) => Ok(DocumentPayloadOutcome::Malformed(error)),
+ None => Ok(DocumentEventOutcome::Ignored),
}
}
@@ -140,7 +140,7 @@ async fn notify_with_retry<S: SoupRealtimeService>(
}
fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {
- match consumer.commit_message(message, CommitMode::Async) {
+ match consumer.inner().commit_message(message, CommitMode::Async) {
Ok(()) => tracing::trace!(
partition = message.partition(),
offset = message.offset(),
@@ -171,12 +171,13 @@ pub async fn run_document_update_consumer<S>(
where
S: SoupRealtimeService,
{
- let consumer = SoupRealtimeKafkaConsumer::from_env(brokers)?;
- consumer
- .subscribe(&subscribed_topics())
+ let consumer = KafkaEventConsumer::<SoupRealtimeConsumerGroup>::from_env(brokers)?;
+ let consumer = KafkaConsumerAdapter::<SoupRealtimeConsumerGroup, ()>::new(consumer)
+ .subscribe::<DeclaredMacroEvent>()
.context("failed to subscribe to document update topic")?;
+ let consumer = SoupRealtimeKafkaConsumer::new(consumer);
tracing::info!(
- topics = ?subscribed_topics(),
+ topics = ?DeclaredMacroEvent::topics(),
group = SoupRealtimeConsumerGroup::GROUP_NAME,
"realtime Soup document consumer listening"
);
@@ -196,63 +197,51 @@ where
continue;
}
};
- tracing::trace!(
- topic = message.topic(),
- partition = message.partition(),
- offset = message.offset(),
- payload_len = message.payload().map_or(0, <[u8]>::len),
- "received realtime Soup input"
- );
-
- let Some(payload) = message.payload().filter(|payload| !payload.is_empty()) else {
- tracing::warn!(
- partition = message.partition(),
- offset = message.offset(),
- "skipping document event with empty payload"
- );
- commit_logged(&consumer, &message);
- continue;
+ let kafka_message = message.inner();
+ let event = match message.decode_payload() {
+ Ok(DeclaredMacroEvent::DocumentMacroEvent(event)) => event,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ topic = kafka_message.topic(),
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
+ "dropping malformed document event"
+ );
+ commit_logged(&consumer, kafka_message);
+ continue;
+ }
};
- let outcome = match process_document_payload(
+ match process_document_event(
&service,
- payload,
- message.partition(),
- message.offset(),
+ &event,
+ kafka_message.partition(),
+ kafka_message.offset(),
)
.await
{
- Ok(outcome) => outcome,
+ Ok(DocumentEventOutcome::Notified | DocumentEventOutcome::Ignored) => {}
Err(error) => {
tracing::error!(
error = ?error,
- topic = message.topic(),
- partition = message.partition(),
- offset = message.offset(),
+ topic = kafka_message.topic(),
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
"realtime Soup fan-out retries exhausted; pausing partition for redelivery"
);
// Kafka commits are cumulative within a partition, so
// pause it before any later record can advance the
// committed offset past this failure.
consumer
- .pause_message_partition(&message)
+ .inner()
+ .pause_message_partition(kafka_message)
.context("failed to pause Kafka partition after fan-out failure")?;
continue;
}
- };
-
- match outcome {
- DocumentPayloadOutcome::Notified | DocumentPayloadOutcome::Ignored => {}
- DocumentPayloadOutcome::Malformed(error) => tracing::error!(
- error = ?error,
- topic = message.topic(),
- partition = message.partition(),
- offset = message.offset(),
- "dropping malformed document event"
- ),
}
- commit_logged(&consumer, &message);
+ commit_logged(&consumer, kafka_message);
}
}
}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
index a00258b697..85c2d787c0 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
@@ -3,12 +3,35 @@ use std::sync::{Arc, Mutex};
use documents::domain::events::{
DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,
- DocumentUpdatedMetadata,
+ DocumentInteractionMetadata, DocumentUpdatedMetadata, InteractionReason,
};
+use macro_event_broker::{Event, EventBrokerError, MacroEventCollection as _, MessageParts};
use macro_user_id::user_id::MacroUserIdStr;
use super::*;
+struct TestMessage {
+ payload: Vec<u8>,
+}
+
+impl MessageParts for TestMessage {
+ fn key(&self) -> Option<&str> {
+ Some(DOCUMENT_ID)
+ }
+
+ fn payload(&self) -> Option<&[u8]> {
+ Some(&self.payload)
+ }
+
+ fn topic(&self) -> &str {
+ "macro.documents"
+ }
+}
+
+fn decode_payload(payload: Vec<u8>) -> Result<DeclaredMacroEvent, EventBrokerError> {
+ DeclaredMacroEvent::decode(&TestMessage { payload })
+}
+
const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";
fn user() -> MacroUserIdStr<'static> {
@@ -54,6 +77,10 @@ fn ignored_events() -> Vec<DocumentTopicEvent> {
project_id: None,
sub_type: None,
}),
+ DocumentTopicEvent::Interaction(DocumentInteractionMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ reason: InteractionReason::Edited,
+ }),
]
}
@@ -86,19 +113,19 @@ fn flaky_service(failures: u32) -> FlakyService {
#[test]
fn subscribes_only_to_documents() {
- assert_eq!(subscribed_topics(), ["macro.documents"]);
+ assert_eq!(DeclaredMacroEvent::topics(), ["macro.documents"]);
}
#[tokio::test]
async fn updated_payload_maps_to_document_entity() {
- let payload = serde_json::to_vec(&updated_event()).expect("serializable event");
+ let event = DocumentMacroEvent::with_event(DOCUMENT_ID, updated_event());
let service = flaky_service(0);
assert!(matches!(
- process_document_payload(&service, &payload, 0, 0)
+ process_document_event(&service, &event, 0, 0)
.await
.expect("processing succeeds"),
- DocumentPayloadOutcome::Notified
+ DocumentEventOutcome::Notified
));
let entities = service.entities.lock().expect("entities lock");
@@ -108,29 +135,23 @@ async fn updated_payload_maps_to_document_entity() {
}
#[tokio::test]
-async fn created_deleted_and_copied_events_are_ignored() {
+async fn non_update_events_are_ignored() {
let service = flaky_service(0);
for event in ignored_events() {
- let payload = serde_json::to_vec(&Event::new(event)).expect("serializable event");
+ let event = DocumentMacroEvent::with_event(DOCUMENT_ID, Event::new(event));
assert!(matches!(
- process_document_payload(&service, &payload, 0, 0)
+ process_document_event(&service, &event, 0, 0)
.await
.expect("processing succeeds"),
- DocumentPayloadOutcome::Ignored
+ DocumentEventOutcome::Ignored
));
}
assert_eq!(service.attempts.load(Ordering::SeqCst), 0);
}
-#[tokio::test]
-async fn malformed_and_unknown_events_are_rejected_without_notifying() {
- let service = flaky_service(0);
- assert!(matches!(
- process_document_payload(&service, b"not json", 0, 0)
- .await
- .expect("poison payload is commit-safe"),
- DocumentPayloadOutcome::Malformed(_)
- ));
+#[test]
+fn malformed_and_unknown_events_are_rejected_by_the_declared_collection() {
+ assert!(decode_payload(b"not json".to_vec()).is_err());
let unknown = serde_json::json!({
"event_id": "00000000-0000-0000-0000-000000000003",
@@ -139,13 +160,7 @@ async fn malformed_and_unknown_events_are_rejected_without_notifying() {
"metadata": { "document_id": DOCUMENT_ID }
});
let payload = serde_json::to_vec(&unknown).expect("serializable JSON");
- assert!(matches!(
- process_document_payload(&service, &payload, 0, 0)
- .await
- .expect("poison payload is commit-safe"),
- DocumentPayloadOutcome::Malformed(_)
- ));
- assert_eq!(service.attempts.load(Ordering::SeqCst), 0);
+ assert!(decode_payload(payload).is_err());
}
#[tokio::test(start_paused = true)]
diff --git a/crates/soup_realtime/src/inbound/mod.rs b/crates/soup_realtime/src/inbound/mod.rs
index 1d73c01b15..e22ac579a2 100644
--- a/crates/soup_realtime/src/inbound/mod.rs
+++ b/crates/soup_realtime/src/inbound/mod.rs
@@ -2,3 +2,5 @@
/// Kafka consumer for document update events.
pub mod kafka_consumer;
+/// Independent Kafka consumer for recipient-targeted Soup messages.
+pub mod soup_consumer;
diff --git a/crates/soup_realtime/src/inbound/soup_consumer.rs b/crates/soup_realtime/src/inbound/soup_consumer.rs
new file mode 100644
index 0000000000..a0dca06f2a
--- /dev/null
+++ b/crates/soup_realtime/src/inbound/soup_consumer.rs
@@ -0,0 +1,75 @@
+//! Independent Kafka consumer for recipient-targeted Soup messages.
+//!
+//! Every process receives every message published after it starts because this
+//! adapter manually assigns all `macro.soup` partitions without joining a
+//! durable consumer group. It does not commit offsets.
+
+#[cfg(test)]
+mod test;
+
+use std::time::Duration;
+
+use kafka_util::{InitialOffset, KafkaEventConsumer, Ungrouped};
+use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};
+use rootcause::prelude::{Report, ResultExt as _};
+
+use crate::domain::models::{SoupMacroEvent, SoupRealtimeMessage};
+
+/// Maximum time to wait for Soup topic metadata during partition assignment.
+const TOPIC_METADATA_TIMEOUT: Duration = Duration::from_secs(10);
+
+type IndependentKafkaConsumer = KafkaConsumerAdapter<Ungrouped, DeclaredMacroEvent>;
+type SoupEventConsumer = MacroEventConsumerService<DeclaredMacroEvent, IndependentKafkaConsumer>;
+
+macro_event_broker::declare_topics!(DeclaredMacroEvent: SoupMacroEvent);
+
+/// Independent consumer of recipient-targeted Soup messages.
+///
+/// This consumer starts at the end of every current `macro.soup` partition, so
+/// it receives only messages published after construction. It does not join a
+/// durable consumer group or persist offsets. Partitions added after
+/// construction require a new consumer so they can be assigned.
+pub struct SoupTopicConsumer {
+ consumer: SoupEventConsumer,
+}
+
+impl SoupTopicConsumer {
+ /// Creates a consumer and assigns every current Soup topic partition.
+ #[tracing::instrument(fields(brokers), err)]
+ pub fn from_env(brokers: &str) -> Result<Self, Report> {
+ let consumer = KafkaEventConsumer::<Ungrouped>::from_env(brokers)
+ .context("failed to create independent realtime Soup consumer")?;
+ let consumer =
+ IndependentKafkaConsumer::new(consumer, InitialOffset::Latest, TOPIC_METADATA_TIMEOUT)
+ .context("failed to assign realtime Soup topic partitions")?;
+
+ tracing::info!(
+ topics = ?DeclaredMacroEvent::topics(),
+ "independent realtime Soup consumer listening"
+ );
+
+ Ok(Self {
+ consumer: SoupEventConsumer::new(consumer),
+ })
+ }
+
+ /// Receives and decodes the next typed realtime Soup event.
+ ///
+ /// This operation is cancel-safe. Missing keys and malformed or unsupported
+ /// payloads are returned as errors; because the consumer is ungrouped, a
+ /// subsequent call proceeds to the next locally assigned record.
+ pub async fn recv(&self) -> Result<SoupRealtimeMessage, Report> {
+ let message = self
+ .consumer
+ .recv()
+ .await
+ .context("failed to receive realtime Soup event")?;
+ let event = message
+ .decode_payload()
+ .context("failed to decode realtime Soup event")?;
+
+ match event {
+ DeclaredMacroEvent::SoupMacroEvent(event) => Ok(event.into_message()),
+ }
+ }
+}
diff --git a/crates/soup_realtime/src/inbound/soup_consumer/test.rs b/crates/soup_realtime/src/inbound/soup_consumer/test.rs
new file mode 100644
index 0000000000..2d5e577f53
--- /dev/null
+++ b/crates/soup_realtime/src/inbound/soup_consumer/test.rs
@@ -0,0 +1,8 @@
+use macro_event_broker::MacroEventCollection as _;
+
+use super::DeclaredMacroEvent;
+
+#[test]
+fn assigns_only_the_typed_soup_topic() {
+ assert_eq!(DeclaredMacroEvent::topics(), ["macro.soup"]);
+}
diff --git a/crates/soup_realtime/src/outbound/kafka_publisher.rs b/crates/soup_realtime/src/outbound/kafka_publisher.rs
index 5b1a8987da..2f59c1904b 100644
--- a/crates/soup_realtime/src/outbound/kafka_publisher.rs
+++ b/crates/soup_realtime/src/outbound/kafka_publisher.rs
@@ -1,29 +1,31 @@
-//! Kafka publisher for versioned realtime Soup messages.
+//! Macro event broker publisher for realtime Soup events.
#[cfg(test)]
mod test;
-use macro_event_broker::EventPublisher;
-use macro_event_topics::MacroSoupRealtimeTopic;
+use macro_event_broker::MacroEventBroker;
use rootcause::prelude::{Report, ResultExt as _};
-use crate::domain::{models::SoupRealtimeMessage, ports::SoupRealtimePublisher};
+use crate::domain::{
+ models::{SoupMacroEvent, SoupRealtimeMessage},
+ ports::SoupRealtimePublisher,
+};
-/// Realtime Soup publisher backed by a byte-oriented event publisher.
-pub struct KafkaSoupRealtimePublisher<P> {
- publisher: P,
+/// Realtime Soup publisher backed by the typed macro event broker service.
+pub struct KafkaSoupRealtimePublisher<B> {
+ broker: B,
}
-impl<P> KafkaSoupRealtimePublisher<P> {
- /// Creates a realtime Soup Kafka publisher.
- pub fn new(publisher: P) -> Self {
- Self { publisher }
+impl<B> KafkaSoupRealtimePublisher<B> {
+ /// Creates a realtime Soup publisher from a macro event broker service.
+ pub fn new(broker: B) -> Self {
+ Self { broker }
}
}
-impl<P> SoupRealtimePublisher for KafkaSoupRealtimePublisher<P>
+impl<B> SoupRealtimePublisher for KafkaSoupRealtimePublisher<B>
where
- P: EventPublisher,
+ B: MacroEventBroker,
{
#[tracing::instrument(
skip(self, message),
@@ -31,13 +33,16 @@ where
err
)]
async fn publish(&self, message: SoupRealtimeMessage) -> Result<(), Report> {
- let payload =
- serde_json::to_vec(&message).context("failed to serialize realtime Soup message")?;
+ let event = SoupMacroEvent::item_updated(message);
+ let publish = self
+ .broker
+ .send_event(&event)
+ .context("failed to dispatch realtime Soup event")?;
- self.publisher
- .publish(MacroSoupRealtimeTopic, message.user_id.as_ref(), &payload)
+ publish
.await
- .context("failed to publish realtime Soup message")?;
+ .context("realtime Soup publish task failed")?
+ .context("failed to publish realtime Soup event")?;
Ok(())
}
diff --git a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs
index a16489484c..a585ef2010 100644
--- a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs
+++ b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs
@@ -1,7 +1,9 @@
use std::sync::{Arc, Mutex};
use chrono::{DateTime, Utc};
-use macro_event_broker::{EventBrokerError, Topic};
+use macro_event_broker::{
+ EventBrokerError, EventPublisher, MacroEvent, MacroEventBrokerService, Topic,
+};
use macro_user_id::user_id::MacroUserIdStr;
use models_soup::{document::SoupDocument, item::SoupItem};
use uuid::Uuid;
@@ -21,17 +23,12 @@ struct RecordingPublisher {
}
impl EventPublisher for RecordingPublisher {
- async fn publish<T: Topic>(
- &self,
- topic: T,
- key: &str,
- payload: &[u8],
- ) -> Result<(), EventBrokerError> {
+ async fn publish<T: Topic>(&self, key: &str, payload: &[u8]) -> Result<(), EventBrokerError> {
self.records
.lock()
.expect("records lock")
.push(PublishedRecord {
- topic: topic.as_str().to_string(),
+ topic: T::TOPIC_STR.to_string(),
key: key.to_string(),
payload: payload.to_vec(),
});
@@ -76,12 +73,13 @@ fn message() -> SoupRealtimeMessage {
}
#[tokio::test]
-async fn publishes_user_keyed_full_message_to_realtime_topic() {
+async fn publishes_typed_recipient_keyed_event_to_soup_topic() {
let records = Arc::new(Mutex::new(Vec::new()));
- let adapter = KafkaSoupRealtimePublisher::new(RecordingPublisher {
+ let broker = MacroEventBrokerService::new(RecordingPublisher {
records: records.clone(),
fail: false,
});
+ let adapter = KafkaSoupRealtimePublisher::new(broker);
adapter.publish(message()).await.expect("publish succeeds");
@@ -92,13 +90,14 @@ async fn publishes_user_keyed_full_message_to_realtime_topic() {
assert_eq!(record.key, "macro|[email protected]");
let json: serde_json::Value = serde_json::from_slice(&record.payload).expect("payload is JSON");
+ assert!(json["event_id"].is_string());
assert_eq!(json["schema_version"], 1);
- assert_eq!(json["user_id"], "macro|[email protected]");
- assert!(json.get("event_id").is_none());
+ assert_eq!(json["event_type"], "soup.item.updated");
+ assert_eq!(json["metadata"]["user_id"], "macro|[email protected]");
- let decoded: SoupRealtimeMessage =
- serde_json::from_slice(&record.payload).expect("payload round-trips");
- assert_eq!(decoded.schema_version, 1);
+ let decoded = SoupMacroEvent::decode(record.key, &record.payload).expect("event round-trips");
+ assert_eq!(decoded.event().schema_version, 1);
+ let decoded = decoded.into_message();
assert_eq!(decoded.user_id.as_ref(), "macro|[email protected]");
match decoded.item {
SoupItem::Document(document) => {
@@ -111,11 +110,12 @@ async fn publishes_user_keyed_full_message_to_realtime_topic() {
}
#[tokio::test]
-async fn propagates_delivery_failures() {
- let adapter = KafkaSoupRealtimePublisher::new(RecordingPublisher {
+async fn propagates_delivery_failures_from_event_broker_service() {
+ let broker = MacroEventBrokerService::new(RecordingPublisher {
records: Arc::new(Mutex::new(Vec::new())),
fail: true,
});
+ let adapter = KafkaSoupRealtimePublisher::new(broker);
adapter
.publish(message())
diff --git a/crates/teams/Cargo.toml b/crates/teams/Cargo.toml
index 8f45d130fb..0b25e0af47 100644
--- a/crates/teams/Cargo.toml
+++ b/crates/teams/Cargo.toml
@@ -38,7 +38,6 @@ ports = [
"dep:model_notifications",
"dep:notification",
"dep:tokio",
- "macro_event_broker/ports",
]
[dependencies]
diff --git a/crates/teams/src/domain/events/test.rs b/crates/teams/src/domain/events/test.rs
index 3416f2d825..ba05f11c5d 100644
--- a/crates/teams/src/domain/events/test.rs
+++ b/crates/teams/src/domain/events/test.rs
@@ -1,5 +1,4 @@
use macro_event_broker::{Event, MacroEvent};
-use macro_event_topics::Topic;
use macro_user_id::user_id::MacroUserIdStr;
use serde_json::{Value, json};
use uuid::Uuid;
@@ -261,7 +260,7 @@ fn every_variant_round_trips() {
assert_eq!(decoded.key(), TEAM_ID);
assert_eq!(decoded.event(), original.event());
- assert_eq!(decoded.topic().as_str(), "macro.teams");
+ assert_eq!(decoded.topic(), "macro.teams");
}
}
@@ -270,7 +269,7 @@ fn constructors_use_teams_topic_bare_uuid_key_and_schema_version_one() {
for event in macro_events() {
assert_eq!(event.key(), TEAM_ID);
assert!(!event.key().starts_with("team|"));
- assert_eq!(event.topic().as_str(), "macro.teams");
+ assert_eq!(event.topic(), "macro.teams");
assert_eq!(event.event().schema_version, 1);
}
}
diff --git a/crates/teams/src/domain/team_service/test.rs b/crates/teams/src/domain/team_service/test.rs
index f74e3485ac..4f6bb36b1b 100644
--- a/crates/teams/src/domain/team_service/test.rs
+++ b/crates/teams/src/domain/team_service/test.rs
@@ -22,7 +22,7 @@ use entity_access::domain::models::{
AdminTeamRole, EntityAccessReceipt, EntityType, MemberTeamRole, OwnerTeamRole,
RequiredPermission,
};
-use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, Topic as _};
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
use macro_user_id::{email::Email, lowercased::Lowercase, user_id::MacroUserIdStr};
use models_pagination::{CreatedAt, Query};
use notification::domain::{
@@ -1219,7 +1219,7 @@ impl MacroEventBroker for RecordingEventBroker {
}
self.events.lock().unwrap().push(PublishedTeamEvent {
- topic: event.topic().as_str(),
+ topic: event.topic(),
key: event.key().to_string(),
envelope: serde_json::to_value(event.event())?,
});
diff --git a/crates/webhook/Cargo.toml b/crates/webhook/Cargo.toml
index c5f68ced18..104b3b8ad8 100644
--- a/crates/webhook/Cargo.toml
+++ b/crates/webhook/Cargo.toml
@@ -6,10 +6,11 @@ version = "0.1.0"
[features]
consumer = [
- "dep:kafka_consumer_util",
+ "dep:kafka_util",
"dep:rdkafka",
"dep:tokio",
"ingestion",
+ "macro_event_broker/outbound",
]
default = ["consumer", "inbound", "outbound", "ports"]
inbound = [
@@ -44,7 +45,7 @@ outbound = [
"dep:url",
"ports",
]
-ports = ["dep:serde_json", "dep:url", "macro_event_broker/ports"]
+ports = ["dep:serde_json", "dep:url"]
worker = ["dep:tokio", "ports"]
[dependencies]
@@ -60,7 +61,7 @@ entity_access = { path = "../entity_access", default-features = false, features
futures = { workspace = true, optional = true }
hex = { workspace = true, optional = true }
hmac = { workspace = true, optional = true }
-kafka_consumer_util = { path = "../kafka_consumer_util", optional = true }
+kafka_util = { path = "../kafka_util", optional = true }
macro_authorization = { path = "../macro_authorization", optional = true, default-features = false, features = [
"axum",
] }
diff --git a/crates/webhook/src/domain/events/test.rs b/crates/webhook/src/domain/events/test.rs
index d64141fd07..09fc241d56 100644
--- a/crates/webhook/src/domain/events/test.rs
+++ b/crates/webhook/src/domain/events/test.rs
@@ -1,6 +1,5 @@
use chrono::{DateTime, Utc};
use macro_event_broker::{Event, MacroEvent};
-use macro_event_topics::Topic;
use macro_user_id::user_id::MacroUserIdStr;
use serde_json::json;
use uuid::Uuid;
@@ -254,7 +253,7 @@ fn constructors_key_events_by_subject_webhook_id() {
for (event, expected_event_type) in cases.into_iter().zip(expected_event_types) {
assert_eq!(event.key(), WEBHOOK_ID);
- assert_eq!(event.topic().as_str(), "macro.webhooks");
+ assert_eq!(event.topic(), "macro.webhooks");
assert_eq!(event.event().schema_version, 1);
assert_eq!(
serde_json::to_value(event.event()).expect("serializable event")["event_type"],
@@ -290,5 +289,5 @@ fn macro_event_round_trips_without_exposing_secrets_or_header_values() {
let decoded = WebhookMacroEvent::decode(original.key(), &payload).expect("decodable event");
assert_eq!(decoded.key(), WEBHOOK_ID);
assert_eq!(decoded.event(), original.event());
- assert_eq!(decoded.topic().as_str(), "macro.webhooks");
+ assert_eq!(decoded.topic(), "macro.webhooks");
}
diff --git a/crates/webhook/src/domain/service_test.rs b/crates/webhook/src/domain/service_test.rs
index 43b0f6273e..686bb68744 100644
--- a/crates/webhook/src/domain/service_test.rs
+++ b/crates/webhook/src/domain/service_test.rs
@@ -8,9 +8,7 @@ use super::{
service::WebhookServiceImpl,
};
use chrono::Utc;
-use macro_event_broker::{
- EventBrokerError, MacroEvent, MacroEventBroker, NoopMacroEventBroker, Topic as _,
-};
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, NoopMacroEventBroker};
use macro_user_id::user_id::MacroUserIdStr;
use serde_json::json;
use std::{
@@ -266,7 +264,7 @@ impl MacroEventBroker for TestEventBroker {
}
self.published.lock().unwrap().push(PublishedEvent {
- topic: event.topic().as_str(),
+ topic: event.topic(),
key: event.key().to_string(),
payload: serde_json::to_value(event.event())?,
});
diff --git a/crates/webhook/src/inbound/kafka_consumer.rs b/crates/webhook/src/inbound/kafka_consumer.rs
index 8327fee250..d2bdb6d455 100644
--- a/crates/webhook/src/inbound/kafka_consumer.rs
+++ b/crates/webhook/src/inbound/kafka_consumer.rs
@@ -1,7 +1,7 @@
//! Kafka consumer that feeds broker events into webhook ingestion.
//!
-//! Subscribes to [`MacroDocumentsTopic`], [`MacroChannelsTopic`], and
-//! [`MacroWebhooksTopic`] and hands every decoded event envelope to a
+//! Subscribes to [`DocumentMacroEvent`], [`ChannelMacroEvent`], and
+//! [`WebhookMacroEvent`] topics and hands every decoded event envelope to a
//! [`WebhookEventIngestionService`].
//!
//! Delivery is at-least-once: an event's offset is committed only after the
@@ -21,13 +21,14 @@
#[cfg(test)]
mod test;
-use crate::domain::{events::WebhookTopicEvent, ingestion::WebhookEventIngestionService};
+use crate::domain::{events::WebhookMacroEvent, ingestion::WebhookEventIngestionService};
use anyhow::Context as _;
-use channels::domain::broker_events::ChannelTopicEvent;
-use documents::domain::events::DocumentTopicEvent;
-use kafka_consumer_util::{GroupName, KafkaEventConsumer};
-use macro_event_broker::{Event, EventBrokerError, Topic as _};
-use macro_event_topics::{MacroChannelsTopic, MacroDocumentsTopic, MacroWebhooksTopic};
+use channels::domain::broker_events::ChannelMacroEvent;
+use documents::domain::events::DocumentMacroEvent;
+use kafka_util::{GroupName, KafkaEventConsumer};
+use macro_event_broker::{
+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
+};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message};
use std::future::Future;
@@ -41,7 +42,15 @@ impl GroupName for WebhookEventIngestionConsumerGroup {
const GROUP_NAME: &'static str = "webhook-event-ingestion";
}
-type WebhookKafkaConsumer = KafkaEventConsumer<WebhookEventIngestionConsumerGroup>;
+type WebhookKafkaAdapter =
+ KafkaConsumerAdapter<WebhookEventIngestionConsumerGroup, DeclaredMacroEvent>;
+type WebhookKafkaConsumer = MacroEventConsumerService<DeclaredMacroEvent, WebhookKafkaAdapter>;
+
+macro_event_broker::declare_topics!(
+ DeclaredMacroEvent: DocumentMacroEvent,
+ ChannelMacroEvent,
+ WebhookMacroEvent,
+);
/// Maximum in-process ingestion attempts per event before the consumer bails
/// out and lets a restart redeliver from the last committed offset.
@@ -53,49 +62,9 @@ const MAX_INGEST_ATTEMPTS: u32 = 5;
/// consumer from its group.
const INGEST_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
-fn subscribed_topics() -> [&'static str; 3] {
- [
- MacroDocumentsTopic.as_str(),
- MacroChannelsTopic.as_str(),
- MacroWebhooksTopic.as_str(),
- ]
-}
-
-/// One decoded event envelope from a topic this consumer subscribes to.
-///
-/// The Kafka message key is not carried: every event's metadata already
-/// contains the entity ids the ingestion service needs.
-#[derive(Debug)]
-pub enum WebhookConsumerEvent {
- /// Event received on [`MacroDocumentsTopic`].
- Documents(Event<DocumentTopicEvent>),
- /// Event received on [`MacroChannelsTopic`].
- Channels(Event<ChannelTopicEvent>),
- /// Event received on [`MacroWebhooksTopic`].
- Webhooks(Event<WebhookTopicEvent>),
-}
-
-impl WebhookConsumerEvent {
- /// Decode one Kafka message into this consumer's event enum.
- pub fn decode(topic: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {
- match topic {
- topic if topic == MacroDocumentsTopic.as_str() => {
- Ok(Self::Documents(Event::decode(payload)?))
- }
- topic if topic == MacroChannelsTopic.as_str() => {
- Ok(Self::Channels(Event::decode(payload)?))
- }
- topic if topic == MacroWebhooksTopic.as_str() => {
- Ok(Self::Webhooks(Event::decode(payload)?))
- }
- unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),
- }
- }
-}
-
/// Commit `message`'s offset, logging the outcome.
fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>) {
- match consumer.commit_message(message, CommitMode::Async) {
+ match consumer.inner().commit_message(message, CommitMode::Async) {
Ok(()) => tracing::trace!(
partition = message.partition(),
offset = message.offset(),
@@ -118,7 +87,7 @@ fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>)
/// the caller must exit without committing so the event is redelivered.
async fn ingest_with_retry<S: WebhookEventIngestionService>(
service: &S,
- event: &WebhookConsumerEvent,
+ event: &DeclaredMacroEvent,
partition: i32,
offset: i64,
) -> anyhow::Result<()> {
@@ -127,14 +96,14 @@ async fn ingest_with_retry<S: WebhookEventIngestionService>(
loop {
tracing::trace!(partition, offset, attempt, "ingesting broker event");
let result = match event {
- WebhookConsumerEvent::Documents(event) => {
- service.ingest_document_event(event.clone()).await
+ DeclaredMacroEvent::DocumentMacroEvent(event) => {
+ service.ingest_document_event(event.event().clone()).await
}
- WebhookConsumerEvent::Channels(event) => {
- service.ingest_channel_event(event.clone()).await
+ DeclaredMacroEvent::ChannelMacroEvent(event) => {
+ service.ingest_channel_event(event.event().clone()).await
}
- WebhookConsumerEvent::Webhooks(event) => {
- service.ingest_webhook_event(event.clone()).await
+ DeclaredMacroEvent::WebhookMacroEvent(event) => {
+ service.ingest_webhook_event(event.event().clone()).await
}
};
match result {
@@ -179,9 +148,9 @@ async fn ingest_with_retry<S: WebhookEventIngestionService>(
/// Run the webhook event consumer until `shutdown` resolves.
///
-/// Connects to `brokers` and subscribes to [`MacroDocumentsTopic`],
-/// [`MacroChannelsTopic`], and [`MacroWebhooksTopic`] under the
-/// `webhook-event-ingestion` consumer group. Every decoded event is fed to
+/// Connects to `brokers` and subscribes to the topics declared by
+/// [`DocumentMacroEvent`], [`ChannelMacroEvent`], and [`WebhookMacroEvent`]
+/// under the `webhook-event-ingestion` consumer group. Every decoded event is fed to
/// `service`, committing each offset only
/// after ingestion succeeds (see `ingest_with_retry` for the retry policy).
/// Returns an error when the consumer cannot be created or subscribed, or when
@@ -197,12 +166,15 @@ pub async fn run_webhook_event_consumer<S>(
where
S: WebhookEventIngestionService,
{
- let consumer = WebhookKafkaConsumer::from_env(brokers)?;
- consumer
- .subscribe(&subscribed_topics())
- .context("failed to subscribe to webhook event topics")?;
+ let consumer = KafkaEventConsumer::<WebhookEventIngestionConsumerGroup>::from_env(brokers)?;
+ let consumer = KafkaConsumerAdapter::<WebhookEventIngestionConsumerGroup, ()>::new(consumer)
+ .subscribe::<DeclaredMacroEvent>()
+ .map_err(|error| {
+ anyhow::anyhow!("failed to subscribe to webhook event topics: {error:?}")
+ })?;
+ let consumer = WebhookKafkaConsumer::new(consumer);
tracing::info!(
- topics = ?subscribed_topics(),
+ topics = ?DeclaredMacroEvent::topics(),
group = WebhookEventIngestionConsumerGroup::GROUP_NAME,
"webhook event consumer listening"
);
@@ -222,36 +194,19 @@ where
continue;
}
};
- tracing::trace!(
- topic = message.topic(),
- partition = message.partition(),
- offset = message.offset(),
- payload_len = message.payload().map_or(0, <[u8]>::len),
- "received kafka message"
- );
-
- let Some(payload) = message.payload() else {
- tracing::warn!(
- partition = message.partition(),
- offset = message.offset(),
- "skipping message with empty payload"
- );
- commit_logged(&consumer, &message);
- continue;
- };
-
- match WebhookConsumerEvent::decode(message.topic(), payload) {
+ let kafka_message = message.inner();
+ match message.decode_payload() {
Ok(event) => {
tracing::trace!(
- partition = message.partition(),
- offset = message.offset(),
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
"decoded broker event"
);
ingest_with_retry(
&service,
&event,
- message.partition(),
- message.offset(),
+ kafka_message.partition(),
+ kafka_message.offset(),
)
.await?;
}
@@ -259,16 +214,16 @@ where
// wedging the partition on a poison message.
Err(e) => tracing::error!(
error = ?e,
- topic = message.topic(),
- partition = message.partition(),
- offset = message.offset(),
+ topic = kafka_message.topic(),
+ partition = kafka_message.partition(),
+ offset = kafka_message.offset(),
"failed to decode broker event"
),
}
// Commit only after the event was ingested or permanently
// rejected: at-least-once, retried across restarts.
- commit_logged(&consumer, &message);
+ commit_logged(&consumer, kafka_message);
}
}
}
diff --git a/crates/webhook/src/inbound/kafka_consumer/test.rs b/crates/webhook/src/inbound/kafka_consumer/test.rs
index 861f11315c..1e456c670f 100644
--- a/crates/webhook/src/inbound/kafka_consumer/test.rs
+++ b/crates/webhook/src/inbound/kafka_consumer/test.rs
@@ -4,13 +4,39 @@ use crate::domain::{
ingestion::WebhookEventIngestionError,
};
use channel_sender::ChannelSender;
-use channels::domain::broker_events::ChannelDeletedMetadata;
-use documents::domain::events::DocumentDeletedMetadata;
+use channels::domain::broker_events::{ChannelDeletedMetadata, ChannelTopicEvent};
+use documents::domain::events::{DocumentDeletedMetadata, DocumentTopicEvent};
+use macro_event_broker::{
+ Event, EventBrokerError, MacroEvent as _, MacroEventCollection as _, MessageParts,
+};
use macro_user_id::user_id::MacroUserIdStr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
use uuid::Uuid;
+struct TestMessage<'a> {
+ topic: &'a str,
+ payload: &'a [u8],
+}
+
+impl MessageParts for TestMessage<'_> {
+ fn key(&self) -> Option<&str> {
+ Some("event-key")
+ }
+
+ fn payload(&self) -> Option<&[u8]> {
+ Some(self.payload)
+ }
+
+ fn topic(&self) -> &str {
+ self.topic
+ }
+}
+
+fn decode_message(topic: &str, payload: &[u8]) -> Result<DeclaredMacroEvent, EventBrokerError> {
+ DeclaredMacroEvent::decode(&TestMessage { topic, payload })
+}
+
fn document_event() -> Event<DocumentTopicEvent> {
Event::new(DocumentTopicEvent::Deleted(DocumentDeletedMetadata {
document_id: "doc_1".to_string(),
@@ -28,10 +54,14 @@ fn webhook_event() -> Event<WebhookTopicEvent> {
}))
}
+fn declared_webhook_event() -> DeclaredMacroEvent {
+ DeclaredMacroEvent::WebhookMacroEvent(WebhookMacroEvent::with_event("wh_1", webhook_event()))
+}
+
#[test]
fn subscribes_to_all_ingestion_topics() {
assert_eq!(
- subscribed_topics(),
+ DeclaredMacroEvent::topics(),
["macro.documents", "macro.channels", "macro.webhooks"]
);
}
@@ -41,12 +71,11 @@ fn decodes_document_events() {
let event = document_event();
let payload = serde_json::to_vec(&event).expect("serializable");
- let decoded =
- WebhookConsumerEvent::decode(MacroDocumentsTopic.as_str(), &payload).expect("decodable");
+ let decoded = decode_message("macro.documents", &payload).expect("decodable");
match decoded {
- WebhookConsumerEvent::Documents(decoded) => assert_eq!(decoded, event),
- WebhookConsumerEvent::Channels(_) | WebhookConsumerEvent::Webhooks(_) => {
+ DeclaredMacroEvent::DocumentMacroEvent(decoded) => assert_eq!(decoded.event(), &event),
+ DeclaredMacroEvent::ChannelMacroEvent(_) | DeclaredMacroEvent::WebhookMacroEvent(_) => {
panic!("decoded into the wrong topic variant")
}
}
@@ -61,12 +90,11 @@ fn decodes_channel_events() {
}));
let payload = serde_json::to_vec(&event).expect("serializable");
- let decoded =
- WebhookConsumerEvent::decode(MacroChannelsTopic.as_str(), &payload).expect("decodable");
+ let decoded = decode_message("macro.channels", &payload).expect("decodable");
match decoded {
- WebhookConsumerEvent::Channels(decoded) => assert_eq!(decoded, event),
- WebhookConsumerEvent::Documents(_) | WebhookConsumerEvent::Webhooks(_) => {
+ DeclaredMacroEvent::ChannelMacroEvent(decoded) => assert_eq!(decoded.event(), &event),
+ DeclaredMacroEvent::DocumentMacroEvent(_) | DeclaredMacroEvent::WebhookMacroEvent(_) => {
panic!("decoded into the wrong topic variant")
}
}
@@ -77,12 +105,11 @@ fn decodes_webhook_events() {
let event = webhook_event();
let payload = serde_json::to_vec(&event).expect("serializable");
- let decoded =
- WebhookConsumerEvent::decode(MacroWebhooksTopic.as_str(), &payload).expect("decodable");
+ let decoded = decode_message("macro.webhooks", &payload).expect("decodable");
match decoded {
- WebhookConsumerEvent::Webhooks(decoded) => assert_eq!(decoded, event),
- WebhookConsumerEvent::Documents(_) | WebhookConsumerEvent::Channels(_) => {
+ DeclaredMacroEvent::WebhookMacroEvent(decoded) => assert_eq!(decoded.event(), &event),
+ DeclaredMacroEvent::DocumentMacroEvent(_) | DeclaredMacroEvent::ChannelMacroEvent(_) => {
panic!("decoded into the wrong topic variant")
}
}
@@ -90,14 +117,17 @@ fn decodes_webhook_events() {
#[test]
fn rejects_unknown_topics() {
- let err = WebhookConsumerEvent::decode("macro.example", b"{}").expect_err("unknown topic");
+ let err = decode_message("macro.example", b"{}")
+ .err()
+ .expect("unknown topic");
assert!(matches!(err, EventBrokerError::UnknownTopic(topic) if topic == "macro.example"));
}
#[test]
fn rejects_malformed_webhook_payloads() {
- let err = WebhookConsumerEvent::decode(MacroWebhooksTopic.as_str(), b"not json")
- .expect_err("malformed payload");
+ let err = decode_message("macro.webhooks", b"not json")
+ .err()
+ .expect("malformed payload");
assert!(matches!(err, EventBrokerError::Serialization(_)));
}
@@ -169,14 +199,9 @@ fn flaky_service(failures: u32, transient: bool) -> (FlakyIngestionService, Arc<
async fn retries_transient_webhook_failures_until_success() {
let (service, attempts) = flaky_service(2, true);
- ingest_with_retry(
- &service,
- &WebhookConsumerEvent::Webhooks(webhook_event()),
- 0,
- 0,
- )
- .await
- .expect("succeeds once the transient failure clears");
+ ingest_with_retry(&service, &declared_webhook_event(), 0, 0)
+ .await
+ .expect("succeeds once the transient failure clears");
assert_eq!(attempts.load(Ordering::SeqCst), 3);
}
@@ -185,14 +210,9 @@ async fn retries_transient_webhook_failures_until_success() {
async fn exhausted_transient_retries_bubble_up_for_redelivery() {
let (service, attempts) = flaky_service(u32::MAX, true);
- ingest_with_retry(
- &service,
- &WebhookConsumerEvent::Webhooks(webhook_event()),
- 0,
- 0,
- )
- .await
- .expect_err("persistent transient failure aborts the consumer without committing");
+ ingest_with_retry(&service, &declared_webhook_event(), 0, 0)
+ .await
+ .expect_err("persistent transient failure aborts the consumer without committing");
assert_eq!(attempts.load(Ordering::SeqCst), MAX_INGEST_ATTEMPTS);
}
@@ -201,14 +221,9 @@ async fn exhausted_transient_retries_bubble_up_for_redelivery() {
async fn permanent_webhook_failures_are_commit_safe_without_retry() {
let (service, attempts) = flaky_service(u32::MAX, false);
- ingest_with_retry(
- &service,
- &WebhookConsumerEvent::Webhooks(webhook_event()),
- 0,
- 0,
- )
- .await
- .expect("permanent failures are skipped so the offset commits");
+ ingest_with_retry(&service, &declared_webhook_event(), 0, 0)
+ .await
+ .expect("permanent failures are skipped so the offset commits");
assert_eq!(attempts.load(Ordering::SeqCst), 1);
}
diff --git a/tooling/xtask/crates/xtask_kafka_topics/src/main.rs b/tooling/xtask/crates/xtask_kafka_topics/src/main.rs
index b6f6d57dfd..b2b6db9777 100644
--- a/tooling/xtask/crates/xtask_kafka_topics/src/main.rs
+++ b/tooling/xtask/crates/xtask_kafka_topics/src/main.rs
@@ -27,8 +27,10 @@ fn run(check: bool) -> Result<()> {
// Sorted for a deterministic file regardless of declaration order. A bare
// array (no disclaimer wrapper) so infra can parse the file directly.
- let topics: BTreeSet<&'static str> =
- macro_event_topics::all_topic_names().into_iter().collect();
+ let topics: BTreeSet<&'static str> = macro_event_topics::all_topic_names()
+ .iter()
+ .copied()
+ .collect();
let new_contents =
serde_json::to_string_pretty(&topics).context("serializing kafka topics")? + "\n";