Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
#![deny(missing_docs)]
//! Environment-aware Kafka producer and consumer transports.
//!
//! These adapters centralize the repository's plaintext-local versus MSK-IAM
//! transport setup. Application inbound adapters retain responsibility for
//! decoding, retries, poison-message handling, and deciding when to commit.
#[cfg(test)]
mod test;
use std::collections::HashMap;
use std::marker::PhantomData;
use std::time::Duration;
use either::Either;
use macro_env::Environment;
use opentelemetry::propagation::{Extractor, Injector};
use rdkafka::consumer::{
CommitMode, Consumer, ConsumerContext, DefaultConsumerContext, StreamConsumer,
};
use rdkafka::error::{KafkaError, KafkaResult};
use rdkafka::message::{BorrowedMessage, Header, Headers, Message, OwnedHeaders};
use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::{ClientConfig, Offset, TopicPartitionList};
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
use uuid::Uuid;
pub use msk_iam::{MskIamClientContext, configure_sasl_iam};
pub use statistics::ConsumerStatistics;
use statistics::StatisticsContext;
mod msk_iam;
mod statistics;
const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";
const MESSAGE_TIMEOUT_MS: &str = "5000";
const SEND_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Default)]
struct TraceHeaderInjector(HashMap<String, String>);
impl Injector for TraceHeaderInjector {
fn set(&mut self, key: &str, value: String) {
self.0.insert(key.to_owned(), value);
}
}
struct TraceHeaderExtractor<'a, H: Headers>(&'a H);
impl<H: Headers> Extractor for TraceHeaderExtractor<'_, H> {
fn get(&self, key: &str) -> Option<&str> {
self.0.iter().find_map(|header| {
(header.key == key)
.then(|| {
header
.value
.and_then(|value| std::str::from_utf8(value).ok())
})
.flatten()
})
}
fn keys(&self) -> Vec<&str> {
self.0.iter().map(|header| header.key).collect()
}
}
fn current_trace_headers() -> OwnedHeaders {
let context = tracing::Span::current().context();
let mut injector = TraceHeaderInjector::default();
opentelemetry::global::get_text_map_propagator(|propagator| {
propagator.inject_context(&context, &mut injector);
});
injector
.0
.into_iter()
.fold(OwnedHeaders::new(), |headers, (key, value)| {
headers.insert(Header {
key: &key,
value: Some(&value),
})
})
}
fn remote_trace_context<M: Message>(message: &M) -> opentelemetry::Context {
opentelemetry::global::get_text_map_propagator(|propagator| {
message
.headers()
.map_or_else(opentelemetry::Context::new, |headers| {
propagator.extract(&TraceHeaderExtractor(headers))
})
})
}
/// Creates a Kafka processing span parented to the message's propagated W3C context.
///
/// The caller should instrument the complete decode, handling, and commit future
/// with this span so the consumer operation represents all record processing.
pub fn consumer_span<M: Message>(message: &M, consumer_group: &'static str) -> tracing::Span {
let span = tracing::info_span!(
"kafka.process",
otel.kind = "consumer",
messaging.system = "kafka",
messaging.operation.name = "process",
messaging.operation.type = "process",
messaging.consumer.group.name = consumer_group,
messaging.destination.name = message.topic(),
messaging.destination.partition.id = %message.partition(),
messaging.kafka.offset = message.offset(),
macro.event.id = tracing::field::Empty,
macro.event.type = tracing::field::Empty,
agent.session.id = tracing::field::Empty,
otel.status_code = tracing::field::Empty,
otel.status_description = tracing::field::Empty,
);
let parent = remote_trace_context(message);
// Setting an empty context is intentional: headerless records must not
// inherit an unrelated process- or task-lifetime ambient span.
let _ = span.set_parent(parent);
span
}
/// Marks a manually-created messaging span as failed.
pub fn record_span_error(span: &tracing::Span, error: &(impl std::fmt::Display + ?Sized)) {
span.record("otel.status_code", "ERROR");
span.record("otel.status_description", tracing::field::display(error));
}
/// Failure to construct an environment-specific Kafka consumer.
#[derive(Debug, thiserror::Error)]
pub enum KafkaConsumerError {
/// Failed to create the unauthenticated local consumer.
#[error("failed to create plaintext Kafka consumer")]
Plaintext(#[source] KafkaError),
/// Failed to create the TLS and MSK-IAM authenticated consumer.
#[error("failed to create MSK IAM Kafka consumer")]
MskIam(#[source] KafkaError),
}
/// Failure to construct an environment-specific Kafka producer.
#[derive(Debug, thiserror::Error)]
pub enum KafkaProducerError {
/// Failed to create the unauthenticated local producer.
#[error("failed to create plaintext Kafka producer")]
Plaintext(#[source] KafkaError),
/// Failed to create the TLS and MSK-IAM authenticated producer.
#[error("failed to create MSK IAM Kafka producer")]
MskIam(#[source] KafkaError),
}
/// Underlying Kafka consumer transport selected from the runtime environment.
struct ConsumerTransport(
Either<
StreamConsumer<StatisticsContext<DefaultConsumerContext>>,
StreamConsumer<StatisticsContext<MskIamClientContext>>,
>,
);
/// Underlying Kafka producer transport selected from the runtime environment.
#[derive(Clone)]
struct ProducerTransport(Either<FutureProducer, FutureProducer<MskIamClientContext>>);
/// Type-level name for a durable Kafka consumer group.
///
/// Defining group names on marker types keeps group identities centralized and
/// prevents consumers from passing arbitrary string group IDs at construction.
pub trait GroupName {
/// Stable Kafka consumer group ID used for partition balancing and offsets.
const GROUP_NAME: &'static str;
/// Where the group starts on a partition it has no committed offset for.
///
/// Partitions the group already committed on resume from that offset either
/// way; this only decides the first read of a partition the group has never
/// consumed, such as one of a topic newly added to its subscription.
/// `Earliest` replays everything the topic still retains there; `Latest`
/// starts at the next record published.
const INITIAL_OFFSET: InitialOffset = InitialOffset::Earliest;
}
/// Marker type for a consumer that does not subscribe or persist offsets.
///
/// librdkafka requires a configured `group.id` for its safe manual-assignment
/// API, so this mode uses a generated internal ID. It never exposes subscription
/// or commit operations and therefore does not create durable group state.
pub struct Ungrouped;
/// Starting position for a partition a consumer has no committed offset for:
/// every partition of an ungrouped consumer, and new partitions of a group.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum InitialOffset {
/// Consume all currently retained records before continuing with new ones.
Earliest,
/// Consume only records published after the partition assignment begins.
Latest,
}
/// Shared Kafka consumer with environment-aware transport and type-safe group behavior.
///
/// Consumers parameterized by a [`GroupName`] may subscribe and commit offsets.
/// [`Ungrouped`] consumers instead manually assign every partition of their
/// requested topics and cannot subscribe or commit.
pub struct KafkaEventConsumer<T> {
consumer: ConsumerTransport,
marker: PhantomData<T>,
}
/// Shared Kafka producer with environment-aware transport.
#[derive(Clone)]
pub struct KafkaEventProducer {
producer: ProducerTransport,
}
fn base_config(brokers: &str) -> ClientConfig {
let mut config = ClientConfig::new();
config.set("bootstrap.servers", brokers);
config
}
fn consumer_config(brokers: &str) -> ClientConfig {
let mut config = base_config(brokers);
config.set("enable.auto.commit", "false");
config
}
fn producer_config(brokers: &str) -> ClientConfig {
let mut config = base_config(brokers);
config.set("message.timeout.ms", MESSAGE_TIMEOUT_MS);
config
}
fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {
let mut config = consumer_config(brokers);
config.set("group.id", T::GROUP_NAME).set(
"auto.offset.reset",
match T::INITIAL_OFFSET {
InitialOffset::Earliest => "earliest",
InitialOffset::Latest => "latest",
},
);
config
}
fn ungrouped_config(brokers: &str) -> ClientConfig {
let mut config = consumer_config(brokers);
let group_id = format!("{UNGROUPED_GROUP_PREFIX}-{}", Uuid::new_v4());
config
.set("group.id", group_id)
.set("enable.auto.offset.store", "false");
config
}
fn create_consumer_from_env<T>(
config: ClientConfig,
observer: Option<ConsumerStatistics>,
) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {
let consumer = match Environment::new_or_prod() {
Environment::Local => Either::Left(
config
.create_with_context(StatisticsContext {
inner: DefaultConsumerContext,
observer,
})
.map_err(KafkaConsumerError::Plaintext)?,
),
Environment::Develop | Environment::Production => {
let config = configure_sasl_iam(config);
Either::Right(
config
.create_with_context(StatisticsContext {
inner: MskIamClientContext::from_env(),
observer,
})
.map_err(KafkaConsumerError::MskIam)?,
)
}
};
Ok(KafkaEventConsumer {
consumer: ConsumerTransport(consumer),
marker: PhantomData,
})
}
fn create_producer_from_env(
config: ClientConfig,
) -> Result<KafkaEventProducer, KafkaProducerError> {
let producer = match Environment::new_or_prod() {
Environment::Local => Either::Left(config.create().map_err(KafkaProducerError::Plaintext)?),
Environment::Develop | Environment::Production => {
let config = configure_sasl_iam(config);
Either::Right(
config
.create_with_context(MskIamClientContext::from_env())
.map_err(KafkaProducerError::MskIam)?,
)
}
};
Ok(KafkaEventProducer {
producer: ProducerTransport(producer),
})
}
/// Polls the consumer once with a no-op waker so OAUTHBEARER installs its
/// initial token before a synchronous broker request (metadata, list offsets)
/// needs a connection.
fn prime_oauth_token<C: ConsumerContext + 'static>(consumer: &StreamConsumer<C>) {
let mut recv = std::pin::pin!(consumer.recv());
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
let _ = std::future::Future::poll(recv.as_mut(), &mut context);
}
fn build_assignment<C, T>(
consumer: &T,
topics: &[&str],
offset: Offset,
metadata_timeout: Duration,
) -> KafkaResult<TopicPartitionList>
where
C: ConsumerContext,
T: Consumer<C>,
{
if topics.is_empty() {
return Err(KafkaError::Subscription(
"at least one topic is required for assignment".to_string(),
));
}
let mut assignment = TopicPartitionList::new();
for topic in topics {
let metadata = consumer.fetch_metadata(Some(topic), metadata_timeout)?;
let topic_metadata = metadata
.topics()
.iter()
.find(|metadata| metadata.name() == *topic)
.ok_or_else(|| {
KafkaError::Subscription(format!(
"metadata response did not include requested topic {topic}"
))
})?;
if let Some(error) = topic_metadata.error() {
return Err(KafkaError::MetadataFetch(error.into()));
}
if topic_metadata.partitions().is_empty() {
return Err(KafkaError::Subscription(format!(
"requested topic {topic} has no partitions"
)));
}
for partition in topic_metadata.partitions() {
if let Some(error) = partition.error() {
return Err(KafkaError::MetadataFetch(error.into()));
}
assignment.add_partition_offset(topic, partition.id(), offset)?;
}
}
Ok(assignment)
}
impl KafkaEventProducer {
/// Creates a producer, selecting plaintext or MSK IAM transport from the runtime environment.
///
/// Producer creation is lazy: no broker connection or IAM token is created
/// until a message is sent.
pub fn from_env(brokers: &str) -> Result<Self, KafkaProducerError> {
create_producer_from_env(producer_config(brokers))
}
/// Sends a keyed payload to `topic` and waits for delivery confirmation.
#[tracing::instrument(
name = "kafka.publish",
err,
skip(self, payload),
fields(
otel.kind = "producer",
messaging.system = "kafka",
messaging.operation.name = "publish",
messaging.operation.type = "send",
messaging.destination.name = topic,
)
)]
pub async fn send(&self, topic: &str, key: &str, payload: &[u8]) -> KafkaResult<()> {
let record = FutureRecord::to(topic)
.key(key)
.payload(payload)
.headers(current_trace_headers());
either::for_both!(&self.producer.0, producer => producer.send(record, SEND_TIMEOUT).await)
.map(|_| ())
.map_err(|(error, _)| error)
}
}
impl<T> KafkaEventConsumer<T> {
/// Receives the next Kafka message.
///
/// `StreamConsumer::recv` is cancel-safe and may be used in `tokio::select!`.
pub async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {
either::for_both!(&self.consumer.0, consumer => consumer.recv().await)
}
/// Pauses the partition containing `message`.
///
/// Grouped consumers can use this to prevent a later cumulative commit
/// from advancing past a failed record. Ungrouped consumers can use it to
/// stop additional delivery from a failed partition.
pub fn pause_message_partition(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {
let mut partitions = TopicPartitionList::new();
partitions.add_partition(message.topic(), message.partition());
either::for_both!(&self.consumer.0, consumer => consumer.pause(&partitions))
}
}
impl<T: GroupName> KafkaEventConsumer<T> {
/// Creates a named-group consumer, selecting plaintext or MSK IAM transport
/// from the runtime environment.
pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {
create_consumer_from_env(grouped_config::<T>(brokers), None)
}
/// Subscribes the consumer to exactly the provided topics.
pub fn subscribe(&self, topics: &[&str]) -> KafkaResult<()> {
either::for_both!(&self.consumer.0, consumer => consumer.subscribe(topics))
}
/// Commits a message using the caller-selected commit mode.
pub fn commit_message(
&self,
message: &BorrowedMessage<'_>,
mode: CommitMode,
) -> KafkaResult<()> {
either::for_both!(&self.consumer.0, consumer => consumer.commit_message(message, mode))
}
}
impl KafkaEventConsumer<Ungrouped> {
/// Creates an ungrouped consumer, selecting plaintext or MSK IAM transport
/// from the runtime environment.
///
/// Call [`Self::assign_topics`] before receiving messages.
pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {
create_consumer_from_env(ungrouped_config(brokers), None)
}
/// Creates an independent consumer with statistics sampled every ten seconds.
/// The observer does not join a group, store offsets, or issue broker calls.
pub fn from_env_with_statistics(
brokers: &str,
observer: ConsumerStatistics,
) -> Result<Self, KafkaConsumerError> {
let mut config = ungrouped_config(brokers);
config.set("statistics.interval.ms", "10000");
create_consumer_from_env(config, Some(observer))
}
/// Manually assigns every current partition of `topics` at `initial_offset`.
///
/// Manual assignment does not join a consumer group, persist offsets, or
/// automatically discover partitions added after this call. Reconnect to
/// discover partition-count changes.
pub fn assign_topics(
&self,
topics: &[&str],
initial_offset: InitialOffset,
metadata_timeout: Duration,
) -> KafkaResult<()> {
either::for_both!(&self.consumer.0, consumer => {
prime_oauth_token(consumer);
let offset = match initial_offset {
InitialOffset::Earliest => Offset::Beginning,
InitialOffset::Latest => Offset::End,
};
let assignment = build_assignment(consumer, topics, offset, metadata_timeout)?;
consumer.assign(&assignment)
})
}
}