Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
//! Kafka consumer that feeds broker events into webhook ingestion.
//!
//! Subscribes to [`DocumentMacroEvent`], [`ChannelMacroEvent`], and
//! [`WebhookMacroEvent`] topics and hands every decoded event envelope to a
//! [`WebhookEventIngestionService`].
//!
//! Delivery is at-least-once: an event's offset is committed only after the
//! ingestion service accepted it or permanently rejected it. Transient
//! ingestion failures (e.g. the database being briefly unavailable while
//! resolving entity access) are retried in-process with exponential backoff;
//! if a transient failure outlives every attempt the consumer exits with an
//! error *without* committing, so the supervising restart loop redelivers the
//! event and retries durably. Undecodable messages are logged and skipped
//! rather than wedging the partition. The transport mirrors
//! [`macro_event_broker::KafkaEventPublisher`]: plaintext for
//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise.
//!
//! For local testing, run with `RUST_LOG=webhook=trace` to see every received
//! message, decoded event, ingestion attempt, and offset commit.
#[cfg(test)]
mod test;
use crate::domain::ingestion::{WebhookEventIngestionError, WebhookEventIngestionService};
use crate::topics::DeclaredMacroEvent;
use anyhow::Context as _;
use kafka_util::{GroupName, InitialOffset, KafkaEventConsumer};
use macro_event_broker::{
KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message};
use std::future::Future;
use std::time::Duration;
use tokio_retry::{RetryIf, strategy::ExponentialBackoff};
/// Consumer group for webhook event ingestion. Offsets are committed under
/// this group, so restarts resume where the previous run left off.
struct WebhookEventIngestionConsumerGroup;
impl GroupName for WebhookEventIngestionConsumerGroup {
const GROUP_NAME: &'static str = "webhook-event-ingestion";
// `macro.messages` joined this group's subscription with no committed
// offset; starting at its head avoids replaying the topic's retention.
const INITIAL_OFFSET: InitialOffset = InitialOffset::Latest;
}
type WebhookKafkaAdapter =
KafkaConsumerAdapter<WebhookEventIngestionConsumerGroup, DeclaredMacroEvent>;
type WebhookKafkaConsumer = MacroEventConsumerService<DeclaredMacroEvent, WebhookKafkaAdapter>;
/// Maximum in-process ingestion attempts per event before the consumer bails
/// out and lets a restart redeliver from the last committed offset.
const MAX_INGEST_ATTEMPTS: u32 = 5;
/// Delay before the first ingestion retry; doubles on each subsequent retry.
/// The worst-case total backoff (1+2+4+8 = 15s) stays well under librdkafka's
/// default `max.poll.interval.ms` (300s), so retrying never evicts this
/// consumer from its group.
const INGEST_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
fn ingest_retry_strategy() -> impl Iterator<Item = Duration> {
ExponentialBackoff::from_millis(2)
.factor(500)
.take((MAX_INGEST_ATTEMPTS - 1) as usize)
}
/// Commit `message`'s offset, logging the outcome.
fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>) {
match consumer.inner().commit_message(message, CommitMode::Async) {
Ok(()) => tracing::trace!(
partition = message.partition(),
offset = message.offset(),
"committed offset"
),
Err(error) => tracing::error!(
error = ?error,
partition = message.partition(),
offset = message.offset(),
"failed to commit offset"
),
}
}
/// Ingest one decoded event, retrying transient failures with exponential
/// backoff.
///
/// Returns `Ok(())` when the event was ingested or permanently rejected (both
/// safe to commit) and `Err` when a transient failure survived every attempt —
/// the caller must exit without committing so the event is redelivered.
async fn ingest_with_retry<S: WebhookEventIngestionService>(
service: &S,
event: &DeclaredMacroEvent,
partition: i32,
offset: i64,
) -> anyhow::Result<()> {
let mut attempt = 0u32;
let result = RetryIf::start(
ingest_retry_strategy(),
|| {
attempt += 1;
async move {
tracing::trace!(partition, offset, attempt, "ingesting broker event");
let result = match event {
DeclaredMacroEvent::DocumentMacroEvent(event) => {
service.ingest_document_event(event.event().clone()).await
}
DeclaredMacroEvent::ChannelMacroEvent(event) => {
service.ingest_channel_event(event.event().clone()).await
}
DeclaredMacroEvent::CallNotificationMacroEvent(event) => {
service
.ingest_call_notification_event(event.event().clone())
.await
}
DeclaredMacroEvent::MessageMacroEvent(event) => {
service.ingest_message_event(event.event().clone()).await
}
DeclaredMacroEvent::WebhookMacroEvent(event) => {
service.ingest_webhook_event(event.event().clone()).await
}
DeclaredMacroEvent::AgentSessionMacroEvent(event) => {
service
.ingest_agent_trigger_event(event.event().clone())
.await
}
DeclaredMacroEvent::AgentSessionLifecycleMacroEvent(event) => {
service
.ingest_agent_session_lifecycle_event(event.event().clone())
.await
}
};
match &result {
Ok(()) => {
tracing::trace!(partition, offset, attempt, "broker event ingested")
}
Err(error) if error.is_transient() && attempt < MAX_INGEST_ATTEMPTS => {
let delay = INGEST_RETRY_BASE_DELAY * 2u32.pow(attempt - 1);
tracing::warn!(
error = ?error,
partition,
offset,
attempt,
delay_secs = delay.as_secs_f32(),
"transient ingestion failure, retrying"
);
}
Err(_) => {}
}
result
}
},
|error: &WebhookEventIngestionError| error.is_transient(),
)
.await;
match result {
Ok(()) => Ok(()),
Err(error) if !error.is_transient() => {
tracing::error!(
error = ?error,
partition,
offset,
"dropping broker event after non-retryable ingestion failure"
);
Ok(())
}
Err(error) => Err(error).with_context(|| {
format!(
"transient ingestion failure persisted after \
{MAX_INGEST_ATTEMPTS} attempts \
(partition {partition} offset {offset})"
)
}),
}
}
/// Run the webhook event consumer until `shutdown` resolves.
///
/// Connects to `brokers` and subscribes to the topics declared by
/// [`DocumentMacroEvent`], [`ChannelMacroEvent`], and [`WebhookMacroEvent`]
/// under the `webhook-event-ingestion` consumer group. Every decoded event is fed to
/// `service`, committing each offset only
/// after ingestion succeeds (see `ingest_with_retry` for the retry policy).
/// Returns an error when the consumer cannot be created or subscribed, or when
/// a transient ingestion failure exhausts its in-process retries; callers
/// should treat that as fatal and restart, which redelivers the uncommitted
/// event. Pass `std::future::pending()` as `shutdown` to run until the process
/// exits.
pub async fn run_webhook_event_consumer<S>(
brokers: &str,
service: S,
shutdown: impl Future<Output = ()> + Send,
) -> anyhow::Result<()>
where
S: WebhookEventIngestionService,
{
let consumer = KafkaEventConsumer::<WebhookEventIngestionConsumerGroup>::from_env(brokers)?;
let consumer = KafkaConsumerAdapter::<WebhookEventIngestionConsumerGroup, ()>::new(consumer)
.subscribe::<DeclaredMacroEvent>()
.map_err(|error| {
anyhow::anyhow!("failed to subscribe to webhook event topics: {error:?}")
})?;
let consumer = WebhookKafkaConsumer::new(consumer);
tracing::info!(
topics = ?DeclaredMacroEvent::topics(),
group = WebhookEventIngestionConsumerGroup::GROUP_NAME,
"webhook event consumer listening"
);
let mut shutdown = std::pin::pin!(shutdown);
loop {
tokio::select! {
_ = &mut shutdown => {
tracing::info!("webhook event consumer shutting down");
break;
}
result = consumer.recv() => {
let message = match result {
Ok(message) => message,
Err(e) => {
tracing::error!(error = ?e, "kafka receive error");
continue;
}
};
let kafka_message = message.inner();
match message.decode_payload() {
Ok(event) => {
tracing::trace!(
partition = kafka_message.partition(),
offset = kafka_message.offset(),
"decoded broker event"
);
ingest_with_retry(
&service,
&event,
kafka_message.partition(),
kafka_message.offset(),
)
.await?;
}
// Undecodable messages are logged and skipped rather than
// wedging the partition on a poison message.
Err(e) => tracing::error!(
error = ?e,
topic = kafka_message.topic(),
partition = kafka_message.partition(),
offset = kafka_message.offset(),
"failed to decode broker event"
),
}
// Commit only after the event was ingested or permanently
// rejected: at-least-once, retried across restarts.
commit_logged(&consumer, kafka_message);
}
}
}
Ok(())
}