Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(webhook): wire up webhook kafka consumer (#4627)

Will Hutchinson12094498b848
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 750f700225..8221395c69 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -5093,12 +5093,91 @@
"workspace-hack"
],
"webhook": [
+ "agent",
+ "ai_toolset",
+ "ai_usage",
+ "attachment",
+ "authentication_service_client",
+ "bot_id",
+ "call",
+ "channel_sender",
+ "channels",
+ "connection",
+ "connection_gateway_client",
+ "connection_gateway_models",
+ "contacts",
"cowlike",
+ "crm",
+ "decode_jwt",
+ "document_sub_type",
+ "documents",
+ "email",
+ "email_db_client",
+ "email_utils",
+ "embedding",
+ "entity_access",
+ "entity_access_db_utils",
+ "entity_access_management",
+ "filter_ast",
+ "foreign_entity",
+ "frecency",
+ "invite_email",
+ "ip_extractor",
+ "item_filters",
+ "last_online_tracker",
+ "lexical_client",
+ "macro_auth",
+ "macro_aws_config",
+ "macro_db_client",
+ "macro_entrypoint",
+ "macro_env",
+ "macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
+ "macro_middleware",
+ "macro_service_urls",
+ "macro_sync_service_jwt",
"macro_user_id",
"macro_uuid",
+ "mention_utils",
+ "model",
+ "model-entity",
"model-error-response",
+ "model_file_type",
+ "model_notifications",
"model_user",
+ "models_ai_projection",
+ "models_bulk_upload",
+ "models_dcs",
+ "models_email",
+ "models_entity_access_management",
+ "models_opensearch",
+ "models_pagination",
+ "models_permissions",
+ "models_properties",
+ "models_search",
+ "models_search_cursor",
+ "models_soup",
+ "models_team",
+ "non_empty",
+ "notification",
+ "opensearch_client",
+ "opensearch_query_builder",
+ "prompt",
+ "properties",
"rate_limit",
+ "remote_env_var",
+ "s3_key",
+ "secretsmanager_client",
+ "share_permission_db_utils",
+ "sqs_client",
+ "sqs_worker",
+ "sync_service",
+ "sync_service_client",
+ "system_properties",
+ "task_dedup",
+ "unfurl",
+ "user_quota",
"webhook",
"workspace-hack"
],
diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nix
index 0f908c52e1..52f6a4e957 100644
--- a/nix/cloud-storage.nix
+++ b/nix/cloud-storage.nix
@@ -719,6 +719,7 @@
sccache
rustToolchain
python3
+ vitejs
]
++ pkgs.lib.optionals isLinux [ mold ];
diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock
index d33a3c111b..b51ddf3956 100644
--- a/rust/cloud-storage/Cargo.lock
+++ b/rust/cloud-storage/Cargo.lock
@@ -14828,15 +14828,23 @@ dependencies = [
"anyhow",
"axum",
"axum-extra",
+ "channel_sender",
+ "channels",
"chrono",
+ "documents",
+ "entity_access",
"hex",
"hmac 0.12.1",
"macro_db_migrator",
+ "macro_env",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_user_id",
"macro_uuid",
"model-error-response",
"model_user",
"rate_limit",
+ "rdkafka",
"reqwest 0.13.4",
"rootcause",
"serde",
@@ -14849,6 +14857,7 @@ dependencies = [
"tracing",
"url",
"utoipa",
+ "uuid",
"workspace-hack",
]
diff --git a/rust/cloud-storage/channels/src/domain/models.rs b/rust/cloud-storage/channels/src/domain/models.rs
index 02767bbc54..ca54d37183 100644
--- a/rust/cloud-storage/channels/src/domain/models.rs
+++ b/rust/cloud-storage/channels/src/domain/models.rs
@@ -5,7 +5,9 @@ use item_filters::ast::{
LiteralTree,
channel::{ChannelLiteral, ChannelThreadLiteral},
};
-use macro_user_id::{email::ReadEmailParts, user_id::MacroUserIdStr};
+#[cfg(any(feature = "list", feature = "outbound"))]
+use macro_user_id::email::ReadEmailParts;
+use macro_user_id::user_id::MacroUserIdStr;
use models_pagination::{CreatedAt, CursorVal, Identify, SortOn};
#[cfg(feature = "list")]
use models_pagination::{Query, SimpleSortMethod};
@@ -1121,9 +1123,11 @@ impl UserName {
}
/// Lookup from Macro user id to display name.
+#[cfg(any(feature = "list", feature = "outbound"))]
pub(crate) type NameLookup = HashMap<MacroUserIdStr<'static>, String>;
/// Produce the human-readable fallback for a Macro user id.
+#[cfg(any(feature = "list", feature = "outbound"))]
pub(crate) fn fallback_user_name(user_id: &MacroUserIdStr<'_>) -> String {
user_id.email_part().local_part().to_string()
}
diff --git a/rust/cloud-storage/document_storage_service/src/main.rs b/rust/cloud-storage/document_storage_service/src/main.rs
index d72fe1989e..9f8b324771 100644
--- a/rust/cloud-storage/document_storage_service/src/main.rs
+++ b/rust/cloud-storage/document_storage_service/src/main.rs
@@ -589,6 +589,29 @@ async fn main() -> anyhow::Result<()> {
webhook_service,
webhook_rate_limiter,
);
+
+ // Webhook event ingestion: consume document and channel broker events and
+ // fan them out to matching webhooks (per-event handlers are stubs for now).
+ let webhook_ingestion_service =
+ webhook::domain::ingestion::WebhookEventIngestionServiceImpl::new(
+ entity_access_service.clone(),
+ );
+ let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();
+ tokio::spawn(async move {
+ loop {
+ if let Err(e) = webhook::inbound::kafka_consumer::run_webhook_event_consumer(
+ &webhook_consumer_brokers,
+ webhook_ingestion_service.clone(),
+ std::future::pending::<()>(),
+ )
+ .await
+ {
+ tracing::error!(error = ?e, "webhook event consumer exited");
+ }
+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ }
+ });
+
let call_internal_state = InternalCallRouterState::new(call_service.clone());
// Create the SQS worker for delete document processing before config is moved.
diff --git a/rust/cloud-storage/webhook/Cargo.toml b/rust/cloud-storage/webhook/Cargo.toml
index c908a16883..f7e2ca3f72 100644
--- a/rust/cloud-storage/webhook/Cargo.toml
+++ b/rust/cloud-storage/webhook/Cargo.toml
@@ -5,7 +5,15 @@ publish = false
version = "0.1.0"
[features]
-default = ["inbound", "outbound", "ports"]
+consumer = [
+ "dep:macro_env",
+ "dep:macro_event_topics",
+ "dep:rdkafka",
+ "dep:tokio",
+ "ingestion",
+ "macro_event_broker/outbound",
+]
+default = ["consumer", "inbound", "outbound", "ports"]
inbound = [
"dep:axum",
"dep:axum-extra",
@@ -17,6 +25,14 @@ inbound = [
"dep:utoipa",
"ports",
]
+ingestion = [
+ "dep:channels",
+ "dep:documents",
+ "dep:entity_access",
+ "dep:macro_event_broker",
+ "dep:uuid",
+ "ports",
+]
outbound = [
"dep:hex",
"dep:hmac",
@@ -34,27 +50,39 @@ ports = ["dep:serde_json", "dep:url"]
anyhow = { workspace = true }
axum = { workspace = true, optional = true }
axum-extra = { workspace = true, optional = true }
+channels = { path = "../channels", default-features = false, optional = true }
chrono = { workspace = true }
+documents = { path = "../documents", default-features = false, optional = true }
+entity_access = { path = "../entity_access", default-features = false, features = [
+ "ports",
+], optional = true }
hex = { workspace = true, optional = true }
hmac = { workspace = true, optional = true }
+macro_env = { path = "../macro_env", optional = true }
+macro_event_broker = { path = "../macro_event_broker", default-features = false, optional = true }
+macro_event_topics = { path = "../macro_event_topics", optional = true }
macro_user_id = { path = "../macro_user_id" }
macro_uuid = { path = "../macro_uuid", optional = true }
model-error-response = { path = "../model-error-response", optional = true }
model_user = { path = "../model_user", optional = true, features = ["axum"] }
rate_limit = { path = "../rate_limit", optional = true, default-features = false }
+rdkafka = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
serde = { workspace = true }
serde_json = { workspace = true, optional = true }
sha2 = { workspace = true, optional = true }
sqlx = { workspace = true, optional = true }
thiserror = { workspace = true }
-tokio = { workspace = true, optional = true }
+tokio = { workspace = true, features = ["macros", "time"], optional = true }
tracing = { workspace = true }
url = { workspace = true, optional = true }
utoipa = { workspace = true, optional = true }
+uuid = { workspace = true, optional = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
+channel_sender = { path = "../channel_sender" }
+tokio = { workspace = true, features = ["test-util"] }
macro_db_migrator = { path = "../macro_db_migrator" }
rate_limit = { path = "../rate_limit", default-features = false, features = [
"axum",
diff --git a/rust/cloud-storage/webhook/src/domain/ingestion.rs b/rust/cloud-storage/webhook/src/domain/ingestion.rs
new file mode 100644
index 0000000000..752c2ff444
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/domain/ingestion.rs
@@ -0,0 +1,381 @@
+//! Webhook event ingestion: broker events in, matching webhooks out.
+//!
+//! [`WebhookEventIngestionService`] is the inbound port driven by broker
+//! consumers (see `crate::inbound::kafka_consumer`). It is intentionally
+//! separate from [`WebhookService`](super::ports::WebhookService), which is
+//! the CRUD surface for webhook creation and edits.
+//!
+//! Event handling is not implemented yet: every handler below is a stub that
+//! only logs the event. The planned flow for each event is:
+//! 1. resolve the users that can see the event's entity via
+//! [`EntityAccessService::get_users_by_entity`] (see
+//! [`WebhookEventIngestionServiceImpl::users_with_access`]);
+//! 2. load those users' active webhooks and match each webhook's `rule`
+//! against the event;
+//! 3. enqueue a delivery for every match.
+
+#[cfg(test)]
+mod test;
+
+use channels::domain::broker_events::{
+ ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,
+ ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata,
+ ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipantAddedMetadata,
+ ChannelParticipantRemovedMetadata, ChannelTopicEvent, ChannelUpdatedMetadata,
+};
+use documents::domain::events::{
+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentTopicEvent,
+ DocumentUpdatedMetadata,
+};
+use entity_access::domain::models::{AccessError, EntityType};
+use entity_access::domain::ports::EntityAccessService;
+use macro_event_broker::Event;
+use macro_user_id::user_id::MacroUserIdStr;
+use std::future::Future;
+use std::sync::Arc;
+use uuid::Uuid;
+
+/// Webhook event ingestion error.
+#[derive(Debug, thiserror::Error)]
+pub enum WebhookEventIngestionError {
+ /// Failed to resolve the users with access to the event's entity.
+ #[error(transparent)]
+ EntityAccess(#[from] AccessError),
+ /// Repository or adapter error.
+ #[error(transparent)]
+ Internal(#[from] anyhow::Error),
+}
+
+impl WebhookEventIngestionError {
+ /// Whether retrying the same event could plausibly succeed.
+ ///
+ /// Database errors from access resolution are transient: the lookup can be
+ /// retried once the database recovers, and the consumer holds off on
+ /// committing the event's offset until it does. Everything else — invalid
+ /// entity ids, unauthorized lookups, adapter bugs — is permanent, and
+ /// retrying would only repeat the failure.
+ pub fn is_transient(&self) -> bool {
+ match self {
+ Self::EntityAccess(AccessError::DatabaseError(_)) => true,
+ Self::EntityAccess(_) | Self::Internal(_) => false,
+ }
+ }
+}
+
+/// Inbound port for ingesting broker events for webhook delivery.
+///
+/// One method per subscribed topic; each takes the decoded event envelope so
+/// implementations keep the `event_id` for delivery idempotency.
+pub trait WebhookEventIngestionService: Clone + Send + Sync + 'static {
+ /// Ingest one `macro.documents` event envelope.
+ fn ingest_document_event(
+ &self,
+ event: Event<DocumentTopicEvent>,
+ ) -> impl Future<Output = Result<(), WebhookEventIngestionError>> + Send;
+
+ /// Ingest one `macro.channels` event envelope.
+ fn ingest_channel_event(
+ &self,
+ event: Event<ChannelTopicEvent>,
+ ) -> impl Future<Output = Result<(), WebhookEventIngestionError>> + Send;
+}
+
+/// Webhook event ingestion service implementation.
+#[derive(Clone)]
+pub struct WebhookEventIngestionServiceImpl<A> {
+ entity_access_service: Arc<A>,
+}
+
+impl<A> WebhookEventIngestionServiceImpl<A> {
+ /// Create a webhook event ingestion service.
+ pub fn new(entity_access_service: Arc<A>) -> Self {
+ Self {
+ entity_access_service,
+ }
+ }
+}
+
+impl<A: EntityAccessService> WebhookEventIngestionServiceImpl<A> {
+ /// Resolve the users that currently have access to an entity.
+ ///
+ /// This is the first step of webhook fan-out: the handlers below will use
+ /// it to find the users whose webhooks should be matched against an event.
+ #[tracing::instrument(skip(self), err)]
+ pub async fn users_with_access(
+ &self,
+ entity_id: &str,
+ entity_type: EntityType,
+ ) -> Result<Vec<MacroUserIdStr<'static>>, WebhookEventIngestionError> {
+ tracing::trace!(entity_id, ?entity_type, "resolving users with access");
+ let users = self
+ .entity_access_service
+ .get_users_by_entity(entity_id, entity_type)
+ .await?;
+ tracing::trace!(
+ entity_id,
+ user_count = users.len(),
+ "resolved users with access"
+ );
+ Ok(users)
+ }
+
+ async fn handle_document_created(
+ &self,
+ event_id: Uuid,
+ metadata: DocumentCreatedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ document_id = %metadata.document_id,
+ "document.created: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_document_updated(
+ &self,
+ event_id: Uuid,
+ metadata: DocumentUpdatedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ document_id = %metadata.document_id,
+ "document.updated: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_document_deleted(
+ &self,
+ event_id: Uuid,
+ metadata: DocumentDeletedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ document_id = %metadata.document_id,
+ "document.deleted: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_document_copied(
+ &self,
+ event_id: Uuid,
+ metadata: DocumentCopiedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ document_id = %metadata.document_id,
+ source_document_id = %metadata.source_document_id,
+ "document.copied: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_created(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelCreatedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ "channel.created: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_updated(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelUpdatedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ "channel.updated: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_deleted(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelDeletedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ "channel.deleted: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_message_posted(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelMessagePostedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ message_id = %metadata.message_id,
+ "channel.message_posted: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_message_patched(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelMessagePatchedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ message_id = %metadata.message_id,
+ "channel.message_patched: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_message_deleted(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelMessageDeletedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ message_id = %metadata.message_id,
+ "channel.message_deleted: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_message_attachment_created(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelMessageAttachmentCreatedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ message_id = %metadata.message_id,
+ "channel.message_attachment_created: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_message_attachment_removed(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelMessageAttachmentRemovedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ message_id = %metadata.message_id,
+ "channel.message_attachment_removed: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_participant_added(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelParticipantAddedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ "channel.participant_added: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+
+ async fn handle_channel_participant_removed(
+ &self,
+ event_id: Uuid,
+ metadata: ChannelParticipantRemovedMetadata,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::debug!(
+ %event_id,
+ channel_id = %metadata.channel_id,
+ "channel.participant_removed: webhook ingestion not implemented yet"
+ );
+ Ok(())
+ }
+}
+
+impl<A: EntityAccessService> WebhookEventIngestionService for WebhookEventIngestionServiceImpl<A> {
+ #[tracing::instrument(skip(self, event), fields(event_id = %event.event_id), err)]
+ async fn ingest_document_event(
+ &self,
+ event: Event<DocumentTopicEvent>,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::trace!(?event, "ingesting document event");
+ let event_id = event.event_id;
+ match event.event {
+ DocumentTopicEvent::Created(metadata) => {
+ self.handle_document_created(event_id, metadata).await
+ }
+ DocumentTopicEvent::Updated(metadata) => {
+ self.handle_document_updated(event_id, metadata).await
+ }
+ DocumentTopicEvent::Deleted(metadata) => {
+ self.handle_document_deleted(event_id, metadata).await
+ }
+ DocumentTopicEvent::Copied(metadata) => {
+ self.handle_document_copied(event_id, metadata).await
+ }
+ }
+ }
+
+ #[tracing::instrument(skip(self, event), fields(event_id = %event.event_id), err)]
+ async fn ingest_channel_event(
+ &self,
+ event: Event<ChannelTopicEvent>,
+ ) -> Result<(), WebhookEventIngestionError> {
+ tracing::trace!(?event, "ingesting channel event");
+ let event_id = event.event_id;
+ match event.event {
+ ChannelTopicEvent::Created(metadata) => {
+ self.handle_channel_created(event_id, metadata).await
+ }
+ ChannelTopicEvent::Updated(metadata) => {
+ self.handle_channel_updated(event_id, metadata).await
+ }
+ ChannelTopicEvent::Deleted(metadata) => {
+ self.handle_channel_deleted(event_id, metadata).await
+ }
+ ChannelTopicEvent::MessagePosted(metadata) => {
+ self.handle_channel_message_posted(event_id, metadata).await
+ }
+ ChannelTopicEvent::MessagePatched(metadata) => {
+ self.handle_channel_message_patched(event_id, metadata)
+ .await
+ }
+ ChannelTopicEvent::MessageDeleted(metadata) => {
+ self.handle_channel_message_deleted(event_id, metadata)
+ .await
+ }
+ ChannelTopicEvent::MessageAttachmentCreated(metadata) => {
+ self.handle_channel_message_attachment_created(event_id, metadata)
+ .await
+ }
+ ChannelTopicEvent::MessageAttachmentRemoved(metadata) => {
+ self.handle_channel_message_attachment_removed(event_id, metadata)
+ .await
+ }
+ ChannelTopicEvent::ParticipantAdded(metadata) => {
+ self.handle_channel_participant_added(event_id, metadata)
+ .await
+ }
+ ChannelTopicEvent::ParticipantRemoved(metadata) => {
+ self.handle_channel_participant_removed(event_id, metadata)
+ .await
+ }
+ }
+ }
+}
diff --git a/rust/cloud-storage/webhook/src/domain/ingestion/test.rs b/rust/cloud-storage/webhook/src/domain/ingestion/test.rs
new file mode 100644
index 0000000000..955bc25612
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/domain/ingestion/test.rs
@@ -0,0 +1,280 @@
+use super::*;
+use channel_sender::ChannelSender;
+use channels::domain::models::ChannelType;
+use chrono::{DateTime, Utc};
+use entity_access::domain::models::{
+ AccessLevel, CallChannelInfo, EntityAccessReceipt, EntityPermission, RequiredPermission,
+ UserTeamInfo,
+};
+use macro_user_id::{lowercased::Lowercase, user_id::MacroUserId};
+
+fn user_id(id: &str) -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from(id.to_string()).expect("valid user id")
+}
+
+fn sender(id: &str) -> ChannelSender<'static> {
+ ChannelSender::try_from(id.to_string()).expect("valid channel sender")
+}
+
+fn timestamp() -> DateTime<Utc> {
+ DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z")
+ .expect("valid timestamp")
+ .with_timezone(&Utc)
+}
+
+/// Mock access service: only [`EntityAccessService::get_users_by_entity`] is
+/// exercised by the ingestion service.
+#[derive(Clone)]
+struct MockAccessService;
+
+impl EntityAccessService for MockAccessService {
+ async fn generate_entity_access_receipt<T: RequiredPermission>(
+ &self,
+ _user_id: &MacroUserId<Lowercase<'_>>,
+ _user_org_id: Option<i64>,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ ) -> Result<EntityAccessReceipt<T>, AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn get_access_level(
+ &self,
+ _user_id: Option<&MacroUserId<Lowercase<'_>>>,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ ) -> Result<Option<AccessLevel>, AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn check_access(
+ &self,
+ _user_id: Option<&MacroUserId<Lowercase<'_>>>,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ _required_level: AccessLevel,
+ ) -> Result<AccessLevel, AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn check_public_access(
+ &self,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ _required_level: AccessLevel,
+ ) -> Result<AccessLevel, AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn get_entity_permission(
+ &self,
+ _user_id: Option<&MacroUserId<Lowercase<'_>>>,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ _user_org_id: Option<i64>,
+ ) -> Result<EntityPermission, AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn get_crm_entity_permission_with_team(
+ &self,
+ _user_id: Option<&MacroUserId<Lowercase<'_>>>,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ ) -> Result<(EntityPermission, Uuid), AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn get_users_by_entity(
+ &self,
+ _entity_id: &str,
+ _entity_type: EntityType,
+ ) -> Result<Vec<MacroUserIdStr<'static>>, AccessError> {
+ Ok(vec![user_id("macro|[email protected]")])
+ }
+
+ async fn get_call_channel(
+ &self,
+ _call_id: &Uuid,
+ ) -> Result<Option<CallChannelInfo>, AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn get_call_channel_by_channel_id(
+ &self,
+ _channel_id: &Uuid,
+ ) -> Result<Option<CallChannelInfo>, AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn get_user_team(
+ &self,
+ _user_id: &MacroUserId<Lowercase<'_>>,
+ ) -> Result<Option<UserTeamInfo>, AccessError> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+}
+
+fn service() -> WebhookEventIngestionServiceImpl<MockAccessService> {
+ WebhookEventIngestionServiceImpl::new(Arc::new(MockAccessService))
+}
+
+fn document_events() -> Vec<DocumentTopicEvent> {
+ vec![
+ DocumentTopicEvent::Created(DocumentCreatedMetadata {
+ document_id: "doc_1".to_string(),
+ owner: user_id("macro|[email protected]"),
+ document_name: "notes".to_string(),
+ file_type: None,
+ project_id: None,
+ sub_type: None,
+ created_at: Some(timestamp()),
+ }),
+ DocumentTopicEvent::Updated(DocumentUpdatedMetadata {
+ document_id: "doc_1".to_string(),
+ owner: user_id("macro|[email protected]"),
+ actor_user_id: Some(user_id("macro|[email protected]")),
+ document_name: Some("renamed".to_string()),
+ previous_project_id: None,
+ project_id: None,
+ file_type: None,
+ share_permission_updated: false,
+ }),
+ DocumentTopicEvent::Deleted(DocumentDeletedMetadata {
+ document_id: "doc_1".to_string(),
+ actor_user_id: Some(user_id("macro|[email protected]")),
+ project_id: None,
+ }),
+ DocumentTopicEvent::Copied(DocumentCopiedMetadata {
+ document_id: "doc_2".to_string(),
+ source_document_id: "doc_1".to_string(),
+ source_version_id: None,
+ owner: user_id("macro|[email protected]"),
+ document_name: "notes (copy)".to_string(),
+ file_type: None,
+ project_id: None,
+ sub_type: None,
+ }),
+ ]
+}
+
+fn channel_events() -> Vec<ChannelTopicEvent> {
+ let channel_id = Uuid::nil();
+ let message_id = Uuid::nil();
+ vec![
+ ChannelTopicEvent::Created(ChannelCreatedMetadata {
+ channel_id,
+ actor: sender("macro|[email protected]"),
+ channel_type: ChannelType::Team,
+ channel_name: Some("general".to_string()),
+ participant_user_ids: vec![user_id("macro|[email protected]")],
+ }),
+ ChannelTopicEvent::Updated(ChannelUpdatedMetadata {
+ channel_id,
+ actor: user_id("macro|[email protected]"),
+ previous_name: Some("general".to_string()),
+ channel_name: Some("announcements".to_string()),
+ }),
+ ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
+ channel_id,
+ actor: sender("macro|[email protected]"),
+ }),
+ ChannelTopicEvent::MessagePosted(ChannelMessagePostedMetadata {
+ channel_id,
+ message_id,
+ thread_id: None,
+ sender: sender("macro|[email protected]"),
+ triggered_by: None,
+ channel_type: ChannelType::Team,
+ content: "hello".to_string(),
+ mentions: vec![],
+ attachments: vec![],
+ created_at: timestamp(),
+ }),
+ ChannelTopicEvent::MessagePatched(ChannelMessagePatchedMetadata {
+ channel_id,
+ message_id,
+ thread_id: None,
+ actor: sender("macro|[email protected]"),
+ content: "hello (edited)".to_string(),
+ edited_at: Some(timestamp()),
+ updated_at: timestamp(),
+ }),
+ ChannelTopicEvent::MessageDeleted(ChannelMessageDeletedMetadata {
+ channel_id,
+ message_id,
+ thread_id: None,
+ actor: sender("macro|[email protected]"),
+ deleted_at: Some(timestamp()),
+ }),
+ ChannelTopicEvent::MessageAttachmentCreated(ChannelMessageAttachmentCreatedMetadata {
+ channel_id,
+ message_id,
+ actor: sender("macro|[email protected]"),
+ attachments: vec![],
+ }),
+ ChannelTopicEvent::MessageAttachmentRemoved(ChannelMessageAttachmentRemovedMetadata {
+ channel_id,
+ message_id,
+ actor: sender("macro|[email protected]"),
+ attachments: vec![],
+ }),
+ ChannelTopicEvent::ParticipantAdded(ChannelParticipantAddedMetadata {
+ channel_id,
+ channel_type: ChannelType::Team,
+ added_by: sender("macro|[email protected]"),
+ added_user_ids: vec![user_id("macro|[email protected]")],
+ }),
+ ChannelTopicEvent::ParticipantRemoved(ChannelParticipantRemovedMetadata {
+ channel_id,
+ channel_type: ChannelType::Team,
+ removed_by: user_id("macro|[email protected]"),
+ removed_user_ids: vec![user_id("macro|[email protected]")],
+ }),
+ ]
+}
+
+#[tokio::test]
+async fn ingests_every_document_event_variant() {
+ let service = service();
+ for event in document_events() {
+ service
+ .ingest_document_event(Event::new(event))
+ .await
+ .expect("stub handlers always succeed");
+ }
+}
+
+#[tokio::test]
+async fn ingests_every_channel_event_variant() {
+ let service = service();
+ for event in channel_events() {
+ service
+ .ingest_channel_event(Event::new(event))
+ .await
+ .expect("stub handlers always succeed");
+ }
+}
+
+#[test]
+fn classifies_transient_errors() {
+ let database_error = WebhookEventIngestionError::EntityAccess(AccessError::DatabaseError(
+ sqlx::Error::PoolTimedOut,
+ ));
+ assert!(database_error.is_transient());
+
+ let unauthorized = WebhookEventIngestionError::EntityAccess(AccessError::Unauthorized);
+ assert!(!unauthorized.is_transient());
+
+ let internal = WebhookEventIngestionError::Internal(anyhow::anyhow!("adapter bug"));
+ assert!(!internal.is_transient());
+}
+
+#[tokio::test]
+async fn users_with_access_delegates_to_entity_access() {
+ let users = service()
+ .users_with_access("doc_1", EntityType::Document)
+ .await
+ .expect("mock resolves users");
+ assert_eq!(users, vec![user_id("macro|[email protected]")]);
+}
diff --git a/rust/cloud-storage/webhook/src/domain/mod.rs b/rust/cloud-storage/webhook/src/domain/mod.rs
index c1065db13e..2aecc98c77 100644
--- a/rust/cloud-storage/webhook/src/domain/mod.rs
+++ b/rust/cloud-storage/webhook/src/domain/mod.rs
@@ -1,5 +1,8 @@
//! Domain layer for webhooks.
+#[cfg(feature = "ingestion")]
+/// Webhook event ingestion service.
+pub mod ingestion;
/// Webhook domain models.
pub mod models;
#[cfg(feature = "ports")]
diff --git a/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs b/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs
new file mode 100644
index 0000000000..03c4b330fd
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs
@@ -0,0 +1,322 @@
+//! Kafka consumer that feeds broker events into webhook ingestion.
+//!
+//! Subscribes to [`MacroDocumentsTopic`] and [`MacroChannelsTopic`] and hands
+//! every decoded event envelope to a [`WebhookEventIngestionService`].
+//!
+//! Delivery is at-least-once: an event's offset is committed only after the
+//! ingestion service accepted it or permanently rejected it. Transient
+//! ingestion failures (e.g. the database being briefly unavailable while
+//! resolving entity access) are retried in-process with exponential backoff;
+//! if a transient failure outlives every attempt the consumer exits with an
+//! error *without* committing, so the supervising restart loop redelivers the
+//! event and retries durably. Undecodable messages are logged and skipped
+//! rather than wedging the partition. The transport mirrors
+//! [`macro_event_broker::KafkaEventPublisher`]: plaintext for
+//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise.
+//!
+//! For local testing, run with `RUST_LOG=webhook=trace` to see every received
+//! message, decoded event, ingestion attempt, and offset commit.
+
+#[cfg(test)]
+mod test;
+
+use crate::domain::ingestion::WebhookEventIngestionService;
+use anyhow::Context as _;
+use channels::domain::broker_events::ChannelTopicEvent;
+use documents::domain::events::DocumentTopicEvent;
+use macro_env::Environment;
+use macro_event_broker::outbound::msk_iam::configure_sasl_iam;
+use macro_event_broker::{Event, EventBrokerError, MskIamClientContext, Topic as _};
+use macro_event_topics::{MacroChannelsTopic, MacroDocumentsTopic};
+use rdkafka::ClientConfig;
+use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};
+use rdkafka::error::KafkaResult;
+use rdkafka::message::{BorrowedMessage, Message};
+use std::future::Future;
+use std::time::Duration;
+
+/// Consumer group id for webhook event ingestion. Offsets are committed under
+/// this group, so restarts resume where the previous run left off.
+const GROUP_ID: &str = "webhook-event-ingestion";
+
+/// Maximum in-process ingestion attempts per event before the consumer bails
+/// out and lets a restart redeliver from the last committed offset.
+const MAX_INGEST_ATTEMPTS: u32 = 5;
+
+/// Delay before the first ingestion retry; doubles on each subsequent retry.
+/// The worst-case total backoff (1+2+4+8 = 15s) stays well under librdkafka's
+/// default `max.poll.interval.ms` (300s), so retrying never evicts this
+/// consumer from its group.
+const INGEST_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
+
+/// One decoded event envelope from a topic this consumer subscribes to.
+///
+/// The Kafka message key is not carried: every event's metadata already
+/// contains the entity ids the ingestion service needs.
+#[derive(Debug)]
+pub enum WebhookConsumerEvent {
+ /// Event received on [`MacroDocumentsTopic`].
+ Documents(Event<DocumentTopicEvent>),
+ /// Event received on [`MacroChannelsTopic`].
+ Channels(Event<ChannelTopicEvent>),
+}
+
+impl WebhookConsumerEvent {
+ /// Decode one Kafka message into this consumer's event enum.
+ pub fn decode(topic: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {
+ match topic {
+ topic if topic == MacroDocumentsTopic.as_str() => {
+ Ok(Self::Documents(Event::decode(payload)?))
+ }
+ topic if topic == MacroChannelsTopic.as_str() => {
+ Ok(Self::Channels(Event::decode(payload)?))
+ }
+ unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),
+ }
+ }
+}
+
+/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).
+enum WebhookKafkaConsumer {
+ /// Unauthenticated plaintext connection (local docker broker).
+ Plaintext(StreamConsumer),
+ /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).
+ MskIam(StreamConsumer<MskIamClientContext>),
+}
+
+impl WebhookKafkaConsumer {
+ /// Build a consumer for the given brokers, choosing the transport from the
+ /// `ENVIRONMENT` variable exactly like the publisher does.
+ fn from_env(brokers: &str) -> anyhow::Result<Self> {
+ let mut config = ClientConfig::new();
+ config
+ .set("bootstrap.servers", brokers)
+ .set("group.id", GROUP_ID)
+ // Offsets are committed manually after successful ingestion.
+ .set("enable.auto.commit", "false")
+ // Start from the beginning of the topics on the first ever run.
+ .set("auto.offset.reset", "earliest");
+
+ let consumer = match Environment::new_or_prod() {
+ Environment::Local => Self::Plaintext(
+ config
+ .create()
+ .context("failed to create plaintext kafka consumer")?,
+ ),
+ Environment::Develop | Environment::Production => {
+ configure_sasl_iam(&mut config);
+ Self::MskIam(
+ config
+ .create_with_context(MskIamClientContext::from_env())
+ .context("failed to create MSK IAM kafka consumer")?,
+ )
+ }
+ };
+
+ Ok(consumer)
+ }
+
+ fn subscribe(&self) -> KafkaResult<()> {
+ let topics = [MacroDocumentsTopic.as_str(), MacroChannelsTopic.as_str()];
+ match self {
+ Self::Plaintext(consumer) => consumer.subscribe(&topics),
+ Self::MskIam(consumer) => consumer.subscribe(&topics),
+ }
+ }
+
+ /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it
+ /// can sit in a `select!` without losing messages.
+ async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {
+ match self {
+ Self::Plaintext(consumer) => consumer.recv().await,
+ Self::MskIam(consumer) => consumer.recv().await,
+ }
+ }
+
+ fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {
+ match self {
+ Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),
+ Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),
+ }
+ }
+
+ /// Commit `message`'s offset, logging the outcome.
+ fn commit_logged(&self, message: &BorrowedMessage<'_>) {
+ match self.commit(message) {
+ Ok(()) => tracing::trace!(
+ partition = message.partition(),
+ offset = message.offset(),
+ "committed offset"
+ ),
+ Err(e) => tracing::error!(
+ error = ?e,
+ partition = message.partition(),
+ offset = message.offset(),
+ "failed to commit offset"
+ ),
+ }
+ }
+}
+
+/// Ingest one decoded event, retrying transient failures with exponential
+/// backoff.
+///
+/// Returns `Ok(())` when the event was ingested or permanently rejected (both
+/// safe to commit) and `Err` when a transient failure survived every attempt —
+/// the caller must exit without committing so the event is redelivered.
+async fn ingest_with_retry<S: WebhookEventIngestionService>(
+ service: &S,
+ event: &WebhookConsumerEvent,
+ partition: i32,
+ offset: i64,
+) -> anyhow::Result<()> {
+ let mut delay = INGEST_RETRY_BASE_DELAY;
+ let mut attempt = 1u32;
+ loop {
+ tracing::trace!(partition, offset, attempt, "ingesting broker event");
+ let result = match event {
+ WebhookConsumerEvent::Documents(event) => {
+ service.ingest_document_event(event.clone()).await
+ }
+ WebhookConsumerEvent::Channels(event) => {
+ service.ingest_channel_event(event.clone()).await
+ }
+ };
+ match result {
+ Ok(()) => {
+ tracing::trace!(partition, offset, attempt, "broker event ingested");
+ return Ok(());
+ }
+ Err(e) if !e.is_transient() => {
+ tracing::error!(
+ error = ?e,
+ partition,
+ offset,
+ "dropping broker event after non-retryable ingestion failure"
+ );
+ return Ok(());
+ }
+ Err(e) if attempt < MAX_INGEST_ATTEMPTS => {
+ tracing::warn!(
+ error = ?e,
+ partition,
+ offset,
+ attempt,
+ delay_secs = delay.as_secs_f32(),
+ "transient ingestion failure, retrying"
+ );
+ tokio::time::sleep(delay).await;
+ delay *= 2;
+ attempt += 1;
+ }
+ Err(e) => {
+ return Err(e).with_context(|| {
+ format!(
+ "transient ingestion failure persisted after \
+ {MAX_INGEST_ATTEMPTS} attempts \
+ (partition {partition} offset {offset})"
+ )
+ });
+ }
+ }
+ }
+}
+
+/// Run the webhook event consumer until `shutdown` resolves.
+///
+/// Connects to `brokers`, subscribes to [`MacroDocumentsTopic`] and
+/// [`MacroChannelsTopic`] under the `webhook-event-ingestion` consumer group,
+/// and feeds every decoded event to `service`, committing each offset only
+/// after ingestion succeeds (see [`ingest_with_retry`] for the retry policy).
+/// Returns an error when the consumer cannot be created or subscribed, or when
+/// a transient ingestion failure exhausts its in-process retries; callers
+/// should treat that as fatal and restart, which redelivers the uncommitted
+/// event. Pass `std::future::pending()` as `shutdown` to run until the process
+/// exits.
+pub async fn run_webhook_event_consumer<S>(
+ brokers: &str,
+ service: S,
+ shutdown: impl Future<Output = ()> + Send,
+) -> anyhow::Result<()>
+where
+ S: WebhookEventIngestionService,
+{
+ let consumer = WebhookKafkaConsumer::from_env(brokers)?;
+ consumer
+ .subscribe()
+ .context("failed to subscribe to webhook event topics")?;
+ tracing::info!(
+ topics = ?[MacroDocumentsTopic.as_str(), MacroChannelsTopic.as_str()],
+ group = GROUP_ID,
+ "webhook event consumer listening"
+ );
+
+ let mut shutdown = std::pin::pin!(shutdown);
+ loop {
+ tokio::select! {
+ _ = &mut shutdown => {
+ tracing::info!("webhook event consumer shutting down");
+ break;
+ }
+ result = consumer.recv() => {
+ let message = match result {
+ Ok(message) => message,
+ Err(e) => {
+ tracing::error!(error = ?e, "kafka receive error");
+ continue;
+ }
+ };
+ tracing::trace!(
+ topic = message.topic(),
+ partition = message.partition(),
+ offset = message.offset(),
+ payload_len = message.payload().map_or(0, <[u8]>::len),
+ "received kafka message"
+ );
+
+ let Some(payload) = message.payload() else {
+ tracing::warn!(
+ partition = message.partition(),
+ offset = message.offset(),
+ "skipping message with empty payload"
+ );
+ consumer.commit_logged(&message);
+ continue;
+ };
+
+ match WebhookConsumerEvent::decode(message.topic(), payload) {
+ Ok(event) => {
+ tracing::trace!(
+ ?event,
+ partition = message.partition(),
+ offset = message.offset(),
+ "decoded broker event"
+ );
+ ingest_with_retry(
+ &service,
+ &event,
+ message.partition(),
+ message.offset(),
+ )
+ .await?;
+ }
+ // Undecodable messages are logged and skipped rather than
+ // wedging the partition on a poison message.
+ Err(e) => tracing::error!(
+ error = ?e,
+ topic = message.topic(),
+ partition = message.partition(),
+ offset = message.offset(),
+ "failed to decode broker event"
+ ),
+ }
+
+ // Commit only after the event was ingested or permanently
+ // rejected: at-least-once, retried across restarts.
+ consumer.commit_logged(&message);
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs b/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs
new file mode 100644
index 0000000000..e8d129f76b
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs
@@ -0,0 +1,162 @@
+use super::*;
+use crate::domain::ingestion::WebhookEventIngestionError;
+use channel_sender::ChannelSender;
+use channels::domain::broker_events::ChannelDeletedMetadata;
+use documents::domain::events::DocumentDeletedMetadata;
+use entity_access::domain::models::AccessError;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU32, Ordering};
+use uuid::Uuid;
+
+fn document_event() -> Event<DocumentTopicEvent> {
+ Event::new(DocumentTopicEvent::Deleted(DocumentDeletedMetadata {
+ document_id: "doc_1".to_string(),
+ actor_user_id: None,
+ project_id: None,
+ }))
+}
+
+#[test]
+fn decodes_document_events() {
+ let event = document_event();
+ let payload = serde_json::to_vec(&event).expect("serializable");
+
+ let decoded =
+ WebhookConsumerEvent::decode(MacroDocumentsTopic.as_str(), &payload).expect("decodable");
+
+ match decoded {
+ WebhookConsumerEvent::Documents(decoded) => assert_eq!(decoded, event),
+ WebhookConsumerEvent::Channels(_) => panic!("decoded into the wrong topic variant"),
+ }
+}
+
+#[test]
+fn decodes_channel_events() {
+ let event = Event::new(ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
+ channel_id: Uuid::nil(),
+ actor: ChannelSender::try_from("macro|[email protected]".to_string())
+ .expect("valid channel sender"),
+ }));
+ let payload = serde_json::to_vec(&event).expect("serializable");
+
+ let decoded =
+ WebhookConsumerEvent::decode(MacroChannelsTopic.as_str(), &payload).expect("decodable");
+
+ match decoded {
+ WebhookConsumerEvent::Channels(decoded) => assert_eq!(decoded, event),
+ WebhookConsumerEvent::Documents(_) => panic!("decoded into the wrong topic variant"),
+ }
+}
+
+#[test]
+fn rejects_unknown_topics() {
+ let err = WebhookConsumerEvent::decode("macro.example", b"{}").expect_err("unknown topic");
+ assert!(matches!(err, EventBrokerError::UnknownTopic(topic) if topic == "macro.example"));
+}
+
+#[test]
+fn rejects_malformed_payloads() {
+ let err = WebhookConsumerEvent::decode(MacroDocumentsTopic.as_str(), b"not json")
+ .expect_err("malformed payload");
+ assert!(matches!(err, EventBrokerError::Serialization(_)));
+}
+
+/// Ingestion service that fails the first `failures` attempts, then succeeds.
+#[derive(Clone)]
+struct FlakyIngestionService {
+ attempts: Arc<AtomicU32>,
+ failures: u32,
+ transient: bool,
+}
+
+impl FlakyIngestionService {
+ fn failure(&self) -> WebhookEventIngestionError {
+ if self.transient {
+ WebhookEventIngestionError::EntityAccess(AccessError::DatabaseError(
+ sqlx::Error::PoolTimedOut,
+ ))
+ } else {
+ WebhookEventIngestionError::Internal(anyhow::anyhow!("permanent failure"))
+ }
+ }
+}
+
+impl WebhookEventIngestionService for FlakyIngestionService {
+ async fn ingest_document_event(
+ &self,
+ _event: Event<DocumentTopicEvent>,
+ ) -> Result<(), WebhookEventIngestionError> {
+ let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;
+ if attempt <= self.failures {
+ Err(self.failure())
+ } else {
+ Ok(())
+ }
+ }
+
+ async fn ingest_channel_event(
+ &self,
+ _event: Event<ChannelTopicEvent>,
+ ) -> Result<(), WebhookEventIngestionError> {
+ Ok(())
+ }
+}
+
+fn flaky_service(failures: u32, transient: bool) -> (FlakyIngestionService, Arc<AtomicU32>) {
+ let attempts = Arc::new(AtomicU32::new(0));
+ let service = FlakyIngestionService {
+ attempts: attempts.clone(),
+ failures,
+ transient,
+ };
+ (service, attempts)
+}
+
+// `start_paused` auto-advances the tokio clock through the backoff sleeps.
+#[tokio::test(start_paused = true)]
+async fn retries_transient_failures_until_success() {
+ let (service, attempts) = flaky_service(2, true);
+
+ ingest_with_retry(
+ &service,
+ &WebhookConsumerEvent::Documents(document_event()),
+ 0,
+ 0,
+ )
+ .await
+ .expect("succeeds once the transient failure clears");
+
+ assert_eq!(attempts.load(Ordering::SeqCst), 3);
+}
+
+#[tokio::test(start_paused = true)]
+async fn exhausted_transient_retries_bubble_up_for_redelivery() {
+ let (service, attempts) = flaky_service(u32::MAX, true);
+
+ ingest_with_retry(
+ &service,
+ &WebhookConsumerEvent::Documents(document_event()),
+ 0,
+ 0,
+ )
+ .await
+ .expect_err("persistent transient failure aborts the consumer without committing");
+
+ assert_eq!(attempts.load(Ordering::SeqCst), MAX_INGEST_ATTEMPTS);
+}
+
+#[tokio::test]
+async fn permanent_failures_are_dropped_without_retry() {
+ let (service, attempts) = flaky_service(u32::MAX, false);
+
+ ingest_with_retry(
+ &service,
+ &WebhookConsumerEvent::Documents(document_event()),
+ 0,
+ 0,
+ )
+ .await
+ .expect("permanent failures are skipped so the offset commits");
+
+ assert_eq!(attempts.load(Ordering::SeqCst), 1);
+}
diff --git a/rust/cloud-storage/webhook/src/inbound/mod.rs b/rust/cloud-storage/webhook/src/inbound/mod.rs
index 33f3d31aba..2b84ac5b7d 100644
--- a/rust/cloud-storage/webhook/src/inbound/mod.rs
+++ b/rust/cloud-storage/webhook/src/inbound/mod.rs
@@ -1,6 +1,10 @@
-//! Inbound HTTP adapters.
+//! Inbound adapters: HTTP handlers and the Kafka event consumer.
+#[cfg(feature = "inbound")]
pub mod axum_router;
-#[cfg(test)]
+#[cfg(all(test, feature = "inbound"))]
mod axum_router_test;
+
+#[cfg(feature = "consumer")]
+pub mod kafka_consumer;
diff --git a/rust/cloud-storage/webhook/src/lib.rs b/rust/cloud-storage/webhook/src/lib.rs
index 116059b185..0804f14701 100644
--- a/rust/cloud-storage/webhook/src/lib.rs
+++ b/rust/cloud-storage/webhook/src/lib.rs
@@ -14,20 +14,24 @@
//! the result in `is_valid`;
//! - validation attempt rate limiting with the existing `rate_limit` crate using
//! the key shape `per-user-validate-webhook:{macro_user_id}:{webhook_id}` and
-//! a limit of 10 attempts per 3600-second window.
+//! a limit of 10 attempts per 3600-second window;
+//! - a Kafka consumer (`inbound::kafka_consumer`, feature `consumer`) that
+//! subscribes to the `macro.documents` and `macro.channels` topics and hands
+//! every event to the `domain::ingestion::WebhookEventIngestionService`
+//! (feature `ingestion`), whose per-event handlers are still stubs.
//!
//! The current implementation intentionally excludes delivery workers, SQS/SNS
-//! infrastructure, event ingestion, redelivery, and auto-pause behavior.
+//! infrastructure, redelivery, and auto-pause behavior.
//!
//! Temporary limitations: IDs use prefixed UUIDv7 strings (`wh_<uuid_v7>`)
//! rather than true ULIDs. Custom headers are stored as JSON in `headers`.
//! Webhook signing secrets are intentionally stored as plaintext in
//! `signing_secret`, matching the existing bot secret storage approach.
-/// Domain models, ports, and service.
+/// Domain models, ports, and services.
pub mod domain;
-#[cfg(feature = "inbound")]
-/// HTTP adapters.
+#[cfg(any(feature = "inbound", feature = "consumer"))]
+/// HTTP and Kafka consumer adapters.
pub mod inbound;
#[cfg(feature = "outbound")]
/// Postgres and HTTP adapters.