Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(webhook): wire up webhook kafka consumer (#4627)
Will Hutchinson12094498b848
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 750f700225..8221395c69 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -5093,12 +5093,91 @@"workspace-hack"],"webhook": [+ "agent",+ "ai_toolset",+ "ai_usage",+ "attachment",+ "authentication_service_client",+ "bot_id",+ "call",+ "channel_sender",+ "channels",+ "connection",+ "connection_gateway_client",+ "connection_gateway_models",+ "contacts","cowlike",+ "crm",+ "decode_jwt",+ "document_sub_type",+ "documents",+ "email",+ "email_db_client",+ "email_utils",+ "embedding",+ "entity_access",+ "entity_access_db_utils",+ "entity_access_management",+ "filter_ast",+ "foreign_entity",+ "frecency",+ "invite_email",+ "ip_extractor",+ "item_filters",+ "last_online_tracker",+ "lexical_client",+ "macro_auth",+ "macro_aws_config",+ "macro_db_client",+ "macro_entrypoint",+ "macro_env",+ "macro_env_var",+ "macro_event_broker",+ "macro_event_topics",+ "macro_middleware",+ "macro_service_urls",+ "macro_sync_service_jwt","macro_user_id","macro_uuid",+ "mention_utils",+ "model",+ "model-entity","model-error-response",+ "model_file_type",+ "model_notifications","model_user",+ "models_ai_projection",+ "models_bulk_upload",+ "models_dcs",+ "models_email",+ "models_entity_access_management",+ "models_opensearch",+ "models_pagination",+ "models_permissions",+ "models_properties",+ "models_search",+ "models_search_cursor",+ "models_soup",+ "models_team",+ "non_empty",+ "notification",+ "opensearch_client",+ "opensearch_query_builder",+ "prompt",+ "properties","rate_limit",+ "remote_env_var",+ "s3_key",+ "secretsmanager_client",+ "share_permission_db_utils",+ "sqs_client",+ "sqs_worker",+ "sync_service",+ "sync_service_client",+ "system_properties",+ "task_dedup",+ "unfurl",+ "user_quota","webhook","workspace-hack"],diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nixindex 0f908c52e1..52f6a4e957 100644--- a/nix/cloud-storage.nix+++ b/nix/cloud-storage.nix@@ -719,6 +719,7 @@sccacherustToolchainpython3+ vitejs]++ pkgs.lib.optionals isLinux [ mold ];diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lockindex d33a3c111b..b51ddf3956 100644--- a/rust/cloud-storage/Cargo.lock+++ b/rust/cloud-storage/Cargo.lock@@ -14828,15 +14828,23 @@ dependencies = ["anyhow","axum","axum-extra",+ "channel_sender",+ "channels","chrono",+ "documents",+ "entity_access","hex","hmac 0.12.1","macro_db_migrator",+ "macro_env",+ "macro_event_broker",+ "macro_event_topics","macro_user_id","macro_uuid","model-error-response","model_user","rate_limit",+ "rdkafka","reqwest 0.13.4","rootcause","serde",@@ -14849,6 +14857,7 @@ dependencies = ["tracing","url","utoipa",+ "uuid","workspace-hack",]diff --git a/rust/cloud-storage/channels/src/domain/models.rs b/rust/cloud-storage/channels/src/domain/models.rsindex 02767bbc54..ca54d37183 100644--- a/rust/cloud-storage/channels/src/domain/models.rs+++ b/rust/cloud-storage/channels/src/domain/models.rs@@ -5,7 +5,9 @@ use item_filters::ast::{LiteralTree,channel::{ChannelLiteral, ChannelThreadLiteral},};-use macro_user_id::{email::ReadEmailParts, user_id::MacroUserIdStr};+#[cfg(any(feature = "list", feature = "outbound"))]+use macro_user_id::email::ReadEmailParts;+use macro_user_id::user_id::MacroUserIdStr;use models_pagination::{CreatedAt, CursorVal, Identify, SortOn};#[cfg(feature = "list")]use models_pagination::{Query, SimpleSortMethod};@@ -1121,9 +1123,11 @@ impl UserName {}/// Lookup from Macro user id to display name.+#[cfg(any(feature = "list", feature = "outbound"))]pub(crate) type NameLookup = HashMap<MacroUserIdStr<'static>, String>;/// Produce the human-readable fallback for a Macro user id.+#[cfg(any(feature = "list", feature = "outbound"))]pub(crate) fn fallback_user_name(user_id: &MacroUserIdStr<'_>) -> String {user_id.email_part().local_part().to_string()}diff --git a/rust/cloud-storage/document_storage_service/src/main.rs b/rust/cloud-storage/document_storage_service/src/main.rsindex d72fe1989e..9f8b324771 100644--- a/rust/cloud-storage/document_storage_service/src/main.rs+++ b/rust/cloud-storage/document_storage_service/src/main.rs@@ -589,6 +589,29 @@ async fn main() -> anyhow::Result<()> {webhook_service,webhook_rate_limiter,);++ // Webhook event ingestion: consume document and channel broker events and+ // fan them out to matching webhooks (per-event handlers are stubs for now).+ let webhook_ingestion_service =+ webhook::domain::ingestion::WebhookEventIngestionServiceImpl::new(+ entity_access_service.clone(),+ );+ let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();+ tokio::spawn(async move {+ loop {+ if let Err(e) = webhook::inbound::kafka_consumer::run_webhook_event_consumer(+ &webhook_consumer_brokers,+ webhook_ingestion_service.clone(),+ std::future::pending::<()>(),+ )+ .await+ {+ tracing::error!(error = ?e, "webhook event consumer exited");+ }+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ }+ });+let call_internal_state = InternalCallRouterState::new(call_service.clone());// Create the SQS worker for delete document processing before config is moved.diff --git a/rust/cloud-storage/webhook/Cargo.toml b/rust/cloud-storage/webhook/Cargo.tomlindex c908a16883..f7e2ca3f72 100644--- a/rust/cloud-storage/webhook/Cargo.toml+++ b/rust/cloud-storage/webhook/Cargo.toml@@ -5,7 +5,15 @@ publish = falseversion = "0.1.0"[features]-default = ["inbound", "outbound", "ports"]+consumer = [+ "dep:macro_env",+ "dep:macro_event_topics",+ "dep:rdkafka",+ "dep:tokio",+ "ingestion",+ "macro_event_broker/outbound",+]+default = ["consumer", "inbound", "outbound", "ports"]inbound = ["dep:axum","dep:axum-extra",@@ -17,6 +25,14 @@ inbound = ["dep:utoipa","ports",]+ingestion = [+ "dep:channels",+ "dep:documents",+ "dep:entity_access",+ "dep:macro_event_broker",+ "dep:uuid",+ "ports",+]outbound = ["dep:hex","dep:hmac",@@ -34,27 +50,39 @@ ports = ["dep:serde_json", "dep:url"]anyhow = { workspace = true }axum = { workspace = true, optional = true }axum-extra = { workspace = true, optional = true }+channels = { path = "../channels", default-features = false, optional = true }chrono = { workspace = true }+documents = { path = "../documents", default-features = false, optional = true }+entity_access = { path = "../entity_access", default-features = false, features = [+ "ports",+], optional = true }hex = { workspace = true, optional = true }hmac = { workspace = true, optional = true }+macro_env = { path = "../macro_env", optional = true }+macro_event_broker = { path = "../macro_event_broker", default-features = false, optional = true }+macro_event_topics = { path = "../macro_event_topics", optional = true }macro_user_id = { path = "../macro_user_id" }macro_uuid = { path = "../macro_uuid", optional = true }model-error-response = { path = "../model-error-response", optional = true }model_user = { path = "../model_user", optional = true, features = ["axum"] }rate_limit = { path = "../rate_limit", optional = true, default-features = false }+rdkafka = { workspace = true, optional = true }reqwest = { workspace = true, optional = true }serde = { workspace = true }serde_json = { workspace = true, optional = true }sha2 = { workspace = true, optional = true }sqlx = { workspace = true, optional = true }thiserror = { workspace = true }-tokio = { workspace = true, optional = true }+tokio = { workspace = true, features = ["macros", "time"], optional = true }tracing = { workspace = true }url = { workspace = true, optional = true }utoipa = { workspace = true, optional = true }+uuid = { workspace = true, optional = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]+channel_sender = { path = "../channel_sender" }+tokio = { workspace = true, features = ["test-util"] }macro_db_migrator = { path = "../macro_db_migrator" }rate_limit = { path = "../rate_limit", default-features = false, features = ["axum",diff --git a/rust/cloud-storage/webhook/src/domain/ingestion.rs b/rust/cloud-storage/webhook/src/domain/ingestion.rsnew file mode 100644index 0000000000..752c2ff444--- /dev/null+++ b/rust/cloud-storage/webhook/src/domain/ingestion.rs@@ -0,0 +1,381 @@+//! Webhook event ingestion: broker events in, matching webhooks out.+//!+//! [`WebhookEventIngestionService`] is the inbound port driven by broker+//! consumers (see `crate::inbound::kafka_consumer`). It is intentionally+//! separate from [`WebhookService`](super::ports::WebhookService), which is+//! the CRUD surface for webhook creation and edits.+//!+//! Event handling is not implemented yet: every handler below is a stub that+//! only logs the event. The planned flow for each event is:+//! 1. resolve the users that can see the event's entity via+//! [`EntityAccessService::get_users_by_entity`] (see+//! [`WebhookEventIngestionServiceImpl::users_with_access`]);+//! 2. load those users' active webhooks and match each webhook's `rule`+//! against the event;+//! 3. enqueue a delivery for every match.++#[cfg(test)]+mod test;++use channels::domain::broker_events::{+ ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,+ ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata,+ ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipantAddedMetadata,+ ChannelParticipantRemovedMetadata, ChannelTopicEvent, ChannelUpdatedMetadata,+};+use documents::domain::events::{+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentTopicEvent,+ DocumentUpdatedMetadata,+};+use entity_access::domain::models::{AccessError, EntityType};+use entity_access::domain::ports::EntityAccessService;+use macro_event_broker::Event;+use macro_user_id::user_id::MacroUserIdStr;+use std::future::Future;+use std::sync::Arc;+use uuid::Uuid;++/// Webhook event ingestion error.+#[derive(Debug, thiserror::Error)]+pub enum WebhookEventIngestionError {+ /// Failed to resolve the users with access to the event's entity.+ #[error(transparent)]+ EntityAccess(#[from] AccessError),+ /// Repository or adapter error.+ #[error(transparent)]+ Internal(#[from] anyhow::Error),+}++impl WebhookEventIngestionError {+ /// Whether retrying the same event could plausibly succeed.+ ///+ /// Database errors from access resolution are transient: the lookup can be+ /// retried once the database recovers, and the consumer holds off on+ /// committing the event's offset until it does. Everything else — invalid+ /// entity ids, unauthorized lookups, adapter bugs — is permanent, and+ /// retrying would only repeat the failure.+ pub fn is_transient(&self) -> bool {+ match self {+ Self::EntityAccess(AccessError::DatabaseError(_)) => true,+ Self::EntityAccess(_) | Self::Internal(_) => false,+ }+ }+}++/// Inbound port for ingesting broker events for webhook delivery.+///+/// One method per subscribed topic; each takes the decoded event envelope so+/// implementations keep the `event_id` for delivery idempotency.+pub trait WebhookEventIngestionService: Clone + Send + Sync + 'static {+ /// Ingest one `macro.documents` event envelope.+ fn ingest_document_event(+ &self,+ event: Event<DocumentTopicEvent>,+ ) -> impl Future<Output = Result<(), WebhookEventIngestionError>> + Send;++ /// Ingest one `macro.channels` event envelope.+ fn ingest_channel_event(+ &self,+ event: Event<ChannelTopicEvent>,+ ) -> impl Future<Output = Result<(), WebhookEventIngestionError>> + Send;+}++/// Webhook event ingestion service implementation.+#[derive(Clone)]+pub struct WebhookEventIngestionServiceImpl<A> {+ entity_access_service: Arc<A>,+}++impl<A> WebhookEventIngestionServiceImpl<A> {+ /// Create a webhook event ingestion service.+ pub fn new(entity_access_service: Arc<A>) -> Self {+ Self {+ entity_access_service,+ }+ }+}++impl<A: EntityAccessService> WebhookEventIngestionServiceImpl<A> {+ /// Resolve the users that currently have access to an entity.+ ///+ /// This is the first step of webhook fan-out: the handlers below will use+ /// it to find the users whose webhooks should be matched against an event.+ #[tracing::instrument(skip(self), err)]+ pub async fn users_with_access(+ &self,+ entity_id: &str,+ entity_type: EntityType,+ ) -> Result<Vec<MacroUserIdStr<'static>>, WebhookEventIngestionError> {+ tracing::trace!(entity_id, ?entity_type, "resolving users with access");+ let users = self+ .entity_access_service+ .get_users_by_entity(entity_id, entity_type)+ .await?;+ tracing::trace!(+ entity_id,+ user_count = users.len(),+ "resolved users with access"+ );+ Ok(users)+ }++ async fn handle_document_created(+ &self,+ event_id: Uuid,+ metadata: DocumentCreatedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ document_id = %metadata.document_id,+ "document.created: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_document_updated(+ &self,+ event_id: Uuid,+ metadata: DocumentUpdatedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ document_id = %metadata.document_id,+ "document.updated: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_document_deleted(+ &self,+ event_id: Uuid,+ metadata: DocumentDeletedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ document_id = %metadata.document_id,+ "document.deleted: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_document_copied(+ &self,+ event_id: Uuid,+ metadata: DocumentCopiedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ document_id = %metadata.document_id,+ source_document_id = %metadata.source_document_id,+ "document.copied: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_created(+ &self,+ event_id: Uuid,+ metadata: ChannelCreatedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ "channel.created: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_updated(+ &self,+ event_id: Uuid,+ metadata: ChannelUpdatedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ "channel.updated: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_deleted(+ &self,+ event_id: Uuid,+ metadata: ChannelDeletedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ "channel.deleted: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_message_posted(+ &self,+ event_id: Uuid,+ metadata: ChannelMessagePostedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ message_id = %metadata.message_id,+ "channel.message_posted: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_message_patched(+ &self,+ event_id: Uuid,+ metadata: ChannelMessagePatchedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ message_id = %metadata.message_id,+ "channel.message_patched: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_message_deleted(+ &self,+ event_id: Uuid,+ metadata: ChannelMessageDeletedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ message_id = %metadata.message_id,+ "channel.message_deleted: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_message_attachment_created(+ &self,+ event_id: Uuid,+ metadata: ChannelMessageAttachmentCreatedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ message_id = %metadata.message_id,+ "channel.message_attachment_created: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_message_attachment_removed(+ &self,+ event_id: Uuid,+ metadata: ChannelMessageAttachmentRemovedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ message_id = %metadata.message_id,+ "channel.message_attachment_removed: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_participant_added(+ &self,+ event_id: Uuid,+ metadata: ChannelParticipantAddedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ "channel.participant_added: webhook ingestion not implemented yet"+ );+ Ok(())+ }++ async fn handle_channel_participant_removed(+ &self,+ event_id: Uuid,+ metadata: ChannelParticipantRemovedMetadata,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::debug!(+ %event_id,+ channel_id = %metadata.channel_id,+ "channel.participant_removed: webhook ingestion not implemented yet"+ );+ Ok(())+ }+}++impl<A: EntityAccessService> WebhookEventIngestionService for WebhookEventIngestionServiceImpl<A> {+ #[tracing::instrument(skip(self, event), fields(event_id = %event.event_id), err)]+ async fn ingest_document_event(+ &self,+ event: Event<DocumentTopicEvent>,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::trace!(?event, "ingesting document event");+ let event_id = event.event_id;+ match event.event {+ DocumentTopicEvent::Created(metadata) => {+ self.handle_document_created(event_id, metadata).await+ }+ DocumentTopicEvent::Updated(metadata) => {+ self.handle_document_updated(event_id, metadata).await+ }+ DocumentTopicEvent::Deleted(metadata) => {+ self.handle_document_deleted(event_id, metadata).await+ }+ DocumentTopicEvent::Copied(metadata) => {+ self.handle_document_copied(event_id, metadata).await+ }+ }+ }++ #[tracing::instrument(skip(self, event), fields(event_id = %event.event_id), err)]+ async fn ingest_channel_event(+ &self,+ event: Event<ChannelTopicEvent>,+ ) -> Result<(), WebhookEventIngestionError> {+ tracing::trace!(?event, "ingesting channel event");+ let event_id = event.event_id;+ match event.event {+ ChannelTopicEvent::Created(metadata) => {+ self.handle_channel_created(event_id, metadata).await+ }+ ChannelTopicEvent::Updated(metadata) => {+ self.handle_channel_updated(event_id, metadata).await+ }+ ChannelTopicEvent::Deleted(metadata) => {+ self.handle_channel_deleted(event_id, metadata).await+ }+ ChannelTopicEvent::MessagePosted(metadata) => {+ self.handle_channel_message_posted(event_id, metadata).await+ }+ ChannelTopicEvent::MessagePatched(metadata) => {+ self.handle_channel_message_patched(event_id, metadata)+ .await+ }+ ChannelTopicEvent::MessageDeleted(metadata) => {+ self.handle_channel_message_deleted(event_id, metadata)+ .await+ }+ ChannelTopicEvent::MessageAttachmentCreated(metadata) => {+ self.handle_channel_message_attachment_created(event_id, metadata)+ .await+ }+ ChannelTopicEvent::MessageAttachmentRemoved(metadata) => {+ self.handle_channel_message_attachment_removed(event_id, metadata)+ .await+ }+ ChannelTopicEvent::ParticipantAdded(metadata) => {+ self.handle_channel_participant_added(event_id, metadata)+ .await+ }+ ChannelTopicEvent::ParticipantRemoved(metadata) => {+ self.handle_channel_participant_removed(event_id, metadata)+ .await+ }+ }+ }+}diff --git a/rust/cloud-storage/webhook/src/domain/ingestion/test.rs b/rust/cloud-storage/webhook/src/domain/ingestion/test.rsnew file mode 100644index 0000000000..955bc25612--- /dev/null+++ b/rust/cloud-storage/webhook/src/domain/ingestion/test.rs@@ -0,0 +1,280 @@+use super::*;+use channel_sender::ChannelSender;+use channels::domain::models::ChannelType;+use chrono::{DateTime, Utc};+use entity_access::domain::models::{+ AccessLevel, CallChannelInfo, EntityAccessReceipt, EntityPermission, RequiredPermission,+ UserTeamInfo,+};+use macro_user_id::{lowercased::Lowercase, user_id::MacroUserId};++fn user_id(id: &str) -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from(id.to_string()).expect("valid user id")+}++fn sender(id: &str) -> ChannelSender<'static> {+ ChannelSender::try_from(id.to_string()).expect("valid channel sender")+}++fn timestamp() -> DateTime<Utc> {+ DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z")+ .expect("valid timestamp")+ .with_timezone(&Utc)+}++/// Mock access service: only [`EntityAccessService::get_users_by_entity`] is+/// exercised by the ingestion service.+#[derive(Clone)]+struct MockAccessService;++impl EntityAccessService for MockAccessService {+ async fn generate_entity_access_receipt<T: RequiredPermission>(+ &self,+ _user_id: &MacroUserId<Lowercase<'_>>,+ _user_org_id: Option<i64>,+ _entity_id: &str,+ _entity_type: EntityType,+ ) -> Result<EntityAccessReceipt<T>, AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn get_access_level(+ &self,+ _user_id: Option<&MacroUserId<Lowercase<'_>>>,+ _entity_id: &str,+ _entity_type: EntityType,+ ) -> Result<Option<AccessLevel>, AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn check_access(+ &self,+ _user_id: Option<&MacroUserId<Lowercase<'_>>>,+ _entity_id: &str,+ _entity_type: EntityType,+ _required_level: AccessLevel,+ ) -> Result<AccessLevel, AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn check_public_access(+ &self,+ _entity_id: &str,+ _entity_type: EntityType,+ _required_level: AccessLevel,+ ) -> Result<AccessLevel, AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn get_entity_permission(+ &self,+ _user_id: Option<&MacroUserId<Lowercase<'_>>>,+ _entity_id: &str,+ _entity_type: EntityType,+ _user_org_id: Option<i64>,+ ) -> Result<EntityPermission, AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn get_crm_entity_permission_with_team(+ &self,+ _user_id: Option<&MacroUserId<Lowercase<'_>>>,+ _entity_id: &str,+ _entity_type: EntityType,+ ) -> Result<(EntityPermission, Uuid), AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn get_users_by_entity(+ &self,+ _entity_id: &str,+ _entity_type: EntityType,+ ) -> Result<Vec<MacroUserIdStr<'static>>, AccessError> {+ Ok(vec![user_id("macro|[email protected]")])+ }++ async fn get_call_channel(+ &self,+ _call_id: &Uuid,+ ) -> Result<Option<CallChannelInfo>, AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn get_call_channel_by_channel_id(+ &self,+ _channel_id: &Uuid,+ ) -> Result<Option<CallChannelInfo>, AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn get_user_team(+ &self,+ _user_id: &MacroUserId<Lowercase<'_>>,+ ) -> Result<Option<UserTeamInfo>, AccessError> {+ unimplemented!("not used by webhook event ingestion")+ }+}++fn service() -> WebhookEventIngestionServiceImpl<MockAccessService> {+ WebhookEventIngestionServiceImpl::new(Arc::new(MockAccessService))+}++fn document_events() -> Vec<DocumentTopicEvent> {+ vec![+ DocumentTopicEvent::Created(DocumentCreatedMetadata {+ document_id: "doc_1".to_string(),+ owner: user_id("macro|[email protected]"),+ document_name: "notes".to_string(),+ file_type: None,+ project_id: None,+ sub_type: None,+ created_at: Some(timestamp()),+ }),+ DocumentTopicEvent::Updated(DocumentUpdatedMetadata {+ document_id: "doc_1".to_string(),+ owner: user_id("macro|[email protected]"),+ actor_user_id: Some(user_id("macro|[email protected]")),+ document_name: Some("renamed".to_string()),+ previous_project_id: None,+ project_id: None,+ file_type: None,+ share_permission_updated: false,+ }),+ DocumentTopicEvent::Deleted(DocumentDeletedMetadata {+ document_id: "doc_1".to_string(),+ actor_user_id: Some(user_id("macro|[email protected]")),+ project_id: None,+ }),+ DocumentTopicEvent::Copied(DocumentCopiedMetadata {+ document_id: "doc_2".to_string(),+ source_document_id: "doc_1".to_string(),+ source_version_id: None,+ owner: user_id("macro|[email protected]"),+ document_name: "notes (copy)".to_string(),+ file_type: None,+ project_id: None,+ sub_type: None,+ }),+ ]+}++fn channel_events() -> Vec<ChannelTopicEvent> {+ let channel_id = Uuid::nil();+ let message_id = Uuid::nil();+ vec![+ ChannelTopicEvent::Created(ChannelCreatedMetadata {+ channel_id,+ actor: sender("macro|[email protected]"),+ channel_type: ChannelType::Team,+ channel_name: Some("general".to_string()),+ participant_user_ids: vec![user_id("macro|[email protected]")],+ }),+ ChannelTopicEvent::Updated(ChannelUpdatedMetadata {+ channel_id,+ actor: user_id("macro|[email protected]"),+ previous_name: Some("general".to_string()),+ channel_name: Some("announcements".to_string()),+ }),+ ChannelTopicEvent::Deleted(ChannelDeletedMetadata {+ channel_id,+ actor: sender("macro|[email protected]"),+ }),+ ChannelTopicEvent::MessagePosted(ChannelMessagePostedMetadata {+ channel_id,+ message_id,+ thread_id: None,+ sender: sender("macro|[email protected]"),+ triggered_by: None,+ channel_type: ChannelType::Team,+ content: "hello".to_string(),+ mentions: vec![],+ attachments: vec![],+ created_at: timestamp(),+ }),+ ChannelTopicEvent::MessagePatched(ChannelMessagePatchedMetadata {+ channel_id,+ message_id,+ thread_id: None,+ actor: sender("macro|[email protected]"),+ content: "hello (edited)".to_string(),+ edited_at: Some(timestamp()),+ updated_at: timestamp(),+ }),+ ChannelTopicEvent::MessageDeleted(ChannelMessageDeletedMetadata {+ channel_id,+ message_id,+ thread_id: None,+ actor: sender("macro|[email protected]"),+ deleted_at: Some(timestamp()),+ }),+ ChannelTopicEvent::MessageAttachmentCreated(ChannelMessageAttachmentCreatedMetadata {+ channel_id,+ message_id,+ actor: sender("macro|[email protected]"),+ attachments: vec![],+ }),+ ChannelTopicEvent::MessageAttachmentRemoved(ChannelMessageAttachmentRemovedMetadata {+ channel_id,+ message_id,+ actor: sender("macro|[email protected]"),+ attachments: vec![],+ }),+ ChannelTopicEvent::ParticipantAdded(ChannelParticipantAddedMetadata {+ channel_id,+ channel_type: ChannelType::Team,+ added_by: sender("macro|[email protected]"),+ added_user_ids: vec![user_id("macro|[email protected]")],+ }),+ ChannelTopicEvent::ParticipantRemoved(ChannelParticipantRemovedMetadata {+ channel_id,+ channel_type: ChannelType::Team,+ removed_by: user_id("macro|[email protected]"),+ removed_user_ids: vec![user_id("macro|[email protected]")],+ }),+ ]+}++#[tokio::test]+async fn ingests_every_document_event_variant() {+ let service = service();+ for event in document_events() {+ service+ .ingest_document_event(Event::new(event))+ .await+ .expect("stub handlers always succeed");+ }+}++#[tokio::test]+async fn ingests_every_channel_event_variant() {+ let service = service();+ for event in channel_events() {+ service+ .ingest_channel_event(Event::new(event))+ .await+ .expect("stub handlers always succeed");+ }+}++#[test]+fn classifies_transient_errors() {+ let database_error = WebhookEventIngestionError::EntityAccess(AccessError::DatabaseError(+ sqlx::Error::PoolTimedOut,+ ));+ assert!(database_error.is_transient());++ let unauthorized = WebhookEventIngestionError::EntityAccess(AccessError::Unauthorized);+ assert!(!unauthorized.is_transient());++ let internal = WebhookEventIngestionError::Internal(anyhow::anyhow!("adapter bug"));+ assert!(!internal.is_transient());+}++#[tokio::test]+async fn users_with_access_delegates_to_entity_access() {+ let users = service()+ .users_with_access("doc_1", EntityType::Document)+ .await+ .expect("mock resolves users");+ assert_eq!(users, vec![user_id("macro|[email protected]")]);+}diff --git a/rust/cloud-storage/webhook/src/domain/mod.rs b/rust/cloud-storage/webhook/src/domain/mod.rsindex c1065db13e..2aecc98c77 100644--- a/rust/cloud-storage/webhook/src/domain/mod.rs+++ b/rust/cloud-storage/webhook/src/domain/mod.rs@@ -1,5 +1,8 @@//! Domain layer for webhooks.+#[cfg(feature = "ingestion")]+/// Webhook event ingestion service.+pub mod ingestion;/// Webhook domain models.pub mod models;#[cfg(feature = "ports")]diff --git a/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs b/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rsnew file mode 100644index 0000000000..03c4b330fd--- /dev/null+++ b/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs@@ -0,0 +1,322 @@+//! Kafka consumer that feeds broker events into webhook ingestion.+//!+//! Subscribes to [`MacroDocumentsTopic`] and [`MacroChannelsTopic`] and hands+//! every decoded event envelope to a [`WebhookEventIngestionService`].+//!+//! Delivery is at-least-once: an event's offset is committed only after the+//! ingestion service accepted it or permanently rejected it. Transient+//! ingestion failures (e.g. the database being briefly unavailable while+//! resolving entity access) are retried in-process with exponential backoff;+//! if a transient failure outlives every attempt the consumer exits with an+//! error *without* committing, so the supervising restart loop redelivers the+//! event and retries durably. Undecodable messages are logged and skipped+//! rather than wedging the partition. The transport mirrors+//! [`macro_event_broker::KafkaEventPublisher`]: plaintext for+//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise.+//!+//! For local testing, run with `RUST_LOG=webhook=trace` to see every received+//! message, decoded event, ingestion attempt, and offset commit.++#[cfg(test)]+mod test;++use crate::domain::ingestion::WebhookEventIngestionService;+use anyhow::Context as _;+use channels::domain::broker_events::ChannelTopicEvent;+use documents::domain::events::DocumentTopicEvent;+use macro_env::Environment;+use macro_event_broker::outbound::msk_iam::configure_sasl_iam;+use macro_event_broker::{Event, EventBrokerError, MskIamClientContext, Topic as _};+use macro_event_topics::{MacroChannelsTopic, MacroDocumentsTopic};+use rdkafka::ClientConfig;+use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};+use rdkafka::error::KafkaResult;+use rdkafka::message::{BorrowedMessage, Message};+use std::future::Future;+use std::time::Duration;++/// Consumer group id for webhook event ingestion. Offsets are committed under+/// this group, so restarts resume where the previous run left off.+const GROUP_ID: &str = "webhook-event-ingestion";++/// Maximum in-process ingestion attempts per event before the consumer bails+/// out and lets a restart redeliver from the last committed offset.+const MAX_INGEST_ATTEMPTS: u32 = 5;++/// Delay before the first ingestion retry; doubles on each subsequent retry.+/// The worst-case total backoff (1+2+4+8 = 15s) stays well under librdkafka's+/// default `max.poll.interval.ms` (300s), so retrying never evicts this+/// consumer from its group.+const INGEST_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);++/// One decoded event envelope from a topic this consumer subscribes to.+///+/// The Kafka message key is not carried: every event's metadata already+/// contains the entity ids the ingestion service needs.+#[derive(Debug)]+pub enum WebhookConsumerEvent {+ /// Event received on [`MacroDocumentsTopic`].+ Documents(Event<DocumentTopicEvent>),+ /// Event received on [`MacroChannelsTopic`].+ Channels(Event<ChannelTopicEvent>),+}++impl WebhookConsumerEvent {+ /// Decode one Kafka message into this consumer's event enum.+ pub fn decode(topic: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {+ match topic {+ topic if topic == MacroDocumentsTopic.as_str() => {+ Ok(Self::Documents(Event::decode(payload)?))+ }+ topic if topic == MacroChannelsTopic.as_str() => {+ Ok(Self::Channels(Event::decode(payload)?))+ }+ unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),+ }+ }+}++/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).+enum WebhookKafkaConsumer {+ /// Unauthenticated plaintext connection (local docker broker).+ Plaintext(StreamConsumer),+ /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).+ MskIam(StreamConsumer<MskIamClientContext>),+}++impl WebhookKafkaConsumer {+ /// Build a consumer for the given brokers, choosing the transport from the+ /// `ENVIRONMENT` variable exactly like the publisher does.+ fn from_env(brokers: &str) -> anyhow::Result<Self> {+ let mut config = ClientConfig::new();+ config+ .set("bootstrap.servers", brokers)+ .set("group.id", GROUP_ID)+ // Offsets are committed manually after successful ingestion.+ .set("enable.auto.commit", "false")+ // Start from the beginning of the topics on the first ever run.+ .set("auto.offset.reset", "earliest");++ let consumer = match Environment::new_or_prod() {+ Environment::Local => Self::Plaintext(+ config+ .create()+ .context("failed to create plaintext kafka consumer")?,+ ),+ Environment::Develop | Environment::Production => {+ configure_sasl_iam(&mut config);+ Self::MskIam(+ config+ .create_with_context(MskIamClientContext::from_env())+ .context("failed to create MSK IAM kafka consumer")?,+ )+ }+ };++ Ok(consumer)+ }++ fn subscribe(&self) -> KafkaResult<()> {+ let topics = [MacroDocumentsTopic.as_str(), MacroChannelsTopic.as_str()];+ match self {+ Self::Plaintext(consumer) => consumer.subscribe(&topics),+ Self::MskIam(consumer) => consumer.subscribe(&topics),+ }+ }++ /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it+ /// can sit in a `select!` without losing messages.+ async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {+ match self {+ Self::Plaintext(consumer) => consumer.recv().await,+ Self::MskIam(consumer) => consumer.recv().await,+ }+ }++ fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {+ match self {+ Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),+ Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),+ }+ }++ /// Commit `message`'s offset, logging the outcome.+ fn commit_logged(&self, message: &BorrowedMessage<'_>) {+ match self.commit(message) {+ Ok(()) => tracing::trace!(+ partition = message.partition(),+ offset = message.offset(),+ "committed offset"+ ),+ Err(e) => tracing::error!(+ error = ?e,+ partition = message.partition(),+ offset = message.offset(),+ "failed to commit offset"+ ),+ }+ }+}++/// Ingest one decoded event, retrying transient failures with exponential+/// backoff.+///+/// Returns `Ok(())` when the event was ingested or permanently rejected (both+/// safe to commit) and `Err` when a transient failure survived every attempt —+/// the caller must exit without committing so the event is redelivered.+async fn ingest_with_retry<S: WebhookEventIngestionService>(+ service: &S,+ event: &WebhookConsumerEvent,+ partition: i32,+ offset: i64,+) -> anyhow::Result<()> {+ let mut delay = INGEST_RETRY_BASE_DELAY;+ let mut attempt = 1u32;+ loop {+ tracing::trace!(partition, offset, attempt, "ingesting broker event");+ let result = match event {+ WebhookConsumerEvent::Documents(event) => {+ service.ingest_document_event(event.clone()).await+ }+ WebhookConsumerEvent::Channels(event) => {+ service.ingest_channel_event(event.clone()).await+ }+ };+ match result {+ Ok(()) => {+ tracing::trace!(partition, offset, attempt, "broker event ingested");+ return Ok(());+ }+ Err(e) if !e.is_transient() => {+ tracing::error!(+ error = ?e,+ partition,+ offset,+ "dropping broker event after non-retryable ingestion failure"+ );+ return Ok(());+ }+ Err(e) if attempt < MAX_INGEST_ATTEMPTS => {+ tracing::warn!(+ error = ?e,+ partition,+ offset,+ attempt,+ delay_secs = delay.as_secs_f32(),+ "transient ingestion failure, retrying"+ );+ tokio::time::sleep(delay).await;+ delay *= 2;+ attempt += 1;+ }+ Err(e) => {+ return Err(e).with_context(|| {+ format!(+ "transient ingestion failure persisted after \+ {MAX_INGEST_ATTEMPTS} attempts \+ (partition {partition} offset {offset})"+ )+ });+ }+ }+ }+}++/// Run the webhook event consumer until `shutdown` resolves.+///+/// Connects to `brokers`, subscribes to [`MacroDocumentsTopic`] and+/// [`MacroChannelsTopic`] under the `webhook-event-ingestion` consumer group,+/// and feeds every decoded event to `service`, committing each offset only+/// after ingestion succeeds (see [`ingest_with_retry`] for the retry policy).+/// Returns an error when the consumer cannot be created or subscribed, or when+/// a transient ingestion failure exhausts its in-process retries; callers+/// should treat that as fatal and restart, which redelivers the uncommitted+/// event. Pass `std::future::pending()` as `shutdown` to run until the process+/// exits.+pub async fn run_webhook_event_consumer<S>(+ brokers: &str,+ service: S,+ shutdown: impl Future<Output = ()> + Send,+) -> anyhow::Result<()>+where+ S: WebhookEventIngestionService,+{+ let consumer = WebhookKafkaConsumer::from_env(brokers)?;+ consumer+ .subscribe()+ .context("failed to subscribe to webhook event topics")?;+ tracing::info!(+ topics = ?[MacroDocumentsTopic.as_str(), MacroChannelsTopic.as_str()],+ group = GROUP_ID,+ "webhook event consumer listening"+ );++ let mut shutdown = std::pin::pin!(shutdown);+ loop {+ tokio::select! {+ _ = &mut shutdown => {+ tracing::info!("webhook event consumer shutting down");+ break;+ }+ result = consumer.recv() => {+ let message = match result {+ Ok(message) => message,+ Err(e) => {+ tracing::error!(error = ?e, "kafka receive error");+ continue;+ }+ };+ tracing::trace!(+ topic = message.topic(),+ partition = message.partition(),+ offset = message.offset(),+ payload_len = message.payload().map_or(0, <[u8]>::len),+ "received kafka message"+ );++ let Some(payload) = message.payload() else {+ tracing::warn!(+ partition = message.partition(),+ offset = message.offset(),+ "skipping message with empty payload"+ );+ consumer.commit_logged(&message);+ continue;+ };++ match WebhookConsumerEvent::decode(message.topic(), payload) {+ Ok(event) => {+ tracing::trace!(+ ?event,+ partition = message.partition(),+ offset = message.offset(),+ "decoded broker event"+ );+ ingest_with_retry(+ &service,+ &event,+ message.partition(),+ message.offset(),+ )+ .await?;+ }+ // Undecodable messages are logged and skipped rather than+ // wedging the partition on a poison message.+ Err(e) => tracing::error!(+ error = ?e,+ topic = message.topic(),+ partition = message.partition(),+ offset = message.offset(),+ "failed to decode broker event"+ ),+ }++ // Commit only after the event was ingested or permanently+ // rejected: at-least-once, retried across restarts.+ consumer.commit_logged(&message);+ }+ }+ }++ Ok(())+}diff --git a/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs b/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rsnew file mode 100644index 0000000000..e8d129f76b--- /dev/null+++ b/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs@@ -0,0 +1,162 @@+use super::*;+use crate::domain::ingestion::WebhookEventIngestionError;+use channel_sender::ChannelSender;+use channels::domain::broker_events::ChannelDeletedMetadata;+use documents::domain::events::DocumentDeletedMetadata;+use entity_access::domain::models::AccessError;+use std::sync::Arc;+use std::sync::atomic::{AtomicU32, Ordering};+use uuid::Uuid;++fn document_event() -> Event<DocumentTopicEvent> {+ Event::new(DocumentTopicEvent::Deleted(DocumentDeletedMetadata {+ document_id: "doc_1".to_string(),+ actor_user_id: None,+ project_id: None,+ }))+}++#[test]+fn decodes_document_events() {+ let event = document_event();+ let payload = serde_json::to_vec(&event).expect("serializable");++ let decoded =+ WebhookConsumerEvent::decode(MacroDocumentsTopic.as_str(), &payload).expect("decodable");++ match decoded {+ WebhookConsumerEvent::Documents(decoded) => assert_eq!(decoded, event),+ WebhookConsumerEvent::Channels(_) => panic!("decoded into the wrong topic variant"),+ }+}++#[test]+fn decodes_channel_events() {+ let event = Event::new(ChannelTopicEvent::Deleted(ChannelDeletedMetadata {+ channel_id: Uuid::nil(),+ actor: ChannelSender::try_from("macro|[email protected]".to_string())+ .expect("valid channel sender"),+ }));+ let payload = serde_json::to_vec(&event).expect("serializable");++ let decoded =+ WebhookConsumerEvent::decode(MacroChannelsTopic.as_str(), &payload).expect("decodable");++ match decoded {+ WebhookConsumerEvent::Channels(decoded) => assert_eq!(decoded, event),+ WebhookConsumerEvent::Documents(_) => panic!("decoded into the wrong topic variant"),+ }+}++#[test]+fn rejects_unknown_topics() {+ let err = WebhookConsumerEvent::decode("macro.example", b"{}").expect_err("unknown topic");+ assert!(matches!(err, EventBrokerError::UnknownTopic(topic) if topic == "macro.example"));+}++#[test]+fn rejects_malformed_payloads() {+ let err = WebhookConsumerEvent::decode(MacroDocumentsTopic.as_str(), b"not json")+ .expect_err("malformed payload");+ assert!(matches!(err, EventBrokerError::Serialization(_)));+}++/// Ingestion service that fails the first `failures` attempts, then succeeds.+#[derive(Clone)]+struct FlakyIngestionService {+ attempts: Arc<AtomicU32>,+ failures: u32,+ transient: bool,+}++impl FlakyIngestionService {+ fn failure(&self) -> WebhookEventIngestionError {+ if self.transient {+ WebhookEventIngestionError::EntityAccess(AccessError::DatabaseError(+ sqlx::Error::PoolTimedOut,+ ))+ } else {+ WebhookEventIngestionError::Internal(anyhow::anyhow!("permanent failure"))+ }+ }+}++impl WebhookEventIngestionService for FlakyIngestionService {+ async fn ingest_document_event(+ &self,+ _event: Event<DocumentTopicEvent>,+ ) -> Result<(), WebhookEventIngestionError> {+ let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;+ if attempt <= self.failures {+ Err(self.failure())+ } else {+ Ok(())+ }+ }++ async fn ingest_channel_event(+ &self,+ _event: Event<ChannelTopicEvent>,+ ) -> Result<(), WebhookEventIngestionError> {+ Ok(())+ }+}++fn flaky_service(failures: u32, transient: bool) -> (FlakyIngestionService, Arc<AtomicU32>) {+ let attempts = Arc::new(AtomicU32::new(0));+ let service = FlakyIngestionService {+ attempts: attempts.clone(),+ failures,+ transient,+ };+ (service, attempts)+}++// `start_paused` auto-advances the tokio clock through the backoff sleeps.+#[tokio::test(start_paused = true)]+async fn retries_transient_failures_until_success() {+ let (service, attempts) = flaky_service(2, true);++ ingest_with_retry(+ &service,+ &WebhookConsumerEvent::Documents(document_event()),+ 0,+ 0,+ )+ .await+ .expect("succeeds once the transient failure clears");++ assert_eq!(attempts.load(Ordering::SeqCst), 3);+}++#[tokio::test(start_paused = true)]+async fn exhausted_transient_retries_bubble_up_for_redelivery() {+ let (service, attempts) = flaky_service(u32::MAX, true);++ ingest_with_retry(+ &service,+ &WebhookConsumerEvent::Documents(document_event()),+ 0,+ 0,+ )+ .await+ .expect_err("persistent transient failure aborts the consumer without committing");++ assert_eq!(attempts.load(Ordering::SeqCst), MAX_INGEST_ATTEMPTS);+}++#[tokio::test]+async fn permanent_failures_are_dropped_without_retry() {+ let (service, attempts) = flaky_service(u32::MAX, false);++ ingest_with_retry(+ &service,+ &WebhookConsumerEvent::Documents(document_event()),+ 0,+ 0,+ )+ .await+ .expect("permanent failures are skipped so the offset commits");++ assert_eq!(attempts.load(Ordering::SeqCst), 1);+}diff --git a/rust/cloud-storage/webhook/src/inbound/mod.rs b/rust/cloud-storage/webhook/src/inbound/mod.rsindex 33f3d31aba..2b84ac5b7d 100644--- a/rust/cloud-storage/webhook/src/inbound/mod.rs+++ b/rust/cloud-storage/webhook/src/inbound/mod.rs@@ -1,6 +1,10 @@-//! Inbound HTTP adapters.+//! Inbound adapters: HTTP handlers and the Kafka event consumer.+#[cfg(feature = "inbound")]pub mod axum_router;-#[cfg(test)]+#[cfg(all(test, feature = "inbound"))]mod axum_router_test;++#[cfg(feature = "consumer")]+pub mod kafka_consumer;diff --git a/rust/cloud-storage/webhook/src/lib.rs b/rust/cloud-storage/webhook/src/lib.rsindex 116059b185..0804f14701 100644--- a/rust/cloud-storage/webhook/src/lib.rs+++ b/rust/cloud-storage/webhook/src/lib.rs@@ -14,20 +14,24 @@//! the result in `is_valid`;//! - validation attempt rate limiting with the existing `rate_limit` crate using//! the key shape `per-user-validate-webhook:{macro_user_id}:{webhook_id}` and-//! a limit of 10 attempts per 3600-second window.+//! a limit of 10 attempts per 3600-second window;+//! - a Kafka consumer (`inbound::kafka_consumer`, feature `consumer`) that+//! subscribes to the `macro.documents` and `macro.channels` topics and hands+//! every event to the `domain::ingestion::WebhookEventIngestionService`+//! (feature `ingestion`), whose per-event handlers are still stubs.//!//! The current implementation intentionally excludes delivery workers, SQS/SNS-//! infrastructure, event ingestion, redelivery, and auto-pause behavior.+//! infrastructure, redelivery, and auto-pause behavior.//!//! Temporary limitations: IDs use prefixed UUIDv7 strings (`wh_<uuid_v7>`)//! rather than true ULIDs. Custom headers are stored as JSON in `headers`.//! Webhook signing secrets are intentionally stored as plaintext in//! `signing_secret`, matching the existing bot secret storage approach.-/// Domain models, ports, and service.+/// Domain models, ports, and services.pub mod domain;-#[cfg(feature = "inbound")]-/// HTTP adapters.+#[cfg(any(feature = "inbound", feature = "consumer"))]+/// HTTP and Kafka consumer adapters.pub mod inbound;#[cfg(feature = "outbound")]/// Postgres and HTTP adapters.