Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/feat/soup realtime (#5068)
Sean Aye9ebbcb91888c
diff --git a/.github/kafka-cluster-topics.json b/.github/kafka-cluster-topics.jsonindex fa43baa313..461a84a529 100644--- a/.github/kafka-cluster-topics.json+++ b/.github/kafka-cluster-topics.json@@ -5,6 +5,7 @@"macro.email","macro.example","macro.projects",+ "macro.soup","macro.teams","macro.webhooks"]diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex a15d516200..6e08fc8a16 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -1507,6 +1507,7 @@"crates/invite_email","crates/ip_extractor","crates/item_filters",+ "crates/kafka_consumer_util","crates/last_online_tracker","crates/lexical_client","crates/lexical_mention_extractor",@@ -2884,6 +2885,14 @@"crates/non_empty","crates/workspace-hack"],+ "kafka_consumer_util": [+ "crates/kafka_consumer_util",+ "crates/macro_env",+ "crates/macro_env_var",+ "crates/macro_event_broker",+ "crates/macro_event_topics",+ "crates/workspace-hack"+ ],"lambda_client": ["crates/lambda_client","crates/workspace-hack"@@ -4956,6 +4965,93 @@"crates/unfurl","crates/workspace-hack"],+ "soup_realtime": [+ "crates/agent",+ "crates/ai_toolset",+ "crates/ai_usage",+ "crates/attachment",+ "crates/authentication_service_client",+ "crates/bot_id",+ "crates/call",+ "crates/channel_sender",+ "crates/channels",+ "crates/connection",+ "crates/connection_gateway_client",+ "crates/connection_gateway_models",+ "crates/contacts",+ "crates/cowlike",+ "crates/crm",+ "crates/document_sub_type",+ "crates/documents",+ "crates/email",+ "crates/email_db_client",+ "crates/email_utils",+ "crates/embedding",+ "crates/entity_access",+ "crates/entity_access_db_utils",+ "crates/entity_access_management",+ "crates/filter_ast",+ "crates/foreign_entity",+ "crates/frecency",+ "crates/invite_email",+ "crates/item_filters",+ "crates/kafka_consumer_util",+ "crates/last_online_tracker",+ "crates/lexical_client",+ "crates/macro_auth",+ "crates/macro_authorization",+ "crates/macro_aws_config",+ "crates/macro_entrypoint",+ "crates/macro_env",+ "crates/macro_env_var",+ "crates/macro_event_broker",+ "crates/macro_event_topics",+ "crates/macro_service_urls",+ "crates/macro_sync_service_jwt",+ "crates/macro_user_id",+ "crates/macro_uuid",+ "crates/mention_utils",+ "crates/model",+ "crates/model-entity",+ "crates/model-error-response",+ "crates/model_file_type",+ "crates/model_notifications",+ "crates/model_user",+ "crates/models_ai_projection",+ "crates/models_bulk_upload",+ "crates/models_email",+ "crates/models_entity_access_management",+ "crates/models_grouping",+ "crates/models_opensearch",+ "crates/models_pagination",+ "crates/models_permissions",+ "crates/models_properties",+ "crates/models_search",+ "crates/models_search_cursor",+ "crates/models_soup",+ "crates/non_empty",+ "crates/notification",+ "crates/opensearch_client",+ "crates/opensearch_query_builder",+ "crates/prompt",+ "crates/properties",+ "crates/rate_limit",+ "crates/readonly_pool",+ "crates/remote_env_var",+ "crates/s3_key",+ "crates/secretsmanager_client",+ "crates/share_permission_db_utils",+ "crates/soup",+ "crates/soup_realtime",+ "crates/sqs_client",+ "crates/sqs_worker",+ "crates/sync_service",+ "crates/sync_service_client",+ "crates/system_properties",+ "crates/task_dedup",+ "crates/unfurl",+ "crates/workspace-hack"+ ],"sqs_client": ["crates/agent","crates/ai_toolset",@@ -5489,6 +5585,7 @@"crates/frecency","crates/invite_email","crates/item_filters",+ "crates/kafka_consumer_util","crates/last_online_tracker","crates/lexical_client","crates/macro_auth",diff --git a/Cargo.lock b/Cargo.lockindex 5728c9dd9f..f87c7393e8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -7539,6 +7539,19 @@ dependencies = ["zeroize",]+[[package]]+name = "kafka_consumer_util"+version = "0.1.0"+dependencies = [+ "either",+ "macro_env",+ "macro_event_broker",+ "rdkafka",+ "thiserror 2.0.18",+ "uuid",+ "workspace-hack",+]+[[package]]name = "lambda_client"version = "0.1.0"@@ -13386,6 +13399,32 @@ dependencies = ["workspace-hack",]+[[package]]+name = "soup_realtime"+version = "0.1.0"+dependencies = [+ "chrono",+ "documents",+ "entity_access",+ "futures",+ "kafka_consumer_util",+ "macro_event_broker",+ "macro_event_topics",+ "macro_user_id",+ "model-entity",+ "models_properties",+ "models_soup",+ "rdkafka",+ "rootcause",+ "serde",+ "serde_json",+ "soup",+ "tokio",+ "tracing",+ "uuid",+ "workspace-hack",+]+[[package]]name = "spin"version = "0.9.8"@@ -15516,9 +15555,9 @@ dependencies = ["futures","hex","hmac 0.12.1",+ "kafka_consumer_util","macro_authorization","macro_db_migrator",- "macro_env","macro_event_broker","macro_event_topics","macro_user_id",diff --git a/Cargo.toml b/Cargo.tomlindex 084f871484..b423ec1478 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -37,6 +37,7 @@ members = ["crates/prompt","crates/properties_service","crates/search_service",+ "crates/soup_realtime","crates/task_dedup","crates/unfurl","crates/workspace-hack",diff --git a/crates/channels/examples/channels_consumer.rs b/crates/channels/examples/channels_consumer.rsindex 0360eb2cea..f12dbbfd10 100644--- a/crates/channels/examples/channels_consumer.rs+++ b/crates/channels/examples/channels_consumer.rs@@ -103,7 +103,7 @@ impl ChannelsConsumer {.context("failed to create plaintext kafka consumer")?,),Environment::Develop | Environment::Production => {- configure_sasl_iam(&mut config);+ let config = configure_sasl_iam(config);Self::MskIam(config.create_with_context(MskIamClientContext::from_env())diff --git a/crates/documents/examples/documents_consumer.rs b/crates/documents/examples/documents_consumer.rsindex 8f4e29168a..66259c1317 100644--- a/crates/documents/examples/documents_consumer.rs+++ b/crates/documents/examples/documents_consumer.rs@@ -103,7 +103,7 @@ impl DocumentsConsumer {.context("failed to create plaintext kafka consumer")?,),Environment::Develop | Environment::Production => {- configure_sasl_iam(&mut config);+ let config = configure_sasl_iam(config);Self::MskIam(config.create_with_context(MskIamClientContext::from_env())diff --git a/crates/kafka_consumer_util/Cargo.toml b/crates/kafka_consumer_util/Cargo.tomlnew file mode 100644index 0000000000..dbdf5d7356--- /dev/null+++ b/crates/kafka_consumer_util/Cargo.toml@@ -0,0 +1,16 @@+[package]+edition = "2024"+name = "kafka_consumer_util"+publish = false+version = "0.1.0"++[dependencies]+either = { workspace = true }+macro_env = { path = "../macro_env" }+macro_event_broker = { path = "../macro_event_broker", default-features = false, features = [+ "kafka",+] }+rdkafka = { workspace = true }+thiserror = { workspace = true }+uuid = { workspace = true }+workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/kafka_consumer_util/README.md b/crates/kafka_consumer_util/README.mdnew file mode 100644index 0000000000..6373185896--- /dev/null+++ b/crates/kafka_consumer_util/README.md@@ -0,0 +1,10 @@+# kafka_consumer_util++Shared environment-aware Kafka consumer transport for backend inbound adapters.++The crate centralizes plaintext-local versus MSK-IAM construction and exposes two type-safe consumption modes:++- Named consumer groups implement `GroupName` and may subscribe, pause, receive, and commit.+- `Ungrouped` consumers use manual partition assignment and may pause and receive, but cannot subscribe or commit.++Ungrouped consumers assign every current partition at an explicit earliest or latest offset. Application adapters remain responsible for decoding, retries, poison-message handling, and refreshing assignments if topic partition counts change.diff --git a/crates/kafka_consumer_util/src/lib.rs b/crates/kafka_consumer_util/src/lib.rsnew file mode 100644index 0000000000..1331329a39--- /dev/null+++ b/crates/kafka_consumer_util/src/lib.rs@@ -0,0 +1,253 @@+#![deny(missing_docs)]+//! Environment-aware Kafka consumer transport.+//!+//! This adapter centralizes the repository's plaintext-local versus MSK-IAM+//! transport setup. Application inbound adapters retain responsibility for+//! decoding, retries, poison-message handling, and deciding when to commit.++#[cfg(test)]+mod test;++use std::marker::PhantomData;+use std::time::Duration;++use either::Either;+use macro_env::Environment;+use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};+use rdkafka::error::{KafkaError, KafkaResult};+use rdkafka::message::{BorrowedMessage, Message as _};+use rdkafka::{ClientConfig, Offset, TopicPartitionList};+use uuid::Uuid;++use macro_event_broker::kafka::msk_iam::{MskIamClientContext, configure_sasl_iam};++const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";++/// Failure to construct an environment-specific Kafka consumer.+#[derive(Debug, thiserror::Error)]+pub enum KafkaConsumerError {+ /// Failed to create the unauthenticated local consumer.+ #[error("failed to create plaintext Kafka consumer")]+ Plaintext(#[source] KafkaError),+ /// Failed to create the TLS and MSK-IAM authenticated consumer.+ #[error("failed to create MSK IAM Kafka consumer")]+ MskIam(#[source] KafkaError),+}++/// Underlying Kafka consumer transport selected from the runtime environment.+struct ConsumerTransport(Either<StreamConsumer, StreamConsumer<MskIamClientContext>>);++/// Type-level name for a durable Kafka consumer group.+///+/// Defining group names on marker types keeps group identities centralized and+/// prevents consumers from passing arbitrary string group IDs at construction.+pub trait GroupName {+ /// Stable Kafka consumer group ID used for partition balancing and offsets.+ const GROUP_NAME: &'static str;+}++/// Marker type for a consumer that does not subscribe or persist offsets.+///+/// librdkafka requires a configured `group.id` for its safe manual-assignment+/// API, so this mode uses a generated internal ID. It never exposes subscription+/// or commit operations and therefore does not create durable group state.+pub struct Ungrouped;++/// Starting position for manually assigned ungrouped topic partitions.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum InitialOffset {+ /// Consume all currently retained records before continuing with new ones.+ Earliest,+ /// Consume only records published after the partition assignment begins.+ Latest,+}++impl InitialOffset {+ fn as_kafka_offset(self) -> Offset {+ match self {+ Self::Earliest => Offset::Beginning,+ Self::Latest => Offset::End,+ }+ }+}++/// Shared Kafka consumer with environment-aware transport and type-safe group behavior.+///+/// Consumers parameterized by a [`GroupName`] may subscribe and commit offsets.+/// [`Ungrouped`] consumers instead manually assign every partition of their+/// requested topics and cannot subscribe or commit.+pub struct KafkaEventConsumer<T> {+ consumer: ConsumerTransport,+ marker: PhantomData<T>,+}++fn base_config(brokers: &str) -> ClientConfig {+ let mut config = ClientConfig::new();+ config+ .set("bootstrap.servers", brokers)+ .set("enable.auto.commit", "false");+ config+}++fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {+ let mut config = base_config(brokers);+ config+ .set("group.id", T::GROUP_NAME)+ .set("auto.offset.reset", "earliest");+ config+}++fn ungrouped_config(brokers: &str) -> ClientConfig {+ let mut config = base_config(brokers);+ let group_id = format!("{UNGROUPED_GROUP_PREFIX}-{}", Uuid::new_v4());+ config+ .set("group.id", group_id)+ .set("enable.auto.offset.store", "false");+ config+}++fn create_from_env<T>(config: ClientConfig) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {+ let consumer = match Environment::new_or_prod() {+ Environment::Local => Either::Left(config.create().map_err(KafkaConsumerError::Plaintext)?),+ Environment::Develop | Environment::Production => {+ let config = configure_sasl_iam(config);+ Either::Right(+ config+ .create_with_context(MskIamClientContext::from_env())+ .map_err(KafkaConsumerError::MskIam)?,+ )+ }+ };++ Ok(KafkaEventConsumer {+ consumer: ConsumerTransport(consumer),+ marker: PhantomData,+ })+}++fn build_assignment<C, T>(+ consumer: &T,+ topics: &[&str],+ initial_offset: InitialOffset,+ metadata_timeout: Duration,+) -> KafkaResult<TopicPartitionList>+where+ C: ConsumerContext,+ T: Consumer<C>,+{+ if topics.is_empty() {+ return Err(KafkaError::Subscription(+ "at least one topic is required for assignment".to_string(),+ ));+ }++ let mut assignment = TopicPartitionList::new();+ for topic in topics {+ let metadata = consumer.fetch_metadata(Some(topic), metadata_timeout)?;+ let topic_metadata = metadata+ .topics()+ .iter()+ .find(|metadata| metadata.name() == *topic)+ .ok_or_else(|| {+ KafkaError::Subscription(format!(+ "metadata response did not include requested topic {topic}"+ ))+ })?;++ if let Some(error) = topic_metadata.error() {+ return Err(KafkaError::MetadataFetch(error.into()));+ }+ if topic_metadata.partitions().is_empty() {+ return Err(KafkaError::Subscription(format!(+ "requested topic {topic} has no partitions"+ )));+ }++ for partition in topic_metadata.partitions() {+ if let Some(error) = partition.error() {+ return Err(KafkaError::MetadataFetch(error.into()));+ }+ assignment.add_partition_offset(+ topic,+ partition.id(),+ initial_offset.as_kafka_offset(),+ )?;+ }+ }++ Ok(assignment)+}++impl<T> KafkaEventConsumer<T> {+ /// Receives the next Kafka message.+ ///+ /// `StreamConsumer::recv` is cancel-safe and may be used in `tokio::select!`.+ pub async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {+ either::for_both!(&self.consumer.0, consumer => consumer.recv().await)+ }++ /// Pauses the partition containing `message`.+ ///+ /// Grouped consumers can use this to prevent a later cumulative commit+ /// from advancing past a failed record. Ungrouped consumers can use it to+ /// stop additional delivery from a failed partition.+ pub fn pause_message_partition(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {+ let mut partitions = TopicPartitionList::new();+ partitions.add_partition(message.topic(), message.partition());+ either::for_both!(&self.consumer.0, consumer => consumer.pause(&partitions))+ }+}++impl<T: GroupName> KafkaEventConsumer<T> {+ /// Creates a named-group consumer, selecting plaintext or MSK IAM transport+ /// from the runtime environment.+ pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {+ create_from_env(grouped_config::<T>(brokers))+ }++ /// Subscribes the consumer to exactly the provided topics.+ pub fn subscribe(&self, topics: &[&str]) -> KafkaResult<()> {+ either::for_both!(&self.consumer.0, consumer => consumer.subscribe(topics))+ }++ /// Commits a message using the caller-selected commit mode.+ pub fn commit_message(+ &self,+ message: &BorrowedMessage<'_>,+ mode: CommitMode,+ ) -> KafkaResult<()> {+ either::for_both!(&self.consumer.0, consumer => consumer.commit_message(message, mode))+ }+}++impl KafkaEventConsumer<Ungrouped> {+ /// Creates an ungrouped consumer, selecting plaintext or MSK IAM transport+ /// from the runtime environment.+ ///+ /// Call [`Self::assign_topics`] before receiving messages.+ pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {+ create_from_env(ungrouped_config(brokers))+ }++ /// Manually assigns every current partition of `topics` at `initial_offset`.+ ///+ /// Manual assignment does not join a consumer group, persist offsets, or+ /// automatically discover partitions added after this call. Callers that+ /// support partition-count changes must refresh the assignment themselves.+ pub fn assign_topics(+ &self,+ topics: &[&str],+ initial_offset: InitialOffset,+ metadata_timeout: Duration,+ ) -> KafkaResult<()> {+ either::for_both!(&self.consumer.0, consumer => {+ let assignment = build_assignment(+ consumer,+ topics,+ initial_offset,+ metadata_timeout,+ )?;+ consumer.assign(&assignment)+ })+ }+}diff --git a/crates/kafka_consumer_util/src/test.rs b/crates/kafka_consumer_util/src/test.rsnew file mode 100644index 0000000000..ca3ed674af--- /dev/null+++ b/crates/kafka_consumer_util/src/test.rs@@ -0,0 +1,41 @@+use super::*;++struct TestConsumerGroup;++impl GroupName for TestConsumerGroup {+ const GROUP_NAME: &'static str = "consumer-group";+}++#[test]+fn grouped_config_uses_named_group_manual_commits_and_earliest_offsets() {+ let config = grouped_config::<TestConsumerGroup>("broker-a:9092,broker-b:9092");++ assert_eq!(+ config.get("bootstrap.servers"),+ Some("broker-a:9092,broker-b:9092")+ );+ assert_eq!(config.get("group.id"), Some("consumer-group"));+ assert_eq!(config.get("enable.auto.commit"), Some("false"));+ assert_eq!(config.get("auto.offset.reset"), Some("earliest"));+}++#[test]+fn ungrouped_config_uses_unique_internal_groups_without_offset_storage() {+ let first = ungrouped_config("broker:9092");+ let second = ungrouped_config("broker:9092");+ let first_group = first.get("group.id").unwrap();+ let second_group = second.get("group.id").unwrap();++ assert!(first_group.starts_with(UNGROUPED_GROUP_PREFIX));+ assert!(second_group.starts_with(UNGROUPED_GROUP_PREFIX));+ assert_ne!(first_group, second_group);+ assert_eq!(first.get("enable.auto.commit"), Some("false"));+ assert_eq!(first.get("enable.auto.offset.store"), Some("false"));+ assert_eq!(first.get("auto.offset.reset"), None);+}++#[test]+fn ungrouped_initial_offsets_are_explicit() {+ assert_eq!(InitialOffset::Earliest.as_kafka_offset(), Offset::Beginning);+ assert_eq!(InitialOffset::Latest.as_kafka_offset(), Offset::End);+}diff --git a/crates/macro_event_broker/Cargo.toml b/crates/macro_event_broker/Cargo.tomlindex e332bb7760..397b42481c 100644--- a/crates/macro_event_broker/Cargo.toml+++ b/crates/macro_event_broker/Cargo.toml@@ -5,15 +5,16 @@ publish = falseversion = "0.1.0"[features]-default = ["ports", "outbound"]-outbound = [+default = ["outbound", "ports"]+kafka = ["dep:aws-msk-iam-sasl-signer","dep:aws-types","dep:macro_env","dep:macro_env_var","dep:rdkafka",- "ports",+ "dep:tokio",]+outbound = ["kafka", "ports"]ports = ["dep:tokio"][dependencies]diff --git a/crates/macro_event_broker/src/kafka/mod.rs b/crates/macro_event_broker/src/kafka/mod.rsnew file mode 100644index 0000000000..a7074dedfc--- /dev/null+++ b/crates/macro_event_broker/src/kafka/mod.rs@@ -0,0 +1,4 @@+//! Kafka transport support shared by inbound and outbound adapters.++/// AWS MSK IAM authentication support for Kafka clients.+pub mod msk_iam;diff --git a/crates/macro_event_broker/src/outbound/msk_iam.rs b/crates/macro_event_broker/src/kafka/msk_iam.rssimilarity index 96%rename from crates/macro_event_broker/src/outbound/msk_iam.rsrename to crates/macro_event_broker/src/kafka/msk_iam.rsindex f8b8043593..de1d18e2f8 100644--- a/crates/macro_event_broker/src/outbound/msk_iam.rs+++ b/crates/macro_event_broker/src/kafka/msk_iam.rs@@ -17,10 +17,11 @@ macro_env_var::maybe_env_var! {/// Apply the TLS + SASL/OAUTHBEARER settings required to authenticate to an/// AWS MSK cluster with IAM auth (the `:9098` bootstrap listener).-pub fn configure_sasl_iam(config: &mut ClientConfig) -> &mut ClientConfig {+pub fn configure_sasl_iam(mut config: ClientConfig) -> ClientConfig {config.set("security.protocol", "SASL_SSL")- .set("sasl.mechanism", "OAUTHBEARER")+ .set("sasl.mechanism", "OAUTHBEARER");+ config}/// Client context that supplies AWS MSK IAM (SASL/OAUTHBEARER) auth tokens,diff --git a/crates/macro_event_broker/src/lib.rs b/crates/macro_event_broker/src/lib.rsindex f96c21aae1..bd60384a92 100644--- a/crates/macro_event_broker/src/lib.rs+++ b/crates/macro_event_broker/src/lib.rs@@ -8,7 +8,7 @@//! [`EventPublisher`](domain::ports::EventPublisher) port, and the//! [`MacroEventBrokerService`](domain::service::MacroEventBrokerService) that ties//! them together. Kafka topic definitions live in the `macro_event_topics` crate.-//! The [`outbound`] layer provides the Kafka adapter implementation.+//! The [`outbound`] layer provides the Kafka publisher adapter./// Domain layer: models, ports, and service.pub mod domain;@@ -23,11 +23,14 @@ pub use macro_event_topics::{pub use domain::ports::{EventPublisher, MacroEventBroker};#[cfg(feature = "ports")]pub use domain::service::{MacroEventBrokerService, NoopMacroEventBroker};+#[cfg(feature = "kafka")]+pub use kafka::msk_iam::MskIamClientContext;#[cfg(feature = "outbound")]pub use outbound::kafka_event_publisher::KafkaEventPublisher;-#[cfg(feature = "outbound")]-pub use outbound::msk_iam::MskIamClientContext;-/// Outbound layer: Kafka adapter for the [`EventPublisher`](domain::ports::EventPublisher) port.+/// Kafka transport support shared by inbound and outbound adapters.+#[cfg(feature = "kafka")]+pub mod kafka;+/// Outbound layer: Kafka publishing adapters.#[cfg(feature = "outbound")]pub mod outbound;diff --git a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rsindex 6f4df4632d..9e46870d93 100644--- a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs+++ b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs@@ -16,7 +16,7 @@ use rdkafka::producer::{FutureProducer, FutureRecord};use crate::domain::models::EventBrokerError;use crate::domain::ports::EventPublisher;-use crate::outbound::msk_iam::{MskIamClientContext, configure_sasl_iam};+use crate::kafka::msk_iam::{MskIamClientContext, configure_sasl_iam};/// How long a record may sit in the producer queue before delivery is considered failed.const MESSAGE_TIMEOUT_MS: &str = "5000";@@ -64,8 +64,7 @@ impl KafkaEventPublisher {Producer::Plaintext(base_config(brokers).create().map_err(create_error)?)}Environment::Develop | Environment::Production => {- let mut config = base_config(brokers);- configure_sasl_iam(&mut config);+ let config = configure_sasl_iam(base_config(brokers));Producer::MskIam(configdiff --git a/crates/macro_event_broker/src/outbound/mod.rs b/crates/macro_event_broker/src/outbound/mod.rsindex 048c8210ef..66675e0544 100644--- a/crates/macro_event_broker/src/outbound/mod.rs+++ b/crates/macro_event_broker/src/outbound/mod.rs@@ -1,4 +1,5 @@/// Kafka adapter implementing the [`EventPublisher`](crate::domain::ports::EventPublisher) port.+#[cfg(feature = "outbound")]pub mod kafka_event_publisher;/// AWS MSK IAM (SASL/OAUTHBEARER) auth support shared by producers and consumers.-pub mod msk_iam;+pub use crate::kafka::msk_iam;diff --git a/crates/macro_event_topics/src/lib.rs b/crates/macro_event_topics/src/lib.rsindex b2295c4b1a..bbbc542250 100644--- a/crates/macro_event_topics/src/lib.rs+++ b/crates/macro_event_topics/src/lib.rs@@ -53,6 +53,8 @@ topics! {MacroBotsTopic => "macro.bots",/// Document lifecycle events (created / updated / deleted / copied).MacroDocumentsTopic => "macro.documents",+ /// User-scoped full Soup items produced from entity updates.+ MacroSoupRealtimeTopic => "macro.soup",/// Project lifecycle events (created, updated, deleted, restored, permanently deleted, and uploaded).MacroProjectsTopic => "macro.projects",/// Team lifecycle, invite, and membership events.diff --git a/crates/macro_event_topics/src/test.rs b/crates/macro_event_topics/src/test.rsindex 593bce01e8..a5e62663ee 100644--- a/crates/macro_event_topics/src/test.rs+++ b/crates/macro_event_topics/src/test.rs@@ -19,6 +19,7 @@ fn all_topic_names_is_non_empty_and_unique() {fn all_topic_names_includes_declared_topics() {assert!(all_topic_names().contains(&MacroExampleTopic.as_str()));assert!(all_topic_names().contains(&MacroDocumentsTopic.as_str()));+ assert!(all_topic_names().contains(&MacroSoupRealtimeTopic.as_str()));assert!(all_topic_names().contains(&MacroProjectsTopic.as_str()));assert!(all_topic_names().contains(&MacroTeamsTopic.as_str()));assert!(all_topic_names().contains(&MacroChannelsTopic.as_str()));diff --git a/crates/projects/examples/projects_consumer.rs b/crates/projects/examples/projects_consumer.rsindex b64f5dcd10..e94972efb9 100644--- a/crates/projects/examples/projects_consumer.rs+++ b/crates/projects/examples/projects_consumer.rs@@ -103,7 +103,7 @@ impl ProjectsConsumer {.context("failed to create plaintext kafka consumer")?,),Environment::Develop | Environment::Production => {- configure_sasl_iam(&mut config);+ let config = configure_sasl_iam(config);Self::MskIam(config.create_with_context(MskIamClientContext::from_env())diff --git a/crates/soup_realtime/Cargo.toml b/crates/soup_realtime/Cargo.tomlnew file mode 100644index 0000000000..2cfe4c4827--- /dev/null+++ b/crates/soup_realtime/Cargo.toml@@ -0,0 +1,52 @@+[package]+edition = "2024"+name = "soup_realtime"+publish = false+version = "0.1.0"++[features]+consumer = [+ "dep:documents",+ "dep:kafka_consumer_util",+ "dep:macro_event_broker",+ "dep:macro_event_topics",+ "dep:rdkafka",+ "dep:tokio",+ "ports",+]+default = ["consumer", "outbound", "ports"]+outbound = [+ "dep:entity_access",+ "dep:macro_event_topics",+ "dep:serde_json",+ "dep:soup",+ "macro_event_broker/ports",+ "ports",+]+ports = ["dep:futures", "dep:rootcause", "dep:tracing"]++[dependencies]+documents = { path = "../documents", default-features = false, optional = true }+entity_access = { path = "../entity_access", default-features = false, features = ["ports"], optional = true }+futures = { workspace = true, optional = true }+kafka_consumer_util = { path = "../kafka_consumer_util", optional = true }+macro_event_broker = { path = "../macro_event_broker", default-features = false, optional = true }+macro_event_topics = { path = "../macro_event_topics", optional = true }+macro_user_id = { path = "../macro_user_id" }+model-entity = { path = "../model-entity" }+models_soup = { path = "../models_soup" }+rdkafka = { workspace = true, optional = true }+rootcause = { workspace = true, optional = true }+serde = { workspace = true }+serde_json = { workspace = true, optional = true }+soup = { path = "../soup", default-features = false, features = ["ports"], optional = true }+tokio = { workspace = true, features = ["time"], optional = true }+tracing = { workspace = true, optional = true }+workspace-hack = { version = "0.1", path = "../workspace-hack" }++[dev-dependencies]+chrono = { workspace = true }+models_properties = { path = "../models_properties" }+serde_json = { workspace = true }+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util", "time"] }+uuid = { workspace = true }diff --git a/crates/soup_realtime/README.md b/crates/soup_realtime/README.mdnew file mode 100644index 0000000000..2aa71b4ac3--- /dev/null+++ b/crates/soup_realtime/README.md@@ -0,0 +1,7 @@+# soup_realtime++Hexagonal components for expanding `document.updated` Kafka events into one full `SoupItem<()>` message per current accessor. The item is loaded once through the first accessor and its user-specific `viewed_at` field is set to `None` before fan-out.++This crate intentionally does not start a consumer or wire itself into a service binary. A later composition root can combine the domain service with the entity-access, Soup repository, Kafka publisher, and Kafka consumer adapters exposed here.++Delivery is at least once. If publication partially succeeds and processing is retried, downstream consumers may receive duplicate messages.diff --git a/crates/soup_realtime/src/domain/mod.rs b/crates/soup_realtime/src/domain/mod.rsnew file mode 100644index 0000000000..a0bdb0b450--- /dev/null+++ b/crates/soup_realtime/src/domain/mod.rs@@ -0,0 +1,12 @@+//! Core realtime Soup contracts and orchestration.++/// Versioned output message models.+pub mod models;++/// Inbound and outbound domain ports.+#[cfg(feature = "ports")]+pub mod ports;++/// Realtime Soup fan-out service.+#[cfg(feature = "ports")]+pub mod service;diff --git a/crates/soup_realtime/src/domain/models.rs b/crates/soup_realtime/src/domain/models.rsnew file mode 100644index 0000000000..af1b0e96fa--- /dev/null+++ b/crates/soup_realtime/src/domain/models.rs@@ -0,0 +1,30 @@+//! Versioned realtime Soup output models.++use macro_user_id::user_id::MacroUserIdStr;+use models_soup::item::SoupItem;+use serde::{Deserialize, Serialize};++/// One full Soup item published to a recipient for realtime delivery.+#[derive(Debug, Serialize, Deserialize)]+pub struct SoupRealtimeMessage {+ /// Version of this message contract.+ pub schema_version: u8,+ /// User to whom the Soup item is addressed.+ pub user_id: MacroUserIdStr<'static>,+ /// Complete Soup item with transient user-specific fields normalized.+ pub item: SoupItem<()>,+}++impl SoupRealtimeMessage {+ /// Current schema version for realtime Soup messages.+ pub const SCHEMA_VERSION: u8 = 1;++ /// Creates a version-one message addressed to a recipient.+ pub fn new(user_id: MacroUserIdStr<'static>, item: SoupItem<()>) -> Self {+ Self {+ schema_version: Self::SCHEMA_VERSION,+ user_id,+ item,+ }+ }+}diff --git a/crates/soup_realtime/src/domain/ports.rs b/crates/soup_realtime/src/domain/ports.rsnew file mode 100644index 0000000000..a67f874ff2--- /dev/null+++ b/crates/soup_realtime/src/domain/ports.rs@@ -0,0 +1,50 @@+//! Ports used by realtime Soup fan-out.++use std::future::Future;++use macro_user_id::user_id::MacroUserIdStr;+use model_entity::Entity;+use models_soup::item::SoupItem;+use rootcause::Report;++use super::models::SoupRealtimeMessage;++/// Inbound use-case port driven by entity update transports.+pub trait SoupRealtimeService: Send + Sync + 'static {+ /// Hydrates one normalized Soup item and publishes it to every current accessor.+ fn notify_users(+ &self,+ entity: Entity<'static>,+ ) -> impl Future<Output = Result<(), Report>> + Send;+}++/// Resolves the users who currently have access to an entity.+pub trait UserAccessExpander: Send + Sync + 'static {+ /// Returns all current user accessors for `entity`.+ ///+ /// Implementations may return duplicates; the domain service deduplicates+ /// recipients before hydration.+ fn expand_user_access(+ &self,+ entity: &Entity<'static>,+ ) -> impl Future<Output = Result<Vec<MacroUserIdStr<'static>>, Report>> + Send;+}++/// Reads a complete Soup item under an individual user's visibility scope.+pub trait SoupItemReader: Send + Sync + 'static {+ /// Reads `entity` through the visibility scope of `user_id`.+ fn read_for_user(+ &self,+ user_id: MacroUserIdStr<'static>,+ entity: &Entity<'static>,+ ) -> impl Future<Output = Result<Option<SoupItem<()>>, Report>> + Send;+}++/// Publishes complete recipient-targeted Soup messages.+pub trait SoupRealtimePublisher: Send + Sync + 'static {+ /// Publishes one realtime Soup message and awaits delivery.+ fn publish(+ &self,+ message: SoupRealtimeMessage,+ ) -> impl Future<Output = Result<(), Report>> + Send;+}diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rsnew file mode 100644index 0000000000..947b495bc1--- /dev/null+++ b/crates/soup_realtime/src/domain/service.rs@@ -0,0 +1,130 @@+//! Realtime Soup hydration and fan-out orchestration.++#[cfg(test)]+mod test;++use std::collections::HashSet;++use futures::{StreamExt as _, stream};+use model_entity::Entity;+use models_soup::item::SoupItem;+use rootcause::prelude::{Report, ResultExt as _};++use super::{+ models::SoupRealtimeMessage,+ ports::{SoupItemReader, SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},+};++/// Maximum number of Kafka publications polled concurrently.+const PUBLISH_CONCURRENCY: usize = 16;++/// Domain service that expands access, hydrates one normalized item, and fans out.+pub struct SoupRealtimeServiceImpl<A, R, P> {+ access_expander: A,+ item_reader: R,+ publisher: P,+}++impl<A, R, P> SoupRealtimeServiceImpl<A, R, P> {+ /// Creates a realtime Soup service from its three outbound capabilities.+ pub fn new(access_expander: A, item_reader: R, publisher: P) -> Self {+ Self {+ access_expander,+ item_reader,+ publisher,+ }+ }+}++impl<A, R, P> SoupRealtimeService for SoupRealtimeServiceImpl<A, R, P>+where+ A: UserAccessExpander,+ R: SoupItemReader,+ P: SoupRealtimePublisher,+{+ #[tracing::instrument(+ skip(self),+ fields(+ entity_type = %entity.entity_type,+ entity_id = %entity.entity_id,+ recipient_count = tracing::field::Empty,+ ),+ err+ )]+ async fn notify_users(&self, entity: Entity<'static>) -> Result<(), Report> {+ let mut users = self+ .access_expander+ .expand_user_access(&entity)+ .await+ .context("failed to expand current user access")?;++ let mut seen = HashSet::with_capacity(users.len());+ users.retain(|user_id| seen.insert(user_id.clone()));+ tracing::Span::current().record("recipient_count", users.len());++ if users.is_empty() {+ return Ok(());+ }++ let hydration_user_id = users[0].clone();+ let item = self+ .item_reader+ .read_for_user(hydration_user_id.clone(), &entity)+ .await+ .context_with(|| {+ format!("failed to hydrate Soup item through accessor {hydration_user_id}")+ })?+ .ok_or_else(|| {+ rootcause::report!(+ "Soup item for {} {} was missing for accessor {}",+ entity.entity_type,+ entity.entity_id,+ hydration_user_id+ )+ })?;++ let hydrated_entity = item.entity();+ if hydrated_entity != entity {+ return Err(rootcause::report!(+ "Soup reader returned {} {} while hydrating {} {} through accessor {}",+ hydrated_entity.entity_type,+ hydrated_entity.entity_id,+ entity.entity_type,+ entity.entity_id,+ hydration_user_id+ ));+ }++ let SoupItem::Document(mut document) = item else {+ return Err(rootcause::report!(+ "realtime Soup fan-out currently supports document items only"+ ));+ };+ document.viewed_at = None;++ let messages = users+ .into_iter()+ .map(|user_id| SoupRealtimeMessage::new(user_id, SoupItem::Document(document.clone())))+ .collect::<Vec<_>>();++ let results = stream::iter(+ messages+ .into_iter()+ .map(|message| self.publisher.publish(message)),+ )+ .buffer_unordered(PUBLISH_CONCURRENCY)+ .collect::<Vec<_>>()+ .await;++ let failure_count = results.iter().filter(|result| result.is_err()).count();+ if let Some(error) = results.into_iter().find_map(Result::err) {+ return Err(error+ .context(format!(+ "{failure_count} realtime Soup publication(s) failed"+ ))+ .into_dynamic());+ }++ Ok(())+ }+}diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rsnew file mode 100644index 0000000000..66e43e9342--- /dev/null+++ b/crates/soup_realtime/src/domain/service/test.rs@@ -0,0 +1,399 @@+use std::collections::{HashMap, HashSet};+use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::{Arc, Mutex};++use chrono::{DateTime, Utc};+use macro_user_id::user_id::MacroUserIdStr;+use model_entity::{Entity, EntityType};+use models_soup::{document::SoupDocument, item::SoupItem};+use uuid::Uuid;++use super::*;+use crate::domain::ports::{SoupItemReader, SoupRealtimePublisher, UserAccessExpander};++const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";+const OTHER_DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000002";++struct FakeAccessExpander {+ users: Vec<MacroUserIdStr<'static>>,+ fail: bool,+ calls: Arc<AtomicUsize>,+}++impl UserAccessExpander for FakeAccessExpander {+ async fn expand_user_access(+ &self,+ _entity: &Entity<'static>,+ ) -> Result<Vec<MacroUserIdStr<'static>>, Report> {+ self.calls.fetch_add(1, Ordering::SeqCst);+ if self.fail {+ Err(rootcause::report!("access unavailable"))+ } else {+ Ok(self.users.clone())+ }+ }+}++enum ReadResponse {+ Item(Box<SoupItem<()>>),+ Missing,+ Failure,+}++struct FakeReader {+ responses: Mutex<HashMap<String, ReadResponse>>,+ calls: Arc<AtomicUsize>,+}++impl SoupItemReader for FakeReader {+ async fn read_for_user(+ &self,+ user_id: MacroUserIdStr<'static>,+ _entity: &Entity<'static>,+ ) -> Result<Option<SoupItem<()>>, Report> {+ self.calls.fetch_add(1, Ordering::SeqCst);+ match self+ .responses+ .lock()+ .expect("responses lock")+ .remove(user_id.as_ref())+ .unwrap_or(ReadResponse::Missing)+ {+ ReadResponse::Item(item) => Ok(Some(*item)),+ ReadResponse::Missing => Ok(None),+ ReadResponse::Failure => Err(rootcause::report!("reader unavailable")),+ }+ }+}++struct FakePublisher {+ messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,+ fail_users: HashSet<String>,+}++impl SoupRealtimePublisher for FakePublisher {+ async fn publish(&self, message: SoupRealtimeMessage) -> Result<(), Report> {+ let should_fail = self.fail_users.contains(message.user_id.as_ref());+ self.messages.lock().expect("messages lock").push(message);+ if should_fail {+ Err(rootcause::report!("publisher unavailable"))+ } else {+ Ok(())+ }+ }+}++struct Harness {+ service: SoupRealtimeServiceImpl<FakeAccessExpander, FakeReader, FakePublisher>,+ access_calls: Arc<AtomicUsize>,+ read_calls: Arc<AtomicUsize>,+ messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,+}++fn user(local: &str) -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from(format!("macro|{local}@example.com")).expect("valid user id")+}++fn timestamp(seconds: i64) -> DateTime<Utc> {+ DateTime::from_timestamp(seconds, 0).expect("valid timestamp")+}++fn document_item(id: &str, name: &str, viewed_at: Option<DateTime<Utc>>) -> SoupItem<()> {+ SoupItem::Document(SoupDocument {+ id: Uuid::parse_str(id).expect("valid document id"),+ document_version_id: 42,+ owner_id: user("owner"),+ name: name.to_string(),+ file_type: Some("md".to_string()),+ sha: Some("document-sha".to_string()),+ project_id: None,+ branched_from_id: None,+ branched_from_version_id: None,+ document_family_id: None,+ created_at: timestamp(1),+ updated_at: timestamp(2),+ viewed_at,+ sub_type: None,+ deleted_at: None,+ extra: (),+ })+}++fn document_entity() -> Entity<'static> {+ EntityType::Document.with_entity_string(DOCUMENT_ID.to_string())+}++fn harness(+ users: Vec<MacroUserIdStr<'static>>,+ responses: HashMap<String, ReadResponse>,+ access_fails: bool,+ fail_users: HashSet<String>,+) -> Harness {+ let access_calls = Arc::new(AtomicUsize::new(0));+ let read_calls = Arc::new(AtomicUsize::new(0));+ let messages = Arc::new(Mutex::new(Vec::new()));+ let service = SoupRealtimeServiceImpl::new(+ FakeAccessExpander {+ users,+ fail: access_fails,+ calls: access_calls.clone(),+ },+ FakeReader {+ responses: Mutex::new(responses),+ calls: read_calls.clone(),+ },+ FakePublisher {+ messages: messages.clone(),+ fail_users,+ },+ );+ Harness {+ service,+ access_calls,+ read_calls,+ messages,+ }+}++fn item_response(+ recipient: &MacroUserIdStr<'static>,+ item: SoupItem<()>,+) -> (String, ReadResponse) {+ (+ recipient.as_ref().to_string(),+ ReadResponse::Item(Box::new(item)),+ )+}++#[tokio::test]+async fn zero_users_skips_reads_and_publications() {+ let harness = harness(Vec::new(), HashMap::new(), false, HashSet::new());++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect("zero recipients is successful");++ assert_eq!(harness.access_calls.load(Ordering::SeqCst), 1);+ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 0);+ assert!(harness.messages.lock().expect("messages lock").is_empty());+}++#[tokio::test]+async fn one_user_receives_one_full_versioned_message() {+ let recipient = user("one");+ let viewed_at = Some(timestamp(3));+ let responses = HashMap::from([item_response(+ &recipient,+ document_item(DOCUMENT_ID, "Full document", viewed_at),+ )]);+ let harness = harness(vec![recipient.clone()], responses, false, HashSet::new());++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect("fan-out succeeds");++ let mut messages = harness.messages.lock().expect("messages lock");+ let message = messages.pop().expect("one message");+ assert!(messages.is_empty());+ assert_eq!(message.schema_version, 1);+ assert_eq!(message.user_id, recipient);+ match message.item {+ SoupItem::Document(document) => {+ assert_eq!(document.name, "Full document");+ assert_eq!(document.document_version_id, 42);+ assert_eq!(document.sha.as_deref(), Some("document-sha"));+ assert_eq!(document.viewed_at, None);+ }+ _ => panic!("expected document item"),+ }+}++#[tokio::test]+async fn three_unique_users_receive_exactly_three_messages() {+ let users = [user("one"), user("two"), user("three")];+ let responses = users+ .iter()+ .map(|recipient| {+ item_response(+ recipient,+ document_item(DOCUMENT_ID, recipient.as_ref(), None),+ )+ })+ .collect();+ let harness = harness(users.to_vec(), responses, false, HashSet::new());++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect("fan-out succeeds");++ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 1);+ let messages = harness.messages.lock().expect("messages lock");+ assert_eq!(messages.len(), 3);+ let recipients: HashSet<_> = messages+ .iter()+ .map(|message| message.user_id.as_ref().to_string())+ .collect();+ assert_eq!(recipients.len(), 3);+ assert!(+ users+ .iter()+ .all(|recipient| recipients.contains(recipient.as_ref()))+ );+}++#[tokio::test]+async fn duplicate_accessors_are_deduplicated() {+ let one = user("one");+ let two = user("two");+ let responses = HashMap::from([+ item_response(&one, document_item(DOCUMENT_ID, "One", None)),+ item_response(&two, document_item(DOCUMENT_ID, "Two", None)),+ ]);+ let harness = harness(+ vec![one.clone(), two, one],+ responses,+ false,+ HashSet::new(),+ );++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect("fan-out succeeds");++ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 1);+ assert_eq!(harness.messages.lock().expect("messages lock").len(), 2);+}++#[tokio::test]+async fn all_recipients_get_the_first_item_with_no_viewed_at() {+ let one = user("one");+ let two = user("two");+ let responses = HashMap::from([+ item_response(+ &one,+ document_item(DOCUMENT_ID, "Representative", Some(timestamp(10))),+ ),+ item_response(+ &two,+ document_item(DOCUMENT_ID, "Should not be read", Some(timestamp(20))),+ ),+ ]);+ let harness = harness(vec![one, two], responses, false, HashSet::new());++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect("fan-out succeeds");++ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 1);+ let messages = harness.messages.lock().expect("messages lock");+ assert_eq!(messages.len(), 2);+ for message in messages.iter() {+ let SoupItem::Document(document) = &message.item else {+ panic!("expected document item")+ };+ assert_eq!(document.name, "Representative");+ assert_eq!(document.viewed_at, None);+ }+}++#[tokio::test]+async fn access_failure_prevents_reads_and_publications() {+ let harness = harness(vec![user("one")], HashMap::new(), true, HashSet::new());++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect_err("access failure propagates");++ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 0);+ assert!(harness.messages.lock().expect("messages lock").is_empty());+}++#[tokio::test]+async fn missing_item_prevents_all_publication() {+ let recipient = user("one");+ let responses = HashMap::from([(recipient.as_ref().to_string(), ReadResponse::Missing)]);+ let harness = harness(vec![recipient], responses, false, HashSet::new());++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect_err("missing item is an error");++ assert!(harness.messages.lock().expect("messages lock").is_empty());+}++#[tokio::test]+async fn mismatched_item_prevents_all_publication() {+ let recipient = user("one");+ let responses = HashMap::from([item_response(+ &recipient,+ document_item(OTHER_DOCUMENT_ID, "Wrong", None),+ )]);+ let harness = harness(vec![recipient], responses, false, HashSet::new());++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect_err("mismatched item is an error");++ assert!(harness.messages.lock().expect("messages lock").is_empty());+}++#[tokio::test]+async fn representative_reader_failure_prevents_all_publication() {+ let one = user("one");+ let two = user("two");+ let responses = HashMap::from([(one.as_ref().to_string(), ReadResponse::Failure)]);+ let harness = harness(vec![one, two], responses, false, HashSet::new());++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect_err("reader failure propagates");++ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 1);+ assert!(harness.messages.lock().expect("messages lock").is_empty());+}++#[tokio::test]+async fn publisher_failure_is_returned_after_all_messages_are_attempted() {+ let users = [user("one"), user("two"), user("three")];+ let responses = users+ .iter()+ .map(|recipient| {+ item_response(+ recipient,+ document_item(DOCUMENT_ID, recipient.as_ref(), None),+ )+ })+ .collect();+ let harness = harness(+ users.to_vec(),+ responses,+ false,+ HashSet::from([users[1].as_ref().to_string()]),+ );++ harness+ .service+ .notify_users(document_entity())+ .await+ .expect_err("publication failure propagates");++ assert_eq!(harness.messages.lock().expect("messages lock").len(), 3);+}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rsnew file mode 100644index 0000000000..7659820494--- /dev/null+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs@@ -0,0 +1,261 @@+//! Kafka consumer for `document.updated` realtime Soup fan-out.+//!+//! Delivery is at least once: offsets are committed only after successful+//! domain processing. Malformed and currently unsupported document events are+//! poison/ignored records and are committed so they cannot wedge a partition.++#[cfg(test)]+mod test;++use std::{future::Future, time::Duration};++use documents::domain::events::DocumentTopicEvent;+use kafka_consumer_util::{GroupName, KafkaEventConsumer};+use macro_event_broker::{Event, EventBrokerError, Topic as _};+use macro_event_topics::MacroDocumentsTopic;+use model_entity::{Entity, EntityType};+use rdkafka::consumer::CommitMode;+use rdkafka::message::{BorrowedMessage, Message as _};+use rootcause::prelude::{Report, ResultExt as _};++use crate::domain::ports::SoupRealtimeService;++/// Consumer group used for document update fan-out offsets.+struct SoupRealtimeConsumerGroup;++impl GroupName for SoupRealtimeConsumerGroup {+ const GROUP_NAME: &'static str = "soup-realtime";+}++type SoupRealtimeKafkaConsumer = KafkaEventConsumer<SoupRealtimeConsumerGroup>;++/// Total service attempts before returning for supervisor-driven redelivery.+const MAX_SERVICE_ATTEMPTS: u32 = 5;+/// Delay before the first retry; each subsequent delay doubles.+const SERVICE_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);++fn subscribed_topics() -> [&'static str; 1] {+ [MacroDocumentsTopic.as_str()]+}++fn entity_from_document_event(event: DocumentTopicEvent) -> Option<Entity<'static>> {+ match event {+ DocumentTopicEvent::Updated(metadata) => {+ Some(EntityType::Document.with_entity_string(metadata.document_id))+ }+ DocumentTopicEvent::Created(_) => {+ tracing::trace!(event_type = "document.created", "ignoring document event");+ None+ }+ DocumentTopicEvent::Deleted(_) => {+ tracing::trace!(event_type = "document.deleted", "ignoring document event");+ None+ }+ DocumentTopicEvent::Copied(_) => {+ tracing::trace!(event_type = "document.copied", "ignoring document event");+ None+ }+ }+}++fn decode_updated_entity(payload: &[u8]) -> Result<Option<Entity<'static>>, EventBrokerError> {+ let event = Event::<DocumentTopicEvent>::decode(payload)?;+ Ok(entity_from_document_event(event.event))+}++/// Commit-safe outcome after processing one non-empty document payload.+enum DocumentPayloadOutcome {+ /// An update was successfully sent through the domain service.+ Notified,+ /// A recognized document event is outside the current scope.+ Ignored,+ /// The payload could not be decoded and should be treated as poison.+ Malformed(EventBrokerError),+}++#[tracing::instrument(skip(service, payload), fields(partition, offset, payload_len = payload.len()), err)]+async fn process_document_payload<S: SoupRealtimeService>(+ service: &S,+ payload: &[u8],+ partition: i32,+ offset: i64,+) -> Result<DocumentPayloadOutcome, Report> {+ match decode_updated_entity(payload) {+ Ok(Some(entity)) => {+ notify_with_retry(service, entity, partition, offset).await?;+ Ok(DocumentPayloadOutcome::Notified)+ }+ Ok(None) => Ok(DocumentPayloadOutcome::Ignored),+ Err(error) => Ok(DocumentPayloadOutcome::Malformed(error)),+ }+}++#[tracing::instrument(+ skip(service),+ fields(+ entity_type = %entity.entity_type,+ entity_id = %entity.entity_id,+ partition,+ offset,+ ),+ err+)]+async fn notify_with_retry<S: SoupRealtimeService>(+ service: &S,+ entity: Entity<'static>,+ partition: i32,+ offset: i64,+) -> Result<(), Report> {+ let mut delay = SERVICE_RETRY_BASE_DELAY;+ let mut attempt = 1u32;++ loop {+ tracing::trace!(attempt, "notifying realtime Soup recipients");+ match service.notify_users(entity.clone()).await {+ Ok(()) => {+ tracing::trace!(attempt, "realtime Soup recipients notified");+ return Ok(());+ }+ Err(error) if attempt < MAX_SERVICE_ATTEMPTS => {+ tracing::warn!(+ error = ?error,+ attempt,+ delay_secs = delay.as_secs_f32(),+ "realtime Soup fan-out failed, retrying"+ );+ tokio::time::sleep(delay).await;+ delay *= 2;+ attempt += 1;+ }+ Err(error) => {+ return Err(error+ .context(format!(+ "realtime Soup fan-out failed after {MAX_SERVICE_ATTEMPTS} attempts \+ (partition {partition} offset {offset})"+ ))+ .into_dynamic());+ }+ }+ }+}++fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {+ match consumer.commit_message(message, CommitMode::Async) {+ Ok(()) => tracing::trace!(+ partition = message.partition(),+ offset = message.offset(),+ "committed realtime Soup input offset"+ ),+ Err(error) => tracing::error!(+ error = ?error,+ partition = message.partition(),+ offset = message.offset(),+ "failed to commit realtime Soup input offset"+ ),+ }+}++/// Runs the document update consumer until `shutdown` resolves.+///+/// The consumer subscribes only to `macro.documents` under the+/// `soup-realtime` group. It commits malformed and recognized-but-ignored+/// events, and commits `document.updated` only after [`SoupRealtimeService`]+/// succeeds. Exhausted service retries return without committing so a future+/// supervisor restart can redeliver the record.+#[tracing::instrument(skip(service, shutdown), fields(brokers), err)]+pub async fn run_document_update_consumer<S>(+ brokers: &str,+ service: S,+ shutdown: impl Future<Output = ()> + Send,+) -> Result<(), Report>+where+ S: SoupRealtimeService,+{+ let consumer = SoupRealtimeKafkaConsumer::from_env(brokers)?;+ consumer+ .subscribe(&subscribed_topics())+ .context("failed to subscribe to document update topic")?;+ tracing::info!(+ topics = ?subscribed_topics(),+ group = SoupRealtimeConsumerGroup::GROUP_NAME,+ "realtime Soup document consumer listening"+ );++ let mut shutdown = std::pin::pin!(shutdown);+ loop {+ tokio::select! {+ _ = &mut shutdown => {+ tracing::info!("realtime Soup document consumer shutting down");+ break;+ }+ result = consumer.recv() => {+ let message = match result {+ Ok(message) => message,+ Err(error) => {+ tracing::error!(error = ?error, "Kafka receive error");+ continue;+ }+ };+ tracing::trace!(+ topic = message.topic(),+ partition = message.partition(),+ offset = message.offset(),+ payload_len = message.payload().map_or(0, <[u8]>::len),+ "received realtime Soup input"+ );++ let Some(payload) = message.payload().filter(|payload| !payload.is_empty()) else {+ tracing::warn!(+ partition = message.partition(),+ offset = message.offset(),+ "skipping document event with empty payload"+ );+ commit_logged(&consumer, &message);+ continue;+ };++ let outcome = match process_document_payload(+ &service,+ payload,+ message.partition(),+ message.offset(),+ )+ .await+ {+ Ok(outcome) => outcome,+ Err(error) => {+ tracing::error!(+ error = ?error,+ topic = message.topic(),+ partition = message.partition(),+ offset = message.offset(),+ "realtime Soup fan-out retries exhausted; pausing partition for redelivery"+ );+ // Kafka commits are cumulative within a partition, so+ // pause it before any later record can advance the+ // committed offset past this failure.+ consumer+ .pause_message_partition(&message)+ .context("failed to pause Kafka partition after fan-out failure")?;+ continue;+ }+ };++ match outcome {+ DocumentPayloadOutcome::Notified | DocumentPayloadOutcome::Ignored => {}+ DocumentPayloadOutcome::Malformed(error) => tracing::error!(+ error = ?error,+ topic = message.topic(),+ partition = message.partition(),+ offset = message.offset(),+ "dropping malformed document event"+ ),+ }++ commit_logged(&consumer, &message);+ }+ }+ }++ Ok(())+}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rsnew file mode 100644index 0000000000..a00258b697--- /dev/null+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs@@ -0,0 +1,177 @@+use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::{Arc, Mutex};++use documents::domain::events::{+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,+ DocumentUpdatedMetadata,+};+use macro_user_id::user_id::MacroUserIdStr;++use super::*;++const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";++fn user() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")+}++fn updated_event() -> Event<DocumentTopicEvent> {+ Event::new(DocumentTopicEvent::Updated(DocumentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: user(),+ actor_user_id: None,+ document_name: Some("Updated".to_string()),+ previous_project_id: None,+ project_id: None,+ file_type: None,+ share_permission_updated: false,+ }))+}++fn ignored_events() -> Vec<DocumentTopicEvent> {+ vec![+ DocumentTopicEvent::Created(DocumentCreatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: user(),+ document_name: "Created".to_string(),+ file_type: None,+ project_id: None,+ sub_type: None,+ created_at: None,+ }),+ DocumentTopicEvent::Deleted(DocumentDeletedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ actor_user_id: None,+ project_id: None,+ }),+ DocumentTopicEvent::Copied(DocumentCopiedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ source_document_id: "00000000-0000-0000-0000-000000000002".to_string(),+ source_version_id: None,+ owner: user(),+ document_name: "Copied".to_string(),+ file_type: None,+ project_id: None,+ sub_type: None,+ }),+ ]+}++#[derive(Clone)]+struct FlakyService {+ attempts: Arc<AtomicU32>,+ failures: u32,+ entities: Arc<Mutex<Vec<Entity<'static>>>>,+}++impl SoupRealtimeService for FlakyService {+ async fn notify_users(&self, entity: Entity<'static>) -> Result<(), Report> {+ self.entities.lock().expect("entities lock").push(entity);+ let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;+ if attempt <= self.failures {+ Err(rootcause::report!("temporary fan-out failure"))+ } else {+ Ok(())+ }+ }+}++fn flaky_service(failures: u32) -> FlakyService {+ FlakyService {+ attempts: Arc::new(AtomicU32::new(0)),+ failures,+ entities: Arc::new(Mutex::new(Vec::new())),+ }+}++#[test]+fn subscribes_only_to_documents() {+ assert_eq!(subscribed_topics(), ["macro.documents"]);+}++#[tokio::test]+async fn updated_payload_maps_to_document_entity() {+ let payload = serde_json::to_vec(&updated_event()).expect("serializable event");+ let service = flaky_service(0);++ assert!(matches!(+ process_document_payload(&service, &payload, 0, 0)+ .await+ .expect("processing succeeds"),+ DocumentPayloadOutcome::Notified+ ));++ let entities = service.entities.lock().expect("entities lock");+ assert_eq!(entities.len(), 1);+ assert_eq!(entities[0].entity_type, EntityType::Document);+ assert_eq!(entities[0].entity_id, DOCUMENT_ID);+}++#[tokio::test]+async fn created_deleted_and_copied_events_are_ignored() {+ let service = flaky_service(0);+ for event in ignored_events() {+ let payload = serde_json::to_vec(&Event::new(event)).expect("serializable event");+ assert!(matches!(+ process_document_payload(&service, &payload, 0, 0)+ .await+ .expect("processing succeeds"),+ DocumentPayloadOutcome::Ignored+ ));+ }+ assert_eq!(service.attempts.load(Ordering::SeqCst), 0);+}++#[tokio::test]+async fn malformed_and_unknown_events_are_rejected_without_notifying() {+ let service = flaky_service(0);+ assert!(matches!(+ process_document_payload(&service, b"not json", 0, 0)+ .await+ .expect("poison payload is commit-safe"),+ DocumentPayloadOutcome::Malformed(_)+ ));++ let unknown = serde_json::json!({+ "event_id": "00000000-0000-0000-0000-000000000003",+ "schema_version": 1,+ "event_type": "document.restored",+ "metadata": { "document_id": DOCUMENT_ID }+ });+ let payload = serde_json::to_vec(&unknown).expect("serializable JSON");+ assert!(matches!(+ process_document_payload(&service, &payload, 0, 0)+ .await+ .expect("poison payload is commit-safe"),+ DocumentPayloadOutcome::Malformed(_)+ ));+ assert_eq!(service.attempts.load(Ordering::SeqCst), 0);+}++#[tokio::test(start_paused = true)]+async fn transient_service_failures_retry_then_succeed() {+ let service = flaky_service(2);+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());++ notify_with_retry(&service, entity, 2, 17)+ .await+ .expect("eventual success");++ assert_eq!(service.attempts.load(Ordering::SeqCst), 3);+ assert_eq!(service.entities.lock().expect("entities lock").len(), 3);+}++#[tokio::test(start_paused = true)]+async fn exhausted_retries_return_for_redelivery() {+ let service = flaky_service(u32::MAX);+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());++ notify_with_retry(&service, entity, 2, 17)+ .await+ .expect_err("persistent failure returns without a commit");++ assert_eq!(+ service.attempts.load(Ordering::SeqCst),+ MAX_SERVICE_ATTEMPTS+ );+}diff --git a/crates/soup_realtime/src/inbound/mod.rs b/crates/soup_realtime/src/inbound/mod.rsnew file mode 100644index 0000000000..1d73c01b15--- /dev/null+++ b/crates/soup_realtime/src/inbound/mod.rs@@ -0,0 +1,4 @@+//! Inbound adapters that drive realtime Soup fan-out.++/// Kafka consumer for document update events.+pub mod kafka_consumer;diff --git a/crates/soup_realtime/src/lib.rs b/crates/soup_realtime/src/lib.rsnew file mode 100644index 0000000000..ab4794b1f9--- /dev/null+++ b/crates/soup_realtime/src/lib.rs@@ -0,0 +1,16 @@+#![deny(missing_docs)]+//! User-scoped realtime Soup fan-out components.+//!+//! The crate exposes domain orchestration plus optional Kafka and infrastructure+//! adapters. It intentionally contains no composition root and starts no tasks.++/// Domain models, ports, and fan-out orchestration.+pub mod domain;++/// Inbound transport adapters.+#[cfg(feature = "consumer")]+pub mod inbound;++/// Outbound infrastructure adapters.+#[cfg(feature = "outbound")]+pub mod outbound;diff --git a/crates/soup_realtime/src/outbound/entity_access.rs b/crates/soup_realtime/src/outbound/entity_access.rsnew file mode 100644index 0000000000..abe77312b1--- /dev/null+++ b/crates/soup_realtime/src/outbound/entity_access.rs@@ -0,0 +1,73 @@+//! Adapter from entity access to current realtime Soup recipients.++#[cfg(test)]+mod test;++use std::future::Future;++use entity_access::domain::{models::AccessError, ports::EntityAccessService};+use macro_user_id::user_id::MacroUserIdStr;+use model_entity::{Entity, EntityType};+use rootcause::prelude::{Report, ResultExt as _};++use crate::domain::ports::UserAccessExpander;++/// Narrow seam over the external entity-access service used by this adapter.+///+/// Every [`EntityAccessService`] implements this trait automatically. Keeping+/// the seam narrow also allows adapter tests to avoid constructing unrelated+/// authorization capabilities.+pub trait EntityAccessUserLookup: Send + Sync + 'static {+ /// Returns all users with current access to the entity ID and type.+ fn get_users_by_entity(+ &self,+ entity_id: &str,+ entity_type: EntityType,+ ) -> impl Future<Output = Result<Vec<MacroUserIdStr<'static>>, AccessError>> + Send;+}++impl<S> EntityAccessUserLookup for S+where+ S: EntityAccessService,+{+ async fn get_users_by_entity(+ &self,+ entity_id: &str,+ entity_type: EntityType,+ ) -> Result<Vec<MacroUserIdStr<'static>>, AccessError> {+ EntityAccessService::get_users_by_entity(self, entity_id, entity_type).await+ }+}++/// Current-access expander backed by an entity-access service.+pub struct EntityAccessExpander<S> {+ service: S,+}++impl<S> EntityAccessExpander<S> {+ /// Creates an expander around an entity-access service.+ pub fn new(service: S) -> Self {+ Self { service }+ }+}++impl<S> UserAccessExpander for EntityAccessExpander<S>+where+ S: EntityAccessUserLookup,+{+ #[tracing::instrument(+ skip(self),+ fields(entity_type = %entity.entity_type, entity_id = %entity.entity_id),+ err+ )]+ async fn expand_user_access(+ &self,+ entity: &Entity<'static>,+ ) -> Result<Vec<MacroUserIdStr<'static>>, Report> {+ self.service+ .get_users_by_entity(entity.entity_id.as_ref(), entity.entity_type)+ .await+ .context("failed to query current entity accessors")+ .map_err(Into::into)+ }+}diff --git a/crates/soup_realtime/src/outbound/entity_access/test.rs b/crates/soup_realtime/src/outbound/entity_access/test.rsnew file mode 100644index 0000000000..2f07634786--- /dev/null+++ b/crates/soup_realtime/src/outbound/entity_access/test.rs@@ -0,0 +1,50 @@+use std::sync::{Arc, Mutex};++use super::*;++struct RecordingLookup {+ calls: Arc<Mutex<Vec<(String, EntityType)>>>,+ users: Vec<MacroUserIdStr<'static>>,+}++impl EntityAccessUserLookup for RecordingLookup {+ async fn get_users_by_entity(+ &self,+ entity_id: &str,+ entity_type: EntityType,+ ) -> Result<Vec<MacroUserIdStr<'static>>, AccessError> {+ self.calls+ .lock()+ .expect("calls lock")+ .push((entity_id.to_string(), entity_type));+ Ok(self.users.clone())+ }+}++fn user(value: &str) -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from(value.to_string()).expect("valid user id")+}++#[tokio::test]+async fn forwards_entity_and_returns_underlying_users() {+ let calls = Arc::new(Mutex::new(Vec::new()));+ let adapter = EntityAccessExpander::new(RecordingLookup {+ calls: calls.clone(),+ users: vec![user("macro|[email protected]"), user("macro|[email protected]")],+ });+ let entity =+ EntityType::Document.with_entity_string("00000000-0000-0000-0000-000000000001".to_string());++ let users = adapter+ .expand_user_access(&entity)+ .await+ .expect("access expansion succeeds");++ assert_eq!(users.len(), 2);+ assert_eq!(users[0].as_ref(), "macro|[email protected]");+ assert_eq!(users[1].as_ref(), "macro|[email protected]");+ assert_eq!(+ calls.lock().expect("calls lock").as_slice(),+ &[(entity.entity_id.to_string(), EntityType::Document)]+ );+}diff --git a/crates/soup_realtime/src/outbound/kafka_publisher.rs b/crates/soup_realtime/src/outbound/kafka_publisher.rsnew file mode 100644index 0000000000..5b1a8987da--- /dev/null+++ b/crates/soup_realtime/src/outbound/kafka_publisher.rs@@ -0,0 +1,44 @@+//! Kafka publisher for versioned realtime Soup messages.++#[cfg(test)]+mod test;++use macro_event_broker::EventPublisher;+use macro_event_topics::MacroSoupRealtimeTopic;+use rootcause::prelude::{Report, ResultExt as _};++use crate::domain::{models::SoupRealtimeMessage, ports::SoupRealtimePublisher};++/// Realtime Soup publisher backed by a byte-oriented event publisher.+pub struct KafkaSoupRealtimePublisher<P> {+ publisher: P,+}++impl<P> KafkaSoupRealtimePublisher<P> {+ /// Creates a realtime Soup Kafka publisher.+ pub fn new(publisher: P) -> Self {+ Self { publisher }+ }+}++impl<P> SoupRealtimePublisher for KafkaSoupRealtimePublisher<P>+where+ P: EventPublisher,+{+ #[tracing::instrument(+ skip(self, message),+ fields(user_id = %message.user_id),+ err+ )]+ async fn publish(&self, message: SoupRealtimeMessage) -> Result<(), Report> {+ let payload =+ serde_json::to_vec(&message).context("failed to serialize realtime Soup message")?;++ self.publisher+ .publish(MacroSoupRealtimeTopic, message.user_id.as_ref(), &payload)+ .await+ .context("failed to publish realtime Soup message")?;++ Ok(())+ }+}diff --git a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs b/crates/soup_realtime/src/outbound/kafka_publisher/test.rsnew file mode 100644index 0000000000..a16489484c--- /dev/null+++ b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs@@ -0,0 +1,124 @@+use std::sync::{Arc, Mutex};++use chrono::{DateTime, Utc};+use macro_event_broker::{EventBrokerError, Topic};+use macro_user_id::user_id::MacroUserIdStr;+use models_soup::{document::SoupDocument, item::SoupItem};+use uuid::Uuid;++use super::*;++#[derive(Debug)]+struct PublishedRecord {+ topic: String,+ key: String,+ payload: Vec<u8>,+}++struct RecordingPublisher {+ records: Arc<Mutex<Vec<PublishedRecord>>>,+ fail: bool,+}++impl EventPublisher for RecordingPublisher {+ async fn publish<T: Topic>(+ &self,+ topic: T,+ key: &str,+ payload: &[u8],+ ) -> Result<(), EventBrokerError> {+ self.records+ .lock()+ .expect("records lock")+ .push(PublishedRecord {+ topic: topic.as_str().to_string(),+ key: key.to_string(),+ payload: payload.to_vec(),+ });+ if self.fail {+ Err(EventBrokerError::Publish("broker unavailable".to_string()))+ } else {+ Ok(())+ }+ }+}++fn user() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")+}++fn timestamp(seconds: i64) -> DateTime<Utc> {+ DateTime::from_timestamp(seconds, 0).expect("valid timestamp")+}++fn message() -> SoupRealtimeMessage {+ SoupRealtimeMessage::new(+ user(),+ SoupItem::Document(SoupDocument {+ id: Uuid::parse_str("00000000-0000-0000-0000-000000000001").expect("valid document id"),+ document_version_id: 9,+ owner_id: user(),+ name: "Published document".to_string(),+ file_type: Some("md".to_string()),+ sha: Some("sha".to_string()),+ project_id: None,+ branched_from_id: None,+ branched_from_version_id: None,+ document_family_id: None,+ created_at: timestamp(1),+ updated_at: timestamp(2),+ viewed_at: Some(timestamp(3)),+ sub_type: None,+ deleted_at: None,+ extra: (),+ }),+ )+}++#[tokio::test]+async fn publishes_user_keyed_full_message_to_realtime_topic() {+ let records = Arc::new(Mutex::new(Vec::new()));+ let adapter = KafkaSoupRealtimePublisher::new(RecordingPublisher {+ records: records.clone(),+ fail: false,+ });++ adapter.publish(message()).await.expect("publish succeeds");++ let mut records = records.lock().expect("records lock");+ let record = records.pop().expect("one record");+ assert!(records.is_empty());+ assert_eq!(record.topic, "macro.soup");+ assert_eq!(record.key, "macro|[email protected]");++ let json: serde_json::Value = serde_json::from_slice(&record.payload).expect("payload is JSON");+ assert_eq!(json["schema_version"], 1);+ assert_eq!(json["user_id"], "macro|[email protected]");+ assert!(json.get("event_id").is_none());++ let decoded: SoupRealtimeMessage =+ serde_json::from_slice(&record.payload).expect("payload round-trips");+ assert_eq!(decoded.schema_version, 1);+ assert_eq!(decoded.user_id.as_ref(), "macro|[email protected]");+ match decoded.item {+ SoupItem::Document(document) => {+ assert_eq!(document.name, "Published document");+ assert_eq!(document.document_version_id, 9);+ assert_eq!(document.viewed_at, Some(timestamp(3)));+ }+ _ => panic!("expected document item"),+ }+}++#[tokio::test]+async fn propagates_delivery_failures() {+ let adapter = KafkaSoupRealtimePublisher::new(RecordingPublisher {+ records: Arc::new(Mutex::new(Vec::new())),+ fail: true,+ });++ adapter+ .publish(message())+ .await+ .expect_err("delivery failure propagates");+}diff --git a/crates/soup_realtime/src/outbound/mod.rs b/crates/soup_realtime/src/outbound/mod.rsnew file mode 100644index 0000000000..01d025d389--- /dev/null+++ b/crates/soup_realtime/src/outbound/mod.rs@@ -0,0 +1,8 @@+//! Outbound adapters used by realtime Soup fan-out.++/// Current entity-access expansion adapter.+pub mod entity_access;+/// Kafka-backed realtime Soup publisher adapter.+pub mod kafka_publisher;+/// User-scoped Soup item reader adapter.+pub mod soup_item_reader;diff --git a/crates/soup_realtime/src/outbound/soup_item_reader.rs b/crates/soup_realtime/src/outbound/soup_item_reader.rsnew file mode 100644index 0000000000..f72c37c6b4--- /dev/null+++ b/crates/soup_realtime/src/outbound/soup_item_reader.rs@@ -0,0 +1,76 @@+//! Adapter for loading complete, user-scoped Soup items.++#[cfg(test)]+mod test;++use model_entity::{Entity, EntityType};+use models_soup::item::SoupItem;+use rootcause::prelude::{Report, ResultExt as _};+use soup::domain::{models::AdvancedSortParams, ports::SoupRepo};++use crate::domain::ports::SoupItemReader;++/// Soup item reader backed by an existing [`SoupRepo`].+pub struct SoupRepoItemReader<R> {+ repo: R,+}++impl<R> SoupRepoItemReader<R> {+ /// Creates a user-scoped item reader around a Soup repository.+ pub fn new(repo: R) -> Self {+ Self { repo }+ }+}++impl<R> SoupItemReader for SoupRepoItemReader<R>+where+ R: SoupRepo,+ R::Err: std::error::Error + Send + Sync + 'static,+{+ #[tracing::instrument(+ skip(self),+ fields(+ user_id = %user_id,+ entity_type = %entity.entity_type,+ entity_id = %entity.entity_id,+ ),+ err+ )]+ async fn read_for_user(+ &self,+ user_id: macro_user_id::user_id::MacroUserIdStr<'static>,+ entity: &Entity<'static>,+ ) -> Result<Option<SoupItem<()>>, Report> {+ if entity.entity_type != EntityType::Document {+ return Err(rootcause::report!(+ "realtime Soup reader does not support entity type {}",+ entity.entity_type+ ));+ }++ let requested_entities = [entity.clone()];+ let items = self+ .repo+ .expanded_soup_by_ids(AdvancedSortParams {+ entities: &requested_entities,+ user_id,+ })+ .await+ .context("failed to read expanded user-scoped Soup item")?;++ let mut matches = items.into_iter().filter(|item| item.entity() == *entity);+ let Some(item) = matches.next() else {+ return Ok(None);+ };++ if matches.next().is_some() {+ return Err(rootcause::report!(+ "expanded Soup query returned duplicate items for {} {}",+ entity.entity_type,+ entity.entity_id+ ));+ }++ Ok(Some(item))+ }+}diff --git a/crates/soup_realtime/src/outbound/soup_item_reader/test.rs b/crates/soup_realtime/src/outbound/soup_item_reader/test.rsnew file mode 100644index 0000000000..c58cc7e8fb--- /dev/null+++ b/crates/soup_realtime/src/outbound/soup_item_reader/test.rs@@ -0,0 +1,213 @@+use std::io;+use std::sync::{Arc, Mutex};++use chrono::{DateTime, Utc};+use macro_user_id::user_id::MacroUserIdStr;+use models_properties::service::property_definition_with_options::PropertyDefinitionWithOptions;+use models_soup::{document::SoupDocument, item::SoupItem};+use soup::domain::{+ models::{+ AdvancedSortParams, GroupedSortRequest, SimpleSortRequest, SoupPropertiesField,+ grouping::ItemGroupingInfo,+ },+ ports::SoupRepo,+};+use uuid::Uuid;++use super::*;++const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";++type RecordedQuery = (String, Vec<(String, EntityType)>);+type RecordedQueries = Arc<Mutex<Vec<RecordedQuery>>>;++struct RecordingSoupRepo {+ calls: RecordedQueries,+ items: Mutex<Option<Result<Vec<SoupItem<()>>, io::Error>>>,+}++impl SoupRepo for RecordingSoupRepo {+ type Err = io::Error;+ type GroupedItems = std::vec::IntoIter<ItemGroupingInfo>;++ async fn expanded_generic_cursor_soup<'a>(+ &self,+ _req: SimpleSortRequest<'a>,+ ) -> Result<Vec<SoupItem<()>>, Self::Err> {+ unreachable!("unexpected generic cursor query")+ }++ async fn unexpanded_generic_cursor_soup<'a>(+ &self,+ _req: SimpleSortRequest<'a>,+ ) -> Result<Vec<SoupItem<()>>, Self::Err> {+ unreachable!("unexpected generic cursor query")+ }++ async fn expanded_soup_by_ids<'a>(+ &self,+ req: AdvancedSortParams<'a>,+ ) -> Result<Vec<SoupItem<()>>, Self::Err> {+ self.calls.lock().expect("calls lock").push((+ req.user_id.as_ref().to_string(),+ req.entities+ .iter()+ .map(|entity| (entity.entity_id.to_string(), entity.entity_type))+ .collect(),+ ));+ self.items+ .lock()+ .expect("items lock")+ .take()+ .expect("one expanded query expected")+ }++ async fn unexpanded_soup_by_ids<'a>(+ &self,+ _req: AdvancedSortParams<'a>,+ ) -> Result<Vec<SoupItem<()>>, Self::Err> {+ unreachable!("unexpected unexpanded query")+ }++ async fn populate_properties<'a>(+ &self,+ _user_id: MacroUserIdStr<'a>,+ _items: Vec<SoupItem<()>>,+ ) -> Result<Vec<SoupItem<SoupPropertiesField>>, Self::Err> {+ unreachable!("unexpected properties query")+ }++ async fn caller_tag_sets<'a>(+ &self,+ _user_id: MacroUserIdStr<'a>,+ ) -> Result<Vec<PropertyDefinitionWithOptions>, Self::Err> {+ unreachable!("unexpected tag query")+ }++ async fn expanded_grouped_cursor_soup<'a>(+ &self,+ _req: GroupedSortRequest<'a>,+ ) -> Result<Self::GroupedItems, Self::Err> {+ unreachable!("unexpected grouped query")+ }+}++fn user() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")+}++fn timestamp(seconds: i64) -> DateTime<Utc> {+ DateTime::from_timestamp(seconds, 0).expect("valid timestamp")+}++fn document_item(id: &str, viewed_at: Option<DateTime<Utc>>) -> SoupItem<()> {+ SoupItem::Document(SoupDocument {+ id: Uuid::parse_str(id).expect("valid document id"),+ document_version_id: 7,+ owner_id: user(),+ name: "Realtime document".to_string(),+ file_type: Some("md".to_string()),+ sha: None,+ project_id: None,+ branched_from_id: None,+ branched_from_version_id: None,+ document_family_id: None,+ created_at: timestamp(1),+ updated_at: timestamp(2),+ viewed_at,+ sub_type: None,+ deleted_at: None,+ extra: (),+ })+}++fn reader_with(+ items: Result<Vec<SoupItem<()>>, io::Error>,+) -> (SoupRepoItemReader<RecordingSoupRepo>, RecordedQueries) {+ let calls = Arc::new(Mutex::new(Vec::new()));+ let repo = RecordingSoupRepo {+ calls: calls.clone(),+ items: Mutex::new(Some(items)),+ };+ (SoupRepoItemReader::new(repo), calls)+}++#[tokio::test]+async fn uses_expanded_query_for_requested_user_and_entity() {+ let viewed_at = Some(timestamp(3));+ let (reader, calls) = reader_with(Ok(vec![document_item(DOCUMENT_ID, viewed_at)]));+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());++ let item = reader+ .read_for_user(user(), &entity)+ .await+ .expect("read succeeds")+ .expect("item exists");++ match item {+ SoupItem::Document(document) => {+ assert_eq!(document.name, "Realtime document");+ assert_eq!(document.viewed_at, viewed_at);+ }+ _ => panic!("expected document item"),+ }+ assert_eq!(+ calls.lock().expect("calls lock").as_slice(),+ &[(+ "macro|[email protected]".to_string(),+ vec![(DOCUMENT_ID.to_string(), EntityType::Document)]+ )]+ );+}++#[tokio::test]+async fn returns_none_when_requested_item_is_absent() {+ let other_id = "00000000-0000-0000-0000-000000000002";+ let (reader, _) = reader_with(Ok(vec![document_item(other_id, None)]));+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());++ assert!(+ reader+ .read_for_user(user(), &entity)+ .await+ .expect("read succeeds")+ .is_none()+ );+}++#[tokio::test]+async fn rejects_duplicate_matching_items() {+ let (reader, _) = reader_with(Ok(vec![+ document_item(DOCUMENT_ID, None),+ document_item(DOCUMENT_ID, Some(timestamp(4))),+ ]));+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());++ reader+ .read_for_user(user(), &entity)+ .await+ .expect_err("duplicate items violate the repository contract");+}++#[tokio::test]+async fn rejects_non_document_entities_without_querying() {+ let (reader, calls) = reader_with(Ok(Vec::new()));+ let entity = EntityType::Project.with_entity_string(DOCUMENT_ID.to_string());++ reader+ .read_for_user(user(), &entity)+ .await+ .expect_err("projects are unsupported");+ assert!(calls.lock().expect("calls lock").is_empty());+}++#[tokio::test]+async fn propagates_repository_failures() {+ let (reader, _) = reader_with(Err(io::Error::other("query unavailable")));+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());++ reader+ .read_for_user(user(), &entity)+ .await+ .expect_err("repository failure propagates");+}diff --git a/crates/webhook/Cargo.toml b/crates/webhook/Cargo.tomlindex 74c9eb2922..c5f68ced18 100644--- a/crates/webhook/Cargo.toml+++ b/crates/webhook/Cargo.toml@@ -6,11 +6,10 @@ version = "0.1.0"[features]consumer = [- "dep:macro_env",+ "dep:kafka_consumer_util","dep:rdkafka","dep:tokio","ingestion",- "macro_event_broker/outbound",]default = ["consumer", "inbound", "outbound", "ports"]inbound = [@@ -61,10 +60,10 @@ entity_access = { path = "../entity_access", default-features = false, featuresfutures = { workspace = true, optional = true }hex = { workspace = true, optional = true }hmac = { workspace = true, optional = true }+kafka_consumer_util = { path = "../kafka_consumer_util", optional = true }macro_authorization = { path = "../macro_authorization", optional = true, default-features = false, features = ["axum",] }-macro_env = { path = "../macro_env", optional = true }macro_event_broker = { path = "../macro_event_broker", default-features = false }macro_event_topics = { path = "../macro_event_topics" }macro_user_id = { path = "../macro_user_id" }diff --git a/crates/webhook/src/inbound/kafka_consumer.rs b/crates/webhook/src/inbound/kafka_consumer.rsindex c656668031..8327fee250 100644--- a/crates/webhook/src/inbound/kafka_consumer.rs+++ b/crates/webhook/src/inbound/kafka_consumer.rs@@ -25,20 +25,23 @@ use crate::domain::{events::WebhookTopicEvent, ingestion::WebhookEventIngestionSuse anyhow::Context as _;use channels::domain::broker_events::ChannelTopicEvent;use documents::domain::events::DocumentTopicEvent;-use macro_env::Environment;-use macro_event_broker::outbound::msk_iam::configure_sasl_iam;-use macro_event_broker::{Event, EventBrokerError, MskIamClientContext, Topic as _};+use kafka_consumer_util::{GroupName, KafkaEventConsumer};+use macro_event_broker::{Event, EventBrokerError, Topic as _};use macro_event_topics::{MacroChannelsTopic, MacroDocumentsTopic, MacroWebhooksTopic};-use rdkafka::ClientConfig;-use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};-use rdkafka::error::KafkaResult;+use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message};use std::future::Future;use std::time::Duration;-/// Consumer group id for webhook event ingestion. Offsets are committed under+/// Consumer group for webhook event ingestion. Offsets are committed under/// this group, so restarts resume where the previous run left off.-const GROUP_ID: &str = "webhook-event-ingestion";+struct WebhookEventIngestionConsumerGroup;++impl GroupName for WebhookEventIngestionConsumerGroup {+ const GROUP_NAME: &'static str = "webhook-event-ingestion";+}++type WebhookKafkaConsumer = KafkaEventConsumer<WebhookEventIngestionConsumerGroup>;/// Maximum in-process ingestion attempts per event before the consumer bails/// out and lets a restart redeliver from the last committed offset.@@ -90,85 +93,20 @@ impl WebhookConsumerEvent {}}-/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).-enum WebhookKafkaConsumer {- /// Unauthenticated plaintext connection (local docker broker).- Plaintext(StreamConsumer),- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).- MskIam(StreamConsumer<MskIamClientContext>),-}--impl WebhookKafkaConsumer {- /// Build a consumer for the given brokers, choosing the transport from the- /// `ENVIRONMENT` variable exactly like the publisher does.- fn from_env(brokers: &str) -> anyhow::Result<Self> {- let mut config = ClientConfig::new();- config- .set("bootstrap.servers", brokers)- .set("group.id", GROUP_ID)- // Offsets are committed manually after successful ingestion.- .set("enable.auto.commit", "false")- // Start from the beginning of the topics on the first ever run.- .set("auto.offset.reset", "earliest");-- let consumer = match Environment::new_or_prod() {- Environment::Local => Self::Plaintext(- config- .create()- .context("failed to create plaintext kafka consumer")?,- ),- Environment::Develop | Environment::Production => {- configure_sasl_iam(&mut config);- Self::MskIam(- config- .create_with_context(MskIamClientContext::from_env())- .context("failed to create MSK IAM kafka consumer")?,- )- }- };-- Ok(consumer)- }-- fn subscribe(&self) -> KafkaResult<()> {- let topics = subscribed_topics();- match self {- Self::Plaintext(consumer) => consumer.subscribe(&topics),- Self::MskIam(consumer) => consumer.subscribe(&topics),- }- }-- /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it- /// can sit in a `select!` without losing messages.- async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {- match self {- Self::Plaintext(consumer) => consumer.recv().await,- Self::MskIam(consumer) => consumer.recv().await,- }- }-- fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {- match self {- Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),- Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),- }- }-- /// Commit `message`'s offset, logging the outcome.- fn commit_logged(&self, message: &BorrowedMessage<'_>) {- match self.commit(message) {- Ok(()) => tracing::trace!(- partition = message.partition(),- offset = message.offset(),- "committed offset"- ),- Err(e) => tracing::error!(- error = ?e,- partition = message.partition(),- offset = message.offset(),- "failed to commit offset"- ),- }+/// Commit `message`'s offset, logging the outcome.+fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>) {+ match consumer.commit_message(message, CommitMode::Async) {+ Ok(()) => tracing::trace!(+ partition = message.partition(),+ offset = message.offset(),+ "committed offset"+ ),+ Err(error) => tracing::error!(+ error = ?error,+ partition = message.partition(),+ offset = message.offset(),+ "failed to commit offset"+ ),}}@@ -261,11 +199,11 @@ where{let consumer = WebhookKafkaConsumer::from_env(brokers)?;consumer- .subscribe()+ .subscribe(&subscribed_topics()).context("failed to subscribe to webhook event topics")?;tracing::info!(topics = ?subscribed_topics(),- group = GROUP_ID,+ group = WebhookEventIngestionConsumerGroup::GROUP_NAME,"webhook event consumer listening");@@ -298,7 +236,7 @@ whereoffset = message.offset(),"skipping message with empty payload");- consumer.commit_logged(&message);+ commit_logged(&consumer, &message);continue;};@@ -330,7 +268,7 @@ where// Commit only after the event was ingested or permanently// rejected: at-least-once, retried across restarts.- consumer.commit_logged(&message);+ commit_logged(&consumer, &message);}}}diff --git a/docs/cache-record-selection-plan.md b/docs/cache-record-selection-plan.mddeleted file mode 100644index b74444f44e..0000000000--- a/docs/cache-record-selection-plan.md+++ /dev/null@@ -1,173 +0,0 @@-# Cache Record Selection Implementation Plan--## Proposed API--```ts-const selection = selectRecords(SoupItemFieldsFragmentDoc);--const page = await readRecords(cacheHost, selection, {- cursor,- limit: 500,-});-```--```ts-type SelectedRecordPage<T> = {- records: T[];- nextCursor: string | null;-};-```--`nextCursor` is present only when another complete matching record exists.--## Implementation plan--### 1. Remove the schema-specific entity index--- Delete `cache-core::entity_index` and its tests.-- Remove bucket, timestamp, search-text, scoring, counting, and snapshot APIs.-- Restore ordinary record persistence in:- - `cache-idb`- - `cache-sqlite`- - `cache-wasm`-- Remove `queryIndexedItems` from WASM, Tauri, worker, host, and protocol layers.-- Remove projected SQLite columns and IndexedDB envelopes/indexes.-- Restore the unreleased feature-branch storage versions rather than supporting migration from them; normalized records are disposable.--### 2. Add generic fragment-based record selection--In `cache-core`:--- Extend document parsing to support named, fragment-only documents.-- Add a `record_selection` module containing:- - Fragment validation.- - Object/interface/union type-condition resolution.- - Validation that selected concrete types are normalized entities.- - Cursor and page models.-- Reject:- - Unknown fragments or schema types.- - Embedded/non-normalized type conditions.- - Unbound variables in fragment fields.- - Invalid or excessive page limits.--Add:--```rust-Engine::read_records(selection, cursor, limit)-```--The engine will:--1. Derive concrete type names from the fragment condition.-2. Iterate matching records in deterministic entity-key order.-3. Apply the fragment projection using normal denormalization.-4. Load linked cold records in batches.-5. Read the effective view, including optimistic layers.-6. Skip records that cannot satisfy the complete fragment.-7. Use one-record lookahead to decide whether to return `nextCursor`.--### 3. Add schema-neutral storage iteration--Replace the specialized index methods with one generic storage capability:--```rust-scan_records(type_names, after, limit)-```--Implement it for:--- `InMemoryStorage`-- SQLite, using ordered primary-key ranges-- IndexedDB, using object-store key cursors--No schema field names or application-level ordering metadata will be persisted.--### 4. Surface the API through every host--Add `readRecords` across:--- `cache-wasm`-- SharedWorker protocol and worker core-- Tauri commands/plugin-- `CacheHost`-- No-op host--Wire input contains only:--- Serialized fragment document-- Fragment name-- Opaque cursor-- Limit--Wire output contains only selected JSON records and `nextCursor`.--### 5. Add the typed frontend selection API--Create `exchange/record-selection.ts` with:--- `selectRecords(fragmentDocument)`-- `readRecords(host, selection, options)`-- Generated result-type inference-- Fragment-name extraction-- Defensive wire-result validation--Export it from `@graphql-cache/index`.--### 6. Replace index notifications--Replace `onEntityIndexChanged` with schema-neutral `onCacheChanged`.--Emit it for:--- Normal writes-- Optimistic writes and settlements-- External invalidations-- Queue refreshes that change the effective view-- Cache clears/resets--Record-selection consumers can invalidate and rerun without cache-core knowing their schema.--### 7. Adapt Quick Access--- Select cached `GraphqlSoupItem` records using the existing generated `SoupItemFieldsFragmentDoc`.-- Export/reuse the existing GraphQL Soup item mapper.-- Load record pages and map them into `QuickAccessItem`s.-- Perform bucket filtering, timestamp sorting, fuzzy search, counts, and visible-page slicing entirely in TypeScript.-- Make `loadMore` increase the locally visible slice after records have been loaded.-- Rename index-specific files, query keys, and flags to record-selection terminology.-- Remove indexed-snapshot mapping and tests.--### 8. Tests and verification--Add coverage for:--- Object and union fragment selection.-- Aliases and nested linked records.-- Cold storage reads.-- Optimistic records and updates.-- Incomplete-record omission.-- Stable, exclusive cursor pagination.-- `nextCursor` lookahead semantics.-- SQLite and IndexedDB parity.-- WASM, worker, Tauri, and no-op host transport.-- TypeScript result inference and malformed responses.-- Quick Access filtering, sorting, searching, counting, and pagination.--Verification:--```bash-cargo fmt-cargo test -p cache-core-cargo test -p cache-sqlite-cargo test -p graphql_cache_plugin-cargo check --target wasm32-unknown-unknown -p cache-idb -p cache-wasm --all-targets-wasm-pack test --headless --chrome crates/client/cache-idb-```--Then run focused web tests and `bun check` from `apps/web`.--Implementation will proceed incrementally: verify the generic Rust API first, then wire and consume it in Quick Access. After successful verification, create the required `jj` revision.--## Architecture check--Hexagonal boundaries were checked: these are client cache/transport adapters with no server-domain authorization or business-policy movement.