Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/feat/soup realtime (#5068)

Sean Aye9ebbcb91888c
diff --git a/.github/kafka-cluster-topics.json b/.github/kafka-cluster-topics.json
index fa43baa313..461a84a529 100644
--- a/.github/kafka-cluster-topics.json
+++ b/.github/kafka-cluster-topics.json
@@ -5,6 +5,7 @@
"macro.email",
"macro.example",
"macro.projects",
+ "macro.soup",
"macro.teams",
"macro.webhooks"
]
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index a15d516200..6e08fc8a16 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -1507,6 +1507,7 @@
"crates/invite_email",
"crates/ip_extractor",
"crates/item_filters",
+ "crates/kafka_consumer_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/lexical_mention_extractor",
@@ -2884,6 +2885,14 @@
"crates/non_empty",
"crates/workspace-hack"
],
+ "kafka_consumer_util": [
+ "crates/kafka_consumer_util",
+ "crates/macro_env",
+ "crates/macro_env_var",
+ "crates/macro_event_broker",
+ "crates/macro_event_topics",
+ "crates/workspace-hack"
+ ],
"lambda_client": [
"crates/lambda_client",
"crates/workspace-hack"
@@ -4956,6 +4965,93 @@
"crates/unfurl",
"crates/workspace-hack"
],
+ "soup_realtime": [
+ "crates/agent",
+ "crates/ai_toolset",
+ "crates/ai_usage",
+ "crates/attachment",
+ "crates/authentication_service_client",
+ "crates/bot_id",
+ "crates/call",
+ "crates/channel_sender",
+ "crates/channels",
+ "crates/connection",
+ "crates/connection_gateway_client",
+ "crates/connection_gateway_models",
+ "crates/contacts",
+ "crates/cowlike",
+ "crates/crm",
+ "crates/document_sub_type",
+ "crates/documents",
+ "crates/email",
+ "crates/email_db_client",
+ "crates/email_utils",
+ "crates/embedding",
+ "crates/entity_access",
+ "crates/entity_access_db_utils",
+ "crates/entity_access_management",
+ "crates/filter_ast",
+ "crates/foreign_entity",
+ "crates/frecency",
+ "crates/invite_email",
+ "crates/item_filters",
+ "crates/kafka_consumer_util",
+ "crates/last_online_tracker",
+ "crates/lexical_client",
+ "crates/macro_auth",
+ "crates/macro_authorization",
+ "crates/macro_aws_config",
+ "crates/macro_entrypoint",
+ "crates/macro_env",
+ "crates/macro_env_var",
+ "crates/macro_event_broker",
+ "crates/macro_event_topics",
+ "crates/macro_service_urls",
+ "crates/macro_sync_service_jwt",
+ "crates/macro_user_id",
+ "crates/macro_uuid",
+ "crates/mention_utils",
+ "crates/model",
+ "crates/model-entity",
+ "crates/model-error-response",
+ "crates/model_file_type",
+ "crates/model_notifications",
+ "crates/model_user",
+ "crates/models_ai_projection",
+ "crates/models_bulk_upload",
+ "crates/models_email",
+ "crates/models_entity_access_management",
+ "crates/models_grouping",
+ "crates/models_opensearch",
+ "crates/models_pagination",
+ "crates/models_permissions",
+ "crates/models_properties",
+ "crates/models_search",
+ "crates/models_search_cursor",
+ "crates/models_soup",
+ "crates/non_empty",
+ "crates/notification",
+ "crates/opensearch_client",
+ "crates/opensearch_query_builder",
+ "crates/prompt",
+ "crates/properties",
+ "crates/rate_limit",
+ "crates/readonly_pool",
+ "crates/remote_env_var",
+ "crates/s3_key",
+ "crates/secretsmanager_client",
+ "crates/share_permission_db_utils",
+ "crates/soup",
+ "crates/soup_realtime",
+ "crates/sqs_client",
+ "crates/sqs_worker",
+ "crates/sync_service",
+ "crates/sync_service_client",
+ "crates/system_properties",
+ "crates/task_dedup",
+ "crates/unfurl",
+ "crates/workspace-hack"
+ ],
"sqs_client": [
"crates/agent",
"crates/ai_toolset",
@@ -5489,6 +5585,7 @@
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
+ "crates/kafka_consumer_util",
"crates/last_online_tracker",
"crates/lexical_client",
"crates/macro_auth",
diff --git a/Cargo.lock b/Cargo.lock
index 5728c9dd9f..f87c7393e8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7539,6 +7539,19 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "kafka_consumer_util"
+version = "0.1.0"
+dependencies = [
+ "either",
+ "macro_env",
+ "macro_event_broker",
+ "rdkafka",
+ "thiserror 2.0.18",
+ "uuid",
+ "workspace-hack",
+]
+
[[package]]
name = "lambda_client"
version = "0.1.0"
@@ -13386,6 +13399,32 @@ dependencies = [
"workspace-hack",
]
+[[package]]
+name = "soup_realtime"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "documents",
+ "entity_access",
+ "futures",
+ "kafka_consumer_util",
+ "macro_event_broker",
+ "macro_event_topics",
+ "macro_user_id",
+ "model-entity",
+ "models_properties",
+ "models_soup",
+ "rdkafka",
+ "rootcause",
+ "serde",
+ "serde_json",
+ "soup",
+ "tokio",
+ "tracing",
+ "uuid",
+ "workspace-hack",
+]
+
[[package]]
name = "spin"
version = "0.9.8"
@@ -15516,9 +15555,9 @@ dependencies = [
"futures",
"hex",
"hmac 0.12.1",
+ "kafka_consumer_util",
"macro_authorization",
"macro_db_migrator",
- "macro_env",
"macro_event_broker",
"macro_event_topics",
"macro_user_id",
diff --git a/Cargo.toml b/Cargo.toml
index 084f871484..b423ec1478 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,6 +37,7 @@ members = [
"crates/prompt",
"crates/properties_service",
"crates/search_service",
+ "crates/soup_realtime",
"crates/task_dedup",
"crates/unfurl",
"crates/workspace-hack",
diff --git a/crates/channels/examples/channels_consumer.rs b/crates/channels/examples/channels_consumer.rs
index 0360eb2cea..f12dbbfd10 100644
--- a/crates/channels/examples/channels_consumer.rs
+++ b/crates/channels/examples/channels_consumer.rs
@@ -103,7 +103,7 @@ impl ChannelsConsumer {
.context("failed to create plaintext kafka consumer")?,
),
Environment::Develop | Environment::Production => {
- configure_sasl_iam(&mut config);
+ let config = configure_sasl_iam(config);
Self::MskIam(
config
.create_with_context(MskIamClientContext::from_env())
diff --git a/crates/documents/examples/documents_consumer.rs b/crates/documents/examples/documents_consumer.rs
index 8f4e29168a..66259c1317 100644
--- a/crates/documents/examples/documents_consumer.rs
+++ b/crates/documents/examples/documents_consumer.rs
@@ -103,7 +103,7 @@ impl DocumentsConsumer {
.context("failed to create plaintext kafka consumer")?,
),
Environment::Develop | Environment::Production => {
- configure_sasl_iam(&mut config);
+ let config = configure_sasl_iam(config);
Self::MskIam(
config
.create_with_context(MskIamClientContext::from_env())
diff --git a/crates/kafka_consumer_util/Cargo.toml b/crates/kafka_consumer_util/Cargo.toml
new file mode 100644
index 0000000000..dbdf5d7356
--- /dev/null
+++ b/crates/kafka_consumer_util/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+edition = "2024"
+name = "kafka_consumer_util"
+publish = false
+version = "0.1.0"
+
+[dependencies]
+either = { workspace = true }
+macro_env = { path = "../macro_env" }
+macro_event_broker = { path = "../macro_event_broker", default-features = false, features = [
+ "kafka",
+] }
+rdkafka = { workspace = true }
+thiserror = { workspace = true }
+uuid = { workspace = true }
+workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/kafka_consumer_util/README.md b/crates/kafka_consumer_util/README.md
new file mode 100644
index 0000000000..6373185896
--- /dev/null
+++ b/crates/kafka_consumer_util/README.md
@@ -0,0 +1,10 @@
+# kafka_consumer_util
+
+Shared environment-aware Kafka consumer transport for backend inbound adapters.
+
+The crate centralizes plaintext-local versus MSK-IAM construction and exposes two type-safe consumption modes:
+
+- Named consumer groups implement `GroupName` and may subscribe, pause, receive, and commit.
+- `Ungrouped` consumers use manual partition assignment and may pause and receive, but cannot subscribe or commit.
+
+Ungrouped consumers assign every current partition at an explicit earliest or latest offset. Application adapters remain responsible for decoding, retries, poison-message handling, and refreshing assignments if topic partition counts change.
diff --git a/crates/kafka_consumer_util/src/lib.rs b/crates/kafka_consumer_util/src/lib.rs
new file mode 100644
index 0000000000..1331329a39
--- /dev/null
+++ b/crates/kafka_consumer_util/src/lib.rs
@@ -0,0 +1,253 @@
+#![deny(missing_docs)]
+//! Environment-aware Kafka consumer transport.
+//!
+//! This adapter centralizes the repository's plaintext-local versus MSK-IAM
+//! transport setup. Application inbound adapters retain responsibility for
+//! decoding, retries, poison-message handling, and deciding when to commit.
+
+#[cfg(test)]
+mod test;
+
+use std::marker::PhantomData;
+use std::time::Duration;
+
+use either::Either;
+use macro_env::Environment;
+use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};
+use rdkafka::error::{KafkaError, KafkaResult};
+use rdkafka::message::{BorrowedMessage, Message as _};
+use rdkafka::{ClientConfig, Offset, TopicPartitionList};
+use uuid::Uuid;
+
+use macro_event_broker::kafka::msk_iam::{MskIamClientContext, configure_sasl_iam};
+
+const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";
+
+/// Failure to construct an environment-specific Kafka consumer.
+#[derive(Debug, thiserror::Error)]
+pub enum KafkaConsumerError {
+ /// Failed to create the unauthenticated local consumer.
+ #[error("failed to create plaintext Kafka consumer")]
+ Plaintext(#[source] KafkaError),
+ /// Failed to create the TLS and MSK-IAM authenticated consumer.
+ #[error("failed to create MSK IAM Kafka consumer")]
+ MskIam(#[source] KafkaError),
+}
+
+/// Underlying Kafka consumer transport selected from the runtime environment.
+struct ConsumerTransport(Either<StreamConsumer, StreamConsumer<MskIamClientContext>>);
+
+/// Type-level name for a durable Kafka consumer group.
+///
+/// Defining group names on marker types keeps group identities centralized and
+/// prevents consumers from passing arbitrary string group IDs at construction.
+pub trait GroupName {
+ /// Stable Kafka consumer group ID used for partition balancing and offsets.
+ const GROUP_NAME: &'static str;
+}
+
+/// Marker type for a consumer that does not subscribe or persist offsets.
+///
+/// librdkafka requires a configured `group.id` for its safe manual-assignment
+/// API, so this mode uses a generated internal ID. It never exposes subscription
+/// or commit operations and therefore does not create durable group state.
+pub struct Ungrouped;
+
+/// Starting position for manually assigned ungrouped topic partitions.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum InitialOffset {
+ /// Consume all currently retained records before continuing with new ones.
+ Earliest,
+ /// Consume only records published after the partition assignment begins.
+ Latest,
+}
+
+impl InitialOffset {
+ fn as_kafka_offset(self) -> Offset {
+ match self {
+ Self::Earliest => Offset::Beginning,
+ Self::Latest => Offset::End,
+ }
+ }
+}
+
+/// Shared Kafka consumer with environment-aware transport and type-safe group behavior.
+///
+/// Consumers parameterized by a [`GroupName`] may subscribe and commit offsets.
+/// [`Ungrouped`] consumers instead manually assign every partition of their
+/// requested topics and cannot subscribe or commit.
+pub struct KafkaEventConsumer<T> {
+ consumer: ConsumerTransport,
+ marker: PhantomData<T>,
+}
+
+fn base_config(brokers: &str) -> ClientConfig {
+ let mut config = ClientConfig::new();
+ config
+ .set("bootstrap.servers", brokers)
+ .set("enable.auto.commit", "false");
+ config
+}
+
+fn grouped_config<T: GroupName>(brokers: &str) -> ClientConfig {
+ let mut config = base_config(brokers);
+ config
+ .set("group.id", T::GROUP_NAME)
+ .set("auto.offset.reset", "earliest");
+ config
+}
+
+fn ungrouped_config(brokers: &str) -> ClientConfig {
+ let mut config = base_config(brokers);
+ let group_id = format!("{UNGROUPED_GROUP_PREFIX}-{}", Uuid::new_v4());
+ config
+ .set("group.id", group_id)
+ .set("enable.auto.offset.store", "false");
+ config
+}
+
+fn create_from_env<T>(config: ClientConfig) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {
+ let consumer = match Environment::new_or_prod() {
+ Environment::Local => Either::Left(config.create().map_err(KafkaConsumerError::Plaintext)?),
+ Environment::Develop | Environment::Production => {
+ let config = configure_sasl_iam(config);
+ Either::Right(
+ config
+ .create_with_context(MskIamClientContext::from_env())
+ .map_err(KafkaConsumerError::MskIam)?,
+ )
+ }
+ };
+
+ Ok(KafkaEventConsumer {
+ consumer: ConsumerTransport(consumer),
+ marker: PhantomData,
+ })
+}
+
+fn build_assignment<C, T>(
+ consumer: &T,
+ topics: &[&str],
+ initial_offset: InitialOffset,
+ metadata_timeout: Duration,
+) -> KafkaResult<TopicPartitionList>
+where
+ C: ConsumerContext,
+ T: Consumer<C>,
+{
+ if topics.is_empty() {
+ return Err(KafkaError::Subscription(
+ "at least one topic is required for assignment".to_string(),
+ ));
+ }
+
+ let mut assignment = TopicPartitionList::new();
+ for topic in topics {
+ let metadata = consumer.fetch_metadata(Some(topic), metadata_timeout)?;
+ let topic_metadata = metadata
+ .topics()
+ .iter()
+ .find(|metadata| metadata.name() == *topic)
+ .ok_or_else(|| {
+ KafkaError::Subscription(format!(
+ "metadata response did not include requested topic {topic}"
+ ))
+ })?;
+
+ if let Some(error) = topic_metadata.error() {
+ return Err(KafkaError::MetadataFetch(error.into()));
+ }
+ if topic_metadata.partitions().is_empty() {
+ return Err(KafkaError::Subscription(format!(
+ "requested topic {topic} has no partitions"
+ )));
+ }
+
+ for partition in topic_metadata.partitions() {
+ if let Some(error) = partition.error() {
+ return Err(KafkaError::MetadataFetch(error.into()));
+ }
+ assignment.add_partition_offset(
+ topic,
+ partition.id(),
+ initial_offset.as_kafka_offset(),
+ )?;
+ }
+ }
+
+ Ok(assignment)
+}
+
+impl<T> KafkaEventConsumer<T> {
+ /// Receives the next Kafka message.
+ ///
+ /// `StreamConsumer::recv` is cancel-safe and may be used in `tokio::select!`.
+ pub async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {
+ either::for_both!(&self.consumer.0, consumer => consumer.recv().await)
+ }
+
+ /// Pauses the partition containing `message`.
+ ///
+ /// Grouped consumers can use this to prevent a later cumulative commit
+ /// from advancing past a failed record. Ungrouped consumers can use it to
+ /// stop additional delivery from a failed partition.
+ pub fn pause_message_partition(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {
+ let mut partitions = TopicPartitionList::new();
+ partitions.add_partition(message.topic(), message.partition());
+ either::for_both!(&self.consumer.0, consumer => consumer.pause(&partitions))
+ }
+}
+
+impl<T: GroupName> KafkaEventConsumer<T> {
+ /// Creates a named-group consumer, selecting plaintext or MSK IAM transport
+ /// from the runtime environment.
+ pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {
+ create_from_env(grouped_config::<T>(brokers))
+ }
+
+ /// Subscribes the consumer to exactly the provided topics.
+ pub fn subscribe(&self, topics: &[&str]) -> KafkaResult<()> {
+ either::for_both!(&self.consumer.0, consumer => consumer.subscribe(topics))
+ }
+
+ /// Commits a message using the caller-selected commit mode.
+ pub fn commit_message(
+ &self,
+ message: &BorrowedMessage<'_>,
+ mode: CommitMode,
+ ) -> KafkaResult<()> {
+ either::for_both!(&self.consumer.0, consumer => consumer.commit_message(message, mode))
+ }
+}
+
+impl KafkaEventConsumer<Ungrouped> {
+ /// Creates an ungrouped consumer, selecting plaintext or MSK IAM transport
+ /// from the runtime environment.
+ ///
+ /// Call [`Self::assign_topics`] before receiving messages.
+ pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {
+ create_from_env(ungrouped_config(brokers))
+ }
+
+ /// Manually assigns every current partition of `topics` at `initial_offset`.
+ ///
+ /// Manual assignment does not join a consumer group, persist offsets, or
+ /// automatically discover partitions added after this call. Callers that
+ /// support partition-count changes must refresh the assignment themselves.
+ pub fn assign_topics(
+ &self,
+ topics: &[&str],
+ initial_offset: InitialOffset,
+ metadata_timeout: Duration,
+ ) -> KafkaResult<()> {
+ either::for_both!(&self.consumer.0, consumer => {
+ let assignment = build_assignment(
+ consumer,
+ topics,
+ initial_offset,
+ metadata_timeout,
+ )?;
+ consumer.assign(&assignment)
+ })
+ }
+}
diff --git a/crates/kafka_consumer_util/src/test.rs b/crates/kafka_consumer_util/src/test.rs
new file mode 100644
index 0000000000..ca3ed674af
--- /dev/null
+++ b/crates/kafka_consumer_util/src/test.rs
@@ -0,0 +1,41 @@
+use super::*;
+
+struct TestConsumerGroup;
+
+impl GroupName for TestConsumerGroup {
+ const GROUP_NAME: &'static str = "consumer-group";
+}
+
+#[test]
+fn grouped_config_uses_named_group_manual_commits_and_earliest_offsets() {
+ let config = grouped_config::<TestConsumerGroup>("broker-a:9092,broker-b:9092");
+
+ assert_eq!(
+ config.get("bootstrap.servers"),
+ Some("broker-a:9092,broker-b:9092")
+ );
+ assert_eq!(config.get("group.id"), Some("consumer-group"));
+ assert_eq!(config.get("enable.auto.commit"), Some("false"));
+ assert_eq!(config.get("auto.offset.reset"), Some("earliest"));
+}
+
+#[test]
+fn ungrouped_config_uses_unique_internal_groups_without_offset_storage() {
+ let first = ungrouped_config("broker:9092");
+ let second = ungrouped_config("broker:9092");
+ let first_group = first.get("group.id").unwrap();
+ let second_group = second.get("group.id").unwrap();
+
+ assert!(first_group.starts_with(UNGROUPED_GROUP_PREFIX));
+ assert!(second_group.starts_with(UNGROUPED_GROUP_PREFIX));
+ assert_ne!(first_group, second_group);
+ assert_eq!(first.get("enable.auto.commit"), Some("false"));
+ assert_eq!(first.get("enable.auto.offset.store"), Some("false"));
+ assert_eq!(first.get("auto.offset.reset"), None);
+}
+
+#[test]
+fn ungrouped_initial_offsets_are_explicit() {
+ assert_eq!(InitialOffset::Earliest.as_kafka_offset(), Offset::Beginning);
+ assert_eq!(InitialOffset::Latest.as_kafka_offset(), Offset::End);
+}
diff --git a/crates/macro_event_broker/Cargo.toml b/crates/macro_event_broker/Cargo.toml
index e332bb7760..397b42481c 100644
--- a/crates/macro_event_broker/Cargo.toml
+++ b/crates/macro_event_broker/Cargo.toml
@@ -5,15 +5,16 @@ publish = false
version = "0.1.0"
[features]
-default = ["ports", "outbound"]
-outbound = [
+default = ["outbound", "ports"]
+kafka = [
"dep:aws-msk-iam-sasl-signer",
"dep:aws-types",
"dep:macro_env",
"dep:macro_env_var",
"dep:rdkafka",
- "ports",
+ "dep:tokio",
]
+outbound = ["kafka", "ports"]
ports = ["dep:tokio"]
[dependencies]
diff --git a/crates/macro_event_broker/src/kafka/mod.rs b/crates/macro_event_broker/src/kafka/mod.rs
new file mode 100644
index 0000000000..a7074dedfc
--- /dev/null
+++ b/crates/macro_event_broker/src/kafka/mod.rs
@@ -0,0 +1,4 @@
+//! Kafka transport support shared by inbound and outbound adapters.
+
+/// AWS MSK IAM authentication support for Kafka clients.
+pub mod msk_iam;
diff --git a/crates/macro_event_broker/src/outbound/msk_iam.rs b/crates/macro_event_broker/src/kafka/msk_iam.rs
similarity index 96%
rename from crates/macro_event_broker/src/outbound/msk_iam.rs
rename to crates/macro_event_broker/src/kafka/msk_iam.rs
index f8b8043593..de1d18e2f8 100644
--- a/crates/macro_event_broker/src/outbound/msk_iam.rs
+++ b/crates/macro_event_broker/src/kafka/msk_iam.rs
@@ -17,10 +17,11 @@ macro_env_var::maybe_env_var! {
/// Apply the TLS + SASL/OAUTHBEARER settings required to authenticate to an
/// AWS MSK cluster with IAM auth (the `:9098` bootstrap listener).
-pub fn configure_sasl_iam(config: &mut ClientConfig) -> &mut ClientConfig {
+pub fn configure_sasl_iam(mut config: ClientConfig) -> ClientConfig {
config
.set("security.protocol", "SASL_SSL")
- .set("sasl.mechanism", "OAUTHBEARER")
+ .set("sasl.mechanism", "OAUTHBEARER");
+ config
}
/// Client context that supplies AWS MSK IAM (SASL/OAUTHBEARER) auth tokens,
diff --git a/crates/macro_event_broker/src/lib.rs b/crates/macro_event_broker/src/lib.rs
index f96c21aae1..bd60384a92 100644
--- a/crates/macro_event_broker/src/lib.rs
+++ b/crates/macro_event_broker/src/lib.rs
@@ -8,7 +8,7 @@
//! [`EventPublisher`](domain::ports::EventPublisher) port, and the
//! [`MacroEventBrokerService`](domain::service::MacroEventBrokerService) that ties
//! them together. Kafka topic definitions live in the `macro_event_topics` crate.
-//! The [`outbound`] layer provides the Kafka adapter implementation.
+//! The [`outbound`] layer provides the Kafka publisher adapter.
/// Domain layer: models, ports, and service.
pub mod domain;
@@ -23,11 +23,14 @@ pub use macro_event_topics::{
pub use domain::ports::{EventPublisher, MacroEventBroker};
#[cfg(feature = "ports")]
pub use domain::service::{MacroEventBrokerService, NoopMacroEventBroker};
+#[cfg(feature = "kafka")]
+pub use kafka::msk_iam::MskIamClientContext;
#[cfg(feature = "outbound")]
pub use outbound::kafka_event_publisher::KafkaEventPublisher;
-#[cfg(feature = "outbound")]
-pub use outbound::msk_iam::MskIamClientContext;
-/// Outbound layer: Kafka adapter for the [`EventPublisher`](domain::ports::EventPublisher) port.
+/// Kafka transport support shared by inbound and outbound adapters.
+#[cfg(feature = "kafka")]
+pub mod kafka;
+/// Outbound layer: Kafka publishing adapters.
#[cfg(feature = "outbound")]
pub mod outbound;
diff --git a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
index 6f4df4632d..9e46870d93 100644
--- a/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
+++ b/crates/macro_event_broker/src/outbound/kafka_event_publisher.rs
@@ -16,7 +16,7 @@ use rdkafka::producer::{FutureProducer, FutureRecord};
use crate::domain::models::EventBrokerError;
use crate::domain::ports::EventPublisher;
-use crate::outbound::msk_iam::{MskIamClientContext, configure_sasl_iam};
+use crate::kafka::msk_iam::{MskIamClientContext, configure_sasl_iam};
/// How long a record may sit in the producer queue before delivery is considered failed.
const MESSAGE_TIMEOUT_MS: &str = "5000";
@@ -64,8 +64,7 @@ impl KafkaEventPublisher {
Producer::Plaintext(base_config(brokers).create().map_err(create_error)?)
}
Environment::Develop | Environment::Production => {
- let mut config = base_config(brokers);
- configure_sasl_iam(&mut config);
+ let config = configure_sasl_iam(base_config(brokers));
Producer::MskIam(
config
diff --git a/crates/macro_event_broker/src/outbound/mod.rs b/crates/macro_event_broker/src/outbound/mod.rs
index 048c8210ef..66675e0544 100644
--- a/crates/macro_event_broker/src/outbound/mod.rs
+++ b/crates/macro_event_broker/src/outbound/mod.rs
@@ -1,4 +1,5 @@
/// Kafka adapter implementing the [`EventPublisher`](crate::domain::ports::EventPublisher) port.
+#[cfg(feature = "outbound")]
pub mod kafka_event_publisher;
/// AWS MSK IAM (SASL/OAUTHBEARER) auth support shared by producers and consumers.
-pub mod msk_iam;
+pub use crate::kafka::msk_iam;
diff --git a/crates/macro_event_topics/src/lib.rs b/crates/macro_event_topics/src/lib.rs
index b2295c4b1a..bbbc542250 100644
--- a/crates/macro_event_topics/src/lib.rs
+++ b/crates/macro_event_topics/src/lib.rs
@@ -53,6 +53,8 @@ topics! {
MacroBotsTopic => "macro.bots",
/// Document lifecycle events (created / updated / deleted / copied).
MacroDocumentsTopic => "macro.documents",
+ /// User-scoped full Soup items produced from entity updates.
+ MacroSoupRealtimeTopic => "macro.soup",
/// Project lifecycle events (created, updated, deleted, restored, permanently deleted, and uploaded).
MacroProjectsTopic => "macro.projects",
/// Team lifecycle, invite, and membership events.
diff --git a/crates/macro_event_topics/src/test.rs b/crates/macro_event_topics/src/test.rs
index 593bce01e8..a5e62663ee 100644
--- a/crates/macro_event_topics/src/test.rs
+++ b/crates/macro_event_topics/src/test.rs
@@ -19,6 +19,7 @@ fn all_topic_names_is_non_empty_and_unique() {
fn all_topic_names_includes_declared_topics() {
assert!(all_topic_names().contains(&MacroExampleTopic.as_str()));
assert!(all_topic_names().contains(&MacroDocumentsTopic.as_str()));
+ assert!(all_topic_names().contains(&MacroSoupRealtimeTopic.as_str()));
assert!(all_topic_names().contains(&MacroProjectsTopic.as_str()));
assert!(all_topic_names().contains(&MacroTeamsTopic.as_str()));
assert!(all_topic_names().contains(&MacroChannelsTopic.as_str()));
diff --git a/crates/projects/examples/projects_consumer.rs b/crates/projects/examples/projects_consumer.rs
index b64f5dcd10..e94972efb9 100644
--- a/crates/projects/examples/projects_consumer.rs
+++ b/crates/projects/examples/projects_consumer.rs
@@ -103,7 +103,7 @@ impl ProjectsConsumer {
.context("failed to create plaintext kafka consumer")?,
),
Environment::Develop | Environment::Production => {
- configure_sasl_iam(&mut config);
+ let config = configure_sasl_iam(config);
Self::MskIam(
config
.create_with_context(MskIamClientContext::from_env())
diff --git a/crates/soup_realtime/Cargo.toml b/crates/soup_realtime/Cargo.toml
new file mode 100644
index 0000000000..2cfe4c4827
--- /dev/null
+++ b/crates/soup_realtime/Cargo.toml
@@ -0,0 +1,52 @@
+[package]
+edition = "2024"
+name = "soup_realtime"
+publish = false
+version = "0.1.0"
+
+[features]
+consumer = [
+ "dep:documents",
+ "dep:kafka_consumer_util",
+ "dep:macro_event_broker",
+ "dep:macro_event_topics",
+ "dep:rdkafka",
+ "dep:tokio",
+ "ports",
+]
+default = ["consumer", "outbound", "ports"]
+outbound = [
+ "dep:entity_access",
+ "dep:macro_event_topics",
+ "dep:serde_json",
+ "dep:soup",
+ "macro_event_broker/ports",
+ "ports",
+]
+ports = ["dep:futures", "dep:rootcause", "dep:tracing"]
+
+[dependencies]
+documents = { path = "../documents", default-features = false, optional = true }
+entity_access = { path = "../entity_access", default-features = false, features = ["ports"], optional = true }
+futures = { workspace = true, optional = true }
+kafka_consumer_util = { path = "../kafka_consumer_util", optional = true }
+macro_event_broker = { path = "../macro_event_broker", default-features = false, optional = true }
+macro_event_topics = { path = "../macro_event_topics", optional = true }
+macro_user_id = { path = "../macro_user_id" }
+model-entity = { path = "../model-entity" }
+models_soup = { path = "../models_soup" }
+rdkafka = { workspace = true, optional = true }
+rootcause = { workspace = true, optional = true }
+serde = { workspace = true }
+serde_json = { workspace = true, optional = true }
+soup = { path = "../soup", default-features = false, features = ["ports"], optional = true }
+tokio = { workspace = true, features = ["time"], optional = true }
+tracing = { workspace = true, optional = true }
+workspace-hack = { version = "0.1", path = "../workspace-hack" }
+
+[dev-dependencies]
+chrono = { workspace = true }
+models_properties = { path = "../models_properties" }
+serde_json = { workspace = true }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util", "time"] }
+uuid = { workspace = true }
diff --git a/crates/soup_realtime/README.md b/crates/soup_realtime/README.md
new file mode 100644
index 0000000000..2aa71b4ac3
--- /dev/null
+++ b/crates/soup_realtime/README.md
@@ -0,0 +1,7 @@
+# soup_realtime
+
+Hexagonal components for expanding `document.updated` Kafka events into one full `SoupItem<()>` message per current accessor. The item is loaded once through the first accessor and its user-specific `viewed_at` field is set to `None` before fan-out.
+
+This crate intentionally does not start a consumer or wire itself into a service binary. A later composition root can combine the domain service with the entity-access, Soup repository, Kafka publisher, and Kafka consumer adapters exposed here.
+
+Delivery is at least once. If publication partially succeeds and processing is retried, downstream consumers may receive duplicate messages.
diff --git a/crates/soup_realtime/src/domain/mod.rs b/crates/soup_realtime/src/domain/mod.rs
new file mode 100644
index 0000000000..a0bdb0b450
--- /dev/null
+++ b/crates/soup_realtime/src/domain/mod.rs
@@ -0,0 +1,12 @@
+//! Core realtime Soup contracts and orchestration.
+
+/// Versioned output message models.
+pub mod models;
+
+/// Inbound and outbound domain ports.
+#[cfg(feature = "ports")]
+pub mod ports;
+
+/// Realtime Soup fan-out service.
+#[cfg(feature = "ports")]
+pub mod service;
diff --git a/crates/soup_realtime/src/domain/models.rs b/crates/soup_realtime/src/domain/models.rs
new file mode 100644
index 0000000000..af1b0e96fa
--- /dev/null
+++ b/crates/soup_realtime/src/domain/models.rs
@@ -0,0 +1,30 @@
+//! Versioned realtime Soup output models.
+
+use macro_user_id::user_id::MacroUserIdStr;
+use models_soup::item::SoupItem;
+use serde::{Deserialize, Serialize};
+
+/// One full Soup item published to a recipient for realtime delivery.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SoupRealtimeMessage {
+ /// Version of this message contract.
+ pub schema_version: u8,
+ /// User to whom the Soup item is addressed.
+ pub user_id: MacroUserIdStr<'static>,
+ /// Complete Soup item with transient user-specific fields normalized.
+ pub item: SoupItem<()>,
+}
+
+impl SoupRealtimeMessage {
+ /// Current schema version for realtime Soup messages.
+ pub const SCHEMA_VERSION: u8 = 1;
+
+ /// Creates a version-one message addressed to a recipient.
+ pub fn new(user_id: MacroUserIdStr<'static>, item: SoupItem<()>) -> Self {
+ Self {
+ schema_version: Self::SCHEMA_VERSION,
+ user_id,
+ item,
+ }
+ }
+}
diff --git a/crates/soup_realtime/src/domain/ports.rs b/crates/soup_realtime/src/domain/ports.rs
new file mode 100644
index 0000000000..a67f874ff2
--- /dev/null
+++ b/crates/soup_realtime/src/domain/ports.rs
@@ -0,0 +1,50 @@
+//! Ports used by realtime Soup fan-out.
+
+use std::future::Future;
+
+use macro_user_id::user_id::MacroUserIdStr;
+use model_entity::Entity;
+use models_soup::item::SoupItem;
+use rootcause::Report;
+
+use super::models::SoupRealtimeMessage;
+
+/// Inbound use-case port driven by entity update transports.
+pub trait SoupRealtimeService: Send + Sync + 'static {
+ /// Hydrates one normalized Soup item and publishes it to every current accessor.
+ fn notify_users(
+ &self,
+ entity: Entity<'static>,
+ ) -> impl Future<Output = Result<(), Report>> + Send;
+}
+
+/// Resolves the users who currently have access to an entity.
+pub trait UserAccessExpander: Send + Sync + 'static {
+ /// Returns all current user accessors for `entity`.
+ ///
+ /// Implementations may return duplicates; the domain service deduplicates
+ /// recipients before hydration.
+ fn expand_user_access(
+ &self,
+ entity: &Entity<'static>,
+ ) -> impl Future<Output = Result<Vec<MacroUserIdStr<'static>>, Report>> + Send;
+}
+
+/// Reads a complete Soup item under an individual user's visibility scope.
+pub trait SoupItemReader: Send + Sync + 'static {
+ /// Reads `entity` through the visibility scope of `user_id`.
+ fn read_for_user(
+ &self,
+ user_id: MacroUserIdStr<'static>,
+ entity: &Entity<'static>,
+ ) -> impl Future<Output = Result<Option<SoupItem<()>>, Report>> + Send;
+}
+
+/// Publishes complete recipient-targeted Soup messages.
+pub trait SoupRealtimePublisher: Send + Sync + 'static {
+ /// Publishes one realtime Soup message and awaits delivery.
+ fn publish(
+ &self,
+ message: SoupRealtimeMessage,
+ ) -> impl Future<Output = Result<(), Report>> + Send;
+}
diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rs
new file mode 100644
index 0000000000..947b495bc1
--- /dev/null
+++ b/crates/soup_realtime/src/domain/service.rs
@@ -0,0 +1,130 @@
+//! Realtime Soup hydration and fan-out orchestration.
+
+#[cfg(test)]
+mod test;
+
+use std::collections::HashSet;
+
+use futures::{StreamExt as _, stream};
+use model_entity::Entity;
+use models_soup::item::SoupItem;
+use rootcause::prelude::{Report, ResultExt as _};
+
+use super::{
+ models::SoupRealtimeMessage,
+ ports::{SoupItemReader, SoupRealtimePublisher, SoupRealtimeService, UserAccessExpander},
+};
+
+/// Maximum number of Kafka publications polled concurrently.
+const PUBLISH_CONCURRENCY: usize = 16;
+
+/// Domain service that expands access, hydrates one normalized item, and fans out.
+pub struct SoupRealtimeServiceImpl<A, R, P> {
+ access_expander: A,
+ item_reader: R,
+ publisher: P,
+}
+
+impl<A, R, P> SoupRealtimeServiceImpl<A, R, P> {
+ /// Creates a realtime Soup service from its three outbound capabilities.
+ pub fn new(access_expander: A, item_reader: R, publisher: P) -> Self {
+ Self {
+ access_expander,
+ item_reader,
+ publisher,
+ }
+ }
+}
+
+impl<A, R, P> SoupRealtimeService for SoupRealtimeServiceImpl<A, R, P>
+where
+ A: UserAccessExpander,
+ R: SoupItemReader,
+ P: SoupRealtimePublisher,
+{
+ #[tracing::instrument(
+ skip(self),
+ fields(
+ entity_type = %entity.entity_type,
+ entity_id = %entity.entity_id,
+ recipient_count = tracing::field::Empty,
+ ),
+ err
+ )]
+ async fn notify_users(&self, entity: Entity<'static>) -> Result<(), Report> {
+ let mut users = self
+ .access_expander
+ .expand_user_access(&entity)
+ .await
+ .context("failed to expand current user access")?;
+
+ let mut seen = HashSet::with_capacity(users.len());
+ users.retain(|user_id| seen.insert(user_id.clone()));
+ tracing::Span::current().record("recipient_count", users.len());
+
+ if users.is_empty() {
+ return Ok(());
+ }
+
+ let hydration_user_id = users[0].clone();
+ let item = self
+ .item_reader
+ .read_for_user(hydration_user_id.clone(), &entity)
+ .await
+ .context_with(|| {
+ format!("failed to hydrate Soup item through accessor {hydration_user_id}")
+ })?
+ .ok_or_else(|| {
+ rootcause::report!(
+ "Soup item for {} {} was missing for accessor {}",
+ entity.entity_type,
+ entity.entity_id,
+ hydration_user_id
+ )
+ })?;
+
+ let hydrated_entity = item.entity();
+ if hydrated_entity != entity {
+ return Err(rootcause::report!(
+ "Soup reader returned {} {} while hydrating {} {} through accessor {}",
+ hydrated_entity.entity_type,
+ hydrated_entity.entity_id,
+ entity.entity_type,
+ entity.entity_id,
+ hydration_user_id
+ ));
+ }
+
+ let SoupItem::Document(mut document) = item else {
+ return Err(rootcause::report!(
+ "realtime Soup fan-out currently supports document items only"
+ ));
+ };
+ document.viewed_at = None;
+
+ let messages = users
+ .into_iter()
+ .map(|user_id| SoupRealtimeMessage::new(user_id, SoupItem::Document(document.clone())))
+ .collect::<Vec<_>>();
+
+ let results = stream::iter(
+ messages
+ .into_iter()
+ .map(|message| self.publisher.publish(message)),
+ )
+ .buffer_unordered(PUBLISH_CONCURRENCY)
+ .collect::<Vec<_>>()
+ .await;
+
+ let failure_count = results.iter().filter(|result| result.is_err()).count();
+ if let Some(error) = results.into_iter().find_map(Result::err) {
+ return Err(error
+ .context(format!(
+ "{failure_count} realtime Soup publication(s) failed"
+ ))
+ .into_dynamic());
+ }
+
+ Ok(())
+ }
+}
diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rs
new file mode 100644
index 0000000000..66e43e9342
--- /dev/null
+++ b/crates/soup_realtime/src/domain/service/test.rs
@@ -0,0 +1,399 @@
+use std::collections::{HashMap, HashSet};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+
+use chrono::{DateTime, Utc};
+use macro_user_id::user_id::MacroUserIdStr;
+use model_entity::{Entity, EntityType};
+use models_soup::{document::SoupDocument, item::SoupItem};
+use uuid::Uuid;
+
+use super::*;
+use crate::domain::ports::{SoupItemReader, SoupRealtimePublisher, UserAccessExpander};
+
+const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";
+const OTHER_DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000002";
+
+struct FakeAccessExpander {
+ users: Vec<MacroUserIdStr<'static>>,
+ fail: bool,
+ calls: Arc<AtomicUsize>,
+}
+
+impl UserAccessExpander for FakeAccessExpander {
+ async fn expand_user_access(
+ &self,
+ _entity: &Entity<'static>,
+ ) -> Result<Vec<MacroUserIdStr<'static>>, Report> {
+ self.calls.fetch_add(1, Ordering::SeqCst);
+ if self.fail {
+ Err(rootcause::report!("access unavailable"))
+ } else {
+ Ok(self.users.clone())
+ }
+ }
+}
+
+enum ReadResponse {
+ Item(Box<SoupItem<()>>),
+ Missing,
+ Failure,
+}
+
+struct FakeReader {
+ responses: Mutex<HashMap<String, ReadResponse>>,
+ calls: Arc<AtomicUsize>,
+}
+
+impl SoupItemReader for FakeReader {
+ async fn read_for_user(
+ &self,
+ user_id: MacroUserIdStr<'static>,
+ _entity: &Entity<'static>,
+ ) -> Result<Option<SoupItem<()>>, Report> {
+ self.calls.fetch_add(1, Ordering::SeqCst);
+ match self
+ .responses
+ .lock()
+ .expect("responses lock")
+ .remove(user_id.as_ref())
+ .unwrap_or(ReadResponse::Missing)
+ {
+ ReadResponse::Item(item) => Ok(Some(*item)),
+ ReadResponse::Missing => Ok(None),
+ ReadResponse::Failure => Err(rootcause::report!("reader unavailable")),
+ }
+ }
+}
+
+struct FakePublisher {
+ messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,
+ fail_users: HashSet<String>,
+}
+
+impl SoupRealtimePublisher for FakePublisher {
+ async fn publish(&self, message: SoupRealtimeMessage) -> Result<(), Report> {
+ let should_fail = self.fail_users.contains(message.user_id.as_ref());
+ self.messages.lock().expect("messages lock").push(message);
+ if should_fail {
+ Err(rootcause::report!("publisher unavailable"))
+ } else {
+ Ok(())
+ }
+ }
+}
+
+struct Harness {
+ service: SoupRealtimeServiceImpl<FakeAccessExpander, FakeReader, FakePublisher>,
+ access_calls: Arc<AtomicUsize>,
+ read_calls: Arc<AtomicUsize>,
+ messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,
+}
+
+fn user(local: &str) -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from(format!("macro|{local}@example.com")).expect("valid user id")
+}
+
+fn timestamp(seconds: i64) -> DateTime<Utc> {
+ DateTime::from_timestamp(seconds, 0).expect("valid timestamp")
+}
+
+fn document_item(id: &str, name: &str, viewed_at: Option<DateTime<Utc>>) -> SoupItem<()> {
+ SoupItem::Document(SoupDocument {
+ id: Uuid::parse_str(id).expect("valid document id"),
+ document_version_id: 42,
+ owner_id: user("owner"),
+ name: name.to_string(),
+ file_type: Some("md".to_string()),
+ sha: Some("document-sha".to_string()),
+ project_id: None,
+ branched_from_id: None,
+ branched_from_version_id: None,
+ document_family_id: None,
+ created_at: timestamp(1),
+ updated_at: timestamp(2),
+ viewed_at,
+ sub_type: None,
+ deleted_at: None,
+ extra: (),
+ })
+}
+
+fn document_entity() -> Entity<'static> {
+ EntityType::Document.with_entity_string(DOCUMENT_ID.to_string())
+}
+
+fn harness(
+ users: Vec<MacroUserIdStr<'static>>,
+ responses: HashMap<String, ReadResponse>,
+ access_fails: bool,
+ fail_users: HashSet<String>,
+) -> Harness {
+ let access_calls = Arc::new(AtomicUsize::new(0));
+ let read_calls = Arc::new(AtomicUsize::new(0));
+ let messages = Arc::new(Mutex::new(Vec::new()));
+ let service = SoupRealtimeServiceImpl::new(
+ FakeAccessExpander {
+ users,
+ fail: access_fails,
+ calls: access_calls.clone(),
+ },
+ FakeReader {
+ responses: Mutex::new(responses),
+ calls: read_calls.clone(),
+ },
+ FakePublisher {
+ messages: messages.clone(),
+ fail_users,
+ },
+ );
+ Harness {
+ service,
+ access_calls,
+ read_calls,
+ messages,
+ }
+}
+
+fn item_response(
+ recipient: &MacroUserIdStr<'static>,
+ item: SoupItem<()>,
+) -> (String, ReadResponse) {
+ (
+ recipient.as_ref().to_string(),
+ ReadResponse::Item(Box::new(item)),
+ )
+}
+
+#[tokio::test]
+async fn zero_users_skips_reads_and_publications() {
+ let harness = harness(Vec::new(), HashMap::new(), false, HashSet::new());
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect("zero recipients is successful");
+
+ assert_eq!(harness.access_calls.load(Ordering::SeqCst), 1);
+ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 0);
+ assert!(harness.messages.lock().expect("messages lock").is_empty());
+}
+
+#[tokio::test]
+async fn one_user_receives_one_full_versioned_message() {
+ let recipient = user("one");
+ let viewed_at = Some(timestamp(3));
+ let responses = HashMap::from([item_response(
+ &recipient,
+ document_item(DOCUMENT_ID, "Full document", viewed_at),
+ )]);
+ let harness = harness(vec![recipient.clone()], responses, false, HashSet::new());
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect("fan-out succeeds");
+
+ let mut messages = harness.messages.lock().expect("messages lock");
+ let message = messages.pop().expect("one message");
+ assert!(messages.is_empty());
+ assert_eq!(message.schema_version, 1);
+ assert_eq!(message.user_id, recipient);
+ match message.item {
+ SoupItem::Document(document) => {
+ assert_eq!(document.name, "Full document");
+ assert_eq!(document.document_version_id, 42);
+ assert_eq!(document.sha.as_deref(), Some("document-sha"));
+ assert_eq!(document.viewed_at, None);
+ }
+ _ => panic!("expected document item"),
+ }
+}
+
+#[tokio::test]
+async fn three_unique_users_receive_exactly_three_messages() {
+ let users = [user("one"), user("two"), user("three")];
+ let responses = users
+ .iter()
+ .map(|recipient| {
+ item_response(
+ recipient,
+ document_item(DOCUMENT_ID, recipient.as_ref(), None),
+ )
+ })
+ .collect();
+ let harness = harness(users.to_vec(), responses, false, HashSet::new());
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect("fan-out succeeds");
+
+ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 1);
+ let messages = harness.messages.lock().expect("messages lock");
+ assert_eq!(messages.len(), 3);
+ let recipients: HashSet<_> = messages
+ .iter()
+ .map(|message| message.user_id.as_ref().to_string())
+ .collect();
+ assert_eq!(recipients.len(), 3);
+ assert!(
+ users
+ .iter()
+ .all(|recipient| recipients.contains(recipient.as_ref()))
+ );
+}
+
+#[tokio::test]
+async fn duplicate_accessors_are_deduplicated() {
+ let one = user("one");
+ let two = user("two");
+ let responses = HashMap::from([
+ item_response(&one, document_item(DOCUMENT_ID, "One", None)),
+ item_response(&two, document_item(DOCUMENT_ID, "Two", None)),
+ ]);
+ let harness = harness(
+ vec![one.clone(), two, one],
+ responses,
+ false,
+ HashSet::new(),
+ );
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect("fan-out succeeds");
+
+ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 1);
+ assert_eq!(harness.messages.lock().expect("messages lock").len(), 2);
+}
+
+#[tokio::test]
+async fn all_recipients_get_the_first_item_with_no_viewed_at() {
+ let one = user("one");
+ let two = user("two");
+ let responses = HashMap::from([
+ item_response(
+ &one,
+ document_item(DOCUMENT_ID, "Representative", Some(timestamp(10))),
+ ),
+ item_response(
+ &two,
+ document_item(DOCUMENT_ID, "Should not be read", Some(timestamp(20))),
+ ),
+ ]);
+ let harness = harness(vec![one, two], responses, false, HashSet::new());
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect("fan-out succeeds");
+
+ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 1);
+ let messages = harness.messages.lock().expect("messages lock");
+ assert_eq!(messages.len(), 2);
+ for message in messages.iter() {
+ let SoupItem::Document(document) = &message.item else {
+ panic!("expected document item")
+ };
+ assert_eq!(document.name, "Representative");
+ assert_eq!(document.viewed_at, None);
+ }
+}
+
+#[tokio::test]
+async fn access_failure_prevents_reads_and_publications() {
+ let harness = harness(vec![user("one")], HashMap::new(), true, HashSet::new());
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect_err("access failure propagates");
+
+ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 0);
+ assert!(harness.messages.lock().expect("messages lock").is_empty());
+}
+
+#[tokio::test]
+async fn missing_item_prevents_all_publication() {
+ let recipient = user("one");
+ let responses = HashMap::from([(recipient.as_ref().to_string(), ReadResponse::Missing)]);
+ let harness = harness(vec![recipient], responses, false, HashSet::new());
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect_err("missing item is an error");
+
+ assert!(harness.messages.lock().expect("messages lock").is_empty());
+}
+
+#[tokio::test]
+async fn mismatched_item_prevents_all_publication() {
+ let recipient = user("one");
+ let responses = HashMap::from([item_response(
+ &recipient,
+ document_item(OTHER_DOCUMENT_ID, "Wrong", None),
+ )]);
+ let harness = harness(vec![recipient], responses, false, HashSet::new());
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect_err("mismatched item is an error");
+
+ assert!(harness.messages.lock().expect("messages lock").is_empty());
+}
+
+#[tokio::test]
+async fn representative_reader_failure_prevents_all_publication() {
+ let one = user("one");
+ let two = user("two");
+ let responses = HashMap::from([(one.as_ref().to_string(), ReadResponse::Failure)]);
+ let harness = harness(vec![one, two], responses, false, HashSet::new());
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect_err("reader failure propagates");
+
+ assert_eq!(harness.read_calls.load(Ordering::SeqCst), 1);
+ assert!(harness.messages.lock().expect("messages lock").is_empty());
+}
+
+#[tokio::test]
+async fn publisher_failure_is_returned_after_all_messages_are_attempted() {
+ let users = [user("one"), user("two"), user("three")];
+ let responses = users
+ .iter()
+ .map(|recipient| {
+ item_response(
+ recipient,
+ document_item(DOCUMENT_ID, recipient.as_ref(), None),
+ )
+ })
+ .collect();
+ let harness = harness(
+ users.to_vec(),
+ responses,
+ false,
+ HashSet::from([users[1].as_ref().to_string()]),
+ );
+
+ harness
+ .service
+ .notify_users(document_entity())
+ .await
+ .expect_err("publication failure propagates");
+
+ assert_eq!(harness.messages.lock().expect("messages lock").len(), 3);
+}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rs
new file mode 100644
index 0000000000..7659820494
--- /dev/null
+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs
@@ -0,0 +1,261 @@
+//! Kafka consumer for `document.updated` realtime Soup fan-out.
+//!
+//! Delivery is at least once: offsets are committed only after successful
+//! domain processing. Malformed and currently unsupported document events are
+//! poison/ignored records and are committed so they cannot wedge a partition.
+
+#[cfg(test)]
+mod test;
+
+use std::{future::Future, time::Duration};
+
+use documents::domain::events::DocumentTopicEvent;
+use kafka_consumer_util::{GroupName, KafkaEventConsumer};
+use macro_event_broker::{Event, EventBrokerError, Topic as _};
+use macro_event_topics::MacroDocumentsTopic;
+use model_entity::{Entity, EntityType};
+use rdkafka::consumer::CommitMode;
+use rdkafka::message::{BorrowedMessage, Message as _};
+use rootcause::prelude::{Report, ResultExt as _};
+
+use crate::domain::ports::SoupRealtimeService;
+
+/// Consumer group used for document update fan-out offsets.
+struct SoupRealtimeConsumerGroup;
+
+impl GroupName for SoupRealtimeConsumerGroup {
+ const GROUP_NAME: &'static str = "soup-realtime";
+}
+
+type SoupRealtimeKafkaConsumer = KafkaEventConsumer<SoupRealtimeConsumerGroup>;
+
+/// Total service attempts before returning for supervisor-driven redelivery.
+const MAX_SERVICE_ATTEMPTS: u32 = 5;
+/// Delay before the first retry; each subsequent delay doubles.
+const SERVICE_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
+
+fn subscribed_topics() -> [&'static str; 1] {
+ [MacroDocumentsTopic.as_str()]
+}
+
+fn entity_from_document_event(event: DocumentTopicEvent) -> Option<Entity<'static>> {
+ match event {
+ DocumentTopicEvent::Updated(metadata) => {
+ Some(EntityType::Document.with_entity_string(metadata.document_id))
+ }
+ DocumentTopicEvent::Created(_) => {
+ tracing::trace!(event_type = "document.created", "ignoring document event");
+ None
+ }
+ DocumentTopicEvent::Deleted(_) => {
+ tracing::trace!(event_type = "document.deleted", "ignoring document event");
+ None
+ }
+ DocumentTopicEvent::Copied(_) => {
+ tracing::trace!(event_type = "document.copied", "ignoring document event");
+ None
+ }
+ }
+}
+
+fn decode_updated_entity(payload: &[u8]) -> Result<Option<Entity<'static>>, EventBrokerError> {
+ let event = Event::<DocumentTopicEvent>::decode(payload)?;
+ Ok(entity_from_document_event(event.event))
+}
+
+/// Commit-safe outcome after processing one non-empty document payload.
+enum DocumentPayloadOutcome {
+ /// An update was successfully sent through the domain service.
+ Notified,
+ /// A recognized document event is outside the current scope.
+ Ignored,
+ /// The payload could not be decoded and should be treated as poison.
+ Malformed(EventBrokerError),
+}
+
+#[tracing::instrument(skip(service, payload), fields(partition, offset, payload_len = payload.len()), err)]
+async fn process_document_payload<S: SoupRealtimeService>(
+ service: &S,
+ payload: &[u8],
+ partition: i32,
+ offset: i64,
+) -> Result<DocumentPayloadOutcome, Report> {
+ match decode_updated_entity(payload) {
+ Ok(Some(entity)) => {
+ notify_with_retry(service, entity, partition, offset).await?;
+ Ok(DocumentPayloadOutcome::Notified)
+ }
+ Ok(None) => Ok(DocumentPayloadOutcome::Ignored),
+ Err(error) => Ok(DocumentPayloadOutcome::Malformed(error)),
+ }
+}
+
+#[tracing::instrument(
+ skip(service),
+ fields(
+ entity_type = %entity.entity_type,
+ entity_id = %entity.entity_id,
+ partition,
+ offset,
+ ),
+ err
+)]
+async fn notify_with_retry<S: SoupRealtimeService>(
+ service: &S,
+ entity: Entity<'static>,
+ partition: i32,
+ offset: i64,
+) -> Result<(), Report> {
+ let mut delay = SERVICE_RETRY_BASE_DELAY;
+ let mut attempt = 1u32;
+
+ loop {
+ tracing::trace!(attempt, "notifying realtime Soup recipients");
+ match service.notify_users(entity.clone()).await {
+ Ok(()) => {
+ tracing::trace!(attempt, "realtime Soup recipients notified");
+ return Ok(());
+ }
+ Err(error) if attempt < MAX_SERVICE_ATTEMPTS => {
+ tracing::warn!(
+ error = ?error,
+ attempt,
+ delay_secs = delay.as_secs_f32(),
+ "realtime Soup fan-out failed, retrying"
+ );
+ tokio::time::sleep(delay).await;
+ delay *= 2;
+ attempt += 1;
+ }
+ Err(error) => {
+ return Err(error
+ .context(format!(
+ "realtime Soup fan-out failed after {MAX_SERVICE_ATTEMPTS} attempts \
+ (partition {partition} offset {offset})"
+ ))
+ .into_dynamic());
+ }
+ }
+ }
+}
+
+fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {
+ match consumer.commit_message(message, CommitMode::Async) {
+ Ok(()) => tracing::trace!(
+ partition = message.partition(),
+ offset = message.offset(),
+ "committed realtime Soup input offset"
+ ),
+ Err(error) => tracing::error!(
+ error = ?error,
+ partition = message.partition(),
+ offset = message.offset(),
+ "failed to commit realtime Soup input offset"
+ ),
+ }
+}
+
+/// Runs the document update consumer until `shutdown` resolves.
+///
+/// The consumer subscribes only to `macro.documents` under the
+/// `soup-realtime` group. It commits malformed and recognized-but-ignored
+/// events, and commits `document.updated` only after [`SoupRealtimeService`]
+/// succeeds. Exhausted service retries return without committing so a future
+/// supervisor restart can redeliver the record.
+#[tracing::instrument(skip(service, shutdown), fields(brokers), err)]
+pub async fn run_document_update_consumer<S>(
+ brokers: &str,
+ service: S,
+ shutdown: impl Future<Output = ()> + Send,
+) -> Result<(), Report>
+where
+ S: SoupRealtimeService,
+{
+ let consumer = SoupRealtimeKafkaConsumer::from_env(brokers)?;
+ consumer
+ .subscribe(&subscribed_topics())
+ .context("failed to subscribe to document update topic")?;
+ tracing::info!(
+ topics = ?subscribed_topics(),
+ group = SoupRealtimeConsumerGroup::GROUP_NAME,
+ "realtime Soup document consumer listening"
+ );
+
+ let mut shutdown = std::pin::pin!(shutdown);
+ loop {
+ tokio::select! {
+ _ = &mut shutdown => {
+ tracing::info!("realtime Soup document consumer shutting down");
+ break;
+ }
+ result = consumer.recv() => {
+ let message = match result {
+ Ok(message) => message,
+ Err(error) => {
+ tracing::error!(error = ?error, "Kafka receive error");
+ continue;
+ }
+ };
+ tracing::trace!(
+ topic = message.topic(),
+ partition = message.partition(),
+ offset = message.offset(),
+ payload_len = message.payload().map_or(0, <[u8]>::len),
+ "received realtime Soup input"
+ );
+
+ let Some(payload) = message.payload().filter(|payload| !payload.is_empty()) else {
+ tracing::warn!(
+ partition = message.partition(),
+ offset = message.offset(),
+ "skipping document event with empty payload"
+ );
+ commit_logged(&consumer, &message);
+ continue;
+ };
+
+ let outcome = match process_document_payload(
+ &service,
+ payload,
+ message.partition(),
+ message.offset(),
+ )
+ .await
+ {
+ Ok(outcome) => outcome,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ topic = message.topic(),
+ partition = message.partition(),
+ offset = message.offset(),
+ "realtime Soup fan-out retries exhausted; pausing partition for redelivery"
+ );
+ // Kafka commits are cumulative within a partition, so
+ // pause it before any later record can advance the
+ // committed offset past this failure.
+ consumer
+ .pause_message_partition(&message)
+ .context("failed to pause Kafka partition after fan-out failure")?;
+ continue;
+ }
+ };
+
+ match outcome {
+ DocumentPayloadOutcome::Notified | DocumentPayloadOutcome::Ignored => {}
+ DocumentPayloadOutcome::Malformed(error) => tracing::error!(
+ error = ?error,
+ topic = message.topic(),
+ partition = message.partition(),
+ offset = message.offset(),
+ "dropping malformed document event"
+ ),
+ }
+
+ commit_logged(&consumer, &message);
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
new file mode 100644
index 0000000000..a00258b697
--- /dev/null
+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
@@ -0,0 +1,177 @@
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::{Arc, Mutex};
+
+use documents::domain::events::{
+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,
+ DocumentUpdatedMetadata,
+};
+use macro_user_id::user_id::MacroUserIdStr;
+
+use super::*;
+
+const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";
+
+fn user() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")
+}
+
+fn updated_event() -> Event<DocumentTopicEvent> {
+ Event::new(DocumentTopicEvent::Updated(DocumentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: user(),
+ actor_user_id: None,
+ document_name: Some("Updated".to_string()),
+ previous_project_id: None,
+ project_id: None,
+ file_type: None,
+ share_permission_updated: false,
+ }))
+}
+
+fn ignored_events() -> Vec<DocumentTopicEvent> {
+ vec![
+ DocumentTopicEvent::Created(DocumentCreatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: user(),
+ document_name: "Created".to_string(),
+ file_type: None,
+ project_id: None,
+ sub_type: None,
+ created_at: None,
+ }),
+ DocumentTopicEvent::Deleted(DocumentDeletedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ actor_user_id: None,
+ project_id: None,
+ }),
+ DocumentTopicEvent::Copied(DocumentCopiedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ source_document_id: "00000000-0000-0000-0000-000000000002".to_string(),
+ source_version_id: None,
+ owner: user(),
+ document_name: "Copied".to_string(),
+ file_type: None,
+ project_id: None,
+ sub_type: None,
+ }),
+ ]
+}
+
+#[derive(Clone)]
+struct FlakyService {
+ attempts: Arc<AtomicU32>,
+ failures: u32,
+ entities: Arc<Mutex<Vec<Entity<'static>>>>,
+}
+
+impl SoupRealtimeService for FlakyService {
+ async fn notify_users(&self, entity: Entity<'static>) -> Result<(), Report> {
+ self.entities.lock().expect("entities lock").push(entity);
+ let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;
+ if attempt <= self.failures {
+ Err(rootcause::report!("temporary fan-out failure"))
+ } else {
+ Ok(())
+ }
+ }
+}
+
+fn flaky_service(failures: u32) -> FlakyService {
+ FlakyService {
+ attempts: Arc::new(AtomicU32::new(0)),
+ failures,
+ entities: Arc::new(Mutex::new(Vec::new())),
+ }
+}
+
+#[test]
+fn subscribes_only_to_documents() {
+ assert_eq!(subscribed_topics(), ["macro.documents"]);
+}
+
+#[tokio::test]
+async fn updated_payload_maps_to_document_entity() {
+ let payload = serde_json::to_vec(&updated_event()).expect("serializable event");
+ let service = flaky_service(0);
+
+ assert!(matches!(
+ process_document_payload(&service, &payload, 0, 0)
+ .await
+ .expect("processing succeeds"),
+ DocumentPayloadOutcome::Notified
+ ));
+
+ let entities = service.entities.lock().expect("entities lock");
+ assert_eq!(entities.len(), 1);
+ assert_eq!(entities[0].entity_type, EntityType::Document);
+ assert_eq!(entities[0].entity_id, DOCUMENT_ID);
+}
+
+#[tokio::test]
+async fn created_deleted_and_copied_events_are_ignored() {
+ let service = flaky_service(0);
+ for event in ignored_events() {
+ let payload = serde_json::to_vec(&Event::new(event)).expect("serializable event");
+ assert!(matches!(
+ process_document_payload(&service, &payload, 0, 0)
+ .await
+ .expect("processing succeeds"),
+ DocumentPayloadOutcome::Ignored
+ ));
+ }
+ assert_eq!(service.attempts.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test]
+async fn malformed_and_unknown_events_are_rejected_without_notifying() {
+ let service = flaky_service(0);
+ assert!(matches!(
+ process_document_payload(&service, b"not json", 0, 0)
+ .await
+ .expect("poison payload is commit-safe"),
+ DocumentPayloadOutcome::Malformed(_)
+ ));
+
+ let unknown = serde_json::json!({
+ "event_id": "00000000-0000-0000-0000-000000000003",
+ "schema_version": 1,
+ "event_type": "document.restored",
+ "metadata": { "document_id": DOCUMENT_ID }
+ });
+ let payload = serde_json::to_vec(&unknown).expect("serializable JSON");
+ assert!(matches!(
+ process_document_payload(&service, &payload, 0, 0)
+ .await
+ .expect("poison payload is commit-safe"),
+ DocumentPayloadOutcome::Malformed(_)
+ ));
+ assert_eq!(service.attempts.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test(start_paused = true)]
+async fn transient_service_failures_retry_then_succeed() {
+ let service = flaky_service(2);
+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());
+
+ notify_with_retry(&service, entity, 2, 17)
+ .await
+ .expect("eventual success");
+
+ assert_eq!(service.attempts.load(Ordering::SeqCst), 3);
+ assert_eq!(service.entities.lock().expect("entities lock").len(), 3);
+}
+
+#[tokio::test(start_paused = true)]
+async fn exhausted_retries_return_for_redelivery() {
+ let service = flaky_service(u32::MAX);
+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());
+
+ notify_with_retry(&service, entity, 2, 17)
+ .await
+ .expect_err("persistent failure returns without a commit");
+
+ assert_eq!(
+ service.attempts.load(Ordering::SeqCst),
+ MAX_SERVICE_ATTEMPTS
+ );
+}
diff --git a/crates/soup_realtime/src/inbound/mod.rs b/crates/soup_realtime/src/inbound/mod.rs
new file mode 100644
index 0000000000..1d73c01b15
--- /dev/null
+++ b/crates/soup_realtime/src/inbound/mod.rs
@@ -0,0 +1,4 @@
+//! Inbound adapters that drive realtime Soup fan-out.
+
+/// Kafka consumer for document update events.
+pub mod kafka_consumer;
diff --git a/crates/soup_realtime/src/lib.rs b/crates/soup_realtime/src/lib.rs
new file mode 100644
index 0000000000..ab4794b1f9
--- /dev/null
+++ b/crates/soup_realtime/src/lib.rs
@@ -0,0 +1,16 @@
+#![deny(missing_docs)]
+//! User-scoped realtime Soup fan-out components.
+//!
+//! The crate exposes domain orchestration plus optional Kafka and infrastructure
+//! adapters. It intentionally contains no composition root and starts no tasks.
+
+/// Domain models, ports, and fan-out orchestration.
+pub mod domain;
+
+/// Inbound transport adapters.
+#[cfg(feature = "consumer")]
+pub mod inbound;
+
+/// Outbound infrastructure adapters.
+#[cfg(feature = "outbound")]
+pub mod outbound;
diff --git a/crates/soup_realtime/src/outbound/entity_access.rs b/crates/soup_realtime/src/outbound/entity_access.rs
new file mode 100644
index 0000000000..abe77312b1
--- /dev/null
+++ b/crates/soup_realtime/src/outbound/entity_access.rs
@@ -0,0 +1,73 @@
+//! Adapter from entity access to current realtime Soup recipients.
+
+#[cfg(test)]
+mod test;
+
+use std::future::Future;
+
+use entity_access::domain::{models::AccessError, ports::EntityAccessService};
+use macro_user_id::user_id::MacroUserIdStr;
+use model_entity::{Entity, EntityType};
+use rootcause::prelude::{Report, ResultExt as _};
+
+use crate::domain::ports::UserAccessExpander;
+
+/// Narrow seam over the external entity-access service used by this adapter.
+///
+/// Every [`EntityAccessService`] implements this trait automatically. Keeping
+/// the seam narrow also allows adapter tests to avoid constructing unrelated
+/// authorization capabilities.
+pub trait EntityAccessUserLookup: Send + Sync + 'static {
+ /// Returns all users with current access to the entity ID and type.
+ fn get_users_by_entity(
+ &self,
+ entity_id: &str,
+ entity_type: EntityType,
+ ) -> impl Future<Output = Result<Vec<MacroUserIdStr<'static>>, AccessError>> + Send;
+}
+
+impl<S> EntityAccessUserLookup for S
+where
+ S: EntityAccessService,
+{
+ async fn get_users_by_entity(
+ &self,
+ entity_id: &str,
+ entity_type: EntityType,
+ ) -> Result<Vec<MacroUserIdStr<'static>>, AccessError> {
+ EntityAccessService::get_users_by_entity(self, entity_id, entity_type).await
+ }
+}
+
+/// Current-access expander backed by an entity-access service.
+pub struct EntityAccessExpander<S> {
+ service: S,
+}
+
+impl<S> EntityAccessExpander<S> {
+ /// Creates an expander around an entity-access service.
+ pub fn new(service: S) -> Self {
+ Self { service }
+ }
+}
+
+impl<S> UserAccessExpander for EntityAccessExpander<S>
+where
+ S: EntityAccessUserLookup,
+{
+ #[tracing::instrument(
+ skip(self),
+ fields(entity_type = %entity.entity_type, entity_id = %entity.entity_id),
+ err
+ )]
+ async fn expand_user_access(
+ &self,
+ entity: &Entity<'static>,
+ ) -> Result<Vec<MacroUserIdStr<'static>>, Report> {
+ self.service
+ .get_users_by_entity(entity.entity_id.as_ref(), entity.entity_type)
+ .await
+ .context("failed to query current entity accessors")
+ .map_err(Into::into)
+ }
+}
diff --git a/crates/soup_realtime/src/outbound/entity_access/test.rs b/crates/soup_realtime/src/outbound/entity_access/test.rs
new file mode 100644
index 0000000000..2f07634786
--- /dev/null
+++ b/crates/soup_realtime/src/outbound/entity_access/test.rs
@@ -0,0 +1,50 @@
+use std::sync::{Arc, Mutex};
+
+use super::*;
+
+struct RecordingLookup {
+ calls: Arc<Mutex<Vec<(String, EntityType)>>>,
+ users: Vec<MacroUserIdStr<'static>>,
+}
+
+impl EntityAccessUserLookup for RecordingLookup {
+ async fn get_users_by_entity(
+ &self,
+ entity_id: &str,
+ entity_type: EntityType,
+ ) -> Result<Vec<MacroUserIdStr<'static>>, AccessError> {
+ self.calls
+ .lock()
+ .expect("calls lock")
+ .push((entity_id.to_string(), entity_type));
+ Ok(self.users.clone())
+ }
+}
+
+fn user(value: &str) -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from(value.to_string()).expect("valid user id")
+}
+
+#[tokio::test]
+async fn forwards_entity_and_returns_underlying_users() {
+ let calls = Arc::new(Mutex::new(Vec::new()));
+ let adapter = EntityAccessExpander::new(RecordingLookup {
+ calls: calls.clone(),
+ users: vec![user("macro|[email protected]"), user("macro|[email protected]")],
+ });
+ let entity =
+ EntityType::Document.with_entity_string("00000000-0000-0000-0000-000000000001".to_string());
+
+ let users = adapter
+ .expand_user_access(&entity)
+ .await
+ .expect("access expansion succeeds");
+
+ assert_eq!(users.len(), 2);
+ assert_eq!(users[0].as_ref(), "macro|[email protected]");
+ assert_eq!(users[1].as_ref(), "macro|[email protected]");
+ assert_eq!(
+ calls.lock().expect("calls lock").as_slice(),
+ &[(entity.entity_id.to_string(), EntityType::Document)]
+ );
+}
diff --git a/crates/soup_realtime/src/outbound/kafka_publisher.rs b/crates/soup_realtime/src/outbound/kafka_publisher.rs
new file mode 100644
index 0000000000..5b1a8987da
--- /dev/null
+++ b/crates/soup_realtime/src/outbound/kafka_publisher.rs
@@ -0,0 +1,44 @@
+//! Kafka publisher for versioned realtime Soup messages.
+
+#[cfg(test)]
+mod test;
+
+use macro_event_broker::EventPublisher;
+use macro_event_topics::MacroSoupRealtimeTopic;
+use rootcause::prelude::{Report, ResultExt as _};
+
+use crate::domain::{models::SoupRealtimeMessage, ports::SoupRealtimePublisher};
+
+/// Realtime Soup publisher backed by a byte-oriented event publisher.
+pub struct KafkaSoupRealtimePublisher<P> {
+ publisher: P,
+}
+
+impl<P> KafkaSoupRealtimePublisher<P> {
+ /// Creates a realtime Soup Kafka publisher.
+ pub fn new(publisher: P) -> Self {
+ Self { publisher }
+ }
+}
+
+impl<P> SoupRealtimePublisher for KafkaSoupRealtimePublisher<P>
+where
+ P: EventPublisher,
+{
+ #[tracing::instrument(
+ skip(self, message),
+ fields(user_id = %message.user_id),
+ err
+ )]
+ async fn publish(&self, message: SoupRealtimeMessage) -> Result<(), Report> {
+ let payload =
+ serde_json::to_vec(&message).context("failed to serialize realtime Soup message")?;
+
+ self.publisher
+ .publish(MacroSoupRealtimeTopic, message.user_id.as_ref(), &payload)
+ .await
+ .context("failed to publish realtime Soup message")?;
+
+ Ok(())
+ }
+}
diff --git a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs
new file mode 100644
index 0000000000..a16489484c
--- /dev/null
+++ b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs
@@ -0,0 +1,124 @@
+use std::sync::{Arc, Mutex};
+
+use chrono::{DateTime, Utc};
+use macro_event_broker::{EventBrokerError, Topic};
+use macro_user_id::user_id::MacroUserIdStr;
+use models_soup::{document::SoupDocument, item::SoupItem};
+use uuid::Uuid;
+
+use super::*;
+
+#[derive(Debug)]
+struct PublishedRecord {
+ topic: String,
+ key: String,
+ payload: Vec<u8>,
+}
+
+struct RecordingPublisher {
+ records: Arc<Mutex<Vec<PublishedRecord>>>,
+ fail: bool,
+}
+
+impl EventPublisher for RecordingPublisher {
+ async fn publish<T: Topic>(
+ &self,
+ topic: T,
+ key: &str,
+ payload: &[u8],
+ ) -> Result<(), EventBrokerError> {
+ self.records
+ .lock()
+ .expect("records lock")
+ .push(PublishedRecord {
+ topic: topic.as_str().to_string(),
+ key: key.to_string(),
+ payload: payload.to_vec(),
+ });
+ if self.fail {
+ Err(EventBrokerError::Publish("broker unavailable".to_string()))
+ } else {
+ Ok(())
+ }
+ }
+}
+
+fn user() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")
+}
+
+fn timestamp(seconds: i64) -> DateTime<Utc> {
+ DateTime::from_timestamp(seconds, 0).expect("valid timestamp")
+}
+
+fn message() -> SoupRealtimeMessage {
+ SoupRealtimeMessage::new(
+ user(),
+ SoupItem::Document(SoupDocument {
+ id: Uuid::parse_str("00000000-0000-0000-0000-000000000001").expect("valid document id"),
+ document_version_id: 9,
+ owner_id: user(),
+ name: "Published document".to_string(),
+ file_type: Some("md".to_string()),
+ sha: Some("sha".to_string()),
+ project_id: None,
+ branched_from_id: None,
+ branched_from_version_id: None,
+ document_family_id: None,
+ created_at: timestamp(1),
+ updated_at: timestamp(2),
+ viewed_at: Some(timestamp(3)),
+ sub_type: None,
+ deleted_at: None,
+ extra: (),
+ }),
+ )
+}
+
+#[tokio::test]
+async fn publishes_user_keyed_full_message_to_realtime_topic() {
+ let records = Arc::new(Mutex::new(Vec::new()));
+ let adapter = KafkaSoupRealtimePublisher::new(RecordingPublisher {
+ records: records.clone(),
+ fail: false,
+ });
+
+ adapter.publish(message()).await.expect("publish succeeds");
+
+ let mut records = records.lock().expect("records lock");
+ let record = records.pop().expect("one record");
+ assert!(records.is_empty());
+ assert_eq!(record.topic, "macro.soup");
+ assert_eq!(record.key, "macro|[email protected]");
+
+ let json: serde_json::Value = serde_json::from_slice(&record.payload).expect("payload is JSON");
+ assert_eq!(json["schema_version"], 1);
+ assert_eq!(json["user_id"], "macro|[email protected]");
+ assert!(json.get("event_id").is_none());
+
+ let decoded: SoupRealtimeMessage =
+ serde_json::from_slice(&record.payload).expect("payload round-trips");
+ assert_eq!(decoded.schema_version, 1);
+ assert_eq!(decoded.user_id.as_ref(), "macro|[email protected]");
+ match decoded.item {
+ SoupItem::Document(document) => {
+ assert_eq!(document.name, "Published document");
+ assert_eq!(document.document_version_id, 9);
+ assert_eq!(document.viewed_at, Some(timestamp(3)));
+ }
+ _ => panic!("expected document item"),
+ }
+}
+
+#[tokio::test]
+async fn propagates_delivery_failures() {
+ let adapter = KafkaSoupRealtimePublisher::new(RecordingPublisher {
+ records: Arc::new(Mutex::new(Vec::new())),
+ fail: true,
+ });
+
+ adapter
+ .publish(message())
+ .await
+ .expect_err("delivery failure propagates");
+}
diff --git a/crates/soup_realtime/src/outbound/mod.rs b/crates/soup_realtime/src/outbound/mod.rs
new file mode 100644
index 0000000000..01d025d389
--- /dev/null
+++ b/crates/soup_realtime/src/outbound/mod.rs
@@ -0,0 +1,8 @@
+//! Outbound adapters used by realtime Soup fan-out.
+
+/// Current entity-access expansion adapter.
+pub mod entity_access;
+/// Kafka-backed realtime Soup publisher adapter.
+pub mod kafka_publisher;
+/// User-scoped Soup item reader adapter.
+pub mod soup_item_reader;
diff --git a/crates/soup_realtime/src/outbound/soup_item_reader.rs b/crates/soup_realtime/src/outbound/soup_item_reader.rs
new file mode 100644
index 0000000000..f72c37c6b4
--- /dev/null
+++ b/crates/soup_realtime/src/outbound/soup_item_reader.rs
@@ -0,0 +1,76 @@
+//! Adapter for loading complete, user-scoped Soup items.
+
+#[cfg(test)]
+mod test;
+
+use model_entity::{Entity, EntityType};
+use models_soup::item::SoupItem;
+use rootcause::prelude::{Report, ResultExt as _};
+use soup::domain::{models::AdvancedSortParams, ports::SoupRepo};
+
+use crate::domain::ports::SoupItemReader;
+
+/// Soup item reader backed by an existing [`SoupRepo`].
+pub struct SoupRepoItemReader<R> {
+ repo: R,
+}
+
+impl<R> SoupRepoItemReader<R> {
+ /// Creates a user-scoped item reader around a Soup repository.
+ pub fn new(repo: R) -> Self {
+ Self { repo }
+ }
+}
+
+impl<R> SoupItemReader for SoupRepoItemReader<R>
+where
+ R: SoupRepo,
+ R::Err: std::error::Error + Send + Sync + 'static,
+{
+ #[tracing::instrument(
+ skip(self),
+ fields(
+ user_id = %user_id,
+ entity_type = %entity.entity_type,
+ entity_id = %entity.entity_id,
+ ),
+ err
+ )]
+ async fn read_for_user(
+ &self,
+ user_id: macro_user_id::user_id::MacroUserIdStr<'static>,
+ entity: &Entity<'static>,
+ ) -> Result<Option<SoupItem<()>>, Report> {
+ if entity.entity_type != EntityType::Document {
+ return Err(rootcause::report!(
+ "realtime Soup reader does not support entity type {}",
+ entity.entity_type
+ ));
+ }
+
+ let requested_entities = [entity.clone()];
+ let items = self
+ .repo
+ .expanded_soup_by_ids(AdvancedSortParams {
+ entities: &requested_entities,
+ user_id,
+ })
+ .await
+ .context("failed to read expanded user-scoped Soup item")?;
+
+ let mut matches = items.into_iter().filter(|item| item.entity() == *entity);
+ let Some(item) = matches.next() else {
+ return Ok(None);
+ };
+
+ if matches.next().is_some() {
+ return Err(rootcause::report!(
+ "expanded Soup query returned duplicate items for {} {}",
+ entity.entity_type,
+ entity.entity_id
+ ));
+ }
+
+ Ok(Some(item))
+ }
+}
diff --git a/crates/soup_realtime/src/outbound/soup_item_reader/test.rs b/crates/soup_realtime/src/outbound/soup_item_reader/test.rs
new file mode 100644
index 0000000000..c58cc7e8fb
--- /dev/null
+++ b/crates/soup_realtime/src/outbound/soup_item_reader/test.rs
@@ -0,0 +1,213 @@
+use std::io;
+use std::sync::{Arc, Mutex};
+
+use chrono::{DateTime, Utc};
+use macro_user_id::user_id::MacroUserIdStr;
+use models_properties::service::property_definition_with_options::PropertyDefinitionWithOptions;
+use models_soup::{document::SoupDocument, item::SoupItem};
+use soup::domain::{
+ models::{
+ AdvancedSortParams, GroupedSortRequest, SimpleSortRequest, SoupPropertiesField,
+ grouping::ItemGroupingInfo,
+ },
+ ports::SoupRepo,
+};
+use uuid::Uuid;
+
+use super::*;
+
+const DOCUMENT_ID: &str = "00000000-0000-0000-0000-000000000001";
+
+type RecordedQuery = (String, Vec<(String, EntityType)>);
+type RecordedQueries = Arc<Mutex<Vec<RecordedQuery>>>;
+
+struct RecordingSoupRepo {
+ calls: RecordedQueries,
+ items: Mutex<Option<Result<Vec<SoupItem<()>>, io::Error>>>,
+}
+
+impl SoupRepo for RecordingSoupRepo {
+ type Err = io::Error;
+ type GroupedItems = std::vec::IntoIter<ItemGroupingInfo>;
+
+ async fn expanded_generic_cursor_soup<'a>(
+ &self,
+ _req: SimpleSortRequest<'a>,
+ ) -> Result<Vec<SoupItem<()>>, Self::Err> {
+ unreachable!("unexpected generic cursor query")
+ }
+
+ async fn unexpanded_generic_cursor_soup<'a>(
+ &self,
+ _req: SimpleSortRequest<'a>,
+ ) -> Result<Vec<SoupItem<()>>, Self::Err> {
+ unreachable!("unexpected generic cursor query")
+ }
+
+ async fn expanded_soup_by_ids<'a>(
+ &self,
+ req: AdvancedSortParams<'a>,
+ ) -> Result<Vec<SoupItem<()>>, Self::Err> {
+ self.calls.lock().expect("calls lock").push((
+ req.user_id.as_ref().to_string(),
+ req.entities
+ .iter()
+ .map(|entity| (entity.entity_id.to_string(), entity.entity_type))
+ .collect(),
+ ));
+ self.items
+ .lock()
+ .expect("items lock")
+ .take()
+ .expect("one expanded query expected")
+ }
+
+ async fn unexpanded_soup_by_ids<'a>(
+ &self,
+ _req: AdvancedSortParams<'a>,
+ ) -> Result<Vec<SoupItem<()>>, Self::Err> {
+ unreachable!("unexpected unexpanded query")
+ }
+
+ async fn populate_properties<'a>(
+ &self,
+ _user_id: MacroUserIdStr<'a>,
+ _items: Vec<SoupItem<()>>,
+ ) -> Result<Vec<SoupItem<SoupPropertiesField>>, Self::Err> {
+ unreachable!("unexpected properties query")
+ }
+
+ async fn caller_tag_sets<'a>(
+ &self,
+ _user_id: MacroUserIdStr<'a>,
+ ) -> Result<Vec<PropertyDefinitionWithOptions>, Self::Err> {
+ unreachable!("unexpected tag query")
+ }
+
+ async fn expanded_grouped_cursor_soup<'a>(
+ &self,
+ _req: GroupedSortRequest<'a>,
+ ) -> Result<Self::GroupedItems, Self::Err> {
+ unreachable!("unexpected grouped query")
+ }
+}
+
+fn user() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")
+}
+
+fn timestamp(seconds: i64) -> DateTime<Utc> {
+ DateTime::from_timestamp(seconds, 0).expect("valid timestamp")
+}
+
+fn document_item(id: &str, viewed_at: Option<DateTime<Utc>>) -> SoupItem<()> {
+ SoupItem::Document(SoupDocument {
+ id: Uuid::parse_str(id).expect("valid document id"),
+ document_version_id: 7,
+ owner_id: user(),
+ name: "Realtime document".to_string(),
+ file_type: Some("md".to_string()),
+ sha: None,
+ project_id: None,
+ branched_from_id: None,
+ branched_from_version_id: None,
+ document_family_id: None,
+ created_at: timestamp(1),
+ updated_at: timestamp(2),
+ viewed_at,
+ sub_type: None,
+ deleted_at: None,
+ extra: (),
+ })
+}
+
+fn reader_with(
+ items: Result<Vec<SoupItem<()>>, io::Error>,
+) -> (SoupRepoItemReader<RecordingSoupRepo>, RecordedQueries) {
+ let calls = Arc::new(Mutex::new(Vec::new()));
+ let repo = RecordingSoupRepo {
+ calls: calls.clone(),
+ items: Mutex::new(Some(items)),
+ };
+ (SoupRepoItemReader::new(repo), calls)
+}
+
+#[tokio::test]
+async fn uses_expanded_query_for_requested_user_and_entity() {
+ let viewed_at = Some(timestamp(3));
+ let (reader, calls) = reader_with(Ok(vec![document_item(DOCUMENT_ID, viewed_at)]));
+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());
+
+ let item = reader
+ .read_for_user(user(), &entity)
+ .await
+ .expect("read succeeds")
+ .expect("item exists");
+
+ match item {
+ SoupItem::Document(document) => {
+ assert_eq!(document.name, "Realtime document");
+ assert_eq!(document.viewed_at, viewed_at);
+ }
+ _ => panic!("expected document item"),
+ }
+ assert_eq!(
+ calls.lock().expect("calls lock").as_slice(),
+ &[(
+ "macro|[email protected]".to_string(),
+ vec![(DOCUMENT_ID.to_string(), EntityType::Document)]
+ )]
+ );
+}
+
+#[tokio::test]
+async fn returns_none_when_requested_item_is_absent() {
+ let other_id = "00000000-0000-0000-0000-000000000002";
+ let (reader, _) = reader_with(Ok(vec![document_item(other_id, None)]));
+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());
+
+ assert!(
+ reader
+ .read_for_user(user(), &entity)
+ .await
+ .expect("read succeeds")
+ .is_none()
+ );
+}
+
+#[tokio::test]
+async fn rejects_duplicate_matching_items() {
+ let (reader, _) = reader_with(Ok(vec![
+ document_item(DOCUMENT_ID, None),
+ document_item(DOCUMENT_ID, Some(timestamp(4))),
+ ]));
+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());
+
+ reader
+ .read_for_user(user(), &entity)
+ .await
+ .expect_err("duplicate items violate the repository contract");
+}
+
+#[tokio::test]
+async fn rejects_non_document_entities_without_querying() {
+ let (reader, calls) = reader_with(Ok(Vec::new()));
+ let entity = EntityType::Project.with_entity_string(DOCUMENT_ID.to_string());
+
+ reader
+ .read_for_user(user(), &entity)
+ .await
+ .expect_err("projects are unsupported");
+ assert!(calls.lock().expect("calls lock").is_empty());
+}
+
+#[tokio::test]
+async fn propagates_repository_failures() {
+ let (reader, _) = reader_with(Err(io::Error::other("query unavailable")));
+ let entity = EntityType::Document.with_entity_string(DOCUMENT_ID.to_string());
+
+ reader
+ .read_for_user(user(), &entity)
+ .await
+ .expect_err("repository failure propagates");
+}
diff --git a/crates/webhook/Cargo.toml b/crates/webhook/Cargo.toml
index 74c9eb2922..c5f68ced18 100644
--- a/crates/webhook/Cargo.toml
+++ b/crates/webhook/Cargo.toml
@@ -6,11 +6,10 @@ version = "0.1.0"
[features]
consumer = [
- "dep:macro_env",
+ "dep:kafka_consumer_util",
"dep:rdkafka",
"dep:tokio",
"ingestion",
- "macro_event_broker/outbound",
]
default = ["consumer", "inbound", "outbound", "ports"]
inbound = [
@@ -61,10 +60,10 @@ entity_access = { path = "../entity_access", default-features = false, features
futures = { workspace = true, optional = true }
hex = { workspace = true, optional = true }
hmac = { workspace = true, optional = true }
+kafka_consumer_util = { path = "../kafka_consumer_util", optional = true }
macro_authorization = { path = "../macro_authorization", optional = true, default-features = false, features = [
"axum",
] }
-macro_env = { path = "../macro_env", optional = true }
macro_event_broker = { path = "../macro_event_broker", default-features = false }
macro_event_topics = { path = "../macro_event_topics" }
macro_user_id = { path = "../macro_user_id" }
diff --git a/crates/webhook/src/inbound/kafka_consumer.rs b/crates/webhook/src/inbound/kafka_consumer.rs
index c656668031..8327fee250 100644
--- a/crates/webhook/src/inbound/kafka_consumer.rs
+++ b/crates/webhook/src/inbound/kafka_consumer.rs
@@ -25,20 +25,23 @@ use crate::domain::{events::WebhookTopicEvent, ingestion::WebhookEventIngestionS
use anyhow::Context as _;
use channels::domain::broker_events::ChannelTopicEvent;
use documents::domain::events::DocumentTopicEvent;
-use macro_env::Environment;
-use macro_event_broker::outbound::msk_iam::configure_sasl_iam;
-use macro_event_broker::{Event, EventBrokerError, MskIamClientContext, Topic as _};
+use kafka_consumer_util::{GroupName, KafkaEventConsumer};
+use macro_event_broker::{Event, EventBrokerError, Topic as _};
use macro_event_topics::{MacroChannelsTopic, MacroDocumentsTopic, MacroWebhooksTopic};
-use rdkafka::ClientConfig;
-use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};
-use rdkafka::error::KafkaResult;
+use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message};
use std::future::Future;
use std::time::Duration;
-/// Consumer group id for webhook event ingestion. Offsets are committed under
+/// Consumer group for webhook event ingestion. Offsets are committed under
/// this group, so restarts resume where the previous run left off.
-const GROUP_ID: &str = "webhook-event-ingestion";
+struct WebhookEventIngestionConsumerGroup;
+
+impl GroupName for WebhookEventIngestionConsumerGroup {
+ const GROUP_NAME: &'static str = "webhook-event-ingestion";
+}
+
+type WebhookKafkaConsumer = KafkaEventConsumer<WebhookEventIngestionConsumerGroup>;
/// Maximum in-process ingestion attempts per event before the consumer bails
/// out and lets a restart redeliver from the last committed offset.
@@ -90,85 +93,20 @@ impl WebhookConsumerEvent {
}
}
-/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).
-enum WebhookKafkaConsumer {
- /// Unauthenticated plaintext connection (local docker broker).
- Plaintext(StreamConsumer),
- /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).
- MskIam(StreamConsumer<MskIamClientContext>),
-}
-
-impl WebhookKafkaConsumer {
- /// Build a consumer for the given brokers, choosing the transport from the
- /// `ENVIRONMENT` variable exactly like the publisher does.
- fn from_env(brokers: &str) -> anyhow::Result<Self> {
- let mut config = ClientConfig::new();
- config
- .set("bootstrap.servers", brokers)
- .set("group.id", GROUP_ID)
- // Offsets are committed manually after successful ingestion.
- .set("enable.auto.commit", "false")
- // Start from the beginning of the topics on the first ever run.
- .set("auto.offset.reset", "earliest");
-
- let consumer = match Environment::new_or_prod() {
- Environment::Local => Self::Plaintext(
- config
- .create()
- .context("failed to create plaintext kafka consumer")?,
- ),
- Environment::Develop | Environment::Production => {
- configure_sasl_iam(&mut config);
- Self::MskIam(
- config
- .create_with_context(MskIamClientContext::from_env())
- .context("failed to create MSK IAM kafka consumer")?,
- )
- }
- };
-
- Ok(consumer)
- }
-
- fn subscribe(&self) -> KafkaResult<()> {
- let topics = subscribed_topics();
- match self {
- Self::Plaintext(consumer) => consumer.subscribe(&topics),
- Self::MskIam(consumer) => consumer.subscribe(&topics),
- }
- }
-
- /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it
- /// can sit in a `select!` without losing messages.
- async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {
- match self {
- Self::Plaintext(consumer) => consumer.recv().await,
- Self::MskIam(consumer) => consumer.recv().await,
- }
- }
-
- fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {
- match self {
- Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),
- Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),
- }
- }
-
- /// Commit `message`'s offset, logging the outcome.
- fn commit_logged(&self, message: &BorrowedMessage<'_>) {
- match self.commit(message) {
- Ok(()) => tracing::trace!(
- partition = message.partition(),
- offset = message.offset(),
- "committed offset"
- ),
- Err(e) => tracing::error!(
- error = ?e,
- partition = message.partition(),
- offset = message.offset(),
- "failed to commit offset"
- ),
- }
+/// Commit `message`'s offset, logging the outcome.
+fn commit_logged(consumer: &WebhookKafkaConsumer, message: &BorrowedMessage<'_>) {
+ match consumer.commit_message(message, CommitMode::Async) {
+ Ok(()) => tracing::trace!(
+ partition = message.partition(),
+ offset = message.offset(),
+ "committed offset"
+ ),
+ Err(error) => tracing::error!(
+ error = ?error,
+ partition = message.partition(),
+ offset = message.offset(),
+ "failed to commit offset"
+ ),
}
}
@@ -261,11 +199,11 @@ where
{
let consumer = WebhookKafkaConsumer::from_env(brokers)?;
consumer
- .subscribe()
+ .subscribe(&subscribed_topics())
.context("failed to subscribe to webhook event topics")?;
tracing::info!(
topics = ?subscribed_topics(),
- group = GROUP_ID,
+ group = WebhookEventIngestionConsumerGroup::GROUP_NAME,
"webhook event consumer listening"
);
@@ -298,7 +236,7 @@ where
offset = message.offset(),
"skipping message with empty payload"
);
- consumer.commit_logged(&message);
+ commit_logged(&consumer, &message);
continue;
};
@@ -330,7 +268,7 @@ where
// Commit only after the event was ingested or permanently
// rejected: at-least-once, retried across restarts.
- consumer.commit_logged(&message);
+ commit_logged(&consumer, &message);
}
}
}
diff --git a/docs/cache-record-selection-plan.md b/docs/cache-record-selection-plan.md
deleted file mode 100644
index b74444f44e..0000000000
--- a/docs/cache-record-selection-plan.md
+++ /dev/null
@@ -1,173 +0,0 @@
-# Cache Record Selection Implementation Plan
-
-## Proposed API
-
-```ts
-const selection = selectRecords(SoupItemFieldsFragmentDoc);
-
-const page = await readRecords(cacheHost, selection, {
- cursor,
- limit: 500,
-});
-```
-
-```ts
-type SelectedRecordPage<T> = {
- records: T[];
- nextCursor: string | null;
-};
-```
-
-`nextCursor` is present only when another complete matching record exists.
-
-## Implementation plan
-
-### 1. Remove the schema-specific entity index
-
-- Delete `cache-core::entity_index` and its tests.
-- Remove bucket, timestamp, search-text, scoring, counting, and snapshot APIs.
-- Restore ordinary record persistence in:
- - `cache-idb`
- - `cache-sqlite`
- - `cache-wasm`
-- Remove `queryIndexedItems` from WASM, Tauri, worker, host, and protocol layers.
-- Remove projected SQLite columns and IndexedDB envelopes/indexes.
-- Restore the unreleased feature-branch storage versions rather than supporting migration from them; normalized records are disposable.
-
-### 2. Add generic fragment-based record selection
-
-In `cache-core`:
-
-- Extend document parsing to support named, fragment-only documents.
-- Add a `record_selection` module containing:
- - Fragment validation.
- - Object/interface/union type-condition resolution.
- - Validation that selected concrete types are normalized entities.
- - Cursor and page models.
-- Reject:
- - Unknown fragments or schema types.
- - Embedded/non-normalized type conditions.
- - Unbound variables in fragment fields.
- - Invalid or excessive page limits.
-
-Add:
-
-```rust
-Engine::read_records(selection, cursor, limit)
-```
-
-The engine will:
-
-1. Derive concrete type names from the fragment condition.
-2. Iterate matching records in deterministic entity-key order.
-3. Apply the fragment projection using normal denormalization.
-4. Load linked cold records in batches.
-5. Read the effective view, including optimistic layers.
-6. Skip records that cannot satisfy the complete fragment.
-7. Use one-record lookahead to decide whether to return `nextCursor`.
-
-### 3. Add schema-neutral storage iteration
-
-Replace the specialized index methods with one generic storage capability:
-
-```rust
-scan_records(type_names, after, limit)
-```
-
-Implement it for:
-
-- `InMemoryStorage`
-- SQLite, using ordered primary-key ranges
-- IndexedDB, using object-store key cursors
-
-No schema field names or application-level ordering metadata will be persisted.
-
-### 4. Surface the API through every host
-
-Add `readRecords` across:
-
-- `cache-wasm`
-- SharedWorker protocol and worker core
-- Tauri commands/plugin
-- `CacheHost`
-- No-op host
-
-Wire input contains only:
-
-- Serialized fragment document
-- Fragment name
-- Opaque cursor
-- Limit
-
-Wire output contains only selected JSON records and `nextCursor`.
-
-### 5. Add the typed frontend selection API
-
-Create `exchange/record-selection.ts` with:
-
-- `selectRecords(fragmentDocument)`
-- `readRecords(host, selection, options)`
-- Generated result-type inference
-- Fragment-name extraction
-- Defensive wire-result validation
-
-Export it from `@graphql-cache/index`.
-
-### 6. Replace index notifications
-
-Replace `onEntityIndexChanged` with schema-neutral `onCacheChanged`.
-
-Emit it for:
-
-- Normal writes
-- Optimistic writes and settlements
-- External invalidations
-- Queue refreshes that change the effective view
-- Cache clears/resets
-
-Record-selection consumers can invalidate and rerun without cache-core knowing their schema.
-
-### 7. Adapt Quick Access
-
-- Select cached `GraphqlSoupItem` records using the existing generated `SoupItemFieldsFragmentDoc`.
-- Export/reuse the existing GraphQL Soup item mapper.
-- Load record pages and map them into `QuickAccessItem`s.
-- Perform bucket filtering, timestamp sorting, fuzzy search, counts, and visible-page slicing entirely in TypeScript.
-- Make `loadMore` increase the locally visible slice after records have been loaded.
-- Rename index-specific files, query keys, and flags to record-selection terminology.
-- Remove indexed-snapshot mapping and tests.
-
-### 8. Tests and verification
-
-Add coverage for:
-
-- Object and union fragment selection.
-- Aliases and nested linked records.
-- Cold storage reads.
-- Optimistic records and updates.
-- Incomplete-record omission.
-- Stable, exclusive cursor pagination.
-- `nextCursor` lookahead semantics.
-- SQLite and IndexedDB parity.
-- WASM, worker, Tauri, and no-op host transport.
-- TypeScript result inference and malformed responses.
-- Quick Access filtering, sorting, searching, counting, and pagination.
-
-Verification:
-
-```bash
-cargo fmt
-cargo test -p cache-core
-cargo test -p cache-sqlite
-cargo test -p graphql_cache_plugin
-cargo check --target wasm32-unknown-unknown -p cache-idb -p cache-wasm --all-targets
-wasm-pack test --headless --chrome crates/client/cache-idb
-```
-
-Then run focused web tests and `bun check` from `apps/web`.
-
-Implementation will proceed incrementally: verify the generic Rust API first, then wire and consume it in Quick Access. After successful verification, create the required `jj` revision.
-
-## Architecture check
-
-Hexagonal boundaries were checked: these are client cache/transport adapters with no server-domain authorization or business-policy movement.