Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(kafka): wire up documents with macro event broker (#4552)
Will Hutchinsond676f8a9131c
diff --git a/.github/kafka-cluster-topics.json b/.github/kafka-cluster-topics.jsonindex f279549082..fa885291eb 100644--- a/.github/kafka-cluster-topics.json+++ b/.github/kafka-cluster-topics.json@@ -1,3 +1,4 @@[+ "macro.documents","macro.example"]diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex e51c4b391d..08c8ca860c 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -62,6 +62,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_queues","macro_service_urls",@@ -204,6 +206,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_queues","macro_service_urls",@@ -345,6 +349,8 @@"macro_env","macro_env_var","macro_env_vars/database_env_vars",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_queues","macro_redis",@@ -639,6 +645,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_queues","macro_service_urls",@@ -1270,6 +1278,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_queues","macro_service_urls",@@ -1388,6 +1398,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_project_utils","macro_queues",@@ -1545,6 +1557,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_service_urls","macro_sync_service_jwt",@@ -1630,6 +1644,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_service_urls","macro_sync_service_jwt",@@ -2250,6 +2266,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_service_urls","macro_sync_service_jwt",@@ -2664,6 +2682,8 @@"workspace-hack"],"macro_event_broker": [+ "macro_env",+ "macro_env_var","macro_event_broker","macro_event_topics","workspace-hack"@@ -2845,6 +2865,8 @@"macro_env","macro_env_var","macro_env_vars/database_env_vars",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_queues","macro_service_urls",@@ -2945,6 +2967,8 @@"macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_queues","macro_service_urls",@@ -3750,6 +3774,8 @@"macro_env","macro_env_var","macro_env_vars/database_env_vars",+ "macro_event_broker",+ "macro_event_topics","macro_middleware","macro_queues","macro_service_urls",diff --git a/infra/packages/resources/src/resources/kafka-cluster.ts b/infra/packages/resources/src/resources/kafka-cluster.tsindex 59945d363a..b2240da9e7 100644--- a/infra/packages/resources/src/resources/kafka-cluster.ts+++ b/infra/packages/resources/src/resources/kafka-cluster.ts@@ -50,6 +50,7 @@ export class KafkaCluster extends pulumi.ComponentResource {public logGroup: aws.cloudwatch.LogGroup;public cluster: aws.msk.Cluster;public topics: Map<string, aws.msk.Topic>;+ public clientAccessPolicy: aws.iam.Policy;constructor(name: string,@@ -222,5 +223,65 @@ export class KafkaCluster extends pulumi.ComponentResource {));}++ // MSK IAM resource ARNs are derived from the cluster ARN:+ // arn:aws:kafka:REGION:ACCT:cluster/NAME/UUID → arn:aws:kafka:REGION:ACCT:topic/NAME/UUID/<topic>+ // so a trailing /* covers every topic (and group) on this cluster.+ const allTopicArns = this.cluster.arn.apply(+ (arn) => `${arn.replace(':cluster/', ':topic/')}/*`+ );+ const allGroupArns = this.cluster.arn.apply(+ (arn) => `${arn.replace(':cluster/', ':group/')}/*`+ );++ // Managed policy granting full client (producer + consumer) access to this+ // cluster and all of its topics and consumer groups. Attach it to the ECS+ // task role of any service that publishes or consumes events. Topic+ // management (create/alter/delete) is deliberately excluded — topics are+ // owned by this component.+ this.clientAccessPolicy = new aws.iam.Policy(+ 'client-access-policy',+ {+ name: `${name}-client-access`,+ description: `Producer + consumer access to the ${name} Kafka cluster and all its topics and consumer groups`,+ policy: {+ Version: '2012-10-17',+ Statement: [+ {+ Sid: 'Cluster',+ Effect: 'Allow',+ Action: [+ 'kafka-cluster:Connect',+ 'kafka-cluster:DescribeCluster',+ // Cluster-scoped by MSK; needed if a producer enables idempotence.+ 'kafka-cluster:WriteDataIdempotently',+ ],+ Resource: [this.cluster.arn],+ },+ {+ Sid: 'Topics',+ Effect: 'Allow',+ Action: [+ 'kafka-cluster:DescribeTopic',+ 'kafka-cluster:ReadData',+ 'kafka-cluster:WriteData',+ ],+ Resource: [allTopicArns],+ },+ {+ Sid: 'Groups',+ Effect: 'Allow',+ Action: [+ 'kafka-cluster:DescribeGroup',+ 'kafka-cluster:AlterGroup',+ ],+ Resource: [allGroupArns],+ },+ ],+ },+ tags: this.tags,+ },+ { parent: this }+ );}}diff --git a/infra/packages/shared/src/index.ts b/infra/packages/shared/src/index.tsindex 0fcad65d1f..edf2ddbe03 100644--- a/infra/packages/shared/src/index.ts+++ b/infra/packages/shared/src/index.ts@@ -50,5 +50,6 @@ export {getAiToolsServiceRoleArns,type AiToolsInfra,} from './ai_tools';+export { getKafkaClusterPolicy } from './kafka_cluster_policy';export * from './service_urls';diff --git a/infra/packages/shared/src/kafka_cluster_policy.ts b/infra/packages/shared/src/kafka_cluster_policy.tsnew file mode 100644index 0000000000..dc45f5ce46--- /dev/null+++ b/infra/packages/shared/src/kafka_cluster_policy.ts@@ -0,0 +1,12 @@+import * as pulumi from '@pulumi/pulumi';+import { stack } from '../../shared';++export function getKafkaClusterPolicy(): pulumi.Output<string> {+ const kafkaClusterStack = new pulumi.StackReference('kafka-cluster-stack', {+ name: `macro-inc/kafka-cluster/${stack}`,+ });++ return kafkaClusterStack+ .getOutput('clientAccessPolicyArn')+ .apply((arn) => arn as string);+}diff --git a/infra/stacks/agent-schedule-service/service.ts b/infra/stacks/agent-schedule-service/service.tsindex 6914a5405e..5cea2f7219 100644--- a/infra/stacks/agent-schedule-service/service.ts+++ b/infra/stacks/agent-schedule-service/service.ts@@ -14,6 +14,7 @@ import {BASE_DOMAIN,CLOUD_TRAIL_SNS_TOPIC_ARN,DopplerEcsEnvironment,+ getKafkaClusterPolicy,stack,} from '../../packages/shared';@@ -171,6 +172,8 @@ export class AgentScheduleService extends pulumi.ComponentResource {secretsManagerPolicy.arn,sqsPolicy.arn,s3Policy.arn,+ // Producer/consumer access to the macro event Kafka cluster.+ getKafkaClusterPolicy(),],tags,},diff --git a/infra/stacks/cloud-storage-service/cloud-storage-service.ts b/infra/stacks/cloud-storage-service/cloud-storage-service.tsindex 3eb72d95bc..021e490f24 100644--- a/infra/stacks/cloud-storage-service/cloud-storage-service.ts+++ b/infra/stacks/cloud-storage-service/cloud-storage-service.ts@@ -17,6 +17,7 @@ import {CLOUD_TRAIL_SNS_TOPIC_ARN,SERVICE_DOMAIN_NAME,DopplerEcsEnvironment,+ getKafkaClusterPolicy,stack,} from '../../packages/shared';@@ -266,6 +267,16 @@ export class CloudStorageService extends pulumi.ComponentResource {{ parent: this, dependsOn: [snsPolicy, this.role] });+ // Producer/consumer access to the macro event Kafka cluster.+ new aws.iam.RolePolicyAttachment(+ `${BASE_NAME}-role-kafka-client-att-${stack}`,+ {+ role: this.role,+ policyArn: getKafkaClusterPolicy(),+ },+ { parent: this, dependsOn: [this.role] }+ );+// ecr imageconst image = new EcrImage(`${BASE_NAME}-ecr-image-${stack}`,diff --git a/infra/stacks/document-cognition-service/document-cognition-service.ts b/infra/stacks/document-cognition-service/document-cognition-service.tsindex f4f2fbee25..beb4ab5986 100644--- a/infra/stacks/document-cognition-service/document-cognition-service.ts+++ b/infra/stacks/document-cognition-service/document-cognition-service.ts@@ -15,6 +15,7 @@ import {BASE_DOMAIN,CLOUD_TRAIL_SNS_TOPIC_ARN,DopplerEcsEnvironment,+ getKafkaClusterPolicy,stack,} from '../../packages/shared';@@ -219,6 +220,8 @@ export class DocumentCognitionService extends pulumi.ComponentResource {secretsManagerPolicy.arn,s3Policy.arn,connectionTablePolicyArn,+ // Producer/consumer access to the macro event Kafka cluster.+ getKafkaClusterPolicy(),],tags: this.tags,},diff --git a/infra/stacks/kafka-cluster/index.ts b/infra/stacks/kafka-cluster/index.tsindex 396e20bedf..ad62e36dd7 100644--- a/infra/stacks/kafka-cluster/index.ts+++ b/infra/stacks/kafka-cluster/index.ts@@ -28,3 +28,17 @@ export const clusterArn = kafkaCluster.cluster.arn;export const bootstrapBrokersSaslIam =kafkaCluster.cluster.bootstrapBrokersSaslIam;export const securityGroupId = kafkaCluster.securityGroup.id;++// Producer + consumer access to the cluster and all its topics/groups. Attach+// to the ECS task role of any service that publishes or consumes events:+//+// const kafkaClusterStack = new pulumi.StackReference('kafka-cluster', {+// name: `macro-inc/kafka-cluster/${stack}`,+// });+// new aws.iam.RolePolicyAttachment('kafka-client-access', {+// role: taskRole.name,+// policyArn: kafkaClusterStack+// .getOutput('clientAccessPolicyArn')+// .apply((arn) => arn as string),+// });+export const clientAccessPolicyArn = kafkaCluster.clientAccessPolicy.arn;diff --git a/infra/stacks/mcp-server/mcp-server.ts b/infra/stacks/mcp-server/mcp-server.tsindex 8aa1faf2d1..38ed97f1cb 100644--- a/infra/stacks/mcp-server/mcp-server.ts+++ b/infra/stacks/mcp-server/mcp-server.ts@@ -14,6 +14,7 @@ import {BASE_DOMAIN,CLOUD_TRAIL_SNS_TOPIC_ARN,DopplerEcsEnvironment,+ getKafkaClusterPolicy,stack,} from '../../packages/shared';@@ -215,6 +216,8 @@ export class McpServer extends pulumi.ComponentResource {secretsManagerPolicy.arn,s3Policy.arn,sqsPolicy.arn,+ // Producer/consumer access to the macro event Kafka cluster.+ getKafkaClusterPolicy(),],tags: this.tags,},diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lockindex e6cdc9cb9e..1600b2643c 100644--- a/rust/cloud-storage/Cargo.lock+++ b/rust/cloud-storage/Cargo.lock@@ -202,6 +202,7 @@ dependencies = ["macro_db_client","macro_env","macro_env_var",+ "macro_event_broker","macro_queues","macro_service_urls","macro_user_id",@@ -1164,6 +1165,24 @@ dependencies = ["fs_extra",]+[[package]]+name = "aws-msk-iam-sasl-signer"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e62f2c07adf3f0bd1ad40414dd30fcf34c595d5f613e36011d5b62eeff14f490"+dependencies = [+ "aws-config",+ "aws-credential-types",+ "aws-sdk-sts",+ "aws-sigv4",+ "aws-types",+ "base64 0.22.1",+ "chrono",+ "futures",+ "thiserror 1.0.69",+ "url",+]+[[package]]name = "aws-runtime"version = "1.7.4"@@ -4022,6 +4041,7 @@ dependencies = ["macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker","macro_middleware","macro_queues","macro_service_urls",@@ -4137,6 +4157,7 @@ dependencies = ["macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker","macro_middleware","macro_project_utils","macro_queues",@@ -4315,7 +4336,10 @@ dependencies = ["macro_aws_config","macro_db_migrator","macro_entrypoint",+ "macro_env","macro_env_var",+ "macro_event_broker",+ "macro_event_topics","macro_service_urls","macro_sync_service_jwt","macro_user_id",@@ -4329,6 +4353,7 @@ dependencies = ["models_permissions","models_properties","non_empty",+ "rdkafka","reqwest 0.13.4","s3_key","schemars 1.2.1",@@ -4342,6 +4367,7 @@ dependencies = ["tokio","tower 0.5.3","tracing",+ "tracing-subscriber","unicode-segmentation","urlencoding","utoipa",@@ -7792,6 +7818,10 @@ dependencies = [name = "macro_event_broker"version = "0.1.0"dependencies = [+ "aws-msk-iam-sasl-signer",+ "aws-types",+ "macro_env",+ "macro_env_var","macro_event_topics","rdkafka","rootcause",@@ -8163,6 +8193,7 @@ dependencies = ["macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker","macro_middleware","macro_queues","macro_service_urls",@@ -15330,10 +15361,11 @@ version = "0.1.0"dependencies = ["ahash","aho-corasick",+ "aws-config","aws-credential-types","aws-runtime",+ "aws-sdk-sts","aws-sigv4",- "aws-smithy-async","aws-smithy-http","aws-smithy-http-client","aws-smithy-runtime",diff --git a/rust/cloud-storage/Cargo.toml b/rust/cloud-storage/Cargo.tomlindex 4d1739a548..daf941b52e 100644--- a/rust/cloud-storage/Cargo.toml+++ b/rust/cloud-storage/Cargo.toml@@ -90,6 +90,7 @@ async-stream = "0.3.6"async-trait = "0.1.86"automod = "1"aws-config = { version = "1.8.16", features = ["behavior-version-latest"] }+aws-msk-iam-sasl-signer = "1.0.1"aws-sdk-dynamodb = "1.111.0"aws-sdk-lambda = "1.122.0"aws-sdk-s3 = "1.132.0"@@ -98,6 +99,7 @@ aws-sdk-sesv2 = "1.118.0"aws-sdk-sns = "1.99.0"aws-sdk-sqs = "1.98.0"aws-smithy-types = "1.4.7"+aws-types = "1"aws_lambda_events = { version = "0.16.0", default-features = false }axum = { version = "0.8", features = ["tower-log"] }axum-extra = { version = "0.10" }diff --git a/rust/cloud-storage/ai_tools/Cargo.toml b/rust/cloud-storage/ai_tools/Cargo.tomlindex d7addaf828..0f97ae5709 100644--- a/rust/cloud-storage/ai_tools/Cargo.toml+++ b/rust/cloud-storage/ai_tools/Cargo.toml@@ -41,6 +41,7 @@ lexical_client = { path = "../lexical_client" }macro_aws_config = { path = "../macro_aws_config" }macro_db_client = { path = "../macro_db_client" }macro_env_var = { path = "../macro_env_var" }+macro_event_broker = { path = "../macro_event_broker" }macro_queues = { path = "../macro_queues" }macro_user_id = { path = "../macro_user_id" }model = { path = "../model" }diff --git a/rust/cloud-storage/ai_tools/src/build_context.rs b/rust/cloud-storage/ai_tools/src/build_context.rsindex fabb7540ed..aa8dac4218 100644--- a/rust/cloud-storage/ai_tools/src/build_context.rs+++ b/rust/cloud-storage/ai_tools/src/build_context.rs@@ -57,6 +57,7 @@ env_var! {DocumentStorageServiceCloudfrontSignerPublicKeyId,DocumentStorageServiceCloudfrontSignerPrivateKeySecretName,DocumentPermissionJwt,+ KafkaBrokers,}}@@ -81,7 +82,8 @@ maybe_env_var! {/// `DOCX_DOCUMENT_UPLOAD_BUCKET`,/// `DOCUMENT_STORAGE_SERVICE_CLOUDFRONT_DISTRIBUTION_URL`,/// `DOCUMENT_STORAGE_SERVICE_CLOUDFRONT_SIGNER_PUBLIC_KEY_ID`,-/// `DOCUMENT_STORAGE_SERVICE_CLOUDFRONT_SIGNER_PRIVATE_KEY_SECRET_NAME`.+/// `DOCUMENT_STORAGE_SERVICE_CLOUDFRONT_SIGNER_PRIVATE_KEY_SECRET_NAME`,+/// `KAFKA_BROKERS`.////// Service URLs are resolved through the `macro_service_urls` crate, and queue/// names through the `macro_queues` crate (both using optional `OVERRIDE_*` env@@ -240,6 +242,10 @@ pub async fn build_tool_service_context_from_env(pool.clone(),properties_service.clone(),);+ let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(+ macro_event_broker::KafkaEventPublisher::new(env.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ );let document_service = documents::domain::service::DocumentServiceImpl {repo: document_repo,cloudfront_config,@@ -254,6 +260,7 @@ pub async fn build_tool_service_context_from_env(foreign_entity_service: ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(pool.clone(),)),+ macro_event_broker,};let document_tool_context = DocumentToolContext::new(diff --git a/rust/cloud-storage/ai_tools/src/tool_context.rs b/rust/cloud-storage/ai_tools/src/tool_context.rsindex a1521f0fb6..56bbd90fd5 100644--- a/rust/cloud-storage/ai_tools/src/tool_context.rs+++ b/rust/cloud-storage/ai_tools/src/tool_context.rs@@ -456,6 +456,7 @@ pub type ToolDocumentService = documents::domain::service::DocumentServiceImpl<NoOpConnectionService,ToolEntityAccessManagementService,ToolForeignEntityService,+ macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,>;/// Type alias for the entity access service implementationdiff --git a/rust/cloud-storage/deny.toml b/rust/cloud-storage/deny.tomlindex 9860ee6ad8..09b227ccaa 100644--- a/rust/cloud-storage/deny.toml+++ b/rust/cloud-storage/deny.toml@@ -190,9 +190,12 @@ allow = [# List of crates to denydeny = [{ crate = "aws-config", wrappers = [+ "aws-msk-iam-sasl-signer", # External crate, requires aws-config directly"macro_aws_config",+ "workspace-hack", # cargo-hakari generated crate], reason = "Use macro_aws_config for AWS configuration in workspace crates" },{ crate = "tracing-subscriber", wrappers = [+ "documents", # Dev logging only"email", # Dev logging only"lambda_runtime_api_client", # External crate allowed to use tracing-subscriber"macro_entrypoint",@@ -230,26 +233,18 @@ deny = [# Certain crates/versions that will be skipped when doing duplicate detection.skip = [# These duplicates are due to complex dependency trees from AWS SDK, async-stripe, axum, and other major dependencies- { crate = "aligned-vec", reason = "Different dependencies require different versions" },- { crate = "base16ct", reason = "Cryptographic dependencies version mismatch" },{ crate = "base64", reason = "AWS SDK and other dependencies require different versions" },{ crate = "bitflags", reason = "Different dependencies require different versions" },{ crate = "core-foundation", reason = "Different dependencies require different versions" },{ crate = "cpufeatures", reason = "Different dependencies require different versions" },- { crate = "crypto-bigint", reason = "AWS cryptographic dependencies version mismatch" },{ crate = "cssparser", reason = "HTML parsing dependencies version mismatch" },{ crate = "darling", reason = "Derive macro dependencies version mismatch" },{ crate = "darling_core", reason = "Derive macro dependencies version mismatch" },{ crate = "darling_macro", reason = "Derive macro dependencies version mismatch" },- { crate = "der", reason = "Cryptographic dependencies version mismatch" },- { crate = "ecdsa", reason = "Cryptographic dependencies version mismatch" },- { crate = "elliptic-curve", reason = "Cryptographic dependencies version mismatch" },{ crate = "event-listener", reason = "Async dependencies version mismatch" },{ crate = "fastrand", reason = "Different dependencies require different versions" },- { crate = "ff", reason = "Cryptographic dependencies version mismatch" },{ crate = "foldhash", reason = "Hash function dependencies" },{ crate = "getrandom", reason = "Random number generation dependencies" },- { crate = "group", reason = "Cryptographic dependencies version mismatch" },{ crate = "h2", reason = "HTTP/2 dependencies version mismatch" },{ crate = "hashbrown", reason = "Different dependencies require different versions" },{ crate = "heck", reason = "Different dependencies require different versions" },@@ -264,38 +259,30 @@ skip = [{ crate = "jni-sys", reason = "Different dependencies require different versions" },{ crate = "jsonwebtoken", reason = "Git fork and crates.io version coexist" },{ crate = "markup5ever", reason = "HTML parsing dependencies" },- { crate = "match_token", reason = "HTML parsing dependencies" },{ crate = "mio", reason = "Different dependencies require different versions" },{ crate = "nix", reason = "Different dependencies require different versions" },{ crate = "nom", reason = "Parser combinator dependencies" },- { crate = "p256", reason = "Cryptographic dependencies version mismatch" },{ crate = "pastey", reason = "Macro dependencies version mismatch" },{ crate = "phf", reason = "Perfect hash function dependencies" },{ crate = "phf_codegen", reason = "Perfect hash function dependencies" },{ crate = "phf_generator", reason = "Perfect hash function dependencies" },{ crate = "phf_shared", reason = "Perfect hash function dependencies" },- { crate = "pkcs8", reason = "Cryptographic dependencies" },{ crate = "prost", reason = "Protocol buffer dependencies version mismatch" },{ crate = "prost-derive", reason = "Protocol buffer dependencies version mismatch" },{ crate = "rand", reason = "Different dependencies require different versions" },{ crate = "rand_chacha", reason = "Random number generation dependencies" },{ crate = "rand_core", reason = "Different dependencies require different versions" },{ crate = "reqwest", reason = "livekit-api pins reqwest 0.12 via services-tokio feature; workspace uses 0.13" },- { crate = "rfc6979", reason = "Cryptographic dependencies version mismatch" },{ crate = "rustc-hash", reason = "Hash function dependencies" },{ crate = "rustls", reason = "TLS dependencies version mismatch" },{ crate = "rustls-webpki", reason = "TLS dependencies version mismatch" },- { crate = "sec1", reason = "Cryptographic dependencies version mismatch" },- { crate = "security-framework", reason = "macOS security framework dependencies" },{ crate = "selectors", reason = "CSS selector dependencies" },{ crate = "serde_qs", reason = "Query string serialization dependencies" },{ crate = "servo_arc", reason = "Arc implementation dependencies" },- { crate = "signature", reason = "Cryptographic signature dependencies" },{ crate = "siphasher", reason = "Hash function dependencies" },{ crate = "smol_str", reason = "Different dependencies require different versions" },{ crate = "socket2", reason = "Different dependencies require different versions" },{ crate = "spin", reason = "Different dependencies require different versions" },- { crate = "spki", reason = "Cryptographic dependencies" },{ crate = "syn", reason = "Procedural macros require different versions" },{ crate = "thiserror", reason = "Error handling dependencies" },{ crate = "thiserror-impl", reason = "Error handling dependencies" },@@ -303,24 +290,16 @@ skip = [{ crate = "tower", reason = "Service framework dependencies" },{ crate = "unicode-width", reason = "Unicode dependencies" },{ crate = "wasi", reason = "WebAssembly system interface dependencies" },- { crate = "windows-link", reason = "Windows dependencies" },- { crate = "windows-registry", reason = "Windows dependencies version mismatch" },- { crate = "windows-result", reason = "Windows dependencies version mismatch" },- { crate = "windows-strings", reason = "Windows dependencies version mismatch" },{ crate = "windows-sys", reason = "Windows dependencies version mismatch" },{ crate = "windows-targets", reason = "Windows dependencies" },{ crate = "windows_aarch64_gnullvm", reason = "Windows dependencies" },{ crate = "windows_aarch64_msvc", reason = "Windows dependencies" },{ crate = "windows_i686_gnu", reason = "Windows dependencies" },- { crate = "windows_i686_gnullvm", reason = "Windows dependencies" },{ crate = "windows_i686_msvc", reason = "Windows dependencies" },{ crate = "windows_x86_64_gnu", reason = "Windows dependencies" },{ crate = "windows_x86_64_gnullvm", reason = "Windows dependencies" },{ crate = "windows_x86_64_msvc", reason = "Windows dependencies" },- { crate = "zerocopy", reason = "Zero-copy serialization dependencies" },{ crate = "zip", reason = "Different dependencies require different versions" },- { crate = "zune-core", reason = "Duplicated dep" },- { crate = "zune-jpeg", reason = "Deuplicated dep" },]# Similarly to `skip` allows you to skip certain crates during duplicate# detection. Unlike skip, it also includes the entire tree of transitivediff --git a/rust/cloud-storage/document_cognition_service/Cargo.toml b/rust/cloud-storage/document_cognition_service/Cargo.tomlindex 819245fd6e..39fd700c32 100644--- a/rust/cloud-storage/document_cognition_service/Cargo.toml+++ b/rust/cloud-storage/document_cognition_service/Cargo.toml@@ -74,6 +74,7 @@ macro_db_client = { path = "../macro_db_client" }macro_entrypoint = { path = "../macro_entrypoint" }macro_env = { path = "../macro_env" }macro_env_var = { path = "../macro_env_var" }+macro_event_broker = { path = "../macro_event_broker" }macro_queues = { path = "../macro_queues" }macro_middleware = { path = "../macro_middleware", default-features = false, features = ["auth",diff --git a/rust/cloud-storage/document_cognition_service/src/api/context/test.rs b/rust/cloud-storage/document_cognition_service/src/api/context/test.rsindex c19f3b9986..0832e17ee3 100644--- a/rust/cloud-storage/document_cognition_service/src/api/context/test.rs+++ b/rust/cloud-storage/document_cognition_service/src/api/context/test.rs@@ -254,6 +254,12 @@ pub async fn test_api_context(pool: sqlx::Pool<sqlx::Postgres>) -> std::sync::Arentity_access_management::outbound::PgRepository::new(pool.clone()),),ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(pool.clone())),+ // Producer creation is lazy: nothing connects to Kafka unless an event+ // is published, so a dummy broker address is safe for tests.+ macro_event_broker::MacroEventBrokerService::new(+ macro_event_broker::KafkaEventPublisher::new("localhost:9092")+ .expect("kafka producer config is valid"),+ ),);let test_lexical_client = LexicalClient::new("test".into(), "http://nofileshere".into());let test_editing_client =diff --git a/rust/cloud-storage/document_cognition_service/src/config.rs b/rust/cloud-storage/document_cognition_service/src/config.rsindex 6519225bd1..4eecaf8b63 100644--- a/rust/cloud-storage/document_cognition_service/src/config.rs+++ b/rust/cloud-storage/document_cognition_service/src/config.rs@@ -21,6 +21,8 @@ env_vars!(pub struct DocumentStorageServiceCloudfrontSignerPrivateKey;pub struct McpCredentialsKeySecretName;pub struct DocumentPermissionJwt;+ /// Comma-separated Kafka bootstrap servers for the macro event broker.+ pub struct KafkaBrokers;);maybe_env_vars!(@@ -71,6 +73,8 @@ pub struct Config {pub ai_editing_worker_url: String,/// JWT secret for minting document permission tokens for the editing worker.pub document_permission_jwt: DocumentPermissionJwt,+ /// Comma-separated Kafka bootstrap servers for the macro event broker.+ pub kafka_brokers: KafkaBrokers,}impl Config {@@ -119,6 +123,7 @@ impl Config {internal_api_key: InternalApiKey::Comptime(""),ai_editing_worker_url: AiEditingWorkerUrl::unwrap_new().to_string(),document_permission_jwt: DocumentPermissionJwt::Comptime("DOCUMENT_PERMISSION_JWT"),+ kafka_brokers: KafkaBrokers::Comptime("localhost:9092"),}}}diff --git a/rust/cloud-storage/document_cognition_service/src/main.rs b/rust/cloud-storage/document_cognition_service/src/main.rsindex 827996c3ec..927f75d439 100644--- a/rust/cloud-storage/document_cognition_service/src/main.rs+++ b/rust/cloud-storage/document_cognition_service/src/main.rs@@ -272,6 +272,10 @@ async fn main() -> anyhow::Result<()> {ai_tools::build_properties_service(db.clone(), entity_access_service.clone());let task_properties_service =ai_tools::build_task_properties_adapter(db.clone(), properties_service.clone());+ let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(+ macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ );let document_service = DocumentServiceImpl::new(document_repo,cloudfront_config,@@ -283,6 +287,7 @@ async fn main() -> anyhow::Result<()> {entity_access_management::outbound::PgRepository::new(db.clone()),),ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ macro_event_broker,);let lexical_client_for_tools = (*lexical_client).clone();let document_tool_context = DocumentToolContext::new(diff --git a/rust/cloud-storage/document_storage_service/Cargo.toml b/rust/cloud-storage/document_storage_service/Cargo.tomlindex 7efcfd1482..1fc217386e 100644--- a/rust/cloud-storage/document_storage_service/Cargo.toml+++ b/rust/cloud-storage/document_storage_service/Cargo.toml@@ -94,6 +94,7 @@ macro_cors = { path = "../macro_cors" }macro_db_client = { path = "../macro_db_client" }macro_entrypoint = { path = "../macro_entrypoint" }macro_env = { path = "../macro_env" }+macro_event_broker = { path = "../macro_event_broker" }macro_service_urls = { path = "../macro_service_urls" }macro_sync_service_jwt = { path = "../macro_sync_service_jwt" }macro_env_var = { path = "../macro_env_var" }diff --git a/rust/cloud-storage/document_storage_service/src/api/context.rs b/rust/cloud-storage/document_storage_service/src/api/context.rsindex 084bc14597..0a0810e12a 100644--- a/rust/cloud-storage/document_storage_service/src/api/context.rs+++ b/rust/cloud-storage/document_storage_service/src/api/context.rs@@ -53,6 +53,8 @@ use favorites::{domain::service::FavoritesServiceImpl, inbound::axum_router::FavoritesRouterState,outbound::pg_favorites_repo::PgFavoritesRepo,};+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};+use foreign_entity::{domain::service::ForeignEntityServiceImpl, inbound::axum_router::ForeignEntityRouterState,outbound::pg_foreign_entity_repo::PgForeignEntityRepo,@@ -209,6 +211,7 @@ pub(crate) type DocumentService = DocumentServiceImpl<ConnectionServiceImpl<EntityAccessService, ConnectionGatewayImpl>,EntityAccessManagementService,ForeignEntityServiceImpl<PgForeignEntityRepo>,+ MacroEventBrokerService<KafkaEventPublisher>,>;/// Type alias for the documents router state.diff --git a/rust/cloud-storage/document_storage_service/src/config.rs b/rust/cloud-storage/document_storage_service/src/config.rsindex 97f39bcbcf..d878e39c7c 100644--- a/rust/cloud-storage/document_storage_service/src/config.rs+++ b/rust/cloud-storage/document_storage_service/src/config.rs@@ -23,6 +23,8 @@ env_vars! {pub struct OpensearchPassword;pub struct GithubSyncAppUrl;pub struct GithubSyncAppClientId;+ /// Comma-separated Kafka bootstrap servers for the macro event broker.+ pub struct KafkaBrokers;pub struct LivekitServerUrl;pub struct LivekitApiKey;pub struct LivekitApiSecret;@@ -92,6 +94,7 @@ pub struct Config {pub opensearch_password: OpensearchPassword,pub github_sync_app_url: GithubSyncAppUrl,pub github_sync_app_client_id: GithubSyncAppClientId,+ pub kafka_brokers: KafkaBrokers,pub livekit_server_url: LivekitServerUrl,pub livekit_api_key: LivekitApiKey,pub livekit_api_secret: LivekitApiSecret,diff --git a/rust/cloud-storage/document_storage_service/src/main.rs b/rust/cloud-storage/document_storage_service/src/main.rsindex e22f32e50d..5961c3e2e3 100644--- a/rust/cloud-storage/document_storage_service/src/main.rs+++ b/rust/cloud-storage/document_storage_service/src/main.rs@@ -71,6 +71,7 @@ use lexical_client::LexicalClient;use macro_auth::middleware::decode_jwt::JwtValidationArgs;use macro_entrypoint::MacroEntrypoint;use macro_env_var::maybe_env_vars;+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use macro_service_urls::{ConnectionGatewayUrl, LexicalServiceUrl, SyncServiceUrl};use macro_sha_count_client::Redis;use notification::domain::service::SqsNotificationIngress;@@ -375,6 +376,11 @@ async fn main() -> anyhow::Result<()> {entity_access_management::outbound::PgRepository::new(db.clone()),);+ let macro_event_broker = MacroEventBrokerService::new(+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ );+let document_service = Arc::new(DocumentServiceImpl::new(document_repo,cloudfront_config,@@ -387,6 +393,7 @@ async fn main() -> anyhow::Result<()> {connection_service,entity_access_management_service.clone(),ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ macro_event_broker,));let foreign_entity_service = Arc::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(diff --git a/rust/cloud-storage/documents/Cargo.toml b/rust/cloud-storage/documents/Cargo.tomlindex 96d08ac889..e01e570b35 100644--- a/rust/cloud-storage/documents/Cargo.toml+++ b/rust/cloud-storage/documents/Cargo.toml@@ -70,7 +70,12 @@ outbound = ["ports",]ports = ["dep:tokio"]-service = ["dep:entity_access_management", "dep:foreign_entity", "ports"]+service = [+ "dep:entity_access_management",+ "dep:foreign_entity",+ "macro_event_broker/ports",+ "ports",+][dependencies]# Core@@ -100,6 +105,8 @@ foreign_entity = { path = "../foreign_entity", default-features = false, featuremacro_aws_config = { path = "../macro_aws_config", features = ["s3"] }macro_entrypoint = { path = "../macro_entrypoint", optional = true }macro_env_var = { path = "../macro_env_var" }+macro_event_broker = { path = "../macro_event_broker", default-features = false }+macro_event_topics = { path = "../macro_event_topics" }macro_sync_service_jwt = { path = "../macro_sync_service_jwt" }macro_user_id = { path = "../macro_user_id" }macro_uuid = { path = "../macro_uuid" }@@ -152,7 +159,12 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }dotenvy = { workspace = true }http-body-util = { workspace = true }macro_db_migrator = { path = "../macro_db_migrator" }+macro_env = { path = "../macro_env" }+# Examples need the Kafka adapter (outbound); the library itself only uses ports.+macro_event_broker = { path = "../macro_event_broker" }mockall = { workspace = true }+rdkafka = { workspace = true }serde_json = { workspace = true }-tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] }tower = { workspace = true }+tracing-subscriber = { workspace = true }diff --git a/rust/cloud-storage/documents/examples/documents_consumer.rs b/rust/cloud-storage/documents/examples/documents_consumer.rsnew file mode 100644index 0000000000..6603e2ed30--- /dev/null+++ b/rust/cloud-storage/documents/examples/documents_consumer.rs@@ -0,0 +1,266 @@+//! Kafka consumer example for the `macro.documents` topic.+//!+//! Connects with the same `KAFKA_BROKERS` env var and environment-driven+//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for+//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),+//! subscribes to [`MacroDocumentsTopic`], and prints every document lifecycle+//! event it receives.+//!+//! The consumer follows the standard poll-loop / worker split: a Kafka poll+//! loop decodes messages and hands them to a processing task over a bounded+//! tokio mpsc channel (backpressure applies when the processor falls behind),+//! offsets are committed only after a successful handoff (at-least-once), and+//! ctrl-c drains the channel before exiting.+//!+//! Run against the local docker broker:+//! ```sh+//! ENVIRONMENT=local KAFKA_BROKERS=localhost:9092 \+//! cargo run -p documents --example documents_consumer+//! ```+//! or against the dev MSK cluster (needs AWS credentials and network access):+//! ```sh+//! ENVIRONMENT=develop AWS_REGION=us-east-1 KAFKA_BROKERS=<bootstrapBrokersSaslIam> \+//! cargo run -p documents --example documents_consumer+//! ```++use anyhow::Context as _;+use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};+use macro_env::Environment;+use macro_event_broker::outbound::msk_iam::configure_sasl_iam;+use macro_event_broker::{EventBrokerError, MacroEvent, MskIamClientContext, Topic as _};+use macro_event_topics::MacroDocumentsTopic;+use rdkafka::ClientConfig;+use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};+use rdkafka::error::KafkaResult;+use rdkafka::message::{BorrowedMessage, Message};+use tokio::sync::mpsc;++macro_env_var::env_var! {+ struct ConsumerEnvVars {+ KafkaBrokers,+ }+}++/// Consumer group id for this example. Offsets are committed under this group,+/// so re-runs resume where the previous run left off.+const GROUP_ID: &str = "documents-consumer-example";++/// Bounded capacity of the channel between the poll loop and the processor.+const CHANNEL_CAPACITY: usize = 128;++/// Consumer-specific event enum for a consumer subscribed to `macro.documents`,+/// following the pattern in `macro_event_broker/examples/example_event.rs`.+enum DocumentsConsumerEvent {+ /// Event received on [`MacroDocumentsTopic`].+ Documents(DocumentMacroEvent),+}++impl DocumentsConsumerEvent {+ /// Decode one Kafka message into this consumer's event enum.+ fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {+ match topic {+ topic if topic == MacroDocumentsTopic.as_str() => {+ Ok(Self::Documents(DocumentMacroEvent::decode(key, payload)?))+ }+ unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),+ }+ }+}++/// A decoded event plus the Kafka coordinates it came from, as handed from the+/// poll loop to the processor over the channel.+struct ReceivedEvent {+ partition: i32,+ offset: i64,+ event: DocumentsConsumerEvent,+}++/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).+enum DocumentsConsumer {+ /// Unauthenticated plaintext connection (local docker broker).+ Plaintext(StreamConsumer),+ /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).+ MskIam(StreamConsumer<MskIamClientContext>),+}++impl DocumentsConsumer {+ /// Build a consumer for the given brokers, choosing the transport from the+ /// `ENVIRONMENT` variable exactly like the publisher does.+ fn from_env(brokers: &str) -> anyhow::Result<Self> {+ let mut config = ClientConfig::new();+ config+ .set("bootstrap.servers", brokers)+ .set("group.id", GROUP_ID)+ // Offsets are committed manually after a successful handoff.+ .set("enable.auto.commit", "false")+ // Start from the beginning of the topic on the first ever run.+ .set("auto.offset.reset", "earliest");++ let consumer = match Environment::new_or_prod() {+ Environment::Local => Self::Plaintext(+ config+ .create()+ .context("failed to create plaintext kafka consumer")?,+ ),+ Environment::Develop | Environment::Production => {+ configure_sasl_iam(&mut config);+ Self::MskIam(+ config+ .create_with_context(MskIamClientContext::from_env())+ .context("failed to create MSK IAM kafka consumer")?,+ )+ }+ };++ Ok(consumer)+ }++ fn subscribe(&self) -> KafkaResult<()> {+ let topics = [MacroDocumentsTopic.as_str()];+ match self {+ Self::Plaintext(consumer) => consumer.subscribe(&topics),+ Self::MskIam(consumer) => consumer.subscribe(&topics),+ }+ }++ /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it+ /// can sit in a `select!` without losing messages.+ async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {+ match self {+ Self::Plaintext(consumer) => consumer.recv().await,+ Self::MskIam(consumer) => consumer.recv().await,+ }+ }++ fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {+ match self {+ Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),+ Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),+ }+ }+}++fn event_type(event: &DocumentTopicEvent) -> &'static str {+ match event {+ DocumentTopicEvent::Created(_) => "document.created",+ DocumentTopicEvent::Updated(_) => "document.updated",+ DocumentTopicEvent::Deleted(_) => "document.deleted",+ DocumentTopicEvent::Copied(_) => "document.copied",+ }+}++/// Print each received event. This stands in for real event handling — swap+/// the body for indexing, notifications, etc.+async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {+ while let Some(received) = events.recv().await {+ let DocumentsConsumerEvent::Documents(event) = received.event;+ let envelope = event.event();++ println!(+ "[partition {} offset {}] {} key={} event_id={} schema_version={}",+ received.partition,+ received.offset,+ event_type(&envelope.event),+ event.key(),+ envelope.event_id,+ envelope.schema_version,+ );+ match serde_json::to_string_pretty(envelope) {+ Ok(json) => println!("{json}"),+ Err(e) => eprintln!("failed to re-serialize event: {e}"),+ }+ }+}++#[tokio::main]+async fn main() -> anyhow::Result<()> {+ // Surface tracing from macro_event_broker (e.g. MSK IAM token refreshes)+ // and librdkafka's internal connection/auth logs (bridged from the `log`+ // crate). Tune with RUST_LOG, e.g. RUST_LOG=debug for librdkafka detail.+ tracing_subscriber::fmt()+ .with_env_filter(+ tracing_subscriber::EnvFilter::try_from_default_env()+ .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),+ )+ .init();++ let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;++ let consumer = DocumentsConsumer::from_env(env.kafka_brokers.as_ref())?;+ consumer+ .subscribe()+ .context("failed to subscribe to documents topic")?;+ println!(+ "listening on topic={} group={} brokers={} (ctrl-c to stop)",+ MacroDocumentsTopic.as_str(),+ GROUP_ID,+ env.kafka_brokers.as_ref(),+ );++ let (events_tx, events_rx) = mpsc::channel::<ReceivedEvent>(CHANNEL_CAPACITY);+ let processor = tokio::spawn(process_events(events_rx));++ let mut shutdown = std::pin::pin!(tokio::signal::ctrl_c());+ loop {+ tokio::select! {+ _ = &mut shutdown => {+ println!("received ctrl-c, shutting down");+ break;+ }+ result = consumer.recv() => {+ let message = match result {+ Ok(message) => message,+ Err(e) => {+ eprintln!("kafka receive error: {e}");+ continue;+ }+ };++ let Some(payload) = message.payload() else {+ eprintln!(+ "skipping message with empty payload at partition {} offset {}",+ message.partition(),+ message.offset(),+ );+ let _ = consumer.commit(&message).inspect_err(|e| {+ eprintln!("failed to commit offset {}: {e}", message.offset());+ });+ continue;+ };+ let key = String::from_utf8_lossy(message.key().unwrap_or_default());++ match DocumentsConsumerEvent::decode(message.topic(), key.as_ref(), payload) {+ Ok(event) => {+ events_tx+ .send(ReceivedEvent {+ partition: message.partition(),+ offset: message.offset(),+ event,+ })+ .await+ .context("processor task closed unexpectedly")?;+ }+ // Undecodable messages are logged and skipped rather than+ // wedging the partition on a poison message.+ Err(e) => eprintln!(+ "failed to decode message at partition {} offset {}: {e}",+ message.partition(),+ message.offset(),+ ),+ }++ // Commit only after the event was handed to the processor:+ // at-least-once delivery (up to the channel buffer on a crash).+ let _ = consumer.commit(&message).inspect_err(|e| {+ eprintln!("failed to commit offset {}: {e}", message.offset());+ });+ }+ }+ }++ // Close the channel and let the processor drain what it already received.+ drop(events_tx);+ processor.await.context("processor task panicked")?;++ Ok(())+}diff --git a/rust/cloud-storage/documents/src/domain.rs b/rust/cloud-storage/documents/src/domain.rsindex 2803df036a..3b3bf94198 100644--- a/rust/cloud-storage/documents/src/domain.rs+++ b/rust/cloud-storage/documents/src/domain.rs@@ -2,6 +2,7 @@pub mod branch_name;pub mod content;+pub mod events;#[cfg(feature = "ports")]pub mod markdown_backfill;diff --git a/rust/cloud-storage/documents/src/domain/events.rs b/rust/cloud-storage/documents/src/domain/events.rsnew file mode 100644index 0000000000..9f6c576580--- /dev/null+++ b/rust/cloud-storage/documents/src/domain/events.rs@@ -0,0 +1,173 @@+//! Kafka event models for the `macro.documents` topic.+//!+//! Follows the canonical pattern in `macro_event_broker/examples/example_event.rs`:+//! per-variant metadata structs, a [`TopicEvent`] enum tagged by `event_type`,+//! and a [`MacroEvent`] wrapper keyed by document id.++use chrono::{DateTime, Utc};+use document_sub_type::DocumentSubType;+use macro_event_broker::{Event, MacroEvent, TopicEvent};+use macro_event_topics::MacroDocumentsTopic;+use macro_user_id::user_id::MacroUserIdStr;+use model::document::FileType;+use serde::{Deserialize, Serialize};++use super::models::FileTypeUpdate;++/// Metadata for [`DocumentTopicEvent::Created`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct DocumentCreatedMetadata {+ /// The id of the created document.+ pub document_id: String,+ /// The owner (creator) of the document.+ pub owner: MacroUserIdStr<'static>,+ /// The document name, with any file extension stripped.+ pub document_name: String,+ /// File type of the document, when known.+ pub file_type: Option<FileType>,+ /// Project the document was created in, when any.+ pub project_id: Option<String>,+ /// Sub type (task / snippet), when any.+ pub sub_type: Option<DocumentSubType>,+ /// Creation timestamp reported by the repository.+ pub created_at: Option<DateTime<Utc>>,+}++/// Metadata for [`DocumentTopicEvent::Updated`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct DocumentUpdatedMetadata {+ /// The id of the updated document.+ pub document_id: String,+ /// The owner of the document.+ pub owner: MacroUserIdStr<'static>,+ /// The authenticated user who performed the update; `None` for+ /// unauthenticated or internal callers.+ pub actor_user_id: Option<MacroUserIdStr<'static>>,+ /// New (cleaned) document name; `None` when unchanged.+ pub document_name: Option<String>,+ /// Project id before the update.+ pub previous_project_id: Option<String>,+ /// New project id; `None` when unchanged, `Some("")` when removed from+ /// its project (mirrors the edit repo args semantics).+ pub project_id: Option<String>,+ /// Requested file type change; `None` when unchanged.+ pub file_type: Option<FileTypeUpdate>,+ /// Whether share permissions were updated.+ pub share_permission_updated: bool,+}++/// Metadata for [`DocumentTopicEvent::Deleted`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct DocumentDeletedMetadata {+ /// The id of the deleted document.+ pub document_id: String,+ /// The authenticated user who deleted the document; `None` for+ /// unauthenticated or internal callers.+ pub actor_user_id: Option<MacroUserIdStr<'static>>,+ /// Project the document belonged to, when any.+ pub project_id: Option<String>,+}++/// Metadata for [`DocumentTopicEvent::Copied`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct DocumentCopiedMetadata {+ /// The id of the newly created copy.+ pub document_id: String,+ /// The id of the document that was copied.+ pub source_document_id: String,+ /// The specific source version copied, when requested.+ pub source_version_id: Option<i64>,+ /// The owner of the new copy (the copier).+ pub owner: MacroUserIdStr<'static>,+ /// The name of the new document.+ pub document_name: String,+ /// File type of the document, when known.+ pub file_type: Option<FileType>,+ /// Project the copy belongs to, when any.+ pub project_id: Option<String>,+ /// Sub type (task / snippet), when any.+ pub sub_type: Option<DocumentSubType>,+}++/// Events that can be published to [`MacroDocumentsTopic`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[serde(tag = "event_type", content = "metadata")]+pub enum DocumentTopicEvent {+ /// A document was created.+ #[serde(rename = "document.created")]+ Created(DocumentCreatedMetadata),+ /// A document's metadata / permissions were updated.+ #[serde(rename = "document.updated")]+ Updated(DocumentUpdatedMetadata),+ /// A document was soft-deleted.+ #[serde(rename = "document.deleted")]+ Deleted(DocumentDeletedMetadata),+ /// A document was copied.+ #[serde(rename = "document.copied")]+ Copied(DocumentCopiedMetadata),+}++impl TopicEvent for DocumentTopicEvent {+ type Topic = MacroDocumentsTopic;++ fn schema_version(&self) -> u8 {+ 1+ }+}++/// Publishable event for [`MacroDocumentsTopic`], keyed by document id.+pub struct DocumentMacroEvent {+ key: String,+ event: Event<DocumentTopicEvent>,+}++impl DocumentMacroEvent {+ /// Build a created event keyed by the new document id.+ pub fn created(key: impl Into<String>, metadata: DocumentCreatedMetadata) -> Self {+ Self::new(key, DocumentTopicEvent::Created(metadata))+ }++ /// Build an updated event keyed by the updated document id.+ pub fn updated(key: impl Into<String>, metadata: DocumentUpdatedMetadata) -> Self {+ Self::new(key, DocumentTopicEvent::Updated(metadata))+ }++ /// Build a deleted event keyed by the deleted document id.+ pub fn deleted(key: impl Into<String>, metadata: DocumentDeletedMetadata) -> Self {+ Self::new(key, DocumentTopicEvent::Deleted(metadata))+ }++ /// Build a copied event keyed by the new document id.+ pub fn copied(key: impl Into<String>, metadata: DocumentCopiedMetadata) -> Self {+ Self::new(key, DocumentTopicEvent::Copied(metadata))+ }++ /// Build an event from a topic-specific event variant.+ pub fn new(key: impl Into<String>, event: DocumentTopicEvent) -> Self {+ Self::with_event(key, Event::new(event))+ }++ /// Build an event from a pre-built envelope.+ pub fn with_event(key: impl Into<String>, event: Event<DocumentTopicEvent>) -> Self {+ Self {+ key: key.into(),+ event,+ }+ }+}++impl MacroEvent for DocumentMacroEvent {+ type EventPayload = DocumentTopicEvent;++ fn key(&self) -> &str {+ &self.key+ }++ fn event(&self) -> &Event<Self::EventPayload> {+ &self.event+ }++ fn from_event(key: String, event: Event<Self::EventPayload>) -> Self {+ Self::with_event(key, event)+ }+}diff --git a/rust/cloud-storage/documents/src/domain/service.rs b/rust/cloud-storage/documents/src/domain/service.rsindex 08850de119..e07f7c3d4b 100644--- a/rust/cloud-storage/documents/src/domain/service.rs+++ b/rust/cloud-storage/documents/src/domain/service.rs@@ -22,6 +22,7 @@ use entity_access::domain::models::{};use foreign_entity::domain::models::{ForeignEntity, SourceId};use foreign_entity::domain::ports::ForeignEntityService;+use macro_event_broker::MacroEventBroker;use macro_user_id::user_id::MacroUserIdStr;use model::document::response::{DocumentResponseMetadata, LocationResponseData};use model::document::{ContentType, DocumentBasic, FileAssociation, FileType, FileTypeExt};@@ -38,6 +39,10 @@ use crate::domain::models::{use super::branch_name::{build_task_branch_name, user_branch_prefix};use super::content::{DocumentContent, DocumentContentLocation, DocumentContentState};+use super::events::{+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentMacroEvent,+ DocumentUpdatedMetadata,+};use super::models::{CloudFrontConfig, CommentThread, CopyDocumentRepoArgs, CreateDocumentRepoArgs,CreateTaskRequest, DocumentError, DocumentTeamShareResponse, EditDocumentRepoArgs,@@ -60,6 +65,7 @@ pub struct DocumentServiceImpl<C: ConnectionService,Eam: EntityAccessManagementService,F: ForeignEntityService,+ B: MacroEventBroker,> {/// Document repositorypub repo: R,@@ -77,6 +83,8 @@ pub struct DocumentServiceImpl<pub entity_access_management_service: Eam,/// Foreign entity servicepub foreign_entity_service: F,+ /// Macro event broker for publishing document lifecycle events+ pub macro_event_broker: B,}fn ready_content_for_file_type(file_type: Option<FileType>) -> DocumentContent {@@ -118,6 +126,15 @@ fn pending_content_for_file_type(file_type: Option<FileType>) -> DocumentContent}}+/// The user id to attribute a document lifecycle event to, when the caller is+/// an authenticated user.+fn event_actor_user_id(auth: &EntityAccessAuth) -> Option<MacroUserIdStr<'static>> {+ match auth {+ EntityAccessAuth::Authenticated(user_id) => Some(user_id.clone()),+ EntityAccessAuth::Unauthenticated | EntityAccessAuth::Internal => None,+ }+}+const GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE: &str = "github_pull_request";const MAX_DOCUMENT_NAME_GRAPHEMES: usize = 200;@@ -188,7 +205,8 @@ impl<C: ConnectionService,Eam: EntityAccessManagementService,F: ForeignEntityService,-> DocumentServiceImpl<R, U, T, C, Eam, F>+ B: MacroEventBroker,+> DocumentServiceImpl<R, U, T, C, Eam, F, B>{/// Create a document service with its repository and external service ports.#[allow(clippy::too_many_arguments)]@@ -201,6 +219,7 @@ impl<connection_service: C,entity_access_management_service: Eam,foreign_entity_service: F,+ macro_event_broker: B,) -> Self {Self {repo,@@ -211,6 +230,7 @@ impl<connection_service,entity_access_management_service,foreign_entity_service,+ macro_event_broker,}}@@ -466,6 +486,17 @@ impl<.await.map_err(|e| DocumentError::Internal(e.into()))}++ /// Publish a document lifecycle event; failures are logged and dropped.+ async fn publish_document_event(&self, event: &DocumentMacroEvent) {+ let _ = self+ .macro_event_broker+ .send_event(event)+ .await+ .inspect_err(|e| {+ tracing::error!(error=?e, "failed to publish document event");+ });+ }}#[cfg(feature = "document_create")]@@ -476,7 +507,8 @@ impl<C: ConnectionService,Eam: EntityAccessManagementService,F: ForeignEntityService,-> DocumentCreationService for DocumentServiceImpl<R, U, T, C, Eam, F>+ B: MacroEventBroker,+> DocumentCreationService for DocumentServiceImpl<R, U, T, C, Eam, F, B>{async fn create_document(&self,@@ -529,7 +561,8 @@ impl<C: ConnectionService,Eam: EntityAccessManagementService,F: ForeignEntityService,-> DocumentService for DocumentServiceImpl<R, U, T, C, Eam, F>+ B: MacroEventBroker,+> DocumentService for DocumentServiceImpl<R, U, T, C, Eam, F, B>{#[tracing::instrument(err, skip(self))]async fn get_document(@@ -691,6 +724,16 @@ impl<tracing::error!(error=?e, "failed to send invalidation event");});+ self.publish_document_event(&DocumentMacroEvent::deleted(+ entity_access_receipt.entity().entity_id.clone(),+ DocumentDeletedMetadata {+ document_id: entity_access_receipt.entity().entity_id.clone(),+ actor_user_id: event_actor_user_id(entity_access_receipt.auth()),+ project_id,+ },+ ))+ .await;+Ok(())}@@ -995,6 +1038,20 @@ impl<let team_task_metadata = self.team_task_metadata_for_document(&document_id).await?;+ self.publish_document_event(&DocumentMacroEvent::created(+ document_metadata.document_id.clone(),+ DocumentCreatedMetadata {+ document_id: document_metadata.document_id.clone(),+ owner: document_metadata.owner.clone(),+ document_name: document_metadata.document_name.clone(),+ file_type,+ project_id: project_id.map(|p| p.to_string()),+ sub_type: document_metadata.sub_type,+ created_at: document_metadata.created_at,+ },+ ))+ .await;+Ok(CreateDocumentResponseData {document_response: DocumentResponse {document_metadata: DocumentResponseMetadataWithContent::new(@@ -1079,13 +1136,15 @@ impl<.document_name.map(|s| FileType::clean_document_name(&s).unwrap_or(s));+ let share_permission_updated = args.share_permission.is_some();+self.repo.edit_document(EditDocumentRepoArgs {document_id: entity_access_receipt.entity().entity_id.clone(),- document_name,+ document_name: document_name.clone(),project_id: args.project_id.clone(),share_permission: args.share_permission,- file_type: args.file_type,+ file_type: args.file_type.clone(),}).await.map_err(|e| DocumentError::Internal(e.into()))?;@@ -1133,6 +1192,21 @@ impl<tracing::error!(error=?e, "failed to send invalidation event");});+ self.publish_document_event(&DocumentMacroEvent::updated(+ entity_access_receipt.entity().entity_id.clone(),+ DocumentUpdatedMetadata {+ document_id: entity_access_receipt.entity().entity_id.clone(),+ owner: document_context.owner.clone(),+ actor_user_id: event_actor_user_id(entity_access_receipt.auth()),+ document_name,+ previous_project_id: document_context.project_id.clone(),+ project_id: args.project_id,+ file_type: args.file_type,+ share_permission_updated,+ },+ ))+ .await;+Ok(())}@@ -1375,6 +1449,21 @@ impl<.team_task_metadata_for_document(&new_document_id).await?;+ self.publish_document_event(&DocumentMacroEvent::copied(+ new_document_id.clone(),+ DocumentCopiedMetadata {+ document_id: new_document_id.clone(),+ source_document_id: original_metadata.document_id.clone(),+ source_version_id: query_version_id,+ owner: user_id.clone(),+ document_name: new_metadata.document_name.clone(),+ file_type,+ project_id: new_metadata.project_id.clone(),+ sub_type: new_metadata.sub_type,+ },+ ))+ .await;+Ok(DocumentResponse {document_metadata: DocumentResponseMetadataWithContent::new(document_response_metadata,diff --git a/rust/cloud-storage/documents/src/domain/service/tests.rs b/rust/cloud-storage/documents/src/domain/service/tests.rsindex 0409595503..d3e3c642c0 100644--- a/rust/cloud-storage/documents/src/domain/service/tests.rs+++ b/rust/cloud-storage/documents/src/domain/service/tests.rs@@ -2,6 +2,7 @@ use foreign_entity::domain::models::{CreateForeignEntity, ForeignEntity, ForeignEntityError, PatchForeignEntity, SourceId,};use foreign_entity::domain::ports::{ForeignEntityListQuery, ForeignEntityService};+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, Topic as _};use macro_user_id::cowlike::CowLike;use model::document::DocumentMetadata;use std::sync::{Arc, Mutex};@@ -293,6 +294,36 @@ impl ForeignEntityService for TestForeignEntityService {}}+/// A document lifecycle event recorded by [`TestEventBroker`].+#[derive(Clone, Debug)]+struct PublishedEvent {+ topic: &'static str,+ key: String,+ payload: serde_json::Value,+}++#[derive(Clone, Default)]+struct TestEventBroker {+ published: Arc<Mutex<Vec<PublishedEvent>>>,+}++impl TestEventBroker {+ fn published(&self) -> Arc<Mutex<Vec<PublishedEvent>>> {+ Arc::clone(&self.published)+ }+}++impl MacroEventBroker for TestEventBroker {+ async fn send_event<E: MacroEvent + ?Sized>(&self, event: &E) -> Result<(), EventBrokerError> {+ self.published.lock().unwrap().push(PublishedEvent {+ topic: event.topic().as_str(),+ key: event.key().to_string(),+ payload: serde_json::to_value(event.event())?,+ });+ Ok(())+ }+}+type TestDocumentService = DocumentServiceImpl<MockDocumentRepo,TestUploadUrlPort,@@ -300,6 +331,7 @@ type TestDocumentService = DocumentServiceImpl<TestConnectionService,TestEntityAccessManagementService,TestForeignEntityService,+ TestEventBroker,>;fn make_test_service(repo: MockDocumentRepo) -> TestDocumentService {@@ -332,9 +364,32 @@ fn make_test_service_with_foreign_entity_service(TestConnectionService,TestEntityAccessManagementService,foreign_entity_service,+ TestEventBroker::default(),)}+/// Build a test service along with a handle to its recording event broker.+fn make_test_service_with_event_broker(+ repo: MockDocumentRepo,+) -> (TestDocumentService, TestEventBroker) {+ let event_broker = TestEventBroker::default();+ let service = DocumentServiceImpl::new(+ repo,+ test_cloudfront_config(),+ sync_service_client::SyncServiceClient::new(+ "test-sync-key".to_string(),+ "http://sync-service.test".to_string(),+ ),+ TestUploadUrlPort,+ TestTaskPropertiesPort,+ TestConnectionService,+ TestEntityAccessManagementService,+ TestForeignEntityService::default(),+ event_broker.clone(),+ );+ (service, event_broker)+}+fn make_foreign_entity(id: uuid::Uuid,foreign_entity_id: &str,@@ -1019,3 +1074,191 @@ async fn test_get_task_github_pull_requests_skips_malformed_keys_before_lookup()}]);}++fn owner_receipt(document_id: &str) -> EntityAccessReceipt<OwnerAccessLevel> {+ let user_id = macro_user_id::user_id::MacroUserIdStr::parse_from_str("macro|[email protected]")+ .unwrap()+ .into_owned();++ EntityAccessReceipt::dangerously_assert_authenticated_user(+ user_id,+ document_id,+ EntityType::Document,+ )+}++fn edit_receipt(document_id: &str) -> EntityAccessReceipt<EditAccessLevel> {+ let user_id = macro_user_id::user_id::MacroUserIdStr::parse_from_str("macro|[email protected]")+ .unwrap()+ .into_owned();++ EntityAccessReceipt::dangerously_assert_authenticated_user(+ user_id,+ document_id,+ EntityType::Document,+ )+}++#[tokio::test]+async fn test_delete_document_publishes_document_deleted_event() {+ let mut repo = make_mock_repo();+ repo.expect_soft_delete_document()+ .withf(|id| id == "doc-1")+ .returning(|_| Box::pin(std::future::ready(Ok(()))));+ repo.expect_update_project_modified()+ .withf(|id| id == "project-1")+ .returning(|_| Box::pin(std::future::ready(Ok(()))));++ let (service, event_broker) = make_test_service_with_event_broker(repo);++ service+ .delete_document(owner_receipt("doc-1"), Some("project-1".to_string()))+ .await+ .unwrap();++ let published = event_broker.published();+ let published = published.lock().unwrap();+ assert_eq!(published.len(), 1);+ let event = &published[0];+ assert_eq!(event.topic, "macro.documents");+ assert_eq!(event.key, "doc-1");+ assert_eq!(event.payload["event_type"], "document.deleted");+ assert_eq!(event.payload["schema_version"], 1);+ assert_eq!(event.payload["metadata"]["document_id"], "doc-1");+ assert_eq!(+ event.payload["metadata"]["actor_user_id"],+ "macro|[email protected]"+ );+ assert_eq!(event.payload["metadata"]["project_id"], "project-1");+ assert!(+ uuid::Uuid::parse_str(event.payload["event_id"].as_str().unwrap()).is_ok(),+ "event_id should be a valid uuid: {:?}",+ event.payload["event_id"]+ );+}++#[tokio::test]+async fn test_delete_document_publishes_no_event_when_repo_fails() {+ let mut repo = make_mock_repo();+ repo.expect_soft_delete_document()+ .withf(|id| id == "doc-1")+ .returning(|_| Box::pin(std::future::ready(Err(anyhow!("db is down")))));++ let (service, event_broker) = make_test_service_with_event_broker(repo);++ let result = service.delete_document(owner_receipt("doc-1"), None).await;++ assert!(result.is_err());+ assert!(event_broker.published().lock().unwrap().is_empty());+}++#[tokio::test]+async fn test_edit_document_publishes_document_updated_event() {+ let mut repo = make_mock_repo();+ repo.expect_edit_document()+ .withf(|args| args.document_id == "doc-1")+ .returning(|_| Box::pin(std::future::ready(Ok(()))));++ let (service, event_broker) = make_test_service_with_event_broker(repo);++ service+ .edit_document(+ edit_receipt("doc-1"),+ task_document_context("doc-1"),+ EditDocumentServiceArgs {+ document_name: Some("New name.md".to_string()),+ project_id: None,+ share_permission: None,+ file_type: None,+ },+ )+ .await+ .unwrap();++ let published = event_broker.published();+ let published = published.lock().unwrap();+ assert_eq!(published.len(), 1);+ let event = &published[0];+ assert_eq!(event.topic, "macro.documents");+ assert_eq!(event.key, "doc-1");+ assert_eq!(event.payload["event_type"], "document.updated");+ assert_eq!(event.payload["metadata"]["document_name"], "New name");+ assert_eq!(event.payload["metadata"]["owner"], "macro|[email protected]");+ assert_eq!(+ event.payload["metadata"]["actor_user_id"],+ "macro|[email protected]"+ );+ assert_eq!(event.payload["metadata"]["share_permission_updated"], false);+ assert_eq!(+ event.payload["metadata"]["previous_project_id"],+ serde_json::Value::Null+ );+ assert_eq!(+ event.payload["metadata"]["project_id"],+ serde_json::Value::Null+ );+}++#[tokio::test]+async fn test_copy_document_publishes_document_copied_event() {+ let mut repo = make_mock_repo();+ let original_metadata = make_test_metadata();+ let original_for_lookup = original_metadata.clone();+ repo.expect_get_document_metadata()+ .withf(|id| id == "doc-1")+ .returning(move |_| Box::pin(std::future::ready(Ok(original_for_lookup.clone()))));+ repo.expect_get_project_owner()+ .withf(|id| id == "project-1")+ .returning(|_| {+ Box::pin(std::future::ready(Ok(+ macro_user_id::user_id::MacroUserIdStr::parse_from_str("macro|[email protected]")+ .unwrap()+ .into_owned(),+ )))+ });+ let mut copied_metadata = make_test_metadata();+ copied_metadata.document_id = "doc-2".to_string();+ copied_metadata.document_name = "copied doc".to_string();+ repo.expect_copy_document()+ .returning(move |_| Box::pin(std::future::ready(Ok(copied_metadata.clone()))));+ repo.expect_get_document_version_id()+ .returning(|_| Box::pin(std::future::ready(Ok((1, true)))));+ repo.expect_get_latest_document_version_id()+ .returning(|_| Box::pin(std::future::ready(Ok((1, true)))));+ repo.expect_set_document_content()+ .returning(|_, _| Box::pin(std::future::ready(Ok(()))));+ repo.expect_get_team_task_metadata()+ .returning(|_| Box::pin(std::future::ready(Ok(None))));++ let (service, event_broker) = make_test_service_with_event_broker(repo);++ let mut document_context = task_document_context("doc-1");+ document_context.file_type = Some("txt".to_string());+ document_context.sub_type = None;++ service+ .copy_document(+ authenticated_receipt("doc-1"),+ document_context,+ macro_user_id::user_id::MacroUserIdStr::parse_from_str("macro|[email protected]")+ .unwrap()+ .into_owned(),+ "copied doc".to_string(),+ None,+ None,+ )+ .await+ .unwrap();++ let published = event_broker.published();+ let published = published.lock().unwrap();+ assert_eq!(published.len(), 1);+ let event = &published[0];+ assert_eq!(event.topic, "macro.documents");+ assert_eq!(event.key, "doc-2");+ assert_eq!(event.payload["event_type"], "document.copied");+ assert_eq!(event.payload["metadata"]["document_id"], "doc-2");+ assert_eq!(event.payload["metadata"]["source_document_id"], "doc-1");+ assert_eq!(event.payload["metadata"]["document_name"], "copied doc");+ assert_eq!(event.payload["metadata"]["owner"], "macro|[email protected]");+}diff --git a/rust/cloud-storage/macro_event_broker/Cargo.toml b/rust/cloud-storage/macro_event_broker/Cargo.tomlindex 5f87f7a422..da942d0d74 100644--- a/rust/cloud-storage/macro_event_broker/Cargo.toml+++ b/rust/cloud-storage/macro_event_broker/Cargo.toml@@ -6,10 +6,21 @@ version = "0.1.0"[features]default = ["ports", "outbound"]-outbound = ["dep:rdkafka", "ports"]+outbound = [+ "dep:aws-msk-iam-sasl-signer",+ "dep:aws-types",+ "dep:macro_env",+ "dep:macro_env_var",+ "dep:rdkafka",+ "ports",+]ports = ["dep:tokio"][dependencies]+aws-msk-iam-sasl-signer = { workspace = true, optional = true }+aws-types = { workspace = true, optional = true }+macro_env = { path = "../macro_env", optional = true }+macro_env_var = { path = "../macro_env_var", optional = true }rootcause = { workspace = true }rdkafka = { workspace = true, optional = true }serde = { workspace = true }diff --git a/rust/cloud-storage/macro_event_broker/src/domain/service.rs b/rust/cloud-storage/macro_event_broker/src/domain/service.rsindex 31a17f232d..f0de7f4018 100644--- a/rust/cloud-storage/macro_event_broker/src/domain/service.rs+++ b/rust/cloud-storage/macro_event_broker/src/domain/service.rs@@ -9,6 +9,7 @@ use crate::domain::models::{EventBrokerError, MacroEvent};use crate::domain::ports::{EventPublisher, MacroEventBroker};/// Orchestrates serializing events and handing them to an [`EventPublisher`].+#[derive(Clone)]pub struct MacroEventBrokerService<P: EventPublisher> {publisher: P,}diff --git a/rust/cloud-storage/macro_event_broker/src/lib.rs b/rust/cloud-storage/macro_event_broker/src/lib.rsindex d081712271..6299331701 100644--- a/rust/cloud-storage/macro_event_broker/src/lib.rs+++ b/rust/cloud-storage/macro_event_broker/src/lib.rs@@ -14,12 +14,16 @@pub mod domain;pub use domain::models::{Event, EventBrokerError, MacroEvent, TopicEvent};-pub use macro_event_topics::{MacroExampleTopic, Topic};+pub use macro_event_topics::{MacroDocumentsTopic, MacroExampleTopic, Topic};#[cfg(feature = "ports")]pub use domain::ports::{EventPublisher, MacroEventBroker};#[cfg(feature = "ports")]pub use domain::service::MacroEventBrokerService;+#[cfg(feature = "outbound")]+pub use outbound::kafka_event_publisher::KafkaEventPublisher;+#[cfg(feature = "outbound")]+pub use outbound::msk_iam::MskIamClientContext;/// Outbound layer: Kafka adapter for the [`EventPublisher`](domain::ports::EventPublisher) port.#[cfg(feature = "outbound")]diff --git a/rust/cloud-storage/macro_event_broker/src/outbound/kafka_event_publisher.rs b/rust/cloud-storage/macro_event_broker/src/outbound/kafka_event_publisher.rsindex 18b259a1a2..6f4df4632d 100644--- a/rust/cloud-storage/macro_event_broker/src/outbound/kafka_event_publisher.rs+++ b/rust/cloud-storage/macro_event_broker/src/outbound/kafka_event_publisher.rs@@ -1,17 +1,22 @@//! Kafka adapter for the [`EventPublisher`] port.//!-//! Wraps an `rdkafka` [`FutureProducer`] to publish events. The broker address is-//! supplied by the caller (typically from environment configuration at the-//! consuming service), so no env wiring lives in this crate yet.+//! Wraps an `rdkafka` [`FutureProducer`] to publish events. The broker address+//! is supplied by the caller (typically from environment configuration at the+//! consuming service); the transport is chosen here from the `ENVIRONMENT`+//! variable: `Local` uses a plaintext connection (the docker-compose broker),+//! while `Develop` / `Production` use TLS + SASL/OAUTHBEARER with AWS MSK IAM+//! auth via [`MskIamClientContext`].use std::time::Duration;+use macro_env::Environment;use macro_event_topics::Topic;use rdkafka::ClientConfig;use rdkafka::producer::{FutureProducer, FutureRecord};use crate::domain::models::EventBrokerError;use crate::domain::ports::EventPublisher;+use crate::outbound::msk_iam::{MskIamClientContext, configure_sasl_iam};/// How long a record may sit in the producer queue before delivery is considered failed.const MESSAGE_TIMEOUT_MS: &str = "5000";@@ -19,19 +24,56 @@ const MESSAGE_TIMEOUT_MS: &str = "5000";/// How long [`EventPublisher::publish`] waits for delivery confirmation before timing out.const SEND_TIMEOUT: Duration = Duration::from_secs(5);+/// The underlying producer, split by transport.+#[derive(Clone)]+enum Producer {+ /// Unauthenticated plaintext connection (local docker broker).+ Plaintext(FutureProducer),+ /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).+ MskIam(FutureProducer<MskIamClientContext>),+}+/// Kafka-backed implementation of [`EventPublisher`].+#[derive(Clone)]pub struct KafkaEventPublisher {- producer: FutureProducer,+ producer: Producer,+}++fn base_config(brokers: &str) -> ClientConfig {+ let mut config = ClientConfig::new();+ config+ .set("bootstrap.servers", brokers)+ .set("message.timeout.ms", MESSAGE_TIMEOUT_MS);+ config+}++fn create_error(e: impl std::fmt::Display) -> EventBrokerError {+ EventBrokerError::Publish(format!("failed to create producer: {e}"))}impl KafkaEventPublisher {- /// Build a producer connected to the given comma-separated `brokers` list.+ /// Build a producer connected to the given comma-separated `brokers` list,+ /// choosing the transport from the `ENVIRONMENT` variable (see the module+ /// docs).+ ///+ /// Producer creation is lazy: nothing connects to Kafka (and no IAM token+ /// is signed) until an event is published.pub fn new(brokers: &str) -> Result<Self, EventBrokerError> {- let producer: FutureProducer = ClientConfig::new()- .set("bootstrap.servers", brokers)- .set("message.timeout.ms", MESSAGE_TIMEOUT_MS)- .create()- .map_err(|e| EventBrokerError::Publish(format!("failed to create producer: {e}")))?;+ let producer = match Environment::new_or_prod() {+ Environment::Local => {+ Producer::Plaintext(base_config(brokers).create().map_err(create_error)?)+ }+ Environment::Develop | Environment::Production => {+ let mut config = base_config(brokers);+ configure_sasl_iam(&mut config);++ Producer::MskIam(+ config+ .create_with_context(MskIamClientContext::from_env())+ .map_err(create_error)?,+ )+ }+ };Ok(Self { producer })}@@ -47,10 +89,11 @@ impl EventPublisher for KafkaEventPublisher {) -> Result<(), EventBrokerError> {let record = FutureRecord::to(topic.as_str()).key(key).payload(payload);- self.producer- .send(record, SEND_TIMEOUT)- .await- .map_err(|(e, _)| EventBrokerError::Publish(e.to_string()))?;+ match &self.producer {+ Producer::Plaintext(producer) => producer.send(record, SEND_TIMEOUT).await,+ Producer::MskIam(producer) => producer.send(record, SEND_TIMEOUT).await,+ }+ .map_err(|(e, _)| EventBrokerError::Publish(e.to_string()))?;Ok(())}diff --git a/rust/cloud-storage/macro_event_broker/src/outbound/mod.rs b/rust/cloud-storage/macro_event_broker/src/outbound/mod.rsindex c0bd839798..048c8210ef 100644--- a/rust/cloud-storage/macro_event_broker/src/outbound/mod.rs+++ b/rust/cloud-storage/macro_event_broker/src/outbound/mod.rs@@ -1,2 +1,4 @@/// Kafka adapter implementing the [`EventPublisher`](crate::domain::ports::EventPublisher) port.pub mod kafka_event_publisher;+/// AWS MSK IAM (SASL/OAUTHBEARER) auth support shared by producers and consumers.+pub mod msk_iam;diff --git a/rust/cloud-storage/macro_event_broker/src/outbound/msk_iam.rs b/rust/cloud-storage/macro_event_broker/src/outbound/msk_iam.rsnew file mode 100644index 0000000000..f8b8043593--- /dev/null+++ b/rust/cloud-storage/macro_event_broker/src/outbound/msk_iam.rs@@ -0,0 +1,93 @@+//! AWS MSK IAM (SASL/OAUTHBEARER) support shared by Kafka producers and consumers.++use aws_msk_iam_sasl_signer::generate_auth_token;+use aws_types::region::Region;+use rdkafka::ClientConfig;+use rdkafka::client::{ClientContext, OAuthToken};+use rdkafka::consumer::ConsumerContext;++/// Region used to sign MSK IAM auth tokens when `AWS_REGION` is unset.+/// Matches the fallback in `macro_aws_config`.+const DEFAULT_AWS_REGION: &str = "us-east-1";++macro_env_var::maybe_env_var! {+ /// AWS region used to sign MSK IAM auth tokens.+ struct AwsRegion;+}++/// Apply the TLS + SASL/OAUTHBEARER settings required to authenticate to an+/// AWS MSK cluster with IAM auth (the `:9098` bootstrap listener).+pub fn configure_sasl_iam(config: &mut ClientConfig) -> &mut ClientConfig {+ config+ .set("security.protocol", "SASL_SSL")+ .set("sasl.mechanism", "OAUTHBEARER")+}++/// Client context that supplies AWS MSK IAM (SASL/OAUTHBEARER) auth tokens,+/// signing with the ambient AWS credentials (e.g. the ECS task role).+///+/// librdkafka invokes [`ClientContext::generate_oauth_token`] when the+/// connection is first established and again before each token expires.+pub struct MskIamClientContext {+ region: Region,+}++impl MskIamClientContext {+ /// Build a context that signs tokens for the region in `AWS_REGION`,+ /// falling back to `us-east-1`.+ pub fn from_env() -> Self {+ let region = AwsRegion::new()+ .and_then(|region| region.value().map(str::to_string))+ .unwrap_or_else(|| DEFAULT_AWS_REGION.to_string());++ Self {+ region: Region::new(region),+ }+ }+}++impl ClientContext for MskIamClientContext {+ const ENABLE_REFRESH_OAUTH_TOKEN: bool = true;++ fn generate_oauth_token(+ &self,+ _oauthbearer_config: Option<&str>,+ ) -> Result<OAuthToken, Box<dyn std::error::Error>> {+ let region = self.region.clone();++ // librdkafka may invoke this callback on a tokio runtime thread (for a+ // `StreamConsumer`, it fires while `recv()` is polled from async code),+ // where blocking on the ambient runtime panics. Sign on a dedicated+ // thread with its own single-threaded runtime instead, so the callback+ // is safe from any calling context. Tokens live ~15 minutes, so the+ // per-refresh thread + runtime cost is negligible.+ let signer = std::thread::spawn(+ move || -> Result<(String, i64), Box<dyn std::error::Error + Send + Sync>> {+ let runtime = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()?;+ Ok(runtime.block_on(generate_auth_token(region))?)+ },+ );++ let (token, expiration_time_ms) = signer+ .join()+ .map_err(|_| "MSK IAM token signer thread panicked")?+ .map_err(|e| -> Box<dyn std::error::Error> { e })+ .inspect_err(|e| {+ tracing::error!(error = %e, "failed to sign MSK IAM auth token");+ })?;++ tracing::info!(expiration_time_ms, "signed MSK IAM auth token");++ Ok(OAuthToken {+ token,+ principal_name: String::new(),+ lifetime_ms: expiration_time_ms,+ })+ }+}++/// Lets consumers (e.g. `StreamConsumer<MskIamClientContext>`) use this+/// context; all behavior is the trait's defaults.+impl ConsumerContext for MskIamClientContext {}diff --git a/rust/cloud-storage/macro_event_topics/src/lib.rs b/rust/cloud-storage/macro_event_topics/src/lib.rsindex d7beb475f7..e0af821a29 100644--- a/rust/cloud-storage/macro_event_topics/src/lib.rs+++ b/rust/cloud-storage/macro_event_topics/src/lib.rs@@ -49,4 +49,6 @@ macro_rules! topics {topics! {/// Example kafka topic.MacroExampleTopic => "macro.example",+ /// Document lifecycle events (created / updated / deleted / copied).+ MacroDocumentsTopic => "macro.documents",}diff --git a/rust/cloud-storage/macro_event_topics/src/test.rs b/rust/cloud-storage/macro_event_topics/src/test.rsindex 066941da61..0e91ab30e3 100644--- a/rust/cloud-storage/macro_event_topics/src/test.rs+++ b/rust/cloud-storage/macro_event_topics/src/test.rs@@ -18,4 +18,5 @@ fn all_topic_names_is_non_empty_and_unique() {#[test]fn all_topic_names_includes_declared_topics() {assert!(all_topic_names().contains(&MacroExampleTopic.as_str()));+ assert!(all_topic_names().contains(&MacroDocumentsTopic.as_str()));}diff --git a/rust/cloud-storage/mcp_service/Cargo.toml b/rust/cloud-storage/mcp_service/Cargo.tomlindex b38e2ba3a0..723314857e 100644--- a/rust/cloud-storage/mcp_service/Cargo.toml+++ b/rust/cloud-storage/mcp_service/Cargo.toml@@ -43,6 +43,7 @@ macro_entrypoint = { path = "../macro_entrypoint" }macro_config = { path = "../macro_config" }macro_env = { path = "../macro_env" }macro_env_var = { path = "../macro_env_var" }+macro_event_broker = { path = "../macro_event_broker" }macro_queues = { path = "../macro_queues" }database_env_vars = { path = "../macro_env_vars/database_env_vars" }doppler-config = { workspace = true }diff --git a/rust/cloud-storage/mcp_service/src/config.rs b/rust/cloud-storage/mcp_service/src/config.rsindex 9b0eb8dfa8..611059801d 100644--- a/rust/cloud-storage/mcp_service/src/config.rs+++ b/rust/cloud-storage/mcp_service/src/config.rs@@ -50,6 +50,8 @@ env_vars! {pub struct AppBaseUrl;/// JWT secret for minting document permission tokens for the editing worker.pub struct DocumentPermissionJwt;+ /// Comma-separated Kafka bootstrap servers for the macro event broker.+ pub struct KafkaBrokers;}/// The configuration parameters for the MCP server.@@ -87,6 +89,7 @@ pub struct Config {/// The internal api keypub internal_api_key: InternalApiKey,pub document_permission_jwt: DocumentPermissionJwt,+ pub kafka_brokers: KafkaBrokers,}impl Config {diff --git a/rust/cloud-storage/mcp_service/src/context.rs b/rust/cloud-storage/mcp_service/src/context.rsindex eb19effe78..10bb48bd8c 100644--- a/rust/cloud-storage/mcp_service/src/context.rs+++ b/rust/cloud-storage/mcp_service/src/context.rs@@ -213,6 +213,10 @@ async fn build_tool_context(ai_tools::build_properties_service(db.clone(), entity_access_service.clone());let task_properties_service =ai_tools::build_task_properties_adapter(db.clone(), properties_service.clone());+ let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(+ macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ );let document_service = documents::domain::service::DocumentServiceImpl {repo: document_repo,cloudfront_config,@@ -225,6 +229,7 @@ async fn build_tool_context(entity_access_management::outbound::PgRepository::new(db.clone()),),foreign_entity_service: ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ macro_event_broker,};let lexical_client_for_tools = (*lexical_client).clone();let document_tool_context = DocumentToolContext::new(diff --git a/rust/cloud-storage/scheduled_action/src/config.rs b/rust/cloud-storage/scheduled_action/src/config.rsindex 8d8e7bf5d9..401e3a49dc 100644--- a/rust/cloud-storage/scheduled_action/src/config.rs+++ b/rust/cloud-storage/scheduled_action/src/config.rs@@ -31,6 +31,8 @@ env_vars! {pub struct DocumentStorageServiceCloudfrontSignerPublicKeyId;/// Secrets Manager secret name for the CloudFront signer private key.pub struct DocumentStorageServiceCloudfrontSignerPrivateKeySecretName;+ /// Comma-separated Kafka bootstrap servers for the macro event broker.+ pub struct KafkaBrokers;}/// The configuration parameters for the agent schedule service.@@ -55,6 +57,7 @@ pub struct Config {DocumentStorageServiceCloudfrontSignerPublicKeyId,pub document_storage_service_cloudfront_signer_private_key_secret_name:DocumentStorageServiceCloudfrontSignerPrivateKeySecretName,+ pub kafka_brokers: KafkaBrokers,/// The internal api keypub internal_api_key: InternalApiKey,}diff --git a/rust/cloud-storage/workspace-hack/Cargo.toml b/rust/cloud-storage/workspace-hack/Cargo.tomlindex aae4b4a9a0..a7522b4d83 100644--- a/rust/cloud-storage/workspace-hack/Cargo.toml+++ b/rust/cloud-storage/workspace-hack/Cargo.toml@@ -17,10 +17,11 @@ publish = false[dependencies]ahash = { version = "0.8" }aho-corasick = { version = "1" }-aws-credential-types = { version = "1", default-features = false, features = ["test-util"] }+aws-config = { version = "1", features = ["behavior-version-latest"] }+aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials", "test-util"] }aws-runtime = { version = "1", default-features = false, features = ["event-stream", "http-02x", "http-1x", "sigv4a"] }+aws-sdk-sts = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] }aws-sigv4 = { version = "1", features = ["http0-compat", "sign-eventstream", "sigv4a"] }-aws-smithy-async = { version = "1", default-features = false, features = ["rt-tokio"] }aws-smithy-http = { version = "0.63", default-features = false, features = ["event-stream"] }aws-smithy-http-client = { version = "1", default-features = false, features = ["legacy-rustls-ring", "rustls-aws-lc"] }aws-smithy-runtime = { version = "1", default-features = false, features = ["client", "default-https-client", "rt-tokio", "tls-rustls"] }@@ -134,10 +135,11 @@ zeroize = { version = "1", features = ["derive"] }[build-dependencies]ahash = { version = "0.8" }aho-corasick = { version = "1" }-aws-credential-types = { version = "1", default-features = false, features = ["test-util"] }+aws-config = { version = "1", features = ["behavior-version-latest"] }+aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials", "test-util"] }aws-runtime = { version = "1", default-features = false, features = ["event-stream", "http-02x", "http-1x", "sigv4a"] }+aws-sdk-sts = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] }aws-sigv4 = { version = "1", features = ["http0-compat", "sign-eventstream", "sigv4a"] }-aws-smithy-async = { version = "1", default-features = false, features = ["rt-tokio"] }aws-smithy-http = { version = "0.63", default-features = false, features = ["event-stream"] }aws-smithy-http-client = { version = "1", default-features = false, features = ["legacy-rustls-ring", "rustls-aws-lc"] }aws-smithy-runtime = { version = "1", default-features = false, features = ["client", "default-https-client", "rt-tokio", "tls-rustls"] }