Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(kafka): wire up documents with macro event broker (#4552)

Will Hutchinsond676f8a9131c
diff --git a/.github/kafka-cluster-topics.json b/.github/kafka-cluster-topics.json
index f279549082..fa885291eb 100644
--- a/.github/kafka-cluster-topics.json
+++ b/.github/kafka-cluster-topics.json
@@ -1,3 +1,4 @@
[
+ "macro.documents",
"macro.example"
]
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index e51c4b391d..08c8ca860c 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -62,6 +62,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_queues",
"macro_service_urls",
@@ -204,6 +206,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_queues",
"macro_service_urls",
@@ -345,6 +349,8 @@
"macro_env",
"macro_env_var",
"macro_env_vars/database_env_vars",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_queues",
"macro_redis",
@@ -639,6 +645,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_queues",
"macro_service_urls",
@@ -1270,6 +1278,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_queues",
"macro_service_urls",
@@ -1388,6 +1398,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_project_utils",
"macro_queues",
@@ -1545,6 +1557,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_service_urls",
"macro_sync_service_jwt",
@@ -1630,6 +1644,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_service_urls",
"macro_sync_service_jwt",
@@ -2250,6 +2266,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_service_urls",
"macro_sync_service_jwt",
@@ -2664,6 +2682,8 @@
"workspace-hack"
],
"macro_event_broker": [
+ "macro_env",
+ "macro_env_var",
"macro_event_broker",
"macro_event_topics",
"workspace-hack"
@@ -2845,6 +2865,8 @@
"macro_env",
"macro_env_var",
"macro_env_vars/database_env_vars",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_queues",
"macro_service_urls",
@@ -2945,6 +2967,8 @@
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_queues",
"macro_service_urls",
@@ -3750,6 +3774,8 @@
"macro_env",
"macro_env_var",
"macro_env_vars/database_env_vars",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_middleware",
"macro_queues",
"macro_service_urls",
diff --git a/infra/packages/resources/src/resources/kafka-cluster.ts b/infra/packages/resources/src/resources/kafka-cluster.ts
index 59945d363a..b2240da9e7 100644
--- a/infra/packages/resources/src/resources/kafka-cluster.ts
+++ b/infra/packages/resources/src/resources/kafka-cluster.ts
@@ -50,6 +50,7 @@ export class KafkaCluster extends pulumi.ComponentResource {
public logGroup: aws.cloudwatch.LogGroup;
public cluster: aws.msk.Cluster;
public topics: Map<string, aws.msk.Topic>;
+ public clientAccessPolicy: aws.iam.Policy;
constructor(
name: string,
@@ -222,5 +223,65 @@ export class KafkaCluster extends pulumi.ComponentResource {
)
);
}
+
+ // MSK IAM resource ARNs are derived from the cluster ARN:
+ // arn:aws:kafka:REGION:ACCT:cluster/NAME/UUID → arn:aws:kafka:REGION:ACCT:topic/NAME/UUID/<topic>
+ // so a trailing /* covers every topic (and group) on this cluster.
+ const allTopicArns = this.cluster.arn.apply(
+ (arn) => `${arn.replace(':cluster/', ':topic/')}/*`
+ );
+ const allGroupArns = this.cluster.arn.apply(
+ (arn) => `${arn.replace(':cluster/', ':group/')}/*`
+ );
+
+ // Managed policy granting full client (producer + consumer) access to this
+ // cluster and all of its topics and consumer groups. Attach it to the ECS
+ // task role of any service that publishes or consumes events. Topic
+ // management (create/alter/delete) is deliberately excluded — topics are
+ // owned by this component.
+ this.clientAccessPolicy = new aws.iam.Policy(
+ 'client-access-policy',
+ {
+ name: `${name}-client-access`,
+ description: `Producer + consumer access to the ${name} Kafka cluster and all its topics and consumer groups`,
+ policy: {
+ Version: '2012-10-17',
+ Statement: [
+ {
+ Sid: 'Cluster',
+ Effect: 'Allow',
+ Action: [
+ 'kafka-cluster:Connect',
+ 'kafka-cluster:DescribeCluster',
+ // Cluster-scoped by MSK; needed if a producer enables idempotence.
+ 'kafka-cluster:WriteDataIdempotently',
+ ],
+ Resource: [this.cluster.arn],
+ },
+ {
+ Sid: 'Topics',
+ Effect: 'Allow',
+ Action: [
+ 'kafka-cluster:DescribeTopic',
+ 'kafka-cluster:ReadData',
+ 'kafka-cluster:WriteData',
+ ],
+ Resource: [allTopicArns],
+ },
+ {
+ Sid: 'Groups',
+ Effect: 'Allow',
+ Action: [
+ 'kafka-cluster:DescribeGroup',
+ 'kafka-cluster:AlterGroup',
+ ],
+ Resource: [allGroupArns],
+ },
+ ],
+ },
+ tags: this.tags,
+ },
+ { parent: this }
+ );
}
}
diff --git a/infra/packages/shared/src/index.ts b/infra/packages/shared/src/index.ts
index 0fcad65d1f..edf2ddbe03 100644
--- a/infra/packages/shared/src/index.ts
+++ b/infra/packages/shared/src/index.ts
@@ -50,5 +50,6 @@ export {
getAiToolsServiceRoleArns,
type AiToolsInfra,
} from './ai_tools';
+export { getKafkaClusterPolicy } from './kafka_cluster_policy';
export * from './service_urls';
diff --git a/infra/packages/shared/src/kafka_cluster_policy.ts b/infra/packages/shared/src/kafka_cluster_policy.ts
new file mode 100644
index 0000000000..dc45f5ce46
--- /dev/null
+++ b/infra/packages/shared/src/kafka_cluster_policy.ts
@@ -0,0 +1,12 @@
+import * as pulumi from '@pulumi/pulumi';
+import { stack } from '../../shared';
+
+export function getKafkaClusterPolicy(): pulumi.Output<string> {
+ const kafkaClusterStack = new pulumi.StackReference('kafka-cluster-stack', {
+ name: `macro-inc/kafka-cluster/${stack}`,
+ });
+
+ return kafkaClusterStack
+ .getOutput('clientAccessPolicyArn')
+ .apply((arn) => arn as string);
+}
diff --git a/infra/stacks/agent-schedule-service/service.ts b/infra/stacks/agent-schedule-service/service.ts
index 6914a5405e..5cea2f7219 100644
--- a/infra/stacks/agent-schedule-service/service.ts
+++ b/infra/stacks/agent-schedule-service/service.ts
@@ -14,6 +14,7 @@ import {
BASE_DOMAIN,
CLOUD_TRAIL_SNS_TOPIC_ARN,
DopplerEcsEnvironment,
+ getKafkaClusterPolicy,
stack,
} from '../../packages/shared';
@@ -171,6 +172,8 @@ export class AgentScheduleService extends pulumi.ComponentResource {
secretsManagerPolicy.arn,
sqsPolicy.arn,
s3Policy.arn,
+ // Producer/consumer access to the macro event Kafka cluster.
+ getKafkaClusterPolicy(),
],
tags,
},
diff --git a/infra/stacks/cloud-storage-service/cloud-storage-service.ts b/infra/stacks/cloud-storage-service/cloud-storage-service.ts
index 3eb72d95bc..021e490f24 100644
--- a/infra/stacks/cloud-storage-service/cloud-storage-service.ts
+++ b/infra/stacks/cloud-storage-service/cloud-storage-service.ts
@@ -17,6 +17,7 @@ import {
CLOUD_TRAIL_SNS_TOPIC_ARN,
SERVICE_DOMAIN_NAME,
DopplerEcsEnvironment,
+ getKafkaClusterPolicy,
stack,
} from '../../packages/shared';
@@ -266,6 +267,16 @@ export class CloudStorageService extends pulumi.ComponentResource {
{ parent: this, dependsOn: [snsPolicy, this.role] }
);
+ // Producer/consumer access to the macro event Kafka cluster.
+ new aws.iam.RolePolicyAttachment(
+ `${BASE_NAME}-role-kafka-client-att-${stack}`,
+ {
+ role: this.role,
+ policyArn: getKafkaClusterPolicy(),
+ },
+ { parent: this, dependsOn: [this.role] }
+ );
+
// ecr image
const image = new EcrImage(
`${BASE_NAME}-ecr-image-${stack}`,
diff --git a/infra/stacks/document-cognition-service/document-cognition-service.ts b/infra/stacks/document-cognition-service/document-cognition-service.ts
index f4f2fbee25..beb4ab5986 100644
--- a/infra/stacks/document-cognition-service/document-cognition-service.ts
+++ b/infra/stacks/document-cognition-service/document-cognition-service.ts
@@ -15,6 +15,7 @@ import {
BASE_DOMAIN,
CLOUD_TRAIL_SNS_TOPIC_ARN,
DopplerEcsEnvironment,
+ getKafkaClusterPolicy,
stack,
} from '../../packages/shared';
@@ -219,6 +220,8 @@ export class DocumentCognitionService extends pulumi.ComponentResource {
secretsManagerPolicy.arn,
s3Policy.arn,
connectionTablePolicyArn,
+ // Producer/consumer access to the macro event Kafka cluster.
+ getKafkaClusterPolicy(),
],
tags: this.tags,
},
diff --git a/infra/stacks/kafka-cluster/index.ts b/infra/stacks/kafka-cluster/index.ts
index 396e20bedf..ad62e36dd7 100644
--- a/infra/stacks/kafka-cluster/index.ts
+++ b/infra/stacks/kafka-cluster/index.ts
@@ -28,3 +28,17 @@ export const clusterArn = kafkaCluster.cluster.arn;
export const bootstrapBrokersSaslIam =
kafkaCluster.cluster.bootstrapBrokersSaslIam;
export const securityGroupId = kafkaCluster.securityGroup.id;
+
+// Producer + consumer access to the cluster and all its topics/groups. Attach
+// to the ECS task role of any service that publishes or consumes events:
+//
+// const kafkaClusterStack = new pulumi.StackReference('kafka-cluster', {
+// name: `macro-inc/kafka-cluster/${stack}`,
+// });
+// new aws.iam.RolePolicyAttachment('kafka-client-access', {
+// role: taskRole.name,
+// policyArn: kafkaClusterStack
+// .getOutput('clientAccessPolicyArn')
+// .apply((arn) => arn as string),
+// });
+export const clientAccessPolicyArn = kafkaCluster.clientAccessPolicy.arn;
diff --git a/infra/stacks/mcp-server/mcp-server.ts b/infra/stacks/mcp-server/mcp-server.ts
index 8aa1faf2d1..38ed97f1cb 100644
--- a/infra/stacks/mcp-server/mcp-server.ts
+++ b/infra/stacks/mcp-server/mcp-server.ts
@@ -14,6 +14,7 @@ import {
BASE_DOMAIN,
CLOUD_TRAIL_SNS_TOPIC_ARN,
DopplerEcsEnvironment,
+ getKafkaClusterPolicy,
stack,
} from '../../packages/shared';
@@ -215,6 +216,8 @@ export class McpServer extends pulumi.ComponentResource {
secretsManagerPolicy.arn,
s3Policy.arn,
sqsPolicy.arn,
+ // Producer/consumer access to the macro event Kafka cluster.
+ getKafkaClusterPolicy(),
],
tags: this.tags,
},
diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock
index e6cdc9cb9e..1600b2643c 100644
--- a/rust/cloud-storage/Cargo.lock
+++ b/rust/cloud-storage/Cargo.lock
@@ -202,6 +202,7 @@ dependencies = [
"macro_db_client",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
"macro_queues",
"macro_service_urls",
"macro_user_id",
@@ -1164,6 +1165,24 @@ dependencies = [
"fs_extra",
]
+[[package]]
+name = "aws-msk-iam-sasl-signer"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e62f2c07adf3f0bd1ad40414dd30fcf34c595d5f613e36011d5b62eeff14f490"
+dependencies = [
+ "aws-config",
+ "aws-credential-types",
+ "aws-sdk-sts",
+ "aws-sigv4",
+ "aws-types",
+ "base64 0.22.1",
+ "chrono",
+ "futures",
+ "thiserror 1.0.69",
+ "url",
+]
+
[[package]]
name = "aws-runtime"
version = "1.7.4"
@@ -4022,6 +4041,7 @@ dependencies = [
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
"macro_middleware",
"macro_queues",
"macro_service_urls",
@@ -4137,6 +4157,7 @@ dependencies = [
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
"macro_middleware",
"macro_project_utils",
"macro_queues",
@@ -4315,7 +4336,10 @@ dependencies = [
"macro_aws_config",
"macro_db_migrator",
"macro_entrypoint",
+ "macro_env",
"macro_env_var",
+ "macro_event_broker",
+ "macro_event_topics",
"macro_service_urls",
"macro_sync_service_jwt",
"macro_user_id",
@@ -4329,6 +4353,7 @@ dependencies = [
"models_permissions",
"models_properties",
"non_empty",
+ "rdkafka",
"reqwest 0.13.4",
"s3_key",
"schemars 1.2.1",
@@ -4342,6 +4367,7 @@ dependencies = [
"tokio",
"tower 0.5.3",
"tracing",
+ "tracing-subscriber",
"unicode-segmentation",
"urlencoding",
"utoipa",
@@ -7792,6 +7818,10 @@ dependencies = [
name = "macro_event_broker"
version = "0.1.0"
dependencies = [
+ "aws-msk-iam-sasl-signer",
+ "aws-types",
+ "macro_env",
+ "macro_env_var",
"macro_event_topics",
"rdkafka",
"rootcause",
@@ -8163,6 +8193,7 @@ dependencies = [
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
"macro_middleware",
"macro_queues",
"macro_service_urls",
@@ -15330,10 +15361,11 @@ version = "0.1.0"
dependencies = [
"ahash",
"aho-corasick",
+ "aws-config",
"aws-credential-types",
"aws-runtime",
+ "aws-sdk-sts",
"aws-sigv4",
- "aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http-client",
"aws-smithy-runtime",
diff --git a/rust/cloud-storage/Cargo.toml b/rust/cloud-storage/Cargo.toml
index 4d1739a548..daf941b52e 100644
--- a/rust/cloud-storage/Cargo.toml
+++ b/rust/cloud-storage/Cargo.toml
@@ -90,6 +90,7 @@ async-stream = "0.3.6"
async-trait = "0.1.86"
automod = "1"
aws-config = { version = "1.8.16", features = ["behavior-version-latest"] }
+aws-msk-iam-sasl-signer = "1.0.1"
aws-sdk-dynamodb = "1.111.0"
aws-sdk-lambda = "1.122.0"
aws-sdk-s3 = "1.132.0"
@@ -98,6 +99,7 @@ aws-sdk-sesv2 = "1.118.0"
aws-sdk-sns = "1.99.0"
aws-sdk-sqs = "1.98.0"
aws-smithy-types = "1.4.7"
+aws-types = "1"
aws_lambda_events = { version = "0.16.0", default-features = false }
axum = { version = "0.8", features = ["tower-log"] }
axum-extra = { version = "0.10" }
diff --git a/rust/cloud-storage/ai_tools/Cargo.toml b/rust/cloud-storage/ai_tools/Cargo.toml
index d7addaf828..0f97ae5709 100644
--- a/rust/cloud-storage/ai_tools/Cargo.toml
+++ b/rust/cloud-storage/ai_tools/Cargo.toml
@@ -41,6 +41,7 @@ lexical_client = { path = "../lexical_client" }
macro_aws_config = { path = "../macro_aws_config" }
macro_db_client = { path = "../macro_db_client" }
macro_env_var = { path = "../macro_env_var" }
+macro_event_broker = { path = "../macro_event_broker" }
macro_queues = { path = "../macro_queues" }
macro_user_id = { path = "../macro_user_id" }
model = { path = "../model" }
diff --git a/rust/cloud-storage/ai_tools/src/build_context.rs b/rust/cloud-storage/ai_tools/src/build_context.rs
index fabb7540ed..aa8dac4218 100644
--- a/rust/cloud-storage/ai_tools/src/build_context.rs
+++ b/rust/cloud-storage/ai_tools/src/build_context.rs
@@ -57,6 +57,7 @@ env_var! {
DocumentStorageServiceCloudfrontSignerPublicKeyId,
DocumentStorageServiceCloudfrontSignerPrivateKeySecretName,
DocumentPermissionJwt,
+ KafkaBrokers,
}
}
@@ -81,7 +82,8 @@ maybe_env_var! {
/// `DOCX_DOCUMENT_UPLOAD_BUCKET`,
/// `DOCUMENT_STORAGE_SERVICE_CLOUDFRONT_DISTRIBUTION_URL`,
/// `DOCUMENT_STORAGE_SERVICE_CLOUDFRONT_SIGNER_PUBLIC_KEY_ID`,
-/// `DOCUMENT_STORAGE_SERVICE_CLOUDFRONT_SIGNER_PRIVATE_KEY_SECRET_NAME`.
+/// `DOCUMENT_STORAGE_SERVICE_CLOUDFRONT_SIGNER_PRIVATE_KEY_SECRET_NAME`,
+/// `KAFKA_BROKERS`.
///
/// Service URLs are resolved through the `macro_service_urls` crate, and queue
/// names through the `macro_queues` crate (both using optional `OVERRIDE_*` env
@@ -240,6 +242,10 @@ pub async fn build_tool_service_context_from_env(
pool.clone(),
properties_service.clone(),
);
+ let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
+ macro_event_broker::KafkaEventPublisher::new(env.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ );
let document_service = documents::domain::service::DocumentServiceImpl {
repo: document_repo,
cloudfront_config,
@@ -254,6 +260,7 @@ pub async fn build_tool_service_context_from_env(
foreign_entity_service: ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(
pool.clone(),
)),
+ macro_event_broker,
};
let document_tool_context = DocumentToolContext::new(
diff --git a/rust/cloud-storage/ai_tools/src/tool_context.rs b/rust/cloud-storage/ai_tools/src/tool_context.rs
index a1521f0fb6..56bbd90fd5 100644
--- a/rust/cloud-storage/ai_tools/src/tool_context.rs
+++ b/rust/cloud-storage/ai_tools/src/tool_context.rs
@@ -456,6 +456,7 @@ pub type ToolDocumentService = documents::domain::service::DocumentServiceImpl<
NoOpConnectionService,
ToolEntityAccessManagementService,
ToolForeignEntityService,
+ macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,
>;
/// Type alias for the entity access service implementation
diff --git a/rust/cloud-storage/deny.toml b/rust/cloud-storage/deny.toml
index 9860ee6ad8..09b227ccaa 100644
--- a/rust/cloud-storage/deny.toml
+++ b/rust/cloud-storage/deny.toml
@@ -190,9 +190,12 @@ allow = [
# List of crates to deny
deny = [
{ crate = "aws-config", wrappers = [
+ "aws-msk-iam-sasl-signer", # External crate, requires aws-config directly
"macro_aws_config",
+ "workspace-hack", # cargo-hakari generated crate
], reason = "Use macro_aws_config for AWS configuration in workspace crates" },
{ crate = "tracing-subscriber", wrappers = [
+ "documents", # Dev logging only
"email", # Dev logging only
"lambda_runtime_api_client", # External crate allowed to use tracing-subscriber
"macro_entrypoint",
@@ -230,26 +233,18 @@ deny = [
# Certain crates/versions that will be skipped when doing duplicate detection.
skip = [
# These duplicates are due to complex dependency trees from AWS SDK, async-stripe, axum, and other major dependencies
- { crate = "aligned-vec", reason = "Different dependencies require different versions" },
- { crate = "base16ct", reason = "Cryptographic dependencies version mismatch" },
{ crate = "base64", reason = "AWS SDK and other dependencies require different versions" },
{ crate = "bitflags", reason = "Different dependencies require different versions" },
{ crate = "core-foundation", reason = "Different dependencies require different versions" },
{ crate = "cpufeatures", reason = "Different dependencies require different versions" },
- { crate = "crypto-bigint", reason = "AWS cryptographic dependencies version mismatch" },
{ crate = "cssparser", reason = "HTML parsing dependencies version mismatch" },
{ crate = "darling", reason = "Derive macro dependencies version mismatch" },
{ crate = "darling_core", reason = "Derive macro dependencies version mismatch" },
{ crate = "darling_macro", reason = "Derive macro dependencies version mismatch" },
- { crate = "der", reason = "Cryptographic dependencies version mismatch" },
- { crate = "ecdsa", reason = "Cryptographic dependencies version mismatch" },
- { crate = "elliptic-curve", reason = "Cryptographic dependencies version mismatch" },
{ crate = "event-listener", reason = "Async dependencies version mismatch" },
{ crate = "fastrand", reason = "Different dependencies require different versions" },
- { crate = "ff", reason = "Cryptographic dependencies version mismatch" },
{ crate = "foldhash", reason = "Hash function dependencies" },
{ crate = "getrandom", reason = "Random number generation dependencies" },
- { crate = "group", reason = "Cryptographic dependencies version mismatch" },
{ crate = "h2", reason = "HTTP/2 dependencies version mismatch" },
{ crate = "hashbrown", reason = "Different dependencies require different versions" },
{ crate = "heck", reason = "Different dependencies require different versions" },
@@ -264,38 +259,30 @@ skip = [
{ crate = "jni-sys", reason = "Different dependencies require different versions" },
{ crate = "jsonwebtoken", reason = "Git fork and crates.io version coexist" },
{ crate = "markup5ever", reason = "HTML parsing dependencies" },
- { crate = "match_token", reason = "HTML parsing dependencies" },
{ crate = "mio", reason = "Different dependencies require different versions" },
{ crate = "nix", reason = "Different dependencies require different versions" },
{ crate = "nom", reason = "Parser combinator dependencies" },
- { crate = "p256", reason = "Cryptographic dependencies version mismatch" },
{ crate = "pastey", reason = "Macro dependencies version mismatch" },
{ crate = "phf", reason = "Perfect hash function dependencies" },
{ crate = "phf_codegen", reason = "Perfect hash function dependencies" },
{ crate = "phf_generator", reason = "Perfect hash function dependencies" },
{ crate = "phf_shared", reason = "Perfect hash function dependencies" },
- { crate = "pkcs8", reason = "Cryptographic dependencies" },
{ crate = "prost", reason = "Protocol buffer dependencies version mismatch" },
{ crate = "prost-derive", reason = "Protocol buffer dependencies version mismatch" },
{ crate = "rand", reason = "Different dependencies require different versions" },
{ crate = "rand_chacha", reason = "Random number generation dependencies" },
{ crate = "rand_core", reason = "Different dependencies require different versions" },
{ crate = "reqwest", reason = "livekit-api pins reqwest 0.12 via services-tokio feature; workspace uses 0.13" },
- { crate = "rfc6979", reason = "Cryptographic dependencies version mismatch" },
{ crate = "rustc-hash", reason = "Hash function dependencies" },
{ crate = "rustls", reason = "TLS dependencies version mismatch" },
{ crate = "rustls-webpki", reason = "TLS dependencies version mismatch" },
- { crate = "sec1", reason = "Cryptographic dependencies version mismatch" },
- { crate = "security-framework", reason = "macOS security framework dependencies" },
{ crate = "selectors", reason = "CSS selector dependencies" },
{ crate = "serde_qs", reason = "Query string serialization dependencies" },
{ crate = "servo_arc", reason = "Arc implementation dependencies" },
- { crate = "signature", reason = "Cryptographic signature dependencies" },
{ crate = "siphasher", reason = "Hash function dependencies" },
{ crate = "smol_str", reason = "Different dependencies require different versions" },
{ crate = "socket2", reason = "Different dependencies require different versions" },
{ crate = "spin", reason = "Different dependencies require different versions" },
- { crate = "spki", reason = "Cryptographic dependencies" },
{ crate = "syn", reason = "Procedural macros require different versions" },
{ crate = "thiserror", reason = "Error handling dependencies" },
{ crate = "thiserror-impl", reason = "Error handling dependencies" },
@@ -303,24 +290,16 @@ skip = [
{ crate = "tower", reason = "Service framework dependencies" },
{ crate = "unicode-width", reason = "Unicode dependencies" },
{ crate = "wasi", reason = "WebAssembly system interface dependencies" },
- { crate = "windows-link", reason = "Windows dependencies" },
- { crate = "windows-registry", reason = "Windows dependencies version mismatch" },
- { crate = "windows-result", reason = "Windows dependencies version mismatch" },
- { crate = "windows-strings", reason = "Windows dependencies version mismatch" },
{ crate = "windows-sys", reason = "Windows dependencies version mismatch" },
{ crate = "windows-targets", reason = "Windows dependencies" },
{ crate = "windows_aarch64_gnullvm", reason = "Windows dependencies" },
{ crate = "windows_aarch64_msvc", reason = "Windows dependencies" },
{ crate = "windows_i686_gnu", reason = "Windows dependencies" },
- { crate = "windows_i686_gnullvm", reason = "Windows dependencies" },
{ crate = "windows_i686_msvc", reason = "Windows dependencies" },
{ crate = "windows_x86_64_gnu", reason = "Windows dependencies" },
{ crate = "windows_x86_64_gnullvm", reason = "Windows dependencies" },
{ crate = "windows_x86_64_msvc", reason = "Windows dependencies" },
- { crate = "zerocopy", reason = "Zero-copy serialization dependencies" },
{ crate = "zip", reason = "Different dependencies require different versions" },
- { crate = "zune-core", reason = "Duplicated dep" },
- { crate = "zune-jpeg", reason = "Deuplicated dep" },
]
# Similarly to `skip` allows you to skip certain crates during duplicate
# detection. Unlike skip, it also includes the entire tree of transitive
diff --git a/rust/cloud-storage/document_cognition_service/Cargo.toml b/rust/cloud-storage/document_cognition_service/Cargo.toml
index 819245fd6e..39fd700c32 100644
--- a/rust/cloud-storage/document_cognition_service/Cargo.toml
+++ b/rust/cloud-storage/document_cognition_service/Cargo.toml
@@ -74,6 +74,7 @@ macro_db_client = { path = "../macro_db_client" }
macro_entrypoint = { path = "../macro_entrypoint" }
macro_env = { path = "../macro_env" }
macro_env_var = { path = "../macro_env_var" }
+macro_event_broker = { path = "../macro_event_broker" }
macro_queues = { path = "../macro_queues" }
macro_middleware = { path = "../macro_middleware", default-features = false, features = [
"auth",
diff --git a/rust/cloud-storage/document_cognition_service/src/api/context/test.rs b/rust/cloud-storage/document_cognition_service/src/api/context/test.rs
index c19f3b9986..0832e17ee3 100644
--- a/rust/cloud-storage/document_cognition_service/src/api/context/test.rs
+++ b/rust/cloud-storage/document_cognition_service/src/api/context/test.rs
@@ -254,6 +254,12 @@ pub async fn test_api_context(pool: sqlx::Pool<sqlx::Postgres>) -> std::sync::Ar
entity_access_management::outbound::PgRepository::new(pool.clone()),
),
ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(pool.clone())),
+ // Producer creation is lazy: nothing connects to Kafka unless an event
+ // is published, so a dummy broker address is safe for tests.
+ macro_event_broker::MacroEventBrokerService::new(
+ macro_event_broker::KafkaEventPublisher::new("localhost:9092")
+ .expect("kafka producer config is valid"),
+ ),
);
let test_lexical_client = LexicalClient::new("test".into(), "http://nofileshere".into());
let test_editing_client =
diff --git a/rust/cloud-storage/document_cognition_service/src/config.rs b/rust/cloud-storage/document_cognition_service/src/config.rs
index 6519225bd1..4eecaf8b63 100644
--- a/rust/cloud-storage/document_cognition_service/src/config.rs
+++ b/rust/cloud-storage/document_cognition_service/src/config.rs
@@ -21,6 +21,8 @@ env_vars!(
pub struct DocumentStorageServiceCloudfrontSignerPrivateKey;
pub struct McpCredentialsKeySecretName;
pub struct DocumentPermissionJwt;
+ /// Comma-separated Kafka bootstrap servers for the macro event broker.
+ pub struct KafkaBrokers;
);
maybe_env_vars!(
@@ -71,6 +73,8 @@ pub struct Config {
pub ai_editing_worker_url: String,
/// JWT secret for minting document permission tokens for the editing worker.
pub document_permission_jwt: DocumentPermissionJwt,
+ /// Comma-separated Kafka bootstrap servers for the macro event broker.
+ pub kafka_brokers: KafkaBrokers,
}
impl Config {
@@ -119,6 +123,7 @@ impl Config {
internal_api_key: InternalApiKey::Comptime(""),
ai_editing_worker_url: AiEditingWorkerUrl::unwrap_new().to_string(),
document_permission_jwt: DocumentPermissionJwt::Comptime("DOCUMENT_PERMISSION_JWT"),
+ kafka_brokers: KafkaBrokers::Comptime("localhost:9092"),
}
}
}
diff --git a/rust/cloud-storage/document_cognition_service/src/main.rs b/rust/cloud-storage/document_cognition_service/src/main.rs
index 827996c3ec..927f75d439 100644
--- a/rust/cloud-storage/document_cognition_service/src/main.rs
+++ b/rust/cloud-storage/document_cognition_service/src/main.rs
@@ -272,6 +272,10 @@ async fn main() -> anyhow::Result<()> {
ai_tools::build_properties_service(db.clone(), entity_access_service.clone());
let task_properties_service =
ai_tools::build_task_properties_adapter(db.clone(), properties_service.clone());
+ let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
+ macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ );
let document_service = DocumentServiceImpl::new(
document_repo,
cloudfront_config,
@@ -283,6 +287,7 @@ async fn main() -> anyhow::Result<()> {
entity_access_management::outbound::PgRepository::new(db.clone()),
),
ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ macro_event_broker,
);
let lexical_client_for_tools = (*lexical_client).clone();
let document_tool_context = DocumentToolContext::new(
diff --git a/rust/cloud-storage/document_storage_service/Cargo.toml b/rust/cloud-storage/document_storage_service/Cargo.toml
index 7efcfd1482..1fc217386e 100644
--- a/rust/cloud-storage/document_storage_service/Cargo.toml
+++ b/rust/cloud-storage/document_storage_service/Cargo.toml
@@ -94,6 +94,7 @@ macro_cors = { path = "../macro_cors" }
macro_db_client = { path = "../macro_db_client" }
macro_entrypoint = { path = "../macro_entrypoint" }
macro_env = { path = "../macro_env" }
+macro_event_broker = { path = "../macro_event_broker" }
macro_service_urls = { path = "../macro_service_urls" }
macro_sync_service_jwt = { path = "../macro_sync_service_jwt" }
macro_env_var = { path = "../macro_env_var" }
diff --git a/rust/cloud-storage/document_storage_service/src/api/context.rs b/rust/cloud-storage/document_storage_service/src/api/context.rs
index 084bc14597..0a0810e12a 100644
--- a/rust/cloud-storage/document_storage_service/src/api/context.rs
+++ b/rust/cloud-storage/document_storage_service/src/api/context.rs
@@ -53,6 +53,8 @@ use favorites::{
domain::service::FavoritesServiceImpl, inbound::axum_router::FavoritesRouterState,
outbound::pg_favorites_repo::PgFavoritesRepo,
};
+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
+
use foreign_entity::{
domain::service::ForeignEntityServiceImpl, inbound::axum_router::ForeignEntityRouterState,
outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
@@ -209,6 +211,7 @@ pub(crate) type DocumentService = DocumentServiceImpl<
ConnectionServiceImpl<EntityAccessService, ConnectionGatewayImpl>,
EntityAccessManagementService,
ForeignEntityServiceImpl<PgForeignEntityRepo>,
+ MacroEventBrokerService<KafkaEventPublisher>,
>;
/// Type alias for the documents router state.
diff --git a/rust/cloud-storage/document_storage_service/src/config.rs b/rust/cloud-storage/document_storage_service/src/config.rs
index 97f39bcbcf..d878e39c7c 100644
--- a/rust/cloud-storage/document_storage_service/src/config.rs
+++ b/rust/cloud-storage/document_storage_service/src/config.rs
@@ -23,6 +23,8 @@ env_vars! {
pub struct OpensearchPassword;
pub struct GithubSyncAppUrl;
pub struct GithubSyncAppClientId;
+ /// Comma-separated Kafka bootstrap servers for the macro event broker.
+ pub struct KafkaBrokers;
pub struct LivekitServerUrl;
pub struct LivekitApiKey;
pub struct LivekitApiSecret;
@@ -92,6 +94,7 @@ pub struct Config {
pub opensearch_password: OpensearchPassword,
pub github_sync_app_url: GithubSyncAppUrl,
pub github_sync_app_client_id: GithubSyncAppClientId,
+ pub kafka_brokers: KafkaBrokers,
pub livekit_server_url: LivekitServerUrl,
pub livekit_api_key: LivekitApiKey,
pub livekit_api_secret: LivekitApiSecret,
diff --git a/rust/cloud-storage/document_storage_service/src/main.rs b/rust/cloud-storage/document_storage_service/src/main.rs
index e22f32e50d..5961c3e2e3 100644
--- a/rust/cloud-storage/document_storage_service/src/main.rs
+++ b/rust/cloud-storage/document_storage_service/src/main.rs
@@ -71,6 +71,7 @@ use lexical_client::LexicalClient;
use macro_auth::middleware::decode_jwt::JwtValidationArgs;
use macro_entrypoint::MacroEntrypoint;
use macro_env_var::maybe_env_vars;
+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use macro_service_urls::{ConnectionGatewayUrl, LexicalServiceUrl, SyncServiceUrl};
use macro_sha_count_client::Redis;
use notification::domain::service::SqsNotificationIngress;
@@ -375,6 +376,11 @@ async fn main() -> anyhow::Result<()> {
entity_access_management::outbound::PgRepository::new(db.clone()),
);
+ let macro_event_broker = MacroEventBrokerService::new(
+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ );
+
let document_service = Arc::new(DocumentServiceImpl::new(
document_repo,
cloudfront_config,
@@ -387,6 +393,7 @@ async fn main() -> anyhow::Result<()> {
connection_service,
entity_access_management_service.clone(),
ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ macro_event_broker,
));
let foreign_entity_service = Arc::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(
diff --git a/rust/cloud-storage/documents/Cargo.toml b/rust/cloud-storage/documents/Cargo.toml
index 96d08ac889..e01e570b35 100644
--- a/rust/cloud-storage/documents/Cargo.toml
+++ b/rust/cloud-storage/documents/Cargo.toml
@@ -70,7 +70,12 @@ outbound = [
"ports",
]
ports = ["dep:tokio"]
-service = ["dep:entity_access_management", "dep:foreign_entity", "ports"]
+service = [
+ "dep:entity_access_management",
+ "dep:foreign_entity",
+ "macro_event_broker/ports",
+ "ports",
+]
[dependencies]
# Core
@@ -100,6 +105,8 @@ foreign_entity = { path = "../foreign_entity", default-features = false, feature
macro_aws_config = { path = "../macro_aws_config", features = ["s3"] }
macro_entrypoint = { path = "../macro_entrypoint", optional = true }
macro_env_var = { path = "../macro_env_var" }
+macro_event_broker = { path = "../macro_event_broker", default-features = false }
+macro_event_topics = { path = "../macro_event_topics" }
macro_sync_service_jwt = { path = "../macro_sync_service_jwt" }
macro_user_id = { path = "../macro_user_id" }
macro_uuid = { path = "../macro_uuid" }
@@ -152,7 +159,12 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }
dotenvy = { workspace = true }
http-body-util = { workspace = true }
macro_db_migrator = { path = "../macro_db_migrator" }
+macro_env = { path = "../macro_env" }
+# Examples need the Kafka adapter (outbound); the library itself only uses ports.
+macro_event_broker = { path = "../macro_event_broker" }
mockall = { workspace = true }
+rdkafka = { workspace = true }
serde_json = { workspace = true }
-tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] }
tower = { workspace = true }
+tracing-subscriber = { workspace = true }
diff --git a/rust/cloud-storage/documents/examples/documents_consumer.rs b/rust/cloud-storage/documents/examples/documents_consumer.rs
new file mode 100644
index 0000000000..6603e2ed30
--- /dev/null
+++ b/rust/cloud-storage/documents/examples/documents_consumer.rs
@@ -0,0 +1,266 @@
+//! Kafka consumer example for the `macro.documents` topic.
+//!
+//! Connects with the same `KAFKA_BROKERS` env var and environment-driven
+//! transport as [`macro_event_broker::KafkaEventPublisher`] (plaintext for
+//! `ENVIRONMENT=local`, TLS + SASL/OAUTHBEARER with MSK IAM otherwise),
+//! subscribes to [`MacroDocumentsTopic`], and prints every document lifecycle
+//! event it receives.
+//!
+//! The consumer follows the standard poll-loop / worker split: a Kafka poll
+//! loop decodes messages and hands them to a processing task over a bounded
+//! tokio mpsc channel (backpressure applies when the processor falls behind),
+//! offsets are committed only after a successful handoff (at-least-once), and
+//! ctrl-c drains the channel before exiting.
+//!
+//! Run against the local docker broker:
+//! ```sh
+//! ENVIRONMENT=local KAFKA_BROKERS=localhost:9092 \
+//! cargo run -p documents --example documents_consumer
+//! ```
+//! or against the dev MSK cluster (needs AWS credentials and network access):
+//! ```sh
+//! ENVIRONMENT=develop AWS_REGION=us-east-1 KAFKA_BROKERS=<bootstrapBrokersSaslIam> \
+//! cargo run -p documents --example documents_consumer
+//! ```
+
+use anyhow::Context as _;
+use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};
+use macro_env::Environment;
+use macro_event_broker::outbound::msk_iam::configure_sasl_iam;
+use macro_event_broker::{EventBrokerError, MacroEvent, MskIamClientContext, Topic as _};
+use macro_event_topics::MacroDocumentsTopic;
+use rdkafka::ClientConfig;
+use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};
+use rdkafka::error::KafkaResult;
+use rdkafka::message::{BorrowedMessage, Message};
+use tokio::sync::mpsc;
+
+macro_env_var::env_var! {
+ struct ConsumerEnvVars {
+ KafkaBrokers,
+ }
+}
+
+/// Consumer group id for this example. Offsets are committed under this group,
+/// so re-runs resume where the previous run left off.
+const GROUP_ID: &str = "documents-consumer-example";
+
+/// Bounded capacity of the channel between the poll loop and the processor.
+const CHANNEL_CAPACITY: usize = 128;
+
+/// Consumer-specific event enum for a consumer subscribed to `macro.documents`,
+/// following the pattern in `macro_event_broker/examples/example_event.rs`.
+enum DocumentsConsumerEvent {
+ /// Event received on [`MacroDocumentsTopic`].
+ Documents(DocumentMacroEvent),
+}
+
+impl DocumentsConsumerEvent {
+ /// Decode one Kafka message into this consumer's event enum.
+ fn decode(topic: &str, key: &str, payload: &[u8]) -> Result<Self, EventBrokerError> {
+ match topic {
+ topic if topic == MacroDocumentsTopic.as_str() => {
+ Ok(Self::Documents(DocumentMacroEvent::decode(key, payload)?))
+ }
+ unknown => Err(EventBrokerError::UnknownTopic(unknown.to_string())),
+ }
+ }
+}
+
+/// A decoded event plus the Kafka coordinates it came from, as handed from the
+/// poll loop to the processor over the channel.
+struct ReceivedEvent {
+ partition: i32,
+ offset: i64,
+ event: DocumentsConsumerEvent,
+}
+
+/// The underlying consumer, split by transport (mirrors `KafkaEventPublisher`).
+enum DocumentsConsumer {
+ /// Unauthenticated plaintext connection (local docker broker).
+ Plaintext(StreamConsumer),
+ /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).
+ MskIam(StreamConsumer<MskIamClientContext>),
+}
+
+impl DocumentsConsumer {
+ /// Build a consumer for the given brokers, choosing the transport from the
+ /// `ENVIRONMENT` variable exactly like the publisher does.
+ fn from_env(brokers: &str) -> anyhow::Result<Self> {
+ let mut config = ClientConfig::new();
+ config
+ .set("bootstrap.servers", brokers)
+ .set("group.id", GROUP_ID)
+ // Offsets are committed manually after a successful handoff.
+ .set("enable.auto.commit", "false")
+ // Start from the beginning of the topic on the first ever run.
+ .set("auto.offset.reset", "earliest");
+
+ let consumer = match Environment::new_or_prod() {
+ Environment::Local => Self::Plaintext(
+ config
+ .create()
+ .context("failed to create plaintext kafka consumer")?,
+ ),
+ Environment::Develop | Environment::Production => {
+ configure_sasl_iam(&mut config);
+ Self::MskIam(
+ config
+ .create_with_context(MskIamClientContext::from_env())
+ .context("failed to create MSK IAM kafka consumer")?,
+ )
+ }
+ };
+
+ Ok(consumer)
+ }
+
+ fn subscribe(&self) -> KafkaResult<()> {
+ let topics = [MacroDocumentsTopic.as_str()];
+ match self {
+ Self::Plaintext(consumer) => consumer.subscribe(&topics),
+ Self::MskIam(consumer) => consumer.subscribe(&topics),
+ }
+ }
+
+ /// Receive the next message. `StreamConsumer::recv` is cancel-safe, so it
+ /// can sit in a `select!` without losing messages.
+ async fn recv(&self) -> KafkaResult<BorrowedMessage<'_>> {
+ match self {
+ Self::Plaintext(consumer) => consumer.recv().await,
+ Self::MskIam(consumer) => consumer.recv().await,
+ }
+ }
+
+ fn commit(&self, message: &BorrowedMessage<'_>) -> KafkaResult<()> {
+ match self {
+ Self::Plaintext(consumer) => consumer.commit_message(message, CommitMode::Async),
+ Self::MskIam(consumer) => consumer.commit_message(message, CommitMode::Async),
+ }
+ }
+}
+
+fn event_type(event: &DocumentTopicEvent) -> &'static str {
+ match event {
+ DocumentTopicEvent::Created(_) => "document.created",
+ DocumentTopicEvent::Updated(_) => "document.updated",
+ DocumentTopicEvent::Deleted(_) => "document.deleted",
+ DocumentTopicEvent::Copied(_) => "document.copied",
+ }
+}
+
+/// Print each received event. This stands in for real event handling — swap
+/// the body for indexing, notifications, etc.
+async fn process_events(mut events: mpsc::Receiver<ReceivedEvent>) {
+ while let Some(received) = events.recv().await {
+ let DocumentsConsumerEvent::Documents(event) = received.event;
+ let envelope = event.event();
+
+ println!(
+ "[partition {} offset {}] {} key={} event_id={} schema_version={}",
+ received.partition,
+ received.offset,
+ event_type(&envelope.event),
+ event.key(),
+ envelope.event_id,
+ envelope.schema_version,
+ );
+ match serde_json::to_string_pretty(envelope) {
+ Ok(json) => println!("{json}"),
+ Err(e) => eprintln!("failed to re-serialize event: {e}"),
+ }
+ }
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+ // Surface tracing from macro_event_broker (e.g. MSK IAM token refreshes)
+ // and librdkafka's internal connection/auth logs (bridged from the `log`
+ // crate). Tune with RUST_LOG, e.g. RUST_LOG=debug for librdkafka detail.
+ tracing_subscriber::fmt()
+ .with_env_filter(
+ tracing_subscriber::EnvFilter::try_from_default_env()
+ .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
+ )
+ .init();
+
+ let env = ConsumerEnvVars::new().context("KAFKA_BROKERS must be set")?;
+
+ let consumer = DocumentsConsumer::from_env(env.kafka_brokers.as_ref())?;
+ consumer
+ .subscribe()
+ .context("failed to subscribe to documents topic")?;
+ println!(
+ "listening on topic={} group={} brokers={} (ctrl-c to stop)",
+ MacroDocumentsTopic.as_str(),
+ GROUP_ID,
+ env.kafka_brokers.as_ref(),
+ );
+
+ let (events_tx, events_rx) = mpsc::channel::<ReceivedEvent>(CHANNEL_CAPACITY);
+ let processor = tokio::spawn(process_events(events_rx));
+
+ let mut shutdown = std::pin::pin!(tokio::signal::ctrl_c());
+ loop {
+ tokio::select! {
+ _ = &mut shutdown => {
+ println!("received ctrl-c, shutting down");
+ break;
+ }
+ result = consumer.recv() => {
+ let message = match result {
+ Ok(message) => message,
+ Err(e) => {
+ eprintln!("kafka receive error: {e}");
+ continue;
+ }
+ };
+
+ let Some(payload) = message.payload() else {
+ eprintln!(
+ "skipping message with empty payload at partition {} offset {}",
+ message.partition(),
+ message.offset(),
+ );
+ let _ = consumer.commit(&message).inspect_err(|e| {
+ eprintln!("failed to commit offset {}: {e}", message.offset());
+ });
+ continue;
+ };
+ let key = String::from_utf8_lossy(message.key().unwrap_or_default());
+
+ match DocumentsConsumerEvent::decode(message.topic(), key.as_ref(), payload) {
+ Ok(event) => {
+ events_tx
+ .send(ReceivedEvent {
+ partition: message.partition(),
+ offset: message.offset(),
+ event,
+ })
+ .await
+ .context("processor task closed unexpectedly")?;
+ }
+ // Undecodable messages are logged and skipped rather than
+ // wedging the partition on a poison message.
+ Err(e) => eprintln!(
+ "failed to decode message at partition {} offset {}: {e}",
+ message.partition(),
+ message.offset(),
+ ),
+ }
+
+ // Commit only after the event was handed to the processor:
+ // at-least-once delivery (up to the channel buffer on a crash).
+ let _ = consumer.commit(&message).inspect_err(|e| {
+ eprintln!("failed to commit offset {}: {e}", message.offset());
+ });
+ }
+ }
+ }
+
+ // Close the channel and let the processor drain what it already received.
+ drop(events_tx);
+ processor.await.context("processor task panicked")?;
+
+ Ok(())
+}
diff --git a/rust/cloud-storage/documents/src/domain.rs b/rust/cloud-storage/documents/src/domain.rs
index 2803df036a..3b3bf94198 100644
--- a/rust/cloud-storage/documents/src/domain.rs
+++ b/rust/cloud-storage/documents/src/domain.rs
@@ -2,6 +2,7 @@
pub mod branch_name;
pub mod content;
+pub mod events;
#[cfg(feature = "ports")]
pub mod markdown_backfill;
diff --git a/rust/cloud-storage/documents/src/domain/events.rs b/rust/cloud-storage/documents/src/domain/events.rs
new file mode 100644
index 0000000000..9f6c576580
--- /dev/null
+++ b/rust/cloud-storage/documents/src/domain/events.rs
@@ -0,0 +1,173 @@
+//! Kafka event models for the `macro.documents` topic.
+//!
+//! Follows the canonical pattern in `macro_event_broker/examples/example_event.rs`:
+//! per-variant metadata structs, a [`TopicEvent`] enum tagged by `event_type`,
+//! and a [`MacroEvent`] wrapper keyed by document id.
+
+use chrono::{DateTime, Utc};
+use document_sub_type::DocumentSubType;
+use macro_event_broker::{Event, MacroEvent, TopicEvent};
+use macro_event_topics::MacroDocumentsTopic;
+use macro_user_id::user_id::MacroUserIdStr;
+use model::document::FileType;
+use serde::{Deserialize, Serialize};
+
+use super::models::FileTypeUpdate;
+
+/// Metadata for [`DocumentTopicEvent::Created`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DocumentCreatedMetadata {
+ /// The id of the created document.
+ pub document_id: String,
+ /// The owner (creator) of the document.
+ pub owner: MacroUserIdStr<'static>,
+ /// The document name, with any file extension stripped.
+ pub document_name: String,
+ /// File type of the document, when known.
+ pub file_type: Option<FileType>,
+ /// Project the document was created in, when any.
+ pub project_id: Option<String>,
+ /// Sub type (task / snippet), when any.
+ pub sub_type: Option<DocumentSubType>,
+ /// Creation timestamp reported by the repository.
+ pub created_at: Option<DateTime<Utc>>,
+}
+
+/// Metadata for [`DocumentTopicEvent::Updated`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DocumentUpdatedMetadata {
+ /// The id of the updated document.
+ pub document_id: String,
+ /// The owner of the document.
+ pub owner: MacroUserIdStr<'static>,
+ /// The authenticated user who performed the update; `None` for
+ /// unauthenticated or internal callers.
+ pub actor_user_id: Option<MacroUserIdStr<'static>>,
+ /// New (cleaned) document name; `None` when unchanged.
+ pub document_name: Option<String>,
+ /// Project id before the update.
+ pub previous_project_id: Option<String>,
+ /// New project id; `None` when unchanged, `Some("")` when removed from
+ /// its project (mirrors the edit repo args semantics).
+ pub project_id: Option<String>,
+ /// Requested file type change; `None` when unchanged.
+ pub file_type: Option<FileTypeUpdate>,
+ /// Whether share permissions were updated.
+ pub share_permission_updated: bool,
+}
+
+/// Metadata for [`DocumentTopicEvent::Deleted`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DocumentDeletedMetadata {
+ /// The id of the deleted document.
+ pub document_id: String,
+ /// The authenticated user who deleted the document; `None` for
+ /// unauthenticated or internal callers.
+ pub actor_user_id: Option<MacroUserIdStr<'static>>,
+ /// Project the document belonged to, when any.
+ pub project_id: Option<String>,
+}
+
+/// Metadata for [`DocumentTopicEvent::Copied`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DocumentCopiedMetadata {
+ /// The id of the newly created copy.
+ pub document_id: String,
+ /// The id of the document that was copied.
+ pub source_document_id: String,
+ /// The specific source version copied, when requested.
+ pub source_version_id: Option<i64>,
+ /// The owner of the new copy (the copier).
+ pub owner: MacroUserIdStr<'static>,
+ /// The name of the new document.
+ pub document_name: String,
+ /// File type of the document, when known.
+ pub file_type: Option<FileType>,
+ /// Project the copy belongs to, when any.
+ pub project_id: Option<String>,
+ /// Sub type (task / snippet), when any.
+ pub sub_type: Option<DocumentSubType>,
+}
+
+/// Events that can be published to [`MacroDocumentsTopic`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "event_type", content = "metadata")]
+pub enum DocumentTopicEvent {
+ /// A document was created.
+ #[serde(rename = "document.created")]
+ Created(DocumentCreatedMetadata),
+ /// A document's metadata / permissions were updated.
+ #[serde(rename = "document.updated")]
+ Updated(DocumentUpdatedMetadata),
+ /// A document was soft-deleted.
+ #[serde(rename = "document.deleted")]
+ Deleted(DocumentDeletedMetadata),
+ /// A document was copied.
+ #[serde(rename = "document.copied")]
+ Copied(DocumentCopiedMetadata),
+}
+
+impl TopicEvent for DocumentTopicEvent {
+ type Topic = MacroDocumentsTopic;
+
+ fn schema_version(&self) -> u8 {
+ 1
+ }
+}
+
+/// Publishable event for [`MacroDocumentsTopic`], keyed by document id.
+pub struct DocumentMacroEvent {
+ key: String,
+ event: Event<DocumentTopicEvent>,
+}
+
+impl DocumentMacroEvent {
+ /// Build a created event keyed by the new document id.
+ pub fn created(key: impl Into<String>, metadata: DocumentCreatedMetadata) -> Self {
+ Self::new(key, DocumentTopicEvent::Created(metadata))
+ }
+
+ /// Build an updated event keyed by the updated document id.
+ pub fn updated(key: impl Into<String>, metadata: DocumentUpdatedMetadata) -> Self {
+ Self::new(key, DocumentTopicEvent::Updated(metadata))
+ }
+
+ /// Build a deleted event keyed by the deleted document id.
+ pub fn deleted(key: impl Into<String>, metadata: DocumentDeletedMetadata) -> Self {
+ Self::new(key, DocumentTopicEvent::Deleted(metadata))
+ }
+
+ /// Build a copied event keyed by the new document id.
+ pub fn copied(key: impl Into<String>, metadata: DocumentCopiedMetadata) -> Self {
+ Self::new(key, DocumentTopicEvent::Copied(metadata))
+ }
+
+ /// Build an event from a topic-specific event variant.
+ pub fn new(key: impl Into<String>, event: DocumentTopicEvent) -> Self {
+ Self::with_event(key, Event::new(event))
+ }
+
+ /// Build an event from a pre-built envelope.
+ pub fn with_event(key: impl Into<String>, event: Event<DocumentTopicEvent>) -> Self {
+ Self {
+ key: key.into(),
+ event,
+ }
+ }
+}
+
+impl MacroEvent for DocumentMacroEvent {
+ type EventPayload = DocumentTopicEvent;
+
+ fn key(&self) -> &str {
+ &self.key
+ }
+
+ fn event(&self) -> &Event<Self::EventPayload> {
+ &self.event
+ }
+
+ fn from_event(key: String, event: Event<Self::EventPayload>) -> Self {
+ Self::with_event(key, event)
+ }
+}
diff --git a/rust/cloud-storage/documents/src/domain/service.rs b/rust/cloud-storage/documents/src/domain/service.rs
index 08850de119..e07f7c3d4b 100644
--- a/rust/cloud-storage/documents/src/domain/service.rs
+++ b/rust/cloud-storage/documents/src/domain/service.rs
@@ -22,6 +22,7 @@ use entity_access::domain::models::{
};
use foreign_entity::domain::models::{ForeignEntity, SourceId};
use foreign_entity::domain::ports::ForeignEntityService;
+use macro_event_broker::MacroEventBroker;
use macro_user_id::user_id::MacroUserIdStr;
use model::document::response::{DocumentResponseMetadata, LocationResponseData};
use model::document::{ContentType, DocumentBasic, FileAssociation, FileType, FileTypeExt};
@@ -38,6 +39,10 @@ use crate::domain::models::{
use super::branch_name::{build_task_branch_name, user_branch_prefix};
use super::content::{DocumentContent, DocumentContentLocation, DocumentContentState};
+use super::events::{
+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentMacroEvent,
+ DocumentUpdatedMetadata,
+};
use super::models::{
CloudFrontConfig, CommentThread, CopyDocumentRepoArgs, CreateDocumentRepoArgs,
CreateTaskRequest, DocumentError, DocumentTeamShareResponse, EditDocumentRepoArgs,
@@ -60,6 +65,7 @@ pub struct DocumentServiceImpl<
C: ConnectionService,
Eam: EntityAccessManagementService,
F: ForeignEntityService,
+ B: MacroEventBroker,
> {
/// Document repository
pub repo: R,
@@ -77,6 +83,8 @@ pub struct DocumentServiceImpl<
pub entity_access_management_service: Eam,
/// Foreign entity service
pub foreign_entity_service: F,
+ /// Macro event broker for publishing document lifecycle events
+ pub macro_event_broker: B,
}
fn ready_content_for_file_type(file_type: Option<FileType>) -> DocumentContent {
@@ -118,6 +126,15 @@ fn pending_content_for_file_type(file_type: Option<FileType>) -> DocumentContent
}
}
+/// The user id to attribute a document lifecycle event to, when the caller is
+/// an authenticated user.
+fn event_actor_user_id(auth: &EntityAccessAuth) -> Option<MacroUserIdStr<'static>> {
+ match auth {
+ EntityAccessAuth::Authenticated(user_id) => Some(user_id.clone()),
+ EntityAccessAuth::Unauthenticated | EntityAccessAuth::Internal => None,
+ }
+}
+
const GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE: &str = "github_pull_request";
const MAX_DOCUMENT_NAME_GRAPHEMES: usize = 200;
@@ -188,7 +205,8 @@ impl<
C: ConnectionService,
Eam: EntityAccessManagementService,
F: ForeignEntityService,
-> DocumentServiceImpl<R, U, T, C, Eam, F>
+ B: MacroEventBroker,
+> DocumentServiceImpl<R, U, T, C, Eam, F, B>
{
/// Create a document service with its repository and external service ports.
#[allow(clippy::too_many_arguments)]
@@ -201,6 +219,7 @@ impl<
connection_service: C,
entity_access_management_service: Eam,
foreign_entity_service: F,
+ macro_event_broker: B,
) -> Self {
Self {
repo,
@@ -211,6 +230,7 @@ impl<
connection_service,
entity_access_management_service,
foreign_entity_service,
+ macro_event_broker,
}
}
@@ -466,6 +486,17 @@ impl<
.await
.map_err(|e| DocumentError::Internal(e.into()))
}
+
+ /// Publish a document lifecycle event; failures are logged and dropped.
+ async fn publish_document_event(&self, event: &DocumentMacroEvent) {
+ let _ = self
+ .macro_event_broker
+ .send_event(event)
+ .await
+ .inspect_err(|e| {
+ tracing::error!(error=?e, "failed to publish document event");
+ });
+ }
}
#[cfg(feature = "document_create")]
@@ -476,7 +507,8 @@ impl<
C: ConnectionService,
Eam: EntityAccessManagementService,
F: ForeignEntityService,
-> DocumentCreationService for DocumentServiceImpl<R, U, T, C, Eam, F>
+ B: MacroEventBroker,
+> DocumentCreationService for DocumentServiceImpl<R, U, T, C, Eam, F, B>
{
async fn create_document(
&self,
@@ -529,7 +561,8 @@ impl<
C: ConnectionService,
Eam: EntityAccessManagementService,
F: ForeignEntityService,
-> DocumentService for DocumentServiceImpl<R, U, T, C, Eam, F>
+ B: MacroEventBroker,
+> DocumentService for DocumentServiceImpl<R, U, T, C, Eam, F, B>
{
#[tracing::instrument(err, skip(self))]
async fn get_document(
@@ -691,6 +724,16 @@ impl<
tracing::error!(error=?e, "failed to send invalidation event");
});
+ self.publish_document_event(&DocumentMacroEvent::deleted(
+ entity_access_receipt.entity().entity_id.clone(),
+ DocumentDeletedMetadata {
+ document_id: entity_access_receipt.entity().entity_id.clone(),
+ actor_user_id: event_actor_user_id(entity_access_receipt.auth()),
+ project_id,
+ },
+ ))
+ .await;
+
Ok(())
}
@@ -995,6 +1038,20 @@ impl<
let team_task_metadata = self.team_task_metadata_for_document(&document_id).await?;
+ self.publish_document_event(&DocumentMacroEvent::created(
+ document_metadata.document_id.clone(),
+ DocumentCreatedMetadata {
+ document_id: document_metadata.document_id.clone(),
+ owner: document_metadata.owner.clone(),
+ document_name: document_metadata.document_name.clone(),
+ file_type,
+ project_id: project_id.map(|p| p.to_string()),
+ sub_type: document_metadata.sub_type,
+ created_at: document_metadata.created_at,
+ },
+ ))
+ .await;
+
Ok(CreateDocumentResponseData {
document_response: DocumentResponse {
document_metadata: DocumentResponseMetadataWithContent::new(
@@ -1079,13 +1136,15 @@ impl<
.document_name
.map(|s| FileType::clean_document_name(&s).unwrap_or(s));
+ let share_permission_updated = args.share_permission.is_some();
+
self.repo
.edit_document(EditDocumentRepoArgs {
document_id: entity_access_receipt.entity().entity_id.clone(),
- document_name,
+ document_name: document_name.clone(),
project_id: args.project_id.clone(),
share_permission: args.share_permission,
- file_type: args.file_type,
+ file_type: args.file_type.clone(),
})
.await
.map_err(|e| DocumentError::Internal(e.into()))?;
@@ -1133,6 +1192,21 @@ impl<
tracing::error!(error=?e, "failed to send invalidation event");
});
+ self.publish_document_event(&DocumentMacroEvent::updated(
+ entity_access_receipt.entity().entity_id.clone(),
+ DocumentUpdatedMetadata {
+ document_id: entity_access_receipt.entity().entity_id.clone(),
+ owner: document_context.owner.clone(),
+ actor_user_id: event_actor_user_id(entity_access_receipt.auth()),
+ document_name,
+ previous_project_id: document_context.project_id.clone(),
+ project_id: args.project_id,
+ file_type: args.file_type,
+ share_permission_updated,
+ },
+ ))
+ .await;
+
Ok(())
}
@@ -1375,6 +1449,21 @@ impl<
.team_task_metadata_for_document(&new_document_id)
.await?;
+ self.publish_document_event(&DocumentMacroEvent::copied(
+ new_document_id.clone(),
+ DocumentCopiedMetadata {
+ document_id: new_document_id.clone(),
+ source_document_id: original_metadata.document_id.clone(),
+ source_version_id: query_version_id,
+ owner: user_id.clone(),
+ document_name: new_metadata.document_name.clone(),
+ file_type,
+ project_id: new_metadata.project_id.clone(),
+ sub_type: new_metadata.sub_type,
+ },
+ ))
+ .await;
+
Ok(DocumentResponse {
document_metadata: DocumentResponseMetadataWithContent::new(
document_response_metadata,
diff --git a/rust/cloud-storage/documents/src/domain/service/tests.rs b/rust/cloud-storage/documents/src/domain/service/tests.rs
index 0409595503..d3e3c642c0 100644
--- a/rust/cloud-storage/documents/src/domain/service/tests.rs
+++ b/rust/cloud-storage/documents/src/domain/service/tests.rs
@@ -2,6 +2,7 @@ use foreign_entity::domain::models::{
CreateForeignEntity, ForeignEntity, ForeignEntityError, PatchForeignEntity, SourceId,
};
use foreign_entity::domain::ports::{ForeignEntityListQuery, ForeignEntityService};
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker, Topic as _};
use macro_user_id::cowlike::CowLike;
use model::document::DocumentMetadata;
use std::sync::{Arc, Mutex};
@@ -293,6 +294,36 @@ impl ForeignEntityService for TestForeignEntityService {
}
}
+/// A document lifecycle event recorded by [`TestEventBroker`].
+#[derive(Clone, Debug)]
+struct PublishedEvent {
+ topic: &'static str,
+ key: String,
+ payload: serde_json::Value,
+}
+
+#[derive(Clone, Default)]
+struct TestEventBroker {
+ published: Arc<Mutex<Vec<PublishedEvent>>>,
+}
+
+impl TestEventBroker {
+ fn published(&self) -> Arc<Mutex<Vec<PublishedEvent>>> {
+ Arc::clone(&self.published)
+ }
+}
+
+impl MacroEventBroker for TestEventBroker {
+ async fn send_event<E: MacroEvent + ?Sized>(&self, event: &E) -> Result<(), EventBrokerError> {
+ self.published.lock().unwrap().push(PublishedEvent {
+ topic: event.topic().as_str(),
+ key: event.key().to_string(),
+ payload: serde_json::to_value(event.event())?,
+ });
+ Ok(())
+ }
+}
+
type TestDocumentService = DocumentServiceImpl<
MockDocumentRepo,
TestUploadUrlPort,
@@ -300,6 +331,7 @@ type TestDocumentService = DocumentServiceImpl<
TestConnectionService,
TestEntityAccessManagementService,
TestForeignEntityService,
+ TestEventBroker,
>;
fn make_test_service(repo: MockDocumentRepo) -> TestDocumentService {
@@ -332,9 +364,32 @@ fn make_test_service_with_foreign_entity_service(
TestConnectionService,
TestEntityAccessManagementService,
foreign_entity_service,
+ TestEventBroker::default(),
)
}
+/// Build a test service along with a handle to its recording event broker.
+fn make_test_service_with_event_broker(
+ repo: MockDocumentRepo,
+) -> (TestDocumentService, TestEventBroker) {
+ let event_broker = TestEventBroker::default();
+ let service = DocumentServiceImpl::new(
+ repo,
+ test_cloudfront_config(),
+ sync_service_client::SyncServiceClient::new(
+ "test-sync-key".to_string(),
+ "http://sync-service.test".to_string(),
+ ),
+ TestUploadUrlPort,
+ TestTaskPropertiesPort,
+ TestConnectionService,
+ TestEntityAccessManagementService,
+ TestForeignEntityService::default(),
+ event_broker.clone(),
+ );
+ (service, event_broker)
+}
+
fn make_foreign_entity(
id: uuid::Uuid,
foreign_entity_id: &str,
@@ -1019,3 +1074,191 @@ async fn test_get_task_github_pull_requests_skips_malformed_keys_before_lookup()
}]
);
}
+
+fn owner_receipt(document_id: &str) -> EntityAccessReceipt<OwnerAccessLevel> {
+ let user_id = macro_user_id::user_id::MacroUserIdStr::parse_from_str("macro|[email protected]")
+ .unwrap()
+ .into_owned();
+
+ EntityAccessReceipt::dangerously_assert_authenticated_user(
+ user_id,
+ document_id,
+ EntityType::Document,
+ )
+}
+
+fn edit_receipt(document_id: &str) -> EntityAccessReceipt<EditAccessLevel> {
+ let user_id = macro_user_id::user_id::MacroUserIdStr::parse_from_str("macro|[email protected]")
+ .unwrap()
+ .into_owned();
+
+ EntityAccessReceipt::dangerously_assert_authenticated_user(
+ user_id,
+ document_id,
+ EntityType::Document,
+ )
+}
+
+#[tokio::test]
+async fn test_delete_document_publishes_document_deleted_event() {
+ let mut repo = make_mock_repo();
+ repo.expect_soft_delete_document()
+ .withf(|id| id == "doc-1")
+ .returning(|_| Box::pin(std::future::ready(Ok(()))));
+ repo.expect_update_project_modified()
+ .withf(|id| id == "project-1")
+ .returning(|_| Box::pin(std::future::ready(Ok(()))));
+
+ let (service, event_broker) = make_test_service_with_event_broker(repo);
+
+ service
+ .delete_document(owner_receipt("doc-1"), Some("project-1".to_string()))
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ let published = published.lock().unwrap();
+ assert_eq!(published.len(), 1);
+ let event = &published[0];
+ assert_eq!(event.topic, "macro.documents");
+ assert_eq!(event.key, "doc-1");
+ assert_eq!(event.payload["event_type"], "document.deleted");
+ assert_eq!(event.payload["schema_version"], 1);
+ assert_eq!(event.payload["metadata"]["document_id"], "doc-1");
+ assert_eq!(
+ event.payload["metadata"]["actor_user_id"],
+ );
+ assert_eq!(event.payload["metadata"]["project_id"], "project-1");
+ assert!(
+ uuid::Uuid::parse_str(event.payload["event_id"].as_str().unwrap()).is_ok(),
+ "event_id should be a valid uuid: {:?}",
+ event.payload["event_id"]
+ );
+}
+
+#[tokio::test]
+async fn test_delete_document_publishes_no_event_when_repo_fails() {
+ let mut repo = make_mock_repo();
+ repo.expect_soft_delete_document()
+ .withf(|id| id == "doc-1")
+ .returning(|_| Box::pin(std::future::ready(Err(anyhow!("db is down")))));
+
+ let (service, event_broker) = make_test_service_with_event_broker(repo);
+
+ let result = service.delete_document(owner_receipt("doc-1"), None).await;
+
+ assert!(result.is_err());
+ assert!(event_broker.published().lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn test_edit_document_publishes_document_updated_event() {
+ let mut repo = make_mock_repo();
+ repo.expect_edit_document()
+ .withf(|args| args.document_id == "doc-1")
+ .returning(|_| Box::pin(std::future::ready(Ok(()))));
+
+ let (service, event_broker) = make_test_service_with_event_broker(repo);
+
+ service
+ .edit_document(
+ edit_receipt("doc-1"),
+ task_document_context("doc-1"),
+ EditDocumentServiceArgs {
+ document_name: Some("New name.md".to_string()),
+ project_id: None,
+ share_permission: None,
+ file_type: None,
+ },
+ )
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ let published = published.lock().unwrap();
+ assert_eq!(published.len(), 1);
+ let event = &published[0];
+ assert_eq!(event.topic, "macro.documents");
+ assert_eq!(event.key, "doc-1");
+ assert_eq!(event.payload["event_type"], "document.updated");
+ assert_eq!(event.payload["metadata"]["document_name"], "New name");
+ assert_eq!(event.payload["metadata"]["owner"], "macro|[email protected]");
+ assert_eq!(
+ event.payload["metadata"]["actor_user_id"],
+ );
+ assert_eq!(event.payload["metadata"]["share_permission_updated"], false);
+ assert_eq!(
+ event.payload["metadata"]["previous_project_id"],
+ serde_json::Value::Null
+ );
+ assert_eq!(
+ event.payload["metadata"]["project_id"],
+ serde_json::Value::Null
+ );
+}
+
+#[tokio::test]
+async fn test_copy_document_publishes_document_copied_event() {
+ let mut repo = make_mock_repo();
+ let original_metadata = make_test_metadata();
+ let original_for_lookup = original_metadata.clone();
+ repo.expect_get_document_metadata()
+ .withf(|id| id == "doc-1")
+ .returning(move |_| Box::pin(std::future::ready(Ok(original_for_lookup.clone()))));
+ repo.expect_get_project_owner()
+ .withf(|id| id == "project-1")
+ .returning(|_| {
+ Box::pin(std::future::ready(Ok(
+ macro_user_id::user_id::MacroUserIdStr::parse_from_str("macro|[email protected]")
+ .unwrap()
+ .into_owned(),
+ )))
+ });
+ let mut copied_metadata = make_test_metadata();
+ copied_metadata.document_id = "doc-2".to_string();
+ copied_metadata.document_name = "copied doc".to_string();
+ repo.expect_copy_document()
+ .returning(move |_| Box::pin(std::future::ready(Ok(copied_metadata.clone()))));
+ repo.expect_get_document_version_id()
+ .returning(|_| Box::pin(std::future::ready(Ok((1, true)))));
+ repo.expect_get_latest_document_version_id()
+ .returning(|_| Box::pin(std::future::ready(Ok((1, true)))));
+ repo.expect_set_document_content()
+ .returning(|_, _| Box::pin(std::future::ready(Ok(()))));
+ repo.expect_get_team_task_metadata()
+ .returning(|_| Box::pin(std::future::ready(Ok(None))));
+
+ let (service, event_broker) = make_test_service_with_event_broker(repo);
+
+ let mut document_context = task_document_context("doc-1");
+ document_context.file_type = Some("txt".to_string());
+ document_context.sub_type = None;
+
+ service
+ .copy_document(
+ authenticated_receipt("doc-1"),
+ document_context,
+ macro_user_id::user_id::MacroUserIdStr::parse_from_str("macro|[email protected]")
+ .unwrap()
+ .into_owned(),
+ "copied doc".to_string(),
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ let published = published.lock().unwrap();
+ assert_eq!(published.len(), 1);
+ let event = &published[0];
+ assert_eq!(event.topic, "macro.documents");
+ assert_eq!(event.key, "doc-2");
+ assert_eq!(event.payload["event_type"], "document.copied");
+ assert_eq!(event.payload["metadata"]["document_id"], "doc-2");
+ assert_eq!(event.payload["metadata"]["source_document_id"], "doc-1");
+ assert_eq!(event.payload["metadata"]["document_name"], "copied doc");
+ assert_eq!(event.payload["metadata"]["owner"], "macro|[email protected]");
+}
diff --git a/rust/cloud-storage/macro_event_broker/Cargo.toml b/rust/cloud-storage/macro_event_broker/Cargo.toml
index 5f87f7a422..da942d0d74 100644
--- a/rust/cloud-storage/macro_event_broker/Cargo.toml
+++ b/rust/cloud-storage/macro_event_broker/Cargo.toml
@@ -6,10 +6,21 @@ version = "0.1.0"
[features]
default = ["ports", "outbound"]
-outbound = ["dep:rdkafka", "ports"]
+outbound = [
+ "dep:aws-msk-iam-sasl-signer",
+ "dep:aws-types",
+ "dep:macro_env",
+ "dep:macro_env_var",
+ "dep:rdkafka",
+ "ports",
+]
ports = ["dep:tokio"]
[dependencies]
+aws-msk-iam-sasl-signer = { workspace = true, optional = true }
+aws-types = { workspace = true, optional = true }
+macro_env = { path = "../macro_env", optional = true }
+macro_env_var = { path = "../macro_env_var", optional = true }
rootcause = { workspace = true }
rdkafka = { workspace = true, optional = true }
serde = { workspace = true }
diff --git a/rust/cloud-storage/macro_event_broker/src/domain/service.rs b/rust/cloud-storage/macro_event_broker/src/domain/service.rs
index 31a17f232d..f0de7f4018 100644
--- a/rust/cloud-storage/macro_event_broker/src/domain/service.rs
+++ b/rust/cloud-storage/macro_event_broker/src/domain/service.rs
@@ -9,6 +9,7 @@ use crate::domain::models::{EventBrokerError, MacroEvent};
use crate::domain::ports::{EventPublisher, MacroEventBroker};
/// Orchestrates serializing events and handing them to an [`EventPublisher`].
+#[derive(Clone)]
pub struct MacroEventBrokerService<P: EventPublisher> {
publisher: P,
}
diff --git a/rust/cloud-storage/macro_event_broker/src/lib.rs b/rust/cloud-storage/macro_event_broker/src/lib.rs
index d081712271..6299331701 100644
--- a/rust/cloud-storage/macro_event_broker/src/lib.rs
+++ b/rust/cloud-storage/macro_event_broker/src/lib.rs
@@ -14,12 +14,16 @@
pub mod domain;
pub use domain::models::{Event, EventBrokerError, MacroEvent, TopicEvent};
-pub use macro_event_topics::{MacroExampleTopic, Topic};
+pub use macro_event_topics::{MacroDocumentsTopic, MacroExampleTopic, Topic};
#[cfg(feature = "ports")]
pub use domain::ports::{EventPublisher, MacroEventBroker};
#[cfg(feature = "ports")]
pub use domain::service::MacroEventBrokerService;
+#[cfg(feature = "outbound")]
+pub use outbound::kafka_event_publisher::KafkaEventPublisher;
+#[cfg(feature = "outbound")]
+pub use outbound::msk_iam::MskIamClientContext;
/// Outbound layer: Kafka adapter for the [`EventPublisher`](domain::ports::EventPublisher) port.
#[cfg(feature = "outbound")]
diff --git a/rust/cloud-storage/macro_event_broker/src/outbound/kafka_event_publisher.rs b/rust/cloud-storage/macro_event_broker/src/outbound/kafka_event_publisher.rs
index 18b259a1a2..6f4df4632d 100644
--- a/rust/cloud-storage/macro_event_broker/src/outbound/kafka_event_publisher.rs
+++ b/rust/cloud-storage/macro_event_broker/src/outbound/kafka_event_publisher.rs
@@ -1,17 +1,22 @@
//! Kafka adapter for the [`EventPublisher`] port.
//!
-//! Wraps an `rdkafka` [`FutureProducer`] to publish events. The broker address is
-//! supplied by the caller (typically from environment configuration at the
-//! consuming service), so no env wiring lives in this crate yet.
+//! Wraps an `rdkafka` [`FutureProducer`] to publish events. The broker address
+//! is supplied by the caller (typically from environment configuration at the
+//! consuming service); the transport is chosen here from the `ENVIRONMENT`
+//! variable: `Local` uses a plaintext connection (the docker-compose broker),
+//! while `Develop` / `Production` use TLS + SASL/OAUTHBEARER with AWS MSK IAM
+//! auth via [`MskIamClientContext`].
use std::time::Duration;
+use macro_env::Environment;
use macro_event_topics::Topic;
use rdkafka::ClientConfig;
use rdkafka::producer::{FutureProducer, FutureRecord};
use crate::domain::models::EventBrokerError;
use crate::domain::ports::EventPublisher;
+use crate::outbound::msk_iam::{MskIamClientContext, configure_sasl_iam};
/// How long a record may sit in the producer queue before delivery is considered failed.
const MESSAGE_TIMEOUT_MS: &str = "5000";
@@ -19,19 +24,56 @@ const MESSAGE_TIMEOUT_MS: &str = "5000";
/// How long [`EventPublisher::publish`] waits for delivery confirmation before timing out.
const SEND_TIMEOUT: Duration = Duration::from_secs(5);
+/// The underlying producer, split by transport.
+#[derive(Clone)]
+enum Producer {
+ /// Unauthenticated plaintext connection (local docker broker).
+ Plaintext(FutureProducer),
+ /// TLS + SASL/OAUTHBEARER with AWS MSK IAM auth (deployed clusters).
+ MskIam(FutureProducer<MskIamClientContext>),
+}
+
/// Kafka-backed implementation of [`EventPublisher`].
+#[derive(Clone)]
pub struct KafkaEventPublisher {
- producer: FutureProducer,
+ producer: Producer,
+}
+
+fn base_config(brokers: &str) -> ClientConfig {
+ let mut config = ClientConfig::new();
+ config
+ .set("bootstrap.servers", brokers)
+ .set("message.timeout.ms", MESSAGE_TIMEOUT_MS);
+ config
+}
+
+fn create_error(e: impl std::fmt::Display) -> EventBrokerError {
+ EventBrokerError::Publish(format!("failed to create producer: {e}"))
}
impl KafkaEventPublisher {
- /// Build a producer connected to the given comma-separated `brokers` list.
+ /// Build a producer connected to the given comma-separated `brokers` list,
+ /// choosing the transport from the `ENVIRONMENT` variable (see the module
+ /// docs).
+ ///
+ /// Producer creation is lazy: nothing connects to Kafka (and no IAM token
+ /// is signed) until an event is published.
pub fn new(brokers: &str) -> Result<Self, EventBrokerError> {
- let producer: FutureProducer = ClientConfig::new()
- .set("bootstrap.servers", brokers)
- .set("message.timeout.ms", MESSAGE_TIMEOUT_MS)
- .create()
- .map_err(|e| EventBrokerError::Publish(format!("failed to create producer: {e}")))?;
+ let producer = match Environment::new_or_prod() {
+ Environment::Local => {
+ Producer::Plaintext(base_config(brokers).create().map_err(create_error)?)
+ }
+ Environment::Develop | Environment::Production => {
+ let mut config = base_config(brokers);
+ configure_sasl_iam(&mut config);
+
+ Producer::MskIam(
+ config
+ .create_with_context(MskIamClientContext::from_env())
+ .map_err(create_error)?,
+ )
+ }
+ };
Ok(Self { producer })
}
@@ -47,10 +89,11 @@ impl EventPublisher for KafkaEventPublisher {
) -> Result<(), EventBrokerError> {
let record = FutureRecord::to(topic.as_str()).key(key).payload(payload);
- self.producer
- .send(record, SEND_TIMEOUT)
- .await
- .map_err(|(e, _)| EventBrokerError::Publish(e.to_string()))?;
+ match &self.producer {
+ Producer::Plaintext(producer) => producer.send(record, SEND_TIMEOUT).await,
+ Producer::MskIam(producer) => producer.send(record, SEND_TIMEOUT).await,
+ }
+ .map_err(|(e, _)| EventBrokerError::Publish(e.to_string()))?;
Ok(())
}
diff --git a/rust/cloud-storage/macro_event_broker/src/outbound/mod.rs b/rust/cloud-storage/macro_event_broker/src/outbound/mod.rs
index c0bd839798..048c8210ef 100644
--- a/rust/cloud-storage/macro_event_broker/src/outbound/mod.rs
+++ b/rust/cloud-storage/macro_event_broker/src/outbound/mod.rs
@@ -1,2 +1,4 @@
/// Kafka adapter implementing the [`EventPublisher`](crate::domain::ports::EventPublisher) port.
pub mod kafka_event_publisher;
+/// AWS MSK IAM (SASL/OAUTHBEARER) auth support shared by producers and consumers.
+pub mod msk_iam;
diff --git a/rust/cloud-storage/macro_event_broker/src/outbound/msk_iam.rs b/rust/cloud-storage/macro_event_broker/src/outbound/msk_iam.rs
new file mode 100644
index 0000000000..f8b8043593
--- /dev/null
+++ b/rust/cloud-storage/macro_event_broker/src/outbound/msk_iam.rs
@@ -0,0 +1,93 @@
+//! AWS MSK IAM (SASL/OAUTHBEARER) support shared by Kafka producers and consumers.
+
+use aws_msk_iam_sasl_signer::generate_auth_token;
+use aws_types::region::Region;
+use rdkafka::ClientConfig;
+use rdkafka::client::{ClientContext, OAuthToken};
+use rdkafka::consumer::ConsumerContext;
+
+/// Region used to sign MSK IAM auth tokens when `AWS_REGION` is unset.
+/// Matches the fallback in `macro_aws_config`.
+const DEFAULT_AWS_REGION: &str = "us-east-1";
+
+macro_env_var::maybe_env_var! {
+ /// AWS region used to sign MSK IAM auth tokens.
+ struct AwsRegion;
+}
+
+/// Apply the TLS + SASL/OAUTHBEARER settings required to authenticate to an
+/// AWS MSK cluster with IAM auth (the `:9098` bootstrap listener).
+pub fn configure_sasl_iam(config: &mut ClientConfig) -> &mut ClientConfig {
+ config
+ .set("security.protocol", "SASL_SSL")
+ .set("sasl.mechanism", "OAUTHBEARER")
+}
+
+/// Client context that supplies AWS MSK IAM (SASL/OAUTHBEARER) auth tokens,
+/// signing with the ambient AWS credentials (e.g. the ECS task role).
+///
+/// librdkafka invokes [`ClientContext::generate_oauth_token`] when the
+/// connection is first established and again before each token expires.
+pub struct MskIamClientContext {
+ region: Region,
+}
+
+impl MskIamClientContext {
+ /// Build a context that signs tokens for the region in `AWS_REGION`,
+ /// falling back to `us-east-1`.
+ pub fn from_env() -> Self {
+ let region = AwsRegion::new()
+ .and_then(|region| region.value().map(str::to_string))
+ .unwrap_or_else(|| DEFAULT_AWS_REGION.to_string());
+
+ Self {
+ region: Region::new(region),
+ }
+ }
+}
+
+impl ClientContext for MskIamClientContext {
+ const ENABLE_REFRESH_OAUTH_TOKEN: bool = true;
+
+ fn generate_oauth_token(
+ &self,
+ _oauthbearer_config: Option<&str>,
+ ) -> Result<OAuthToken, Box<dyn std::error::Error>> {
+ let region = self.region.clone();
+
+ // librdkafka may invoke this callback on a tokio runtime thread (for a
+ // `StreamConsumer`, it fires while `recv()` is polled from async code),
+ // where blocking on the ambient runtime panics. Sign on a dedicated
+ // thread with its own single-threaded runtime instead, so the callback
+ // is safe from any calling context. Tokens live ~15 minutes, so the
+ // per-refresh thread + runtime cost is negligible.
+ let signer = std::thread::spawn(
+ move || -> Result<(String, i64), Box<dyn std::error::Error + Send + Sync>> {
+ let runtime = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()?;
+ Ok(runtime.block_on(generate_auth_token(region))?)
+ },
+ );
+
+ let (token, expiration_time_ms) = signer
+ .join()
+ .map_err(|_| "MSK IAM token signer thread panicked")?
+ .map_err(|e| -> Box<dyn std::error::Error> { e })
+ .inspect_err(|e| {
+ tracing::error!(error = %e, "failed to sign MSK IAM auth token");
+ })?;
+
+ tracing::info!(expiration_time_ms, "signed MSK IAM auth token");
+
+ Ok(OAuthToken {
+ token,
+ principal_name: String::new(),
+ lifetime_ms: expiration_time_ms,
+ })
+ }
+}
+
+/// Lets consumers (e.g. `StreamConsumer<MskIamClientContext>`) use this
+/// context; all behavior is the trait's defaults.
+impl ConsumerContext for MskIamClientContext {}
diff --git a/rust/cloud-storage/macro_event_topics/src/lib.rs b/rust/cloud-storage/macro_event_topics/src/lib.rs
index d7beb475f7..e0af821a29 100644
--- a/rust/cloud-storage/macro_event_topics/src/lib.rs
+++ b/rust/cloud-storage/macro_event_topics/src/lib.rs
@@ -49,4 +49,6 @@ macro_rules! topics {
topics! {
/// Example kafka topic.
MacroExampleTopic => "macro.example",
+ /// Document lifecycle events (created / updated / deleted / copied).
+ MacroDocumentsTopic => "macro.documents",
}
diff --git a/rust/cloud-storage/macro_event_topics/src/test.rs b/rust/cloud-storage/macro_event_topics/src/test.rs
index 066941da61..0e91ab30e3 100644
--- a/rust/cloud-storage/macro_event_topics/src/test.rs
+++ b/rust/cloud-storage/macro_event_topics/src/test.rs
@@ -18,4 +18,5 @@ fn all_topic_names_is_non_empty_and_unique() {
#[test]
fn all_topic_names_includes_declared_topics() {
assert!(all_topic_names().contains(&MacroExampleTopic.as_str()));
+ assert!(all_topic_names().contains(&MacroDocumentsTopic.as_str()));
}
diff --git a/rust/cloud-storage/mcp_service/Cargo.toml b/rust/cloud-storage/mcp_service/Cargo.toml
index b38e2ba3a0..723314857e 100644
--- a/rust/cloud-storage/mcp_service/Cargo.toml
+++ b/rust/cloud-storage/mcp_service/Cargo.toml
@@ -43,6 +43,7 @@ macro_entrypoint = { path = "../macro_entrypoint" }
macro_config = { path = "../macro_config" }
macro_env = { path = "../macro_env" }
macro_env_var = { path = "../macro_env_var" }
+macro_event_broker = { path = "../macro_event_broker" }
macro_queues = { path = "../macro_queues" }
database_env_vars = { path = "../macro_env_vars/database_env_vars" }
doppler-config = { workspace = true }
diff --git a/rust/cloud-storage/mcp_service/src/config.rs b/rust/cloud-storage/mcp_service/src/config.rs
index 9b0eb8dfa8..611059801d 100644
--- a/rust/cloud-storage/mcp_service/src/config.rs
+++ b/rust/cloud-storage/mcp_service/src/config.rs
@@ -50,6 +50,8 @@ env_vars! {
pub struct AppBaseUrl;
/// JWT secret for minting document permission tokens for the editing worker.
pub struct DocumentPermissionJwt;
+ /// Comma-separated Kafka bootstrap servers for the macro event broker.
+ pub struct KafkaBrokers;
}
/// The configuration parameters for the MCP server.
@@ -87,6 +89,7 @@ pub struct Config {
/// The internal api key
pub internal_api_key: InternalApiKey,
pub document_permission_jwt: DocumentPermissionJwt,
+ pub kafka_brokers: KafkaBrokers,
}
impl Config {
diff --git a/rust/cloud-storage/mcp_service/src/context.rs b/rust/cloud-storage/mcp_service/src/context.rs
index eb19effe78..10bb48bd8c 100644
--- a/rust/cloud-storage/mcp_service/src/context.rs
+++ b/rust/cloud-storage/mcp_service/src/context.rs
@@ -213,6 +213,10 @@ async fn build_tool_context(
ai_tools::build_properties_service(db.clone(), entity_access_service.clone());
let task_properties_service =
ai_tools::build_task_properties_adapter(db.clone(), properties_service.clone());
+ let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
+ macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ );
let document_service = documents::domain::service::DocumentServiceImpl {
repo: document_repo,
cloudfront_config,
@@ -225,6 +229,7 @@ async fn build_tool_context(
entity_access_management::outbound::PgRepository::new(db.clone()),
),
foreign_entity_service: ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ macro_event_broker,
};
let lexical_client_for_tools = (*lexical_client).clone();
let document_tool_context = DocumentToolContext::new(
diff --git a/rust/cloud-storage/scheduled_action/src/config.rs b/rust/cloud-storage/scheduled_action/src/config.rs
index 8d8e7bf5d9..401e3a49dc 100644
--- a/rust/cloud-storage/scheduled_action/src/config.rs
+++ b/rust/cloud-storage/scheduled_action/src/config.rs
@@ -31,6 +31,8 @@ env_vars! {
pub struct DocumentStorageServiceCloudfrontSignerPublicKeyId;
/// Secrets Manager secret name for the CloudFront signer private key.
pub struct DocumentStorageServiceCloudfrontSignerPrivateKeySecretName;
+ /// Comma-separated Kafka bootstrap servers for the macro event broker.
+ pub struct KafkaBrokers;
}
/// The configuration parameters for the agent schedule service.
@@ -55,6 +57,7 @@ pub struct Config {
DocumentStorageServiceCloudfrontSignerPublicKeyId,
pub document_storage_service_cloudfront_signer_private_key_secret_name:
DocumentStorageServiceCloudfrontSignerPrivateKeySecretName,
+ pub kafka_brokers: KafkaBrokers,
/// The internal api key
pub internal_api_key: InternalApiKey,
}
diff --git a/rust/cloud-storage/workspace-hack/Cargo.toml b/rust/cloud-storage/workspace-hack/Cargo.toml
index aae4b4a9a0..a7522b4d83 100644
--- a/rust/cloud-storage/workspace-hack/Cargo.toml
+++ b/rust/cloud-storage/workspace-hack/Cargo.toml
@@ -17,10 +17,11 @@ publish = false
[dependencies]
ahash = { version = "0.8" }
aho-corasick = { version = "1" }
-aws-credential-types = { version = "1", default-features = false, features = ["test-util"] }
+aws-config = { version = "1", features = ["behavior-version-latest"] }
+aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials", "test-util"] }
aws-runtime = { version = "1", default-features = false, features = ["event-stream", "http-02x", "http-1x", "sigv4a"] }
+aws-sdk-sts = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-sigv4 = { version = "1", features = ["http0-compat", "sign-eventstream", "sigv4a"] }
-aws-smithy-async = { version = "1", default-features = false, features = ["rt-tokio"] }
aws-smithy-http = { version = "0.63", default-features = false, features = ["event-stream"] }
aws-smithy-http-client = { version = "1", default-features = false, features = ["legacy-rustls-ring", "rustls-aws-lc"] }
aws-smithy-runtime = { version = "1", default-features = false, features = ["client", "default-https-client", "rt-tokio", "tls-rustls"] }
@@ -134,10 +135,11 @@ zeroize = { version = "1", features = ["derive"] }
[build-dependencies]
ahash = { version = "0.8" }
aho-corasick = { version = "1" }
-aws-credential-types = { version = "1", default-features = false, features = ["test-util"] }
+aws-config = { version = "1", features = ["behavior-version-latest"] }
+aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials", "test-util"] }
aws-runtime = { version = "1", default-features = false, features = ["event-stream", "http-02x", "http-1x", "sigv4a"] }
+aws-sdk-sts = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-sigv4 = { version = "1", features = ["http0-compat", "sign-eventstream", "sigv4a"] }
-aws-smithy-async = { version = "1", default-features = false, features = ["rt-tokio"] }
aws-smithy-http = { version = "0.63", default-features = false, features = ["event-stream"] }
aws-smithy-http-client = { version = "1", default-features = false, features = ["legacy-rustls-ring", "rustls-aws-lc"] }
aws-smithy-runtime = { version = "1", default-features = false, features = ["client", "default-https-client", "rt-tokio", "tls-rustls"] }